#include <fsc_CollectorRedunCopy.h>


Definition at line 32 of file fsc_CollectorRedunCopy.h.
Public Member Functions | |
| fsc_CollectorRedunCopy (fsc_RedunCopyResult_v &a_results, vector< UInt32_t > &a_unavailFilesPerVol, fsc_MediumStatus_v &a_medStat, ivd_MediaKey_t a_origMedKey, vector< UInt32_t > a_origMedVol, bool a_best_effort) | |
| virtual | ~fsc_CollectorRedunCopy (void) |
| virtual bool | ProcRecGeneration (fsc_Generation_t &a_gen, UInt32_t a_status) |
| Called when new file in process. | |
| virtual bool | PostRecCopy (fsc_Copy_t &a_copy) |
| Called after each copy processed. | |
| virtual bool | ProcRecSplit (fsc_Split_t &a_split) |
| Called when generation or copy has splits. | |
| virtual bool | ProcRecMedPosMgr (fsc_MediaPosition_t &a_medPos) |
| Called when media position is known. | |
| virtual bool | PostCopiesProc () |
| Called when all copies are preocessed. | |
Static Private Member Functions | |
| static UInt64_t | KeyGen (ivd_MediaKey_t a_mediaKey, ivd_MedVolNum_t a_medVolNum) |
| Key to searc element in stx_cache. | |
Private Attributes | |
| log_CLASSID_m | |
| fsc_RedunCopyResult_v & | m_results |
| const fsc_MediumStatus_v | m_medStat |
| const ivd_MediaKey_t | m_origMedKey |
| vector< UInt32_t > | m_origMedVol |
| UInt32_t | m_currentVolume |
| vector< UInt32_t > & | m_unavailFilesPerVol |
| const Int32_t | m_origPoolID |
| const cmn_Path | m_redunCopyDir |
| stx_Cache | m_cache |
| MedVolume file storage. | |
| UInt32_t | m_numOfRedundCopies |
| UInt32_t | m_numOfUnavaillCopies |
| UInt32_t | m_numOfPurgedCopies |
| UInt32_t | m_numOfSkippedCopies |
| ivd_MigrationID_t | m_migID |
| ivd_FileSize_t | m_streamSize |
| stream size stored in generation | |
| UInt8_t | m_dataType |
| each generation set this member | |
| ivd_FileSize_t | m_dataOffset |
| generation and MedPos set to 0, split set its value | |
| ivd_FileSize_t | m_dataSize |
| generation and MedPos set to m_streamSize, split set its value | |
| ivd_CopiesPos_v_t | m_cpyMedPos_v |
| each generation clear and fill its media position | |
| ivd_CopiesPos_v_i | m_copyIter |
| ivd_MediaPos_v_i | m_medPosIter |
| bool | m_best_effort |
| do redundant copy collection on best effort base | |
| bool | m_doRedunCopy |
| Set if copy exists on original medium, reset for each gen. | |
| fsc_CollectorRedunCopy::fsc_CollectorRedunCopy | ( | fsc_RedunCopyResult_v & | a_results, | |
| vector< UInt32_t > & | a_unavailFilesPerVol, | |||
| fsc_MediumStatus_v & | a_medStat, | |||
| ivd_MediaKey_t | a_origMedKey, | |||
| vector< UInt32_t > | a_origMedVol, | |||
| bool | a_best_effort | |||
| ) |
Definition at line 42 of file fsc_CollectorRedunCopy.cpp.
References cmn_CleanDir(), dbg_DETAIL, log_DBG_m, log_FUNC_m, and m_redunCopyDir.
00049 : 00050 m_results(a_results), 00051 m_medStat(a_medStat), 00052 m_origMedKey(a_origMedKey), 00053 m_origMedVol(a_origMedVol), 00054 m_currentVolume(0), 00055 m_unavailFilesPerVol(a_unavailFilesPerVol), 00056 m_origPoolID(a_medStat[a_origMedKey].mediaPoolKey), 00057 m_redunCopyDir(fsc_GetRedunCopyDir(a_origMedKey)), 00058 m_cache(1024), // small amount of different media is possible 00059 m_numOfRedundCopies(0), 00060 m_numOfUnavaillCopies(0), 00061 m_numOfSkippedCopies(0), 00062 m_numOfPurgedCopies(0), 00063 m_best_effort(a_best_effort) 00064 { 00065 log_FUNC_m(fsc_CollectorRedunCopy); 00066 log_DBG_m(dbg_DETAIL, "Create dir " << m_redunCopyDir); 00067 cmn_CleanDir(m_redunCopyDir); 00068 } //============================================================================//

| fsc_CollectorRedunCopy::~fsc_CollectorRedunCopy | ( | void | ) | [virtual] |
Definition at line 71 of file fsc_CollectorRedunCopy.cpp.
References log_WriteEvent(), m_best_effort, m_numOfPurgedCopies, m_numOfRedundCopies, m_numOfSkippedCopies, m_numOfUnavaillCopies, and m_origMedKey.
00071 { 00072 00073 00074 ostringstream sstr; 00075 sstr << "Finished FSC redundant copy collect for medium : " << m_origMedKey 00076 << ", Collected = " << m_numOfRedundCopies 00077 << ", Skipped (already copied) = " << m_numOfSkippedCopies; 00078 if( m_best_effort){ 00079 sstr << ", Purged (on best effort base) = " << m_numOfPurgedCopies; 00080 } 00081 sstr << ", WRN Unavailable = " << m_numOfUnavaillCopies; 00082 log_WriteEvent(sstr.str(), "REDUNDANT COPY"); 00083 }

| bool fsc_CollectorRedunCopy::ProcRecGeneration | ( | fsc_Generation_t & | a_gen, | |
| UInt32_t | a_status | |||
| ) | [virtual] |
Called when new file in process.
Called for each generation.
Reimplemented from fsc_RawCollector.
Definition at line 86 of file fsc_CollectorRedunCopy.cpp.
References fsc_NextEntity_t::dataType, dlc_Copy, fsc_NextEntity_t::dlcType, log_FUNC_m, m_copyIter, m_cpyMedPos_v, m_dataOffset, m_dataSize, m_dataType, m_doRedunCopy, m_migID, m_streamSize, fsc_Generation_t::migrationID, fsc_Generation_t::nextEntity, fsc_Vector_t::numOfElement, fsc_Generation_t::size, and fsc_NextEntity_t::vector.
00087 { 00088 log_FUNC_m(ProcRecGeneration); 00089 00090 m_migID = a_gen.migrationID; 00091 m_streamSize = m_dataSize = a_gen.size; 00092 m_dataType = a_gen.nextEntity.dataType; 00093 m_dataOffset = 0; 00094 m_doRedunCopy = false; 00095 m_cpyMedPos_v.clear(); 00096 if (a_gen.nextEntity.dlcType == dlc_Copy) { 00097 m_cpyMedPos_v.resize(a_gen.nextEntity.vector.numOfElement); 00098 } 00099 else { 00100 m_cpyMedPos_v.resize(1); 00101 } 00102 m_copyIter = m_cpyMedPos_v.begin(); // point to first copy 00103 return true; 00104 }
| bool fsc_CollectorRedunCopy::PostRecCopy | ( | fsc_Copy_t & | a_copy | ) | [virtual] |
Called after each copy processed.
Reimplemented from fsc_RawCollector.
Definition at line 107 of file fsc_CollectorRedunCopy.cpp.
References log_FUNC_m, and m_copyIter.
00107 { 00108 log_FUNC_m(PostRecCopy); 00109 ++m_copyIter; // next copy 00110 return true; 00111 }
| bool fsc_CollectorRedunCopy::ProcRecSplit | ( | fsc_Split_t & | a_split | ) | [virtual] |
Called when generation or copy has splits.
Reimplemented from fsc_RawCollector.
Definition at line 114 of file fsc_CollectorRedunCopy.cpp.
References fsc_Split_t::dataOffset, fsc_Split_t::dataSize, log_FUNC_m, m_dataOffset, and m_dataSize.
00114 { 00115 log_FUNC_m(ProcRecSplit); 00116 m_dataOffset = a_split.dataOffset; 00117 m_dataSize = a_split.dataSize; 00118 return true; 00119 }
| bool fsc_CollectorRedunCopy::ProcRecMedPosMgr | ( | fsc_MediaPosition_t & | a_medPos | ) | [virtual] |
Called when media position is known.
Reimplemented from fsc_RawCollector.
Definition at line 122 of file fsc_CollectorRedunCopy.cpp.
References fsc_MediaPosition_t::blockOffset, ivd_MediaPos_t::blockOffset, ivd_MediaPos_t::dataType, dbg_DETAIL, log_DBG_m, log_FUNC_m, m_copyIter, m_dataOffset, m_dataSize, m_doRedunCopy, m_origMedKey, m_origMedVol, m_streamSize, fsc_MediaPosition_t::mediaKey, ivd_MediaPos_t::mediaKey, fsc_MediaPosition_t::medVolNum, ivd_MediaPos_t::medVolNum, ivd_MediaPos_t::splitOffset, and ivd_MediaPos_t::splitSize.
00122 { 00123 log_FUNC_m(ProcRecMedPosMgr); 00124 00125 ivd_MediaPos_t medPos; 00126 00127 medPos.dataType = 0; // doesn't matter 00128 medPos.medVolNum = a_medPos.medVolNum; 00129 medPos.mediaKey = a_medPos.mediaKey; 00130 medPos.blockOffset= a_medPos.blockOffset; 00131 medPos.splitOffset= m_dataOffset; 00132 medPos.splitSize = m_dataSize; 00133 00134 m_copyIter->push_back(medPos); 00135 00136 if (!m_doRedunCopy) { 00137 00138 for (UInt32_t i(0); i < m_origMedVol.size(); i++){ 00139 log_DBG_m(dbg_DETAIL, "m_origMedVol[i] " << m_origMedVol[i] << 00140 " medPos.medVolNum " << medPos.medVolNum); 00141 if ( (m_origMedVol[i] == medPos.medVolNum) && 00142 (m_origMedKey == medPos.mediaKey) ) { 00143 m_doRedunCopy = true; 00144 } 00145 } 00146 00147 log_DBG_m(dbg_DETAIL, " a_medPos.mediaKey == m_origMedKey " 00148 << a_medPos.mediaKey << " == " << m_origMedKey 00149 << " Do redundant copy " << boolalpha << m_doRedunCopy); 00150 } 00151 m_dataSize = m_streamSize; 00152 m_dataOffset = 0; 00153 return true; 00154 }
| bool fsc_CollectorRedunCopy::PostCopiesProc | ( | ) | [virtual] |
Called when all copies are preocessed.
Find best copy.
Scan through all copy. Check if copy is useful. Copy is useful if has all splits and all media are usable. Copy is better if lays on faster medium or contains less splits. Count all copies on origin pool. Allow only one redundant copy per pool.
Reimplemented from fsc_RawCollector.
Definition at line 164 of file fsc_CollectorRedunCopy.cpp.
References ivd_FileLocationData_t::blockOffset, ivd_MediaPos_t::blockOffset, dbg_DETAIL, ivd_FileLocationData_t::fileID, stx_Cache::Get(), stx_Cache::Insert(), KeyGen(), log_DBG_m, log_FUNC_m, log_WriteEvent(), log_WRN_m, m_best_effort, m_cache, m_cpyMedPos_v, fsc_RawCollector::m_curFileID, m_currentVolume, m_doRedunCopy, m_medStat, m_migID, m_numOfPurgedCopies, m_numOfRedundCopies, m_numOfSkippedCopies, m_numOfUnavaillCopies, m_origMedKey, m_origPoolID, m_results, m_streamSize, m_unavailFilesPerVol, ivd_MediaPos_t::mediaKey, ivd_MediaPos_t::medVolNum, ivd_FileLocationData_t::migrationID, mt_DISK, NULL, rmdb_MEDIUM_UNUSABLE, ivd_MediaPos_t::splitOffset, ivd_FileLocationData_t::splitSize, ivd_MediaPos_t::splitSize, and fsc_RedunCopyDataVolStorage::WriteToBuffer().
00164 { 00165 log_FUNC_m(PostCopiesProc); 00166 00167 if (m_doRedunCopy) { 00168 Int32_t goodCopiesPerPool = 0; 00169 Int32_t bestCopyMedType = 0; 00170 Int32_t bestCopySplitCnt = 0; 00171 log_DBG_m(dbg_DETAIL, m_cpyMedPos_v.size() << " generation's copies foud." ); 00172 ivd_CopiesPos_v_i bestCopyIter = m_cpyMedPos_v.end(); 00173 ivd_CopiesPos_v_i copyIter = m_cpyMedPos_v.begin(); 00174 for (; copyIter != m_cpyMedPos_v.end(); ++copyIter) { 00175 // check if contain all splits 00176 ivd_MediaPos_v_i mpIter = copyIter->begin(); 00177 00178 ivd_MediaKey_t medKey = mpIter->mediaKey; 00179 if (medKey == m_origMedKey) { 00180 //this is the original copy which we required copyContent 00181 m_currentVolume = mpIter->medVolNum; 00182 log_DBG_m(dbg_DETAIL, "m_currentVolume " << m_currentVolume); 00183 } 00184 00185 Int32_t mediaPoolKey = m_medStat[medKey].mediaPoolKey; 00186 Int32_t slowestMedType = m_medStat[medKey].medType; 00187 Int64_t offset = 0; 00188 log_DBG_m(dbg_DETAIL, "medKey " << medKey); 00189 log_DBG_m(dbg_DETAIL, "mediaPoolKey " << mediaPoolKey); 00190 log_DBG_m(dbg_DETAIL, "slowestMedType " << slowestMedType); 00191 log_DBG_m(dbg_DETAIL, "m_streamSize " << m_streamSize ); 00192 00193 log_DBG_m(dbg_DETAIL, copyIter->size() << " split's found." ); 00194 ivd_MediaPos_v_i mpEnd = copyIter->end(); 00195 for (; mpIter != mpEnd; ++mpIter) { 00196 Int32_t medStatus = m_medStat[mpIter->mediaKey].status; 00197 log_DBG_m(dbg_DETAIL, "offset == mpIter->splitOffset " 00198 << offset << " == " << mpIter->splitOffset 00199 << ", medStatus " << medStatus 00200 << ", rmdb_MEDIUM_UNUSABLE = " << rmdb_MEDIUM_UNUSABLE ); 00201 if ( offset == mpIter->splitOffset 00202 && (medStatus & rmdb_MEDIUM_UNUSABLE) == 0) { 00203 offset += mpIter->splitSize; 00204 Int32_t medType = m_medStat[mpIter->mediaKey].medType; 00205 log_DBG_m(dbg_DETAIL, "slowestMedType " << slowestMedType 00206 << ", medType " << medType 00207 << ", mt_DISK " << mt_DISK); 00208 if ( slowestMedType == 0 00209 || medType != mt_DISK) { 00210 slowestMedType = m_medStat[mpIter->mediaKey].medType; 00211 } 00212 } 00213 else { // 00214 log_DBG_m(dbg_DETAIL, "Next copy.") 00215 goto nextCopy; 00216 } 00217 } 00218 00219 log_DBG_m(dbg_DETAIL, "m_streamSize == offset " << 00220 m_streamSize << " == " << offset ); 00221 if (m_streamSize == offset) { 00222 log_DBG_m(dbg_DETAIL, "mediaPoolKey == m_origPoolID " 00223 << mediaPoolKey << " == " << m_origPoolID); 00224 // << ", copyFromOrigMed " << copyFromOrigMed); 00225 if ( mediaPoolKey == m_origPoolID) { 00226 ++goodCopiesPerPool; 00227 } 00228 log_DBG_m(dbg_DETAIL, "goodCopiesPerPool " << goodCopiesPerPool); 00229 log_DBG_m(dbg_DETAIL, "bestCopyIter == m_cpyMedPos_v.end() " << boolalpha << (bestCopyIter == m_cpyMedPos_v.end()) 00230 << ", bestCopyMedType < slowestMedType " 00231 << bestCopyMedType << " < " << slowestMedType); 00232 if ( (bestCopyIter == m_cpyMedPos_v.end()) 00233 || (bestCopyMedType < slowestMedType) 00234 || ( (bestCopyMedType == slowestMedType) 00235 && bestCopySplitCnt > Int32_t(copyIter->size()) 00236 ) 00237 ) { 00238 bestCopyIter = copyIter; 00239 bestCopyMedType = slowestMedType; 00240 bestCopySplitCnt = copyIter->size(); 00241 } 00242 } 00243 nextCopy: ; 00244 } // for (; copyIter != m_cpyMedPos_v.end(); ++copyIter) 00245 log_DBG_m(dbg_DETAIL, "Good copies per pool " << goodCopiesPerPool); 00246 00247 00248 if (goodCopiesPerPool < 2) { // no redundant copy yet 00249 if (bestCopyIter == m_cpyMedPos_v.end()) { 00250 if(m_best_effort){ 00251 m_numOfPurgedCopies++; 00252 ostringstream sstr; 00253 sstr << "Missing generation copy skipped on best effort base for :" 00254 << "\n fileID " << m_curFileID 00255 << "\n migrationID " << m_migID 00256 << "\n stream size " << m_streamSize; 00257 log_WriteEvent(sstr.str(), "COPY CONTENT"); 00258 } 00259 else { 00260 m_numOfUnavaillCopies++; 00261 // warning no data source available 00262 log_WRN_m("No useful copy for : " 00263 << "\n fileID " << m_curFileID 00264 << "\n migrationID " << m_migID 00265 << "\n stream size " << m_streamSize); 00266 } 00267 if (m_currentVolume > 0){ 00268 while (m_unavailFilesPerVol.size() < m_currentVolume) { 00269 m_unavailFilesPerVol.push_back(0); 00270 } 00271 m_unavailFilesPerVol[m_currentVolume-1]++; 00272 00273 log_DBG_m(dbg_DETAIL, "Increasing counter for unavailable files " 00274 " for volume " << m_currentVolume 00275 << " to " << m_unavailFilesPerVol[m_currentVolume-1]); 00276 } 00277 00278 return true; 00279 } 00280 m_numOfRedundCopies++; 00281 log_DBG_m(dbg_DETAIL, "Got useful copy for : " 00282 "\n fileID " << m_curFileID 00283 << "\n migrationID " << m_migID 00284 << "\n stream size " << m_streamSize 00285 << "\n on media: "); 00286 00287 ivd_MediaPos_v_i mpIter = bestCopyIter->begin(); 00288 ivd_MediaPos_v_i mpEnd = bestCopyIter->end(); 00289 for (; mpIter != mpEnd; ++mpIter) { 00290 ivd_MediaPos_t &medPos = *mpIter; 00291 log_DBG_m(dbg_DETAIL, " medKey " << medPos.mediaKey 00292 << "\n medVolNum " << medPos.medVolNum 00293 << "\n blockOffset" << medPos.blockOffset 00294 << "\n splitOffset" << medPos.splitOffset 00295 << "\n splitSize " << medPos.splitSize); 00296 UInt64_t key = KeyGen(medPos.mediaKey, medPos.medVolNum); 00297 00298 fsc_RedunCopyDataVolStorage *dataStor 00299 = (fsc_RedunCopyDataVolStorage*)(m_cache.Get(key)); 00300 if (dataStor == NULL) { 00301 dataStor = new fsc_RedunCopyDataVolStorageWriter( 00302 key, 00303 medPos.mediaKey, 00304 medPos.medVolNum, 00305 m_results); 00306 m_cache.Insert(dataStor); 00307 } 00308 ivd_FileLocationData_t locData; 00309 locData.blockOffset = medPos.blockOffset; 00310 locData.fileID = m_curFileID; 00311 locData.migrationID = m_migID; 00312 locData.splitSize = medPos.splitSize; 00313 dataStor->WriteToBuffer(locData); 00314 } 00315 } 00316 else { 00317 m_numOfSkippedCopies++; 00318 } 00319 } 00320 return true; 00321 }

| static UInt64_t fsc_CollectorRedunCopy::KeyGen | ( | ivd_MediaKey_t | a_mediaKey, | |
| ivd_MedVolNum_t | a_medVolNum | |||
| ) | [inline, static, private] |
Key to searc element in stx_cache.
Definition at line 69 of file fsc_CollectorRedunCopy.h.
Referenced by PostCopiesProc().
00070 { 00071 return UInt64_t(a_mediaKey << 6) + a_medVolNum; 00072 };

fsc_CollectorRedunCopy::log_CLASSID_m [private] |
const fsc_MediumStatus_v fsc_CollectorRedunCopy::m_medStat [private] |
const ivd_MediaKey_t fsc_CollectorRedunCopy::m_origMedKey [private] |
Definition at line 76 of file fsc_CollectorRedunCopy.h.
Referenced by PostCopiesProc(), ProcRecMedPosMgr(), and ~fsc_CollectorRedunCopy().
vector<UInt32_t> fsc_CollectorRedunCopy::m_origMedVol [private] |
vector<UInt32_t>& fsc_CollectorRedunCopy::m_unavailFilesPerVol [private] |
const Int32_t fsc_CollectorRedunCopy::m_origPoolID [private] |
const cmn_Path fsc_CollectorRedunCopy::m_redunCopyDir [private] |
stx_Cache fsc_CollectorRedunCopy::m_cache [private] |
MedVolume file storage.
Definition at line 84 of file fsc_CollectorRedunCopy.h.
Referenced by PostCopiesProc().
Definition at line 87 of file fsc_CollectorRedunCopy.h.
Referenced by PostCopiesProc(), and ~fsc_CollectorRedunCopy().
Definition at line 89 of file fsc_CollectorRedunCopy.h.
Referenced by PostCopiesProc(), and ~fsc_CollectorRedunCopy().
Definition at line 91 of file fsc_CollectorRedunCopy.h.
Referenced by PostCopiesProc(), and ~fsc_CollectorRedunCopy().
Definition at line 93 of file fsc_CollectorRedunCopy.h.
Referenced by PostCopiesProc(), and ~fsc_CollectorRedunCopy().
Definition at line 95 of file fsc_CollectorRedunCopy.h.
Referenced by PostCopiesProc(), and ProcRecGeneration().
stream size stored in generation
Definition at line 98 of file fsc_CollectorRedunCopy.h.
Referenced by PostCopiesProc(), ProcRecGeneration(), and ProcRecMedPosMgr().
UInt8_t fsc_CollectorRedunCopy::m_dataType [private] |
each generation set this member
Definition at line 101 of file fsc_CollectorRedunCopy.h.
Referenced by ProcRecGeneration().
generation and MedPos set to 0, split set its value
Definition at line 104 of file fsc_CollectorRedunCopy.h.
Referenced by ProcRecGeneration(), ProcRecMedPosMgr(), and ProcRecSplit().
generation and MedPos set to m_streamSize, split set its value
Definition at line 107 of file fsc_CollectorRedunCopy.h.
Referenced by ProcRecGeneration(), ProcRecMedPosMgr(), and ProcRecSplit().
each generation clear and fill its media position
Definition at line 110 of file fsc_CollectorRedunCopy.h.
Referenced by PostCopiesProc(), and ProcRecGeneration().
Definition at line 111 of file fsc_CollectorRedunCopy.h.
Referenced by PostRecCopy(), ProcRecGeneration(), and ProcRecMedPosMgr().
Definition at line 112 of file fsc_CollectorRedunCopy.h.
bool fsc_CollectorRedunCopy::m_best_effort [private] |
do redundant copy collection on best effort base
Definition at line 115 of file fsc_CollectorRedunCopy.h.
Referenced by PostCopiesProc(), and ~fsc_CollectorRedunCopy().
bool fsc_CollectorRedunCopy::m_doRedunCopy [private] |
Set if copy exists on original medium, reset for each gen.
Definition at line 118 of file fsc_CollectorRedunCopy.h.
Referenced by PostCopiesProc(), ProcRecGeneration(), and ProcRecMedPosMgr().
1.5.6