fsc_RawScan Class Reference
[FSC]

#include <fsc_RawScan.h>

Collaboration diagram for fsc_RawScan:

[legend]

List of all members.

---

Detailed Description

Definition at line 29 of file fsc_RawScan.h.

Public Member Functions
	fsc_RawScan (bool a_pmThread, fsc_RawCollector &a_collector, char a_clientPathSeparator=PATH_SEPARATOR)
	~fsc_RawScan (void)
void	ScanAllFileID ()
	ScanAllFileID() method sequentialy reads fileID from 1 forward.
void	DirTreeWalk (ivd_RecordIDX_t a_fileID, cmn_Path a_path)
	DirTreeWalk() method start from root reads all members and continue for each directory.
Public Attributes
	log_CLASSID_m
Private Member Functions
void	ScanAllGeneration (fsc_DataL_t &a_dataL)
void	ScanAllCopy (ivd_RecordIDX_t a_idx, UInt32_t a_size)
void	ScanAllSplit (ivd_RecordIDX_t a_idx, UInt32_t a_size)
Private Attributes
const bool	m_pmThread
	set if scan is run in PM thread
fsc_RawCollector &	m_collector
const char	m_clientPathSeparator
Friends
class	ut_fsc_RawScan

---

Constructor & Destructor Documentation

fsc_RawScan::fsc_RawScan	(	bool	a_pmThread,
		fsc_RawCollector &	a_collector,
		char	a_clientPathSeparator = PATH_SEPARATOR
	)

Definition at line 48 of file fsc_RawScan.cpp.

References log_FUNC_m.

00051     :
00052     m_pmThread(a_pmThread),
00053     m_collector(a_collector),
00054     m_clientPathSeparator(a_clientPathSeparator)
00055 {
00056     log_FUNC_m(fsc_RawScan);
00057 }
//============================================================================//

fsc_RawScan::~fsc_RawScan

(

void

)

[inline]

Definition at line 34 of file fsc_RawScan.h.

{};

---

Member Function Documentation

void fsc_RawScan::ScanAllFileID

(

)

ScanAllFileID() method sequentialy reads fileID from 1 forward.

Definition at line 60 of file fsc_RawScan.cpp.

References cmn_HexDump(), dbg_DETAIL, dbg_NORM, g_dataLRF_p, g_fscUpdate_x, fsc_DataL_t::generationIdx, log_DBG_m, log_FUNC_m, m_collector, m_pmThread, fsc_RawCollector::ProcFileID(), fsc_RawCollector::ProcRecDataL(), fio_RelFile::ReadRec(), and ScanAllGeneration().

Referenced by i_FSC_i::CollectByVolume(), i_FSC_i::DeleteEntries(), main(), i_FSC_i::RedundantCopy(), and i_FSC_i::ReorgScan().

                                {
    log_FUNC_m(ScanAllFileID);
    const int dataLBuffSize = 1024;
    // remove old files and dirs
    log_DBG_m(dbg_NORM, " Fast read of dataL. buff size " <<
                        dataLBuffSize << " of records.\n");

//    fsc_RawMedVolPathCollector m_collector(string("cosmos"), 3, 1);

    fsc_DataL_t dataL[dataLBuffSize];
    log_DBG_m(dbg_DETAIL,  "allocated dataL:" << &dataL <<
        "  last element address: " << &dataL[dataLBuffSize-1]);
    ivd_RecordIDX_t fileID = 1;
    if (m_pmThread) {
        while (fileID < g_dataLRF_p->FirstNewIDX()) {
            // lock to preven changes, but unlock every now and then
            // to prevet FSC update starvation
            cmn_MutexLock l(g_fscUpdate_x);
            log_DBG_m(dbg_DETAIL, "fileID: "); /*<< fileID <<
                " allocated dataL:" << &dataL);*/

            int read = g_dataLRF_p->ReadRec(fileID, &dataL, dataLBuffSize);

            log_DBG_m(dbg_DETAIL, "Read " << read << " records." << endl);
            register fsc_DataL_t *ip = dataL;
            register fsc_DataL_t *ep = &dataL[read];
            for (; ip < ep; ++ip, ++fileID ) {
                if (ip->generationIdx > 0) {
                    log_DBG_m(dbg_DETAIL, "Got DataL from = " << fileID << endl <<
                        cmn_HexDump(ip, sizeof(fsc_DataL_t), sizeof(fsc_DataL_t), false) );
                    m_collector.ProcFileID(fileID);
                    fsc_nsElement_t nsElement(fileID);
                    m_collector.ProcRecDataL(nsElement);
                    ScanAllGeneration(*ip);
                }
            }
        }
    }
    else { // no lock need
        while (fileID < g_dataLRF_p->FirstNewIDX()) {
            int read = g_dataLRF_p->ReadRec(fileID, &dataL, dataLBuffSize);
            log_DBG_m(dbg_DETAIL, "Read " << read << " records." << endl);
            register fsc_DataL_t *ip = dataL;
            register fsc_DataL_t *ep = &dataL[read];
            for (; ip < ep; ++ip, ++fileID ) {
                if (ip->generationIdx > 0) {
                    log_DBG_m(dbg_DETAIL, "Got DataL from = " << fileID << endl <<
                        cmn_HexDump(ip, sizeof(fsc_DataL_t), sizeof(fsc_DataL_t), false) );
                    m_collector.ProcFileID(fileID);
                    fsc_nsElement_t nsElement(fileID);
                    m_collector.ProcRecDataL(nsElement);
                    ScanAllGeneration(*ip);
                }
            }
        }
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void fsc_RawScan::DirTreeWalk	(	ivd_RecordIDX_t	a_fileID,
		cmn_Path	a_path
	)

DirTreeWalk() method start from root reads all members and continue for each directory.

Definition at line 360 of file fsc_RawScan.cpp.

References dbg_DETAIL, fsc_nsElement::Dump(), g_fscUpdate_x, fsc_nsDirEntryFormattedVec::GetFirstMemPtr(), fsc_nsDirEntryVec::GetNextDirEntryVecPtr(), fsc_nsDirEntryFormattedVec::GetNextMemPtr(), fsc_nsElement::GetStandAttrPtr(), log_DBG_m, log_FUNC_m, log_NOTE_m, m_clientPathSeparator, m_collector, fsc_nsElement_t::memberIdx, fsc_nsElement_t::migID, fsc_nsElement_t::nameIdx, fsc_nsElement_t::nameType, df_StandardAttr_t::NetToHost(), nit_NAME, NULL, fsc_nsElement_t::numOfMember, fsc_nsElement_t::numOfName, path, and fsc_RawCollector::ProcDirElement().

Referenced by i_FSC_i::DelExpiredFiles().

                                                                       {
    log_FUNC_m(DirTreeWalk);
    fsc_nsElement_t nsElement(a_fileID);
    log_DBG_m(dbg_DETAIL, "Got nsElement of fileID " << a_fileID
                       << " on path " << a_path);
    log_DBG_m(dbg_DETAIL, "nameType " << (int)nsElement.nameType
                       << ", memIdx " << nsElement.memberIdx
                       << ", mem in 1st vector " << (int)nsElement.numOfMember
                       << " migID " << nsElement.migID);
    if (   nsElement.nameType == nit_NAME
        || (  (a_fileID == 0) // or root with members. NOTE root has no name
           && (nsElement.memberIdx))) {
        fsc_nsString name(nsElement.numOfName, nsElement.nameIdx);
        log_DBG_m(dbg_DETAIL, "name '" << name << "'");
        if (nsElement.memberIdx) { // is a directory
            // lock FSC, not for update, but to preven chances by others
            fsc_nsDirEntryVec *members_p = NULL;
            {
                cmn_MutexLock l(g_fscUpdate_x);
                members_p = new fsc_nsDirEntryVec(nsElement.numOfMember, nsElement.memberIdx);
                fsc_nsDirEntryVec *p = members_p;
                // read all vectors to make a vector chain
                while (NULL != (p = p->GetNextDirEntryVecPtr())) {
                    // empty
                }
            }
            if (members_p) {
//                cmn_Path path(a_path + name);
                fsc_nsDirEntryVec *p = members_p;
                // scan through all vectors is a little tricky
                // there are many fsc_nsDirEntryVec linked in a list.
                // Each contains directory members as fileIDs
                // So function getNextMemPtr return a pointer to fileID
                // if pointer is NULL then end of the vector is reached
                // at this point get next vector and if no vector exist then
                // all members are read
                while (p != NULL) {
                    ivd_RecordIDX_t *el = p->GetFirstMemPtr();
                    while (el != NULL) {
                        string path(a_path);
                        if (path.size() > 0) {
                            path += m_clientPathSeparator + name;
                        }
                        else {
                            path = name;
                        }
                        DirTreeWalk(*el, path );
                        p->GetNextMemPtr(el);
                    }
                    p = p->GetNextDirEntryVecPtr();
                }

                delete members_p;
            }
        }
        else  {
            fsc_nsElement Element(a_fileID);
            df_StandardAttr_t  *attr = Element.GetStandAttrPtr();
            if (attr == NULL) {
                ostringstream sstr;
                Element.Dump(sstr);
                log_NOTE_m("FileID " << a_fileID << " has no attributes stored."
                       << sstr.str());
                return;
            }
            attr->NetToHost();
            m_collector.ProcDirElement(nsElement, *attr, a_path, name);
        }
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void fsc_RawScan::ScanAllGeneration

(

fsc_DataL_t &

a_dataL

)

[private]

Definition at line 119 of file fsc_RawScan.cpp.

References assert, cmn_HexDump(), cmn_Num2Str(), dbg_DETAIL, dlc_Copy, dlc_MediaPos, dlc_Split, fsc_maxNumOfMember_c, fsc_maxRFVecSize_c, g_generationRF_p, fsc_DataL_t::generationIdx, fsc_RawCollector::GetNSNameType(), fio_RelFile::Idx2Pos(), ie_INVALID_ARG, log_DBG_m, log_FUNC_m, m_collector, mvds_DELETED, mvds_LASTGENERATION, mvds_NONSC, nit_DELETED, nit_NAME, nit_NSCMISSING, fsc_DataL_t::numOfAllGen, fsc_DataL_t::numOfElement, fio_File::PosRead(), fsc_RawCollector::PostCopiesProc(), fsc_RawCollector::ProcRecGeneration(), fsc_RawCollector::ProcRecMedPosMgr(), fsc_RawCollector::ProcVecGeneration(), ScanAllCopy(), and ScanAllSplit().

Referenced by ScanAllFileID().

                                                        {
    log_FUNC_m(ScanAllGeneration);
    //
    Int32_t numOfGen = a_dataL.numOfAllGen;
    UInt32_t numOfGenInVec = a_dataL.numOfElement;
    ivd_FileRetSize_t read_result;

    if (numOfGen == 0 && numOfGenInVec == 0){
        // not a single element -- should not happen at all
        log_DBG_m(dbg_DETAIL, "Generation does not have a single element: Ignored");
        return;
    }

    if (numOfGen == 0){
        // backward compatybility see bug 3990
        numOfGen = a_dataL.numOfElement;
    }

    if (numOfGenInVec == 0){
       // we are at a vector boundary so fsc_maxRFVecSize_c (256) elements are
       // in the vector
       numOfGenInVec = fsc_maxRFVecSize_c ;  // last ( fsc_maxRFVecSize_c ths )
                                             // element contains the link
    }

    vector<fsc_Generation_t> generation(numOfGen);
    log_DBG_m(dbg_DETAIL, "Vector generation allocated: " << generation.size());

    register fsc_Generation_t *ip = &generation[0];

    ivd_FilePosition_t pos = g_generationRF_p->Idx2Pos(a_dataL.generationIdx);
    log_DBG_m(dbg_DETAIL, "Seeking to position: " << pos <<
        " reading a_dataL.numOfElement * sizeof(fsc_Generation_t):" <<
        numOfGenInVec * sizeof(fsc_Generation_t) << " byte = " <<
        numOfGenInVec << " records of " <<
        sizeof(fsc_Generation_t) << " byte" );

    // read first vector
    read_result = g_generationRF_p->PosRead(pos,
                              ip,
                              numOfGenInVec * sizeof(fsc_Generation_t));

    // UNCOMMENT BELOW IF YOU LIKE TO DEBUG
    //log_DBG_m(dbg_DETAIL, "P1: read " << read_result << " byte " <<
    //                     (Int32_t) numOfGenInVec << " Records have been read  Dump: " << endl <<
    //                      cmn_HexDump(ip, numOfGenInVec * sizeof(fsc_Generation_t),
    //                                  sizeof(fsc_Generation_t), false ) );

    if (numOfGen >= fsc_maxRFVecSize_c) { // follow chain of generation's vector
        numOfGenInVec -= 1; // Last entry contains the link to the next vector so read one less
        numOfGen -= numOfGenInVec;
        ip += numOfGenInVec;
        log_DBG_m(dbg_DETAIL, " numOfGen:" << (Int32_t) numOfGen << " fsc_maxRFVecSize_c:" << fsc_maxRFVecSize_c  );
        while (numOfGen > fsc_maxRFVecSize_c) {
            // NOTE: the structure is named fio_UnusedRec_t, in fact thie points to the next USED vector.
            pos = g_generationRF_p->Idx2Pos(reinterpret_cast<fio_UnusedRec_t*>(ip)->next);
            log_DBG_m(dbg_DETAIL, "Seeking to position: " << pos <<
                      " reading a_dataL.numOfElement * sizeof(fsc_Generation_t):" <<
                      fsc_maxRFVecSize_c * sizeof(fsc_Generation_t) << " byte = " <<
                      fsc_maxRFVecSize_c << " records of " <<
                      sizeof(fsc_Generation_t) << " byte" );

            // read all vectors except last
            // 256 elements, last elem contain link to next vector
            read_result = g_generationRF_p->PosRead(pos,
                                      ip,
                                      fsc_maxRFVecSize_c * sizeof(fsc_Generation_t));
            // UNCOMMENT BELOW IF YOU LIKE TO DEBUG
            //log_DBG_m(dbg_DETAIL, "P2: read  " << read_result <<
            //                      "byte" << (Int32_t) fsc_maxRFVecSize_c <<
            //                      " Records have been read  Dump: " << endl <<
            //                      cmn_HexDump(ip, fsc_maxRFVecSize_c * sizeof(fsc_Generation_t),
            //                                  sizeof(fsc_Generation_t), false ) );
            numOfGen -= fsc_maxNumOfMember_c;
            ip       += fsc_maxNumOfMember_c;
            log_DBG_m(dbg_DETAIL, " numOfGen:" << (Int32_t) numOfGen << " fsc_maxRFVecSize_c:" << fsc_maxRFVecSize_c  );
        }
        pos = g_generationRF_p->Idx2Pos(reinterpret_cast<fio_UnusedRec_t*>(ip)->next);
        log_DBG_m(dbg_DETAIL, "Seeking to position: " << pos <<
                  " reading a_dataL.numOfElement * sizeof(fsc_Generation_t):" <<
                  numOfGen * sizeof(fsc_Generation_t) << " byte = " <<
                  numOfGen << " records of " <<
                  sizeof(fsc_Generation_t) << " byte" );
        // read last vector without last element
        read_result = g_generationRF_p->PosRead(pos,
                                  ip,
                                  numOfGen * sizeof(fsc_Generation_t));

        // UNCOMMENT BELOW IF YOU LIKE TO DEBUG
        //log_DBG_m(dbg_DETAIL, "P3: read" << read_result << " byte " <<
        //                      numOfGen << " Records have been read  Dump: " << endl <<
        //                      cmn_HexDump(ip, numOfGen * sizeof(fsc_Generation_t),
        //                      sizeof(fsc_Generation_t), false ) );

        // the assert happens if we do not read fsc_maxNumOfMember_c (255) members
        // the last vector allways has one less records than the maximum, because
        // it doen not contain a pointer to the next record.
        assert(numOfGen == fsc_maxNumOfMember_c);
    }

        log_DBG_m(dbg_DETAIL, "R Generations from = " << a_dataL.generationIdx
                           << " size = " << a_dataL.numOfAllGen << endl <<
            cmn_HexDump(&generation[0],
                        a_dataL.numOfAllGen * sizeof(fsc_Generation_t),
                        sizeof(fsc_Generation_t),
                        false)
            );

    // for each generation
    m_collector.ProcVecGeneration(generation);


    ip = &generation[0];
    register fsc_Generation_t *ep = &generation[a_dataL.numOfAllGen];
    int genCount = 1;
    for (;ip < ep; ++ip ) {
        if (genCount == 1) {
            if (m_collector.GetNSNameType() == nit_NAME) {
                m_collector.ProcRecGeneration(*ip, mvds_LASTGENERATION);
            }
            else if (m_collector.GetNSNameType() == nit_NSCMISSING) {
                m_collector.ProcRecGeneration(*ip, mvds_NONSC);
            }
            else {
                m_collector.ProcRecGeneration(*ip, 0);
            }
        }
        else if (  genCount == 2
            && m_collector.GetNSNameType() == nit_DELETED) {
                m_collector.ProcRecGeneration(*ip, mvds_DELETED | mvds_LASTGENERATION);
            }
        else {
            m_collector.ProcRecGeneration(*ip, 0);
        }
        genCount++;

        switch (ip->nextEntity.dlcType) {
            case  dlc_MediaPos : {
                m_collector.ProcRecMedPosMgr(ip->nextEntity.GetMedPos());
                break;
            }
            case  dlc_Copy : {
                ScanAllCopy(ip->nextEntity.vector.entryIdx,
                            ip->nextEntity.vector.numOfElement);
                break;
            }
            case  dlc_Split : {
                ScanAllSplit(ip->nextEntity.vector.entryIdx,
                             ip->nextEntity.vector.numOfElement);
                break;
            }
            default : {
                log_FUNC_m(ScanAllGeneration);
                throw ivd_InternalError(ie_INVALID_ARG,
                    "Wrong dlcType =" + cmn_Num2Str((int)ip->nextEntity.dlcType), true);
                break;
            }
        }
        m_collector.PostCopiesProc();
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void fsc_RawScan::ScanAllCopy	(	ivd_RecordIDX_t	a_idx,
		UInt32_t	a_size
	)			[private]

Definition at line 282 of file fsc_RawScan.cpp.

References alloca(), cmn_Num2Str(), dlc_MediaPos, dlc_NotUsed, dlc_Split, fsc_NextEntity_t::dlcType, fsc_Vector_t::entryIdx, g_copyRF_p, fsc_NextEntity_t::GetMedPos(), fio_RelFile::Idx2Pos(), ie_INVALID_ARG, log_FUNC_m, m_collector, fsc_Copy_t::nextEntity, fsc_Vector_t::numOfElement, fio_File::PosRead(), fsc_RawCollector::PostRecCopy(), fsc_RawCollector::ProcRecCopy(), fsc_RawCollector::ProcRecMedPosMgr(), ScanAllSplit(), and fsc_NextEntity_t::vector.

Referenced by ScanAllGeneration().

                                                                    {
    log_FUNC_m(ScanAllCopy);
//    fsc_Copy_t copy[256]; // allocate only a_size to prevent constructor call
    fsc_Copy_t *copy = reinterpret_cast<fsc_Copy_t*>
                        (alloca(a_size * sizeof(fsc_Copy_t)));

    // direct call can be use if this is a single process
    //static_cast<cmn_File*>(g_copyRF_p)->
    g_copyRF_p->PosRead(g_copyRF_p->Idx2Pos(a_idx),
                        copy,
                        a_size * sizeof(fsc_Copy_t));
//    g_copyRF_p->ReadRec(a_idx, copy, a_size);
//        log_DBG_m(dbg_DETAIL, "R copies from = " << a_idx + (ip - copy) << endl <<
//            cmn_HexDump(copy, a_size * sizeof(fsc_Copy_t), sizeof(fsc_Copy_t), false) );

    // for each generation

    register fsc_Copy_t *ip = copy;
    register fsc_Copy_t *ep = &copy[a_size];
    for (; ip < ep; ++ip ) {
        m_collector.ProcRecCopy(*ip);
        switch (ip->nextEntity.dlcType) {
        case  dlc_NotUsed : continue;
        case  dlc_MediaPos : {
                m_collector.ProcRecMedPosMgr(ip->nextEntity.GetMedPos());
                break;
            }
        case  dlc_Split : {
            ScanAllSplit(ip->nextEntity.vector.entryIdx,
                         ip->nextEntity.vector.numOfElement);
                break;
            }
        default :  {
                log_FUNC_m(ScanAllCopy);
                throw ivd_InternalError(ie_INVALID_ARG,
                    "Wrong dlcType =" + cmn_Num2Str((int)ip->nextEntity.dlcType), true);
                break;
            }
        }
        m_collector.PostRecCopy(*ip);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void fsc_RawScan::ScanAllSplit	(	ivd_RecordIDX_t	a_idx,
		UInt32_t	a_size
	)			[private]

Definition at line 326 of file fsc_RawScan.cpp.

References alloca(), cmn_Num2Str(), dlc_MediaPos, fsc_NextEntity_t::dlcType, g_splitRF_p, fsc_NextEntity_t::GetMedPos(), fio_RelFile::Idx2Pos(), ie_INVALID_ARG, log_FUNC_m, m_collector, fsc_Split_t::nextEntity, fio_File::PosRead(), fsc_RawCollector::ProcRecMedPosMgr(), and fsc_RawCollector::ProcRecSplit().

Referenced by ScanAllCopy(), and ScanAllGeneration().

                                                                     {
    log_FUNC_m(ScanAllSplit);
//    fsc_Split_t split[256]; // allocate only a_size to prevent constructor call
    fsc_Split_t *split = reinterpret_cast<fsc_Split_t*>
                                (alloca(a_size * sizeof(fsc_Split_t)));
    g_splitRF_p->PosRead(g_splitRF_p->Idx2Pos(a_idx),
                        split,
                        a_size * sizeof(fsc_Split_t));
//    g_splitRF_p->ReadRec(a_idx, split, a_size);
//    log_DBG_m(dbg_DETAIL, "R Splits from = " << a_idx << endl <<
//    cmn_HexDump(split, a_size * sizeof(fsc_Split_t), sizeof(fsc_Split_t), false) );

    // for each generation
    register fsc_Split_t *ip = split;
    register fsc_Split_t *ep = &split[a_size];
    for (; ip < ep; ++ip ) {
        m_collector.ProcRecSplit(*ip);

        switch (ip->nextEntity.dlcType) {
            case  dlc_MediaPos : {
                m_collector.ProcRecMedPosMgr(ip->nextEntity.GetMedPos());
                break;
            }
            default : {
                log_FUNC_m(ScanAllSplit);
                throw ivd_InternalError(ie_INVALID_ARG,
                    "Wrong dlcType =" + cmn_Num2Str((int)ip->nextEntity.dlcType), true);
                break;
            }
        }
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

---

Friends And Related Function Documentation

friend class ut_fsc_RawScan [friend]

Definition at line 30 of file fsc_RawScan.h.

---

Member Data Documentation

fsc_RawScan::log_CLASSID_m

Definition at line 42 of file fsc_RawScan.h.

const bool fsc_RawScan::m_pmThread [private]

set if scan is run in PM thread

Definition at line 49 of file fsc_RawScan.h.

Referenced by ScanAllFileID().

fsc_RawCollector& fsc_RawScan::m_collector [private]

Definition at line 50 of file fsc_RawScan.h.

Referenced by DirTreeWalk(), ScanAllCopy(), ScanAllFileID(), ScanAllGeneration(), and ScanAllSplit().

const char fsc_RawScan::m_clientPathSeparator [private]

Definition at line 51 of file fsc_RawScan.h.

Referenced by DirTreeWalk().

---

The documentation for this class was generated from the following files:

• fsc_RawScan.h
• fsc_RawScan.cpp