TreeWalk Class Reference
[Classes for managing events]

#include <treewalk.h>

Inheritance diagram for TreeWalk:

[legend]

Collaboration diagram for TreeWalk:

[legend]

List of all members.

---

Detailed Description

Definition at line 109 of file treewalk.h.

Public Member Functions
	TreeWalk (const string &a_strPartName, ui_MsgWriter &a_msgWrite, Int32_t a_numFilesPerBatch, Int32_t a_sysLoadPct, bool &a_abort)
virtual	~TreeWalk ()
void	Report ()
	TreeWalk (const cmn_Path &a_path)
virtual	~TreeWalk ()
bool	IsIgnorable (ivd_FileType_e a_type, const string &a_name)
void	Walk ()
Public Attributes
	log_CLASSID_m
Protected Member Functions
void	create_subdirs (UInt32_t beg, UInt32_t end)
virtual void	PreprocessDir (const cmn_Path &a_dir, UInt32_t a_depth)
virtual void	ProcessEntry (const cmn_Path &a_dir, const string &a_name, ivd_FileType_e a_type)
virtual bool	HandleError (const cmn_Path &a_dir, const ivd_SysError &a_error)
Private Types
typedef vector< DirectoryEntry >	DirectoryVec_t
Private Member Functions
virtual void	CheckDir (bool a_mountpoint, const cmn_Path &a_dirPath, ivd_GenInode_t a_fileInode)
	When check dir is called then process current dir is checked directory.
void	GenerateDirTree ()
Private Attributes
const string	m_strPartName
ui_MsgWriter &	m_msgWrite
cmn_Time	m_startTime
Int32_t	m_sysLoadPct
	Check ivdfs with FSC, prepare package every m_numFilesPerBatch number of files read.
UInt64_t	m_simLinkCounter
UInt64_t	m_filesCounter
UInt64_t	m_missingCount
UInt64_t	m_ownerMismatchCount
UInt64_t	m_suspectCount
UInt64_t	m_prolongedCount
UInt64_t	m_notRelCandidate
UInt64_t	m_missingCopyCount
FileInfoCache	m_fic
DirectoryVec_t	m_dirStack
Classes
class	DirectoryEntry

---

Member Typedef Documentation

typedef vector<DirectoryEntry> TreeWalk::DirectoryVec_t [private]

Definition at line 56 of file TreeWalk.h.

---

Constructor & Destructor Documentation

TreeWalk::TreeWalk	(	const string &	a_strPartName,
		ui_MsgWriter &	a_msgWrite,
		Int32_t	a_numFilesPerBatch,
		Int32_t	a_sysLoadPct,
		bool &	a_abort
	)

Definition at line 156 of file treewalk.cpp.

References dbg_DETAIL, ui_MsgWriter::DisplayMessage(), cmn_TreeWalk::GetStackSize(), log_DBG_m, log_FUNC_A_m, log_WriteEvent(), m_msgWrite, m_startTime, m_strPartName, and cmn_Time::Refresh().

00161     :
00162     cmn_TreeWalk(g_cmn.dirs.tmp + ( a_strPartName + "_hsmfs-fsc_stack"), a_abort),
00163     m_strPartName(a_strPartName),
00164     m_msgWrite(a_msgWrite),
00165     m_sysLoadPct(a_sysLoadPct),
00166     m_simLinkCounter(0),
00167     m_filesCounter(0),
00168     m_missingCount(0),
00169     m_ownerMismatchCount(0),
00170     m_suspectCount(0),
00171     m_prolongedCount(0),
00172     m_notRelCandidate(0),
00173     m_missingCopyCount(0),
00174     m_fic(a_msgWrite, a_numFilesPerBatch, a_sysLoadPct, m_missingCount, m_missingCopyCount, m_ownerMismatchCount)
00175 {
00176     log_FUNC_A_m(TreeWalk, "partName: " << a_strPartName);
00177     
00178     log_DBG_m(dbg_DETAIL, 
00179         "Task is allowed to occupy " << a_sysLoadPct << "% of system."
00180         << " Package holds " << a_numFilesPerBatch << " entries.");
00181 
00182     if (GetStackSize() == 0) {
00183         ostringstream sstrs;
00184         sstrs << "Started FSC vs. HSMFS check. Partition " << m_strPartName;
00185         m_msgWrite.DisplayMessage(sstrs.str());
00186         log_WriteEvent(sstrs.str());
00187     }
00188     else {
00189         ostringstream sstrs;
00190         sstrs << "Started FSC vs. HSMFS check (continue from previous run). Partition " << m_strPartName;
00191         m_msgWrite.DisplayMessage(sstrs.str());
00192         log_WriteEvent(sstrs.str());
00193     }
00194 
00195     m_startTime.Refresh();
00196 }
//============================================================================//

Here is the call graph for this function:

TreeWalk::~TreeWalk

(

)

[virtual]

Definition at line 200 of file treewalk.cpp.

References FileInfoCache::CheckFiles(), dbg_DETAIL, log_DBG_m, log_FUNC_m, m_fic, and Report().

                    {
    log_FUNC_m(~TreeWalk);
    log_DBG_m(dbg_DETAIL,  "Check remaining files.");
    m_fic.CheckFiles();
    Report();
}

Here is the call graph for this function:

TreeWalk::TreeWalk

(

const cmn_Path &

a_path

)

Definition at line 50 of file TreeWalk.cpp.

References log_FUNC_A_m, and m_dirStack.

                                         {

    log_FUNC_A_m(TreeWalk, "a_path=" << a_path);

    m_dirStack.push_back(DirectoryEntry(a_path, 1));
}

virtual TreeWalk::~TreeWalk

(

)

[inline, virtual]

Definition at line 30 of file TreeWalk.h.

{};

---

Member Function Documentation

void TreeWalk::Report

(

)

Definition at line 394 of file treewalk.cpp.

References dbg_DETAIL, ui_MsgWriter::DisplayMessage(), log_DBG_m, log_FUNC_m, log_WriteEvent(), cmn_TreeWalk::m_abort, m_filesCounter, m_missingCopyCount, m_missingCount, m_msgWrite, m_notRelCandidate, m_ownerMismatchCount, m_prolongedCount, m_simLinkCounter, and m_suspectCount.

Referenced by ~TreeWalk().

                      {
    log_FUNC_m(Report);
    ostringstream sstrs;
    sstrs << (m_abort ? "Aborted" : "Finished" )
      << " (symlink: " << m_simLinkCounter
      << ", checked: " << m_filesCounter
      << ", missing: " << m_missingCount
      << ", owner mismatch: " << m_ownerMismatchCount
      << ", num. of copies mismatch: " << m_missingCopyCount
      << ", suspicious: " << m_suspectCount
      << ", partially recalled: " << m_prolongedCount 
      << ", not release candidate: " << m_notRelCandidate
      << ")." << endl;
    log_DBG_m(dbg_DETAIL, sstrs.str());
    log_WriteEvent(sstrs.str());
    m_msgWrite.DisplayMessage(sstrs.str());
}

Here is the call graph for this function:

Here is the caller graph for this function:

void TreeWalk::CheckDir	(	bool	a_mountpoint,
		const cmn_Path &	a_dirPath,
		ivd_GenInode_t	a_fileInode
	)			[private, virtual]

When check dir is called then process current dir is checked directory.

Implements cmn_TreeWalk.

Definition at line 207 of file treewalk.cpp.

References FileInfoCache::AddFile(), FileInfoCache::CheckFiles(), dbg_DETAIL, dbg_LOW, dbg_NORM, hsm_FileHeader::DecrRef(), ui_MsgWriter::DisplayMessage(), cmn_TreeWalk::DoWalk(), ivd_FS_File::e_Cache, g_File2hdbID_p, g_fs_api_p, g_hsm_FHCache_p, hsm_FHcache::GetFH(), cmn_TreeWalk::GetLastVector(), cmn_Time::GetTimeInMicroSec(), ie_ABORT, ift_DIR, ift_SYMLINK, ivd_Error, IVD_PRINT_ID_FS, ivd_USleep, log_DBG_m, log_FUNC_m, cmn_TreeWalk::m_abort, ivd_FS_FileProperty_t::m_dirtyFlag, m_fic, ivd_FS_FileProperty_t::m_fileID, m_filesCounter, ivd_FS_FileProperty_t::m_fileSize, fname_inode_t::m_fileType, ivd_FS_FileProperty_t::m_Inode, fname_inode_t::m_iNode, m_msgWrite, fname_inode_t::m_name, m_notRelCandidate, ivd_FS_FileProperty_t::m_offlineFlag, m_prolongedCount, m_simLinkCounter, m_startTime, m_suspectCount, m_sysLoadPct, ivd_FS_FileProperty_t::m_type, NULL, fio_RelFile::ReadRec(), cmn_Time::Refresh(), cmn_TreeWalk::SaveStack(), and cmn_Time::Time2hmsm().

                                                        {
    log_FUNC_m(CheckDir);

    if (m_abort) {
        m_fic.CheckFiles();
        SaveStack();
        throw ivd_Error(ie_ABORT, " FSC Check");
    }

    log_DBG_m(dbg_DETAIL,  "Check in directory " << a_dirPath << endl << 
                           "a_fileInode" << a_fileInode);

    ivd_FileID_t ownerFID = 0;

    log_DBG_m(dbg_LOW, "*** @owner: " << &(ownerFID) );

    // if not mountpoint (inode == 0), get FileID of parent (owner)
    if ( !a_mountpoint) {
        auto_ptr<ivd_FS_File> owner_ap;
        owner_ap.reset(new ivd_FS_File(*g_fs_api_p, a_dirPath ));
        try {
            //HPUX - The file needs to be opened ( to be in the kernel cache )
            owner_ap->Open(ivd_FS_File::e_Cache, a_dirPath);
            ivd_FS_FileProperty_t ownerProp;
            owner_ap->GetProperties(ownerProp);
            ownerFID = ownerProp.m_fileID;
            log_DBG_m(dbg_LOW, "ownerFID:" << ownerFID);
        }
        catch (ivd_SysError &se) {  // can't get ownerInfo, end of dir scan
            log_DBG_m(dbg_LOW, "Syserror:" << se);
            return;
        }
    }

    while (!GetLastVector().empty()) {
        
        fname_inode_t &f = GetLastVector().back();
        
        log_DBG_m(dbg_NORM, "File to check: " << f.m_name
                            << ", m_iNode:" << f.m_iNode 
                            << ", m_fileType: "<< f.m_fileType);

        if (f.m_fileType == ift_SYMLINK) { // sym link are not migratted so skip it.
            log_DBG_m(dbg_DETAIL,  "SymLink, skip it. " << f.m_name);
            GetLastVector().pop_back();
            ++m_simLinkCounter;
            continue;
        }
        
        ivd_FS_FileProperty_t fileProp;
        
        { // Start of separate block to decrease stack usage
            auto_ptr<ivd_FS_File> file_ap;
            file_ap.reset(new ivd_FS_File(*g_fs_api_p, f.m_iNode));

            try {
                //The file needs to be opened ( to be in the kernel cache )
                file_ap->Open(ivd_FS_File::e_Cache,( a_dirPath + f.m_name));
                file_ap->GetProperties(fileProp);
                file_ap->Close();
            }
            catch (ivd_SysError ) {  // can't get fileInfo, skip it
                GetLastVector().pop_back();
                continue;
            }

            m_filesCounter++;

            // Check if file is active in HSM, otherwise check it in FSC
            hsm_FileHeader  *hsmFH = g_hsm_FHCache_p->GetFH(fileProp.m_Inode);

            log_DBG_m(dbg_DETAIL, "file info: inode "
                << IVD_PRINT_ID_FS(fileProp.m_Inode)
                << " fileId "   << fileProp.m_fileID
                << " fileProp.m_type: " << fileProp.m_type
                << " fileType from dirlist " << f.m_fileType);

            if (hsmFH == NULL) {
                string fullName;
                fullName = a_dirPath + f.m_name;
                //m_msgWrite.DisplayMessage("checking: "+fullName);

                if (  fileProp.m_dirtyFlag
                   || (fileProp.m_fileID == 0)) { // when EA are not set yet.
                    ostringstream sstr;
                    sstr << fileProp.m_fileID << " 0 " // migID=0
                         << fullName << endl;
                    if (f.m_fileType == ift_DIR) {  
                        sstr << "    Directory is not migrated. (It is not in HSM lists. Maybe it is in event queue.)";
                    }
                    else {
                        sstr << "    File is dirty. (It is not in HSM lists. Maybe it is in event queue.)";
                    }
                    m_msgWrite.DisplayMessage(sstr.str());
                    m_suspectCount++;            
                }
                else {
//                    file_ap->SetFullPath(fullName);
                    if (   fileProp.m_type == 1 // it is file
                        && file_ap->IsOfflineProlonged()) { 
                        ostringstream sstr;
                        sstr << fileProp.m_fileID << " 0 " // migID=0
                            << fullName << endl
                            << "Offline file not fully released/recalled.";
                        m_msgWrite.DisplayMessage(sstr.str());
                        m_prolongedCount++;
                    }
                    else if (  fileProp.m_type == 1   // it is file
                            && fileProp.m_fileID > 0) {
                        
                        log_DBG_m(dbg_DETAIL, "Check if file is online and size > 0. " 
                                            << ", dirty: " << boolalpha << fileProp.m_dirtyFlag
                                            << ", offline: " << boolalpha << fileProp.m_offlineFlag
                                            << ", size: " << fileProp.m_fileSize);
                        if (  !fileProp.m_dirtyFlag
                           && !fileProp.m_offlineFlag    // file is online
                           &&  fileProp.m_fileSize > 0) {
                           
                            auto_ptr<hdb_file2hdbID_t> file2hdbID_ap;
                            file2hdbID_ap.reset(new hdb_file2hdbID_t());

                            g_File2hdbID_p->ReadRec(fileProp.m_fileID, file2hdbID_ap.get() );
                            
                            log_DBG_m(dbg_DETAIL, "Check if file is on release list. "
                                                << *file2hdbID_ap);
                            if (file2hdbID_ap->relCandIdx == 0) { // not on release candidte list
                                ostringstream sstr;
                                sstr << fileProp.m_fileID << " 0 " // migID=0
                                    << fullName << endl
                                    << "Online file not on release candidate list.";
                                m_msgWrite.DisplayMessage(sstr.str());
                                m_notRelCandidate++;
                            }
                        }
                    }

                    auto_ptr<FileInfo> fileInfo_ap;
                    fileInfo_ap.reset(new FileInfo(a_dirPath, "", 0, 0, ownerFID));

                    fileInfo_ap->m_name   = f.m_name;
                    fileInfo_ap->m_size   = fileProp.m_fileSize;
                    fileInfo_ap->m_fileID = fileProp.m_fileID;
                    // TODO get Stat of file
                    if (m_fic.AddFile(*fileInfo_ap)) {
                        SaveStack();
                        cmn_Time now;
    //                    log_DBG_m(dbg_DETAIL, "Current time " << now.Time2YMDhmsm() );
    //                    log_DBG_m(dbg_DETAIL, "Start   time " << m_startTime.Time2YMDhmsm() );
                        now -= m_startTime;
                        log_DBG_m(dbg_DETAIL, "To check last package of entries take " << now.Time2hmsm());
                        if (m_sysLoadPct > 0 && m_sysLoadPct < 100) {
                            UInt64_t microSec    = now.GetTimeInMicroSec();
                            if (microSec > 600000000) { // do not wait more than 10min
                                microSec = 600000000;
                            }
                            log_DBG_m(dbg_DETAIL, "Wait for " << microSec << " micro sec to continue.");
                            ivd_USleep(microSec / m_sysLoadPct * 100 - microSec);
                        }
                        m_startTime.Refresh();
                    }
                }
            }
            else { // dereference, cause cache Increment it
                log_DBG_m(dbg_DETAIL, "File found in HSM cache " << a_dirPath + f.m_name);
                hsmFH->DecrRef();
            }
        } // End of block to decrease stack usage

        if (f.m_fileType == ift_DIR) {
            log_DBG_m(dbg_DETAIL, "file is a directory. Call DoWalk():" <<
            a_dirPath + f.m_name << ", inode: " << IVD_PRINT_ID_FS(fileProp.m_Inode));
            
            if (!DoWalk(a_dirPath + f.m_name, fileProp.m_Inode, false)) {
                // DoWalk was unable to cnage to this directory
                // so skip all other entries.
                // !!!! Why skip other entries? !!!!
                return;
            };
        }
        GetLastVector().pop_back();
    }
}

Here is the call graph for this function:

bool TreeWalk::IsIgnorable	(	ivd_FileType_e	a_type,
		const string &	a_name
	)

Definition at line 58 of file TreeWalk.cpp.

References ift_DIR.

Referenced by GenerateDirTree(), and Walk().

                                                                      {

    if (a_type == ift_DIR && (a_name == "." || a_name == "..")) {
        return true;
    }

    return false;
}

Here is the caller graph for this function:

void TreeWalk::Walk

(

)

Definition at line 121 of file TreeWalk.cpp.

References dbg_LOW, GenerateDirTree(), ivd_BaseException::GetFriendly(), cmn_FastDirLst::GetNextName(), HandleError(), ift_DIR, IsIgnorable(), log_DBG_m, log_FUNC_m, m_dirStack, TreeWalk::DirectoryEntry::m_path, and ProcessEntry().

Referenced by _tmain().

                     {

    log_FUNC_m(Walk);

    DirectoryEntry dir;
    string fName;
    ivd_GenInode_t inodeInfo;
    ivd_FileType_e fType;

    /* First let's get the directories */
    GenerateDirTree();

    /* Now it's all about looping the dirs and list their content */
    while (m_dirStack.empty() != true) {

        dir = m_dirStack.back();
        m_dirStack.pop_back();
        cmn_FastDirLst dirDes(dir.m_path);

        while (true) {
            try {
                fName = dirDes.GetNextName(inodeInfo, fType);
            }
            catch (const ivd_SysError &a_error) {

                log_DBG_m(dbg_LOW,
                    "Can't read entry in directory " << dir.m_path
                    << ". Error: " << a_error.GetFriendly());

                if (HandleError(dir.m_path, a_error) != true) {
                    throw;
                }
            }

            if (fName == "") {
                break;
            }

            if (! IsIgnorable(fType,fName) ) {
                ProcessEntry(dir.m_path, fName, fType);
            }
        }
        /*
            We've just finished supplying ProcessEntry() with files,
            now we pass the directory in which the files resided, so in case
            ProcessEntry unlinked the files, the dir is now empty
        */
        ProcessEntry(dir.m_path, "", ift_DIR);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

void TreeWalk::create_subdirs	(	UInt32_t	beg,
		UInt32_t	end
	)			[protected]

virtual void TreeWalk::PreprocessDir	(	const cmn_Path &	a_dir,
		UInt32_t	a_depth
	)			[inline, protected, virtual]

Reimplemented in CreateTree.

Definition at line 39 of file TreeWalk.h.

Referenced by GenerateDirTree().

{ };

Here is the caller graph for this function:

virtual void TreeWalk::ProcessEntry	(	const cmn_Path &	a_dir,
		const string &	a_name,
		ivd_FileType_e	a_type
	)			[inline, protected, virtual]

Reimplemented in RecAccess, RecOpen, and RecRemove.

Definition at line 40 of file TreeWalk.h.

Referenced by Walk().

{  };

Here is the caller graph for this function:

virtual bool TreeWalk::HandleError	(	const cmn_Path &	a_dir,
		const ivd_SysError &	a_error
	)			[inline, protected, virtual]

Definition at line 41 of file TreeWalk.h.

Referenced by GenerateDirTree(), and Walk().

{ return true; };

Here is the caller graph for this function:

void TreeWalk::GenerateDirTree

(

)

[private]

Definition at line 67 of file TreeWalk.cpp.

References dbg_LOW, ivd_BaseException::GetFriendly(), cmn_FastDirLst::GetNextName(), HandleError(), ift_DIR, IsIgnorable(), log_DBG_m, log_FUNC_m, TreeWalk::DirectoryEntry::m_depth, m_dirStack, TreeWalk::DirectoryEntry::m_path, PreprocessDir(), and predicate::type.

Referenced by Walk().

                               {

    log_FUNC_m(GenerateDirTree);

    unsigned long i;
    unsigned long dirCount = m_dirStack.size();
    /*
        This is used to indicate the stack size before the for loop starts, we
        couldn't use m_dirStack.size() as the stack size changes in every iteration
    */
    unsigned long stSize;

    DirectoryEntry dir;
    string fName;
    ivd_GenInode_t inode;
    ivd_FileType_e type;
    
    while (dirCount != 0){
        for (stSize = m_dirStack.size(),i = stSize - dirCount, 
            dirCount = 0; i < stSize; i++) {
        
            dir = m_dirStack.at(i);
            cmn_FastDirLst cmnDir(dir.m_path);
    
            PreprocessDir(dir.m_path, dir.m_depth);
            while (true) {
            
                try {
                    fName = cmnDir.GetNextName(inode, type);
                }
                catch (const ivd_SysError &a_error) {
                    log_DBG_m(dbg_LOW,
                        "Can't read entry in directory " << dir.m_path
                        << ". Error: " << a_error.GetFriendly());

                    if (HandleError(dir.m_path, a_error) != true) {
                        throw;
                    }
                }
            
                if (fName == "") {
                    break;
                }

                if( ! IsIgnorable(type, fName) && type == ift_DIR ) {
                    ++dirCount;
                    m_dirStack.push_back(
                        DirectoryEntry(dir.m_path + fName, dir.m_depth+1) );
                }
            }
        }
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

---

Member Data Documentation

TreeWalk::log_CLASSID_m

Reimplemented from cmn_TreeWalk.

Reimplemented in CreateTree, RecAccess, RecOpen, and RecRemove.

Definition at line 122 of file treewalk.h.

const string TreeWalk::m_strPartName [private]

Definition at line 125 of file treewalk.h.

Referenced by TreeWalk().

ui_MsgWriter& TreeWalk::m_msgWrite [private]

Definition at line 126 of file treewalk.h.

Referenced by CheckDir(), Report(), and TreeWalk().

cmn_Time TreeWalk::m_startTime [private]

Definition at line 127 of file treewalk.h.

Referenced by CheckDir(), and TreeWalk().

Int32_t TreeWalk::m_sysLoadPct [private]

Check ivdfs with FSC, prepare package every m_numFilesPerBatch number of files read.

After each package is checked slep so much that chack process will have only m_sysLoadPct system percetage impact. Hint: sleep for (time for last package) * (100 div a_sysLoadPct - 1) To work with int arithmetic: (time for last package) div a_sysLoadPct * 100 - (time for last package) (time for last package) must bee in proper units to fit integer in seconds if task take more than 10 s othervice in miliseconds

table: % faktor to multiply last package time 100 0 full speed 80 0.25 50 0.5 40 1.5 25 3.0 10 9.0 5 19.0 0 0 full speed

Definition at line 150 of file treewalk.h.

Referenced by CheckDir().

UInt64_t TreeWalk::m_simLinkCounter [private]

Definition at line 155 of file treewalk.h.

Referenced by CheckDir(), and Report().

UInt64_t TreeWalk::m_filesCounter [private]

Definition at line 156 of file treewalk.h.

Referenced by CheckDir(), and Report().

UInt64_t TreeWalk::m_missingCount [private]

Definition at line 157 of file treewalk.h.

Referenced by Report().

UInt64_t TreeWalk::m_ownerMismatchCount [private]

Definition at line 158 of file treewalk.h.

Referenced by Report().

UInt64_t TreeWalk::m_suspectCount [private]

Definition at line 159 of file treewalk.h.

Referenced by CheckDir(), and Report().

UInt64_t TreeWalk::m_prolongedCount [private]

Definition at line 160 of file treewalk.h.

Referenced by CheckDir(), and Report().

UInt64_t TreeWalk::m_notRelCandidate [private]

Definition at line 161 of file treewalk.h.

Referenced by CheckDir(), and Report().

UInt64_t TreeWalk::m_missingCopyCount [private]

Definition at line 162 of file treewalk.h.

Referenced by Report().

FileInfoCache TreeWalk::m_fic [private]

Definition at line 163 of file treewalk.h.

Referenced by CheckDir(), and ~TreeWalk().

DirectoryVec_t TreeWalk::m_dirStack [private]

Definition at line 60 of file TreeWalk.h.

Referenced by GenerateDirTree(), TreeWalk(), and Walk().

---

The documentation for this class was generated from the following files:

• treewalk.h
• TreeWalk.h
• treewalk.cpp
• TreeWalk.cpp