From 44fbdec051fb0d8473c9afac96c89080db9d1c54 Mon Sep 17 00:00:00 2001 From: Andreas Joachim Peters Date: Tue, 11 Jun 2024 13:44:45 +0200 Subject: [PATCH] XrdApps::JCache: fix indention/formatting --- .../app/XrdClCacheCleaner.cc | 175 +++--- .../XrdClJCachePlugin/cache/IntervalTree.hh | 115 ++-- .../XrdClJCachePlugin/cache/Journal.cc | 165 +++--- .../XrdClJCachePlugin/cache/Journal.hh | 113 ++-- src/XrdApps/XrdClJCachePlugin/cache/RbTree.hh | 359 +++++------- src/XrdApps/XrdClJCachePlugin/file/Art.hh | 137 ++--- .../XrdClJCachePlugin/file/CacheStats.hh | 515 ++++++++++-------- .../XrdClJCachePlugin/file/TimeBench.hh | 117 ++-- .../XrdClJCachePlugin/file/XrdClJCacheFile.cc | 353 +++++------- .../XrdClJCachePlugin/file/XrdClJCacheFile.hh | 138 ++--- .../handler/XrdClJCachePgReadHandler.hh | 55 +- .../handler/XrdClJCacheReadHandler.hh | 55 +- .../handler/XrdClJCacheReadVHandler.hh | 97 ++-- .../plugin/XrdClJCachePlugin.cc | 13 +- .../plugin/XrdClJCachePlugin.hh | 154 +++--- .../vector/XrdClVectorCache.cc | 353 ++++++------ .../vector/XrdClVectorCache.hh | 46 +- 17 files changed, 1407 insertions(+), 1553 deletions(-) diff --git a/src/XrdApps/XrdClJCachePlugin/app/XrdClCacheCleaner.cc b/src/XrdApps/XrdClJCachePlugin/app/XrdClCacheCleaner.cc index f33c5a82fac..c3d156ccf43 100644 --- a/src/XrdApps/XrdClJCachePlugin/app/XrdClCacheCleaner.cc +++ b/src/XrdApps/XrdClJCachePlugin/app/XrdClCacheCleaner.cc @@ -21,109 +21,122 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. -#include -#include -#include -#include -#include #include -#include -#include -#include +#include #include +#include #include +#include +#include +#include +#include +#include +#include namespace fs = std::filesystem; void printCurrentTime() { - auto now = std::chrono::system_clock::now(); - auto now_ns = std::chrono::time_point_cast(now); - auto now_ns_since_epoch = now_ns.time_since_epoch(); - auto seconds = std::chrono::duration_cast(now_ns_since_epoch); - auto nanoseconds = std::chrono::duration_cast(now_ns_since_epoch - seconds); - - auto now_t = std::chrono::system_clock::to_time_t(now); - struct tm* tm = std::localtime(&now_t); - - int year = tm->tm_year - 100; // tm_year represents years since 1900 - - std::cout << std::setfill('0') << std::setw(2) << year << std::setfill('0') << std::setw(4) << (tm->tm_mon + 1) * 100 + tm->tm_mday << " "; - std::cout << std::setw(2) << std::setfill('0') << tm->tm_hour << ":" << std::setw(2) << tm->tm_min << ":" << std::setw(2) << tm->tm_sec << " "; - std::cout << "time=" << seconds.count() << "." << std::setw(9) << std::setfill('0') << nanoseconds.count() << " "; + auto now = std::chrono::system_clock::now(); + auto now_ns = std::chrono::time_point_cast(now); + auto now_ns_since_epoch = now_ns.time_since_epoch(); + auto seconds = + std::chrono::duration_cast(now_ns_since_epoch); + auto nanoseconds = std::chrono::duration_cast( + now_ns_since_epoch - seconds); + + auto now_t = std::chrono::system_clock::to_time_t(now); + struct tm *tm = std::localtime(&now_t); + + int year = tm->tm_year - 100; // tm_year represents years since 1900 + + std::cout << std::setfill('0') << std::setw(2) << year << std::setfill('0') + << std::setw(4) << (tm->tm_mon + 1) * 100 + tm->tm_mday << " "; + std::cout << std::setw(2) << std::setfill('0') << tm->tm_hour << ":" + << std::setw(2) << tm->tm_min << ":" << std::setw(2) << tm->tm_sec + << " "; + std::cout << "time=" << seconds.count() << "." << std::setw(9) + << std::setfill('0') << nanoseconds.count() << " "; } -time_t getLastAccessTime(const fs::path& filePath) { - struct stat fileInfo; - if (stat(filePath.c_str(), &fileInfo) != 0) { - return -1; // Error occurred - } - return fileInfo.st_atime; +time_t getLastAccessTime(const fs::path &filePath) { + struct stat fileInfo; + if (stat(filePath.c_str(), &fileInfo) != 0) { + return -1; // Error occurred + } + return fileInfo.st_atime; } -long long getDirectorySize(const fs::path& directory) { - long long totalSize = 0; - for (const auto& entry : fs::recursive_directory_iterator(directory)) { - if (fs::is_regular_file(entry)) { - totalSize += fs::file_size(entry); - } +long long getDirectorySize(const fs::path &directory) { + long long totalSize = 0; + for (const auto &entry : fs::recursive_directory_iterator(directory)) { + if (fs::is_regular_file(entry)) { + totalSize += fs::file_size(entry); } - return totalSize; + } + return totalSize; } -std::vector> getFilesByAccessTime(const fs::path& directory) { - std::vector> fileList; - for (const auto& entry : fs::recursive_directory_iterator(directory)) { - if (fs::is_regular_file(entry)) { - auto accessTime = getLastAccessTime(entry.path()); - fileList.emplace_back(accessTime, entry.path()); - } +std::vector> +getFilesByAccessTime(const fs::path &directory) { + std::vector> fileList; + for (const auto &entry : fs::recursive_directory_iterator(directory)) { + if (fs::is_regular_file(entry)) { + auto accessTime = getLastAccessTime(entry.path()); + fileList.emplace_back(accessTime, entry.path()); } - std::sort(fileList.begin(), fileList.end()); - return fileList; + } + std::sort(fileList.begin(), fileList.end()); + return fileList; } -void cleanDirectory(const fs::path& directory, long long highWatermark, long long lowWatermark) { - long long currentSize = getDirectorySize(directory); - if (currentSize <= highWatermark) { - printCurrentTime(); - std::cout << "Directory size is within the limit. No action needed." << std::endl; - return; - } +void cleanDirectory(const fs::path &directory, long long highWatermark, + long long lowWatermark) { + long long currentSize = getDirectorySize(directory); + if (currentSize <= highWatermark) { + printCurrentTime(); + std::cout << "Directory size is within the limit. No action needed." + << std::endl; + return; + } + + auto files = getFilesByAccessTime(directory); - auto files = getFilesByAccessTime(directory); - - for (const auto& [accessTime, filePath] : files) { - if (currentSize <= lowWatermark) { - break; - } - long long fileSize = fs::file_size(filePath); - try { - fs::remove(filePath); - currentSize -= fileSize; - printCurrentTime(); - std::cout << "Deleted: " << filePath << " (Size: " << fileSize << " bytes)" << std::endl; - } catch (const std::exception& e) { - printCurrentTime(); - std::cerr << "Error deleting " << filePath << ": " << e.what() << std::endl; - } + for (const auto &[accessTime, filePath] : files) { + if (currentSize <= lowWatermark) { + break; } + long long fileSize = fs::file_size(filePath); + try { + fs::remove(filePath); + currentSize -= fileSize; + printCurrentTime(); + std::cout << "Deleted: " << filePath << " (Size: " << fileSize + << " bytes)" << std::endl; + } catch (const std::exception &e) { + printCurrentTime(); + std::cerr << "Error deleting " << filePath << ": " << e.what() + << std::endl; + } + } } -int main(int argc, char* argv[]) { - if (argc != 5) { - std::cerr << "Usage: " << argv[0] << " " << std::endl; - return 1; - } +int main(int argc, char *argv[]) { + if (argc != 5) { + std::cerr << "Usage: " << argv[0] + << " " + << std::endl; + return 1; + } - fs::path directory = argv[1]; - long long highWatermark = std::stoll(argv[2]); - long long lowWatermark = std::stoll(argv[3]); - int interval = std::stoi(argv[4]); + fs::path directory = argv[1]; + long long highWatermark = std::stoll(argv[2]); + long long lowWatermark = std::stoll(argv[3]); + int interval = std::stoi(argv[4]); - while (true) { - cleanDirectory(directory, highWatermark, lowWatermark); - std::this_thread::sleep_for(std::chrono::seconds(interval)); - } + while (true) { + cleanDirectory(directory, highWatermark, lowWatermark); + std::this_thread::sleep_for(std::chrono::seconds(interval)); + } - return 0; + return 0; } diff --git a/src/XrdApps/XrdClJCachePlugin/cache/IntervalTree.hh b/src/XrdApps/XrdClJCachePlugin/cache/IntervalTree.hh index aa9bc81f904..e281c8525bb 100644 --- a/src/XrdApps/XrdClJCachePlugin/cache/IntervalTree.hh +++ b/src/XrdApps/XrdClJCachePlugin/cache/IntervalTree.hh @@ -1,7 +1,7 @@ //------------------------------------------------------------------------------ // Copyright (c) 2024 by European Organization for Nuclear Research (CERN) // Author: Andreas-Joachim Peters -// Michal Simon +// Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // @@ -27,79 +27,68 @@ /*----------------------------------------------------------------------------*/ #include "RbTree.hh" /*----------------------------------------------------------------------------*/ -#include -#include -#include -#include #include #include +#include +#include +#include +#include /*----------------------------------------------------------------------------*/ -template -class interval_node_t -{ +template class interval_node_t { public: - - template friend class interval_tree; - template friend class rbtree; + template friend class interval_tree; + template friend class rbtree; friend class IntervalTreeTest; public: - - interval_node_t(I low, I high, const V& value) : - low(low), high(high), value(value), key(this->low), max(high), colour(RED), - parent(nullptr) { } + interval_node_t(I low, I high, const V &value) + : low(low), high(high), value(value), key(this->low), max(high), + colour(RED), parent(nullptr) {} const I low; const I high; V value; private: - const I& key; + const I &key; I max; colour_t colour; - interval_node_t* parent; + interval_node_t *parent; std::unique_ptr left; std::unique_ptr right; }; -template -class interval_tree : public rbtree< I, V, interval_node_t > -{ +template +class interval_tree : public rbtree> { private: - typedef interval_node_t N; typedef typename rbtree::leaf_node_t leaf_node_t; - std::unique_ptr make_node(I low, I high, const V& value) - { + std::unique_ptr make_node(I low, I high, const V &value) { return std::unique_ptr(new N(low, high, value)); } public: - typedef typename rbtree::iterator iterator; struct less { - bool operator()(const iterator& x, const iterator& y) const - { + bool operator()(const iterator &x, const iterator &y) const { return x->low < y->low; } }; - virtual ~interval_tree() { } + virtual ~interval_tree() {} - void insert(I low, I high, const V& value) - { + void insert(I low, I high, const V &value) { insert_into(low, high, value, this->tree_root); } - void erase(I low, I high) - { - std::unique_ptr& node = this->find_in(low, this->tree_root); + void erase(I low, I high) { + std::unique_ptr &node = this->find_in(low, this->tree_root); if (!node || node->low != low || node->high != high) { return; @@ -108,21 +97,18 @@ public: this->erase_node(node); } - std::set query(I low, I high) - { + std::set query(I low, I high) { std::set result; query(low, high, this->tree_root, result); return result; } private: - using rbtree::insert; using rbtree::erase; using rbtree::find; - static bool overlaps(I low, I high, const N* node) - { + static bool overlaps(I low, I high, const N *node) { int64_t s1 = low + high; int64_t d1 = high - low; int64_t s2 = node->low + node->high; @@ -130,9 +116,8 @@ private: return std::abs(s2 - s1) < d1 + d2; } - static void query(I low, I high, std::unique_ptr& node, - std::set& result) - { + static void query(I low, I high, std::unique_ptr &node, + std::set &result) { // base case if (!node) { return; @@ -157,9 +142,8 @@ private: } } - void insert_into(I low, I high, const V& value, std::unique_ptr& node, - N* parent = nullptr) - { + void insert_into(I low, I high, const V &value, std::unique_ptr &node, + N *parent = nullptr) { if (!node) { node = make_node(low, high, value); node->parent = parent; @@ -180,8 +164,7 @@ private: } } - void erase_node(std::unique_ptr& node) - { + void erase_node(std::unique_ptr &node) { if (!node) { return; } @@ -191,8 +174,8 @@ private: // 1. look for the in-order successor // 2. replace the node with the in-order successor // 3. erase the in-order successor - N* n = node.get(); - std::unique_ptr& successor = this->find_successor(node); + N *n = node.get(); + std::unique_ptr &successor = this->find_successor(node); this->swap_successor(node, successor); // we don't update max since in erase_node we @@ -201,12 +184,12 @@ private: // 'successor' unique pointer holds now the node if (successor.get() == n) { erase_node(successor); - }// otherwise the successor was the right child of node, + } // otherwise the successor was the right child of node, // hence node should be now the right child of 'node' // unoique pointer else if (node->right.get() == n) { erase_node(node->right); - }// there are no other cases so anything else is wrong + } // there are no other cases so anything else is wrong else { throw std::logic_error("Bad rbtree swap."); } @@ -217,8 +200,8 @@ private: // node has at most one child // in this case simply replace the node with the // single child or null if there are no children - N* parent = node->parent; - std::unique_ptr& child = node->left ? node->left : node->right; + N *parent = node->parent; + std::unique_ptr &child = node->left ? node->left : node->right; colour_t old_colour = node->colour; if (child) { @@ -243,17 +226,16 @@ private: this->rb_erase_case1(leaf_node_t(parent)); } } else if (node) - // if the node was red it has to have two BLACK children - // and since at most one of those children is a non-leaf - // child actually both have to be leafs (null) in order - // to satisfy the red-black tree invariant + // if the node was red it has to have two BLACK children + // and since at most one of those children is a non-leaf + // child actually both have to be leafs (null) in order + // to satisfy the red-black tree invariant { throw rb_invariant_error(); } } - void update_max(N* node, I new_high) - { + void update_max(N *node, I new_high) { while (node) { if (new_high > node->max) { node->max = new_high; @@ -264,16 +246,14 @@ private: } } - void update_max(N* node) - { + void update_max(N *node) { while (node) { set_max(node); node = node->parent; } } - void set_max(N* node) - { + void set_max(N *node) { if (!node->left && !node->right) { node->max = node->high; return; @@ -285,20 +265,19 @@ private: return; } - node->max = std::max(node->high, std::max(node->left->max, node->right->max)); + node->max = + std::max(node->high, std::max(node->left->max, node->right->max)); } - virtual void right_rotation(N* node) - { - N* pivot = node->left.get(); + virtual void right_rotation(N *node) { + N *pivot = node->left.get(); rbtree::right_rotation(node); set_max(node); // set first max for node since now it's lower in the tree set_max(pivot); } - virtual void left_rotation(N* node) - { - N* pivot = node->right.get(); + virtual void left_rotation(N *node) { + N *pivot = node->right.get(); rbtree::left_rotation(node); set_max(node); // set first max for node since now it's lower in the tree set_max(pivot); diff --git a/src/XrdApps/XrdClJCachePlugin/cache/Journal.cc b/src/XrdApps/XrdClJCachePlugin/cache/Journal.cc index 9bcff9c50f0..c164236e9b5 100644 --- a/src/XrdApps/XrdClJCachePlugin/cache/Journal.cc +++ b/src/XrdApps/XrdClJCachePlugin/cache/Journal.cc @@ -1,7 +1,7 @@ //------------------------------------------------------------------------------ // Copyright (c) 2024 by European Organization for Nuclear Research (CERN) // Author: Andreas-Joachim Peters -// Michal Simon +// Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // @@ -22,24 +22,21 @@ // granted to it by virtue of its status as an Intergovernmental Organization // or submit itself to any jurisdiction. - /*----------------------------------------------------------------------------*/ #include "Journal.hh" /*----------------------------------------------------------------------------*/ #include +#include #include -#include -#include #include -#include +#include +#include /*----------------------------------------------------------------------------*/ //------------------------------------------------------------------------------ //! Journal Constructor //------------------------------------------------------------------------------ -Journal::Journal() : cachesize(0), - max_offset(0), fd(-1) -{ +Journal::Journal() : cachesize(0), max_offset(0), fd(-1) { std::lock_guard guard(mtx); jheader.magic = JOURNAL_MAGIC; jheader.mtime = 0; @@ -53,8 +50,7 @@ Journal::Journal() : cachesize(0), //------------------------------------------------------------------------------ //! Journal Destructor //------------------------------------------------------------------------------ -Journal::~Journal() -{ +Journal::~Journal() { std::lock_guard guard(mtx); if (fd > 0) { int rc = close(fd); @@ -70,48 +66,50 @@ Journal::~Journal() //------------------------------------------------------------------------------ //! Routine to read a journal header //------------------------------------------------------------------------------ -void Journal::read_jheader() -{ +void Journal::read_jheader() { jheader_t fheader; auto hr = ::pread64(fd, &fheader, sizeof(jheader), 0); - if ( (hr>0) && - ( (hr != sizeof(jheader)) || - (fheader.magic != JOURNAL_MAGIC)) ) { - std::cerr << "warning: inconsistent journal header found (I) - purging path:" << path << std::endl; + if ((hr > 0) && + ((hr != sizeof(jheader)) || (fheader.magic != JOURNAL_MAGIC))) { + std::cerr + << "warning: inconsistent journal header found (I) - purging path:" + << path << std::endl; reset(); return; } // TODO: understand why the mtime is +-1s - if ( ( abs(fheader.mtime - jheader.mtime) > 1 ) - || (fheader.mtime_nsec != jheader.mtime_nsec) - || (fheader.filesize != jheader.filesize) ) { + if ((abs(fheader.mtime - jheader.mtime) > 1) || + (fheader.mtime_nsec != jheader.mtime_nsec) || + (fheader.filesize != jheader.filesize)) { if (fheader.mtime) { - std::cerr << "warning: remote file change detected - purging path:" << path << std::endl; - std::cerr << fheader.mtime << ":" << jheader.mtime << " " << fheader.mtime_nsec << ":" << jheader.mtime_nsec << " " << fheader.filesize << ":" << jheader.filesize << std::endl; + std::cerr << "warning: remote file change detected - purging path:" + << path << std::endl; + std::cerr << fheader.mtime << ":" << jheader.mtime << " " + << fheader.mtime_nsec << ":" << jheader.mtime_nsec << " " + << fheader.filesize << ":" << jheader.filesize << std::endl; } reset(); return; } - } +} //------------------------------------------------------------------------------ //! Routine to write a journal header //------------------------------------------------------------------------------ -int Journal::write_jheader() -{ +int Journal::write_jheader() { auto hw = ::pwrite64(fd, &jheader, sizeof(jheader), 0); - if ( (hw != sizeof(jheader)) ) { - std::cerr << "warning: failed to write journal header - purging path:" << path << std::endl; + if ((hw != sizeof(jheader))) { + std::cerr << "warning: failed to write journal header - purging path:" + << path << std::endl; return -errno; } return 0; - } +} //------------------------------------------------------------------------------ //! Routine to read a journal //------------------------------------------------------------------------------ -int Journal::read_journal() -{ +int Journal::read_journal() { journal.clear(); const size_t bufsize = sizeof(header_t); char buffer[bufsize]; @@ -123,29 +121,30 @@ int Journal::read_journal() do { bytesRead = ::pread(fd, buffer, bufsize, totalBytesRead); if (bytesRead < (ssize_t)bufsize) { - if (bytesRead == 0 && (totalBytesRead==journalsize)) { + if (bytesRead == 0 && (totalBytesRead == journalsize)) { break; } else { - std::cerr << "warning: inconsistent journal found - purging path:" << path << std::endl; + std::cerr << "warning: inconsistent journal found - purging path:" + << path << std::endl; reset(); return 0; } } - header_t* header = reinterpret_cast(buffer); + header_t *header = reinterpret_cast(buffer); journal.insert(header->offset, header->offset + header->size, - totalBytesRead); + totalBytesRead); totalBytesRead += header->size; // size of the fragment - totalBytesRead += bytesRead; // size of the header + totalBytesRead += bytesRead; // size of the header } while (pos < bytesRead); - + return totalBytesRead; } //------------------------------------------------------------------------------ //! Journal attach //------------------------------------------------------------------------------ -int Journal::attach(const std::string& lpath, uint64_t mtime, uint64_t mtime_nsec, uint64_t size) -{ +int Journal::attach(const std::string &lpath, uint64_t mtime, + uint64_t mtime_nsec, uint64_t size) { std::lock_guard guard(mtx); path = lpath; jheader.mtime = mtime; @@ -188,8 +187,7 @@ int Journal::attach(const std::string& lpath, uint64_t mtime, uint64_t mtime_nse //------------------------------------------------------------------------------ //! Journal detach //------------------------------------------------------------------------------ -int Journal::detach() -{ +int Journal::detach() { std::lock_guard guard(mtx); return 0; } @@ -197,8 +195,7 @@ int Journal::detach() //------------------------------------------------------------------------------ //! Journal unlink //------------------------------------------------------------------------------ -int Journal::unlink() -{ +int Journal::unlink() { std::lock_guard guard(mtx); struct stat buf; int rc = stat(path.c_str(), &buf); @@ -212,14 +209,13 @@ int Journal::unlink() //------------------------------------------------------------------------------ //! Journal pread //------------------------------------------------------------------------------ -ssize_t Journal::pread(void* buf, size_t count, off_t offset, bool& eof) -{ - if (fd<0) { +ssize_t Journal::pread(void *buf, size_t count, off_t offset, bool &eof) { + if (fd < 0) { return 0; } // rewrite reads to not go over EOF! - if ( (off_t)(offset + count) > (off_t) jheader.filesize ) { + if ((off_t)(offset + count) > (off_t)jheader.filesize) { if ((off_t)jheader.filesize > offset) { count = (off_t)jheader.filesize - offset; } else { @@ -236,12 +232,11 @@ ssize_t Journal::pread(void* buf, size_t count, off_t offset, bool& eof) return 0; } - char* buffer = reinterpret_cast(buf); + char *buffer = reinterpret_cast(buf); uint64_t off = offset; uint64_t bytesRead = 0; - - for (auto& itr : result) { + for (auto &itr : result) { if (itr->low <= off && off < itr->high) { // read from cache uint64_t cacheoff = itr->value + sizeof(header_t) + (off - itr->low); @@ -274,10 +269,10 @@ ssize_t Journal::pread(void* buf, size_t count, off_t offset, bool& eof) //------------------------------------------------------------------------------ //! Journal process intersection //------------------------------------------------------------------------------ -void Journal::process_intersection(interval_tree& - to_write, interval_tree::iterator itr, - std::vector& updates) -{ +void Journal::process_intersection( + interval_tree &to_write, + interval_tree::iterator itr, + std::vector &updates) { auto result = to_write.query(itr->low, itr->high); if (result.empty()) { @@ -288,7 +283,8 @@ void Journal::process_intersection(interval_tree& throw std::logic_error("Journal: overlapping journal entries"); } - const interval_tree::iterator to_wrt = *result.begin(); + const interval_tree::iterator to_wrt = + *result.begin(); // the intersection uint64_t low = std::max(to_wrt->low, itr->low); uint64_t high = std::min(to_wrt->high, itr->high); @@ -296,12 +292,12 @@ void Journal::process_intersection(interval_tree& chunk_t update; update.offset = offset_for_update(itr->value, low - itr->low); update.size = high - low; - update.buff = static_cast(to_wrt->value) + (low - to_wrt->low); + update.buff = static_cast(to_wrt->value) + (low - to_wrt->low); updates.push_back(std::move(update)); // update the 'to write' intervals uint64_t wrtlow = to_wrt->low; uint64_t wrthigh = to_wrt->high; - const void* wrtbuff = to_wrt->value; + const void *wrtbuff = to_wrt->value; to_write.erase(wrtlow, wrthigh); // the intersection overlaps with the given @@ -312,7 +308,7 @@ void Journal::process_intersection(interval_tree& if (high < wrthigh) { // the remaining right-hand-side interval - const char* buff = static_cast(wrtbuff) + (high - wrtlow); + const char *buff = static_cast(wrtbuff) + (high - wrtlow); to_write.insert(high, wrthigh, buff); } @@ -323,15 +319,14 @@ void Journal::process_intersection(interval_tree& } //------------------------------------------------------------------------------ -//! Journal update +//! Journal update //------------------------------------------------------------------------------ -int Journal::update_cache(std::vector& updates) -{ +int Journal::update_cache(std::vector &updates) { // make sure we are updating the cache in ascending order std::sort(updates.begin(), updates.end()); int rc = 0; - for (auto& u : updates) { + for (auto &u : updates) { rc = ::pwrite(fd, u.buff, u.size, u.offset); // TODO is it safe to assume it will write it all @@ -346,9 +341,8 @@ int Journal::update_cache(std::vector& updates) //------------------------------------------------------------------------------ //! Journal pwrite //------------------------------------------------------------------------------ -ssize_t Journal::pwrite(const void* buf, size_t count, off_t offset) -{ - if (fd<0) { +ssize_t Journal::pwrite(const void *buf, size_t count, off_t offset) { + if (fd < 0) { return 0; } @@ -357,7 +351,7 @@ ssize_t Journal::pwrite(const void* buf, size_t count, off_t offset) return 0; } - interval_tree to_write; + interval_tree to_write; std::vector updates; to_write.insert(offset, offset + count, buf); auto res = journal.query(offset, offset + count); @@ -372,7 +366,7 @@ ssize_t Journal::pwrite(const void* buf, size_t count, off_t offset) return -1; } - interval_tree::iterator itr; + interval_tree::iterator itr; for (itr = to_write.begin(); itr != to_write.end(); ++itr) { uint64_t size = itr->high - itr->low; @@ -382,7 +376,7 @@ ssize_t Journal::pwrite(const void* buf, size_t count, off_t offset) iovec iov[2]; iov[0].iov_base = &header; iov[0].iov_len = sizeof(header_t); - iov[1].iov_base = const_cast(itr->value); + iov[1].iov_base = const_cast(itr->value); iov[1].iov_len = size; rc = ::pwrite(fd, iov[0].iov_base, iov[0].iov_len, cachesize); @@ -397,7 +391,7 @@ ssize_t Journal::pwrite(const void* buf, size_t count, off_t offset) cachesize += sizeof(header_t) + size; } - if ((ssize_t)(offset + count) > max_offset) { + if ((ssize_t)(offset + count) > max_offset) { max_offset = offset + count; } @@ -407,9 +401,8 @@ ssize_t Journal::pwrite(const void* buf, size_t count, off_t offset) //------------------------------------------------------------------------------ //! Journal data sync //------------------------------------------------------------------------------ -int Journal::sync() -{ - if (fd<0) { +int Journal::sync() { + if (fd < 0) { return -1; } return ::fdatasync(fd); @@ -418,8 +411,7 @@ int Journal::sync() //------------------------------------------------------------------------------ //! Journal get size //------------------------------------------------------------------------------ -size_t Journal::size() -{ +size_t Journal::size() { std::lock_guard guard(mtx); return cachesize; } @@ -427,8 +419,7 @@ size_t Journal::size() //------------------------------------------------------------------------------ //! Journal get max offset in the journal //------------------------------------------------------------------------------ -off_t Journal::get_max_offset() -{ +off_t Journal::get_max_offset() { std::lock_guard guard(mtx); return max_offset; } @@ -436,12 +427,11 @@ off_t Journal::get_max_offset() //------------------------------------------------------------------------------ //! Journal reset //------------------------------------------------------------------------------ -int Journal::reset() -{ +int Journal::reset() { journal.clear(); - int retc=0; - if (fd>=0) { - retc = ftruncate(fd,0); + int retc = 0; + if (fd >= 0) { + retc = ftruncate(fd, 0); retc |= write_jheader(); } cachesize = 0; @@ -449,8 +439,7 @@ int Journal::reset() return retc; } -std::string Journal::dump() -{ +std::string Journal::dump() { std::lock_guard guard(mtx); std::string out; out += "fd="; @@ -465,18 +454,16 @@ std::string Journal::dump() //------------------------------------------------------------------------------ //! Journal get chunks //------------------------------------------------------------------------------ -std::vector Journal::get_chunks(off_t offset, - size_t size) -{ +std::vector Journal::get_chunks(off_t offset, size_t size) { auto result = journal.query(offset, offset + size); std::vector ret; - for (auto& itr : result) { - uint64_t off = (off_t) itr->low < (off_t) offset ? offset : itr->low; - uint64_t count = itr->high < offset + size ? itr->high - off : offset + size - - off; + for (auto &itr : result) { + uint64_t off = (off_t)itr->low < (off_t)offset ? offset : itr->low; + uint64_t count = + itr->high < offset + size ? itr->high - off : offset + size - off; uint64_t cacheoff = itr->value + sizeof(header_t) + (off - itr->low); - std::unique_ptr buffer(new char[count]); + std::unique_ptr buffer(new char[count]); ssize_t rc = ::pread(fd, buffer.get(), count, cacheoff); if (rc < 0) { diff --git a/src/XrdApps/XrdClJCachePlugin/cache/Journal.hh b/src/XrdApps/XrdClJCachePlugin/cache/Journal.hh index 936660de751..405bc9a20cb 100644 --- a/src/XrdApps/XrdClJCachePlugin/cache/Journal.hh +++ b/src/XrdApps/XrdClJCachePlugin/cache/Journal.hh @@ -1,7 +1,7 @@ //------------------------------------------------------------------------------ // Copyright (c) 2024 by European Organization for Nuclear Research (CERN) // Author: Andreas-Joachim Peters -// Michal Simon +// Michal Simon //------------------------------------------------------------------------------ // This file is part of the XRootD software suite. // @@ -26,18 +26,18 @@ /*----------------------------------------------------------------------------*/ #include "IntervalTree.hh" /*----------------------------------------------------------------------------*/ +#include +#include #include #include -#include -#include /*----------------------------------------------------------------------------*/ -class Journal -{ +class Journal { static constexpr uint64_t JOURNAL_MAGIC = 0xcafecafecafecafe; struct jheader_t { jheader_t() { - mtime = mtime_nsec = filesize = placeholder1 = placeholder2 = placeholder3 = placeholder4 = magic = 0; + mtime = mtime_nsec = filesize = placeholder1 = placeholder2 = + placeholder3 = placeholder4 = magic = 0; } uint64_t magic; uint64_t mtime; @@ -48,48 +48,45 @@ class Journal uint64_t placeholder3; uint64_t placeholder4; }; - + struct header_t { uint64_t offset; uint64_t size; }; public: - struct chunk_t { - chunk_t() : offset(0), size(0), buff(0) { } + chunk_t() : offset(0), size(0), buff(0) {} /* constructor - no ownership of underlying buffer */ - chunk_t(off_t offset, size_t size, const void* buff) : offset(offset), - size(size), buff(buff) { } + chunk_t(off_t offset, size_t size, const void *buff) + : offset(offset), size(size), buff(buff) {} /* constructor - with ownership of underlying buffer */ - chunk_t(off_t offset, size_t size, std::unique_ptr buff) : - offset(offset), size(size), buffOwnership(std::move(buff)), - buff((const void*) buffOwnership.get()) {} + chunk_t(off_t offset, size_t size, std::unique_ptr buff) + : offset(offset), size(size), buffOwnership(std::move(buff)), + buff((const void *)buffOwnership.get()) {} off_t offset; size_t size; std::unique_ptr buffOwnership; - const void* buff; + const void *buff; - bool operator<(const chunk_t& u) const - { - return offset < u.offset; - } + bool operator<(const chunk_t &u) const { return offset < u.offset; } }; Journal(); virtual ~Journal(); // base class interface - int attach(const std::string& path, uint64_t mtime, uint64_t mtime_nsec, uint64_t size); + int attach(const std::string &path, uint64_t mtime, uint64_t mtime_nsec, + uint64_t size); int detach(); int unlink(); - ssize_t pread(void* buf, size_t count, off_t offset, bool& eof); - ssize_t pwrite(const void* buf, size_t count, off_t offset); + ssize_t pread(void *buf, size_t count, off_t offset, bool &eof); + ssize_t pwrite(const void *buf, size_t count, off_t offset); int sync(); @@ -102,23 +99,23 @@ public: std::vector get_chunks(off_t offset, size_t size); std::string dump(); -private: - void process_intersection(interval_tree& write, - interval_tree::iterator acr, std::vector& updates); +private: + void process_intersection(interval_tree &write, + interval_tree::iterator acr, + std::vector &updates); - static uint64_t offset_for_update(uint64_t offset, uint64_t shift) - { + static uint64_t offset_for_update(uint64_t offset, uint64_t shift) { return offset + sizeof(header_t) + shift; } - int update_cache(std::vector& updates); + int update_cache(std::vector &updates); int read_journal(); - + jheader_t jheader; int write_jheader(); void read_jheader(); - + std::string path; size_t cachesize; off_t max_offset; @@ -128,43 +125,43 @@ private: interval_tree journal; std::mutex mtx; int flags; - }; class JournalManager { private: - std::map> journals; - std::mutex jMutex; + std::map> journals; + std::mutex jMutex; public: JournalManager() {} virtual ~JournalManager() {} - - // Attach method: creates or retrieves a Journal object by key - std::shared_ptr attach(const std::string &key) { - std::lock_guard guard(jMutex); - auto it = journals.find(key); - if (it == journals.end()) { - // Create a new Journal object if it doesn't exist - auto journal = std::make_shared(); - journals[key] = journal; - return journal; - } else { - // Return the existing Journal object - return it->second; - } + + // Attach method: creates or retrieves a Journal object by key + std::shared_ptr attach(const std::string &key) { + std::lock_guard guard(jMutex); + auto it = journals.find(key); + if (it == journals.end()) { + // Create a new Journal object if it doesn't exist + auto journal = std::make_shared(); + journals[key] = journal; + return journal; + } else { + // Return the existing Journal object + return it->second; } + } - // Detach method: checks reference count and removes Journal object if necessary - void detach(const std::string &key) { - std::lock_guard guard(jMutex); - auto it = journals.find(key); - if (it != journals.end()) { - if (it->second.use_count() == 1) { - // Only one reference exists, so erase the entry from the map - journals.erase(it); - } - // If more than one reference exists, do nothing - } + // Detach method: checks reference count and removes Journal object if + // necessary + void detach(const std::string &key) { + std::lock_guard guard(jMutex); + auto it = journals.find(key); + if (it != journals.end()) { + if (it->second.use_count() == 1) { + // Only one reference exists, so erase the entry from the map + journals.erase(it); + } + // If more than one reference exists, do nothing } + } }; diff --git a/src/XrdApps/XrdClJCachePlugin/cache/RbTree.hh b/src/XrdApps/XrdClJCachePlugin/cache/RbTree.hh index 5d9514759b8..fa0043d0381 100644 --- a/src/XrdApps/XrdClJCachePlugin/cache/RbTree.hh +++ b/src/XrdApps/XrdClJCachePlugin/cache/RbTree.hh @@ -27,62 +27,47 @@ #include #include -class rb_invariant_error : public std::exception -{ +class rb_invariant_error : public std::exception { public: + rb_invariant_error() {} - rb_invariant_error() { } - - virtual const char* what() const throw() - { + virtual const char *what() const throw() { return "Red-black tree invariant violation!"; } - }; -enum colour_t { - RED = true, - BLACK = false -}; +enum colour_t { RED = true, BLACK = false }; -template -class node_t -{ - template friend class rbtree; +template class node_t { + template friend class rbtree; friend class RBTreeTest; public: - - node_t(const K& key, const V& value) : key(key), value(value), colour(RED), - parent(nullptr) { } + node_t(const K &key, const V &value) + : key(key), value(value), colour(RED), parent(nullptr) {} const K key; V value; private: colour_t colour; - node_t* parent; + node_t *parent; std::unique_ptr left; std::unique_ptr right; }; -template > -class rbtree -{ +template > class rbtree { friend class RBTreeTest; friend class IntervalTreeTest; protected: - - static std::unique_ptr make_node(const K& key, const V& value) - { + static std::unique_ptr make_node(const K &key, const V &value) { return std::unique_ptr(new N(key, value)); } - static void swap_right_child(std::unique_ptr& node, - std::unique_ptr& successor) - { + static void swap_right_child(std::unique_ptr &node, + std::unique_ptr &successor) { std::swap(node->colour, successor->colour); // first do the obvious std::swap(node->left, successor->left); @@ -96,10 +81,10 @@ protected: } // now gather remaining pointers - N* p = node->parent; - N* n = node.release(); - N* s = successor.release(); - N* s_right = s->right.release(); + N *p = node->parent; + N *n = node.release(); + N *s = successor.release(); + N *s_right = s->right.release(); // and finally reassign those pointers s->parent = p; node.reset(s); @@ -112,9 +97,8 @@ protected: } } - static void swap_successor(std::unique_ptr& node, - std::unique_ptr& successor) - { + static void swap_successor(std::unique_ptr &node, + std::unique_ptr &successor) { // first check if successor is a direct child of node, // since it is the in-order successor it can be only // the right child @@ -160,77 +144,45 @@ protected: struct leaf_node_t { - leaf_node_t(N* parent) : colour(BLACK), parent(parent) { } + leaf_node_t(N *parent) : colour(BLACK), parent(parent) {} - leaf_node_t(const leaf_node_t& leaf) : colour(leaf.colour), - parent(leaf.parent) { } + leaf_node_t(const leaf_node_t &leaf) + : colour(leaf.colour), parent(leaf.parent) {} - leaf_node_t& operator=(const leaf_node_t& leaf) - { + leaf_node_t &operator=(const leaf_node_t &leaf) { colour = leaf.colour; parent = leaf.parent; return *this; } - leaf_node_t* operator->() - { - return this; - } + leaf_node_t *operator->() { return this; } - leaf_node_t& operator*() - { - return *this; - } + leaf_node_t &operator*() { return *this; } - operator bool() const - { - return true; - } + operator bool() const { return true; } - bool operator==(N* node) const - { - return node == nullptr; - } + bool operator==(N *node) const { return node == nullptr; } colour_t colour; - N* parent; + N *parent; }; public: - - class iterator - { + class iterator { public: + iterator(N *node = 0) : node(node) {} - iterator(N* node = 0) : node(node) { } - - N* operator->() - { - return node; - } + N *operator->() { return node; } - N& operator*() - { - return *node; - } + N &operator*() { return *node; } - const N* operator->() const - { - return node; - } + const N *operator->() const { return node; } - const N& operator*() const - { - return *node; - } + const N &operator*() const { return *node; } - operator bool() const - { - return bool(node); - } + operator bool() const { return bool(node); } - iterator& operator++() - { + iterator &operator++() { if (!node) { return *this; } @@ -245,7 +197,7 @@ public: return *this; } - N* parent = node->parent; + N *parent = node->parent; while (parent && is_right(node)) { node = parent; @@ -256,62 +208,46 @@ public: return *this; } - bool operator!=(const iterator& itr) - { - return node != itr.node; - } + bool operator!=(const iterator &itr) { return node != itr.node; } private: - - N* node; + N *node; }; - rbtree() : tree_size(0) { } + rbtree() : tree_size(0) {} - virtual ~rbtree() { } + virtual ~rbtree() {} - void insert(const K& key, const V& value) - { + void insert(const K &key, const V &value) { insert_into(key, value, tree_root); } - void erase(const K& key) - { - std::unique_ptr& node = find_in(key, tree_root); + void erase(const K &key) { + std::unique_ptr &node = find_in(key, tree_root); erase_node(node); } - void clear() - { + void clear() { tree_root.reset(); tree_size = 0; } - iterator find(const K& key) - { - const std::unique_ptr& n = find_in(key, tree_root); + iterator find(const K &key) { + const std::unique_ptr &n = find_in(key, tree_root); return iterator(n.get()); } - const iterator find(const K& key) const - { - const std::unique_ptr& n = find_in(key, tree_root); + const iterator find(const K &key) const { + const std::unique_ptr &n = find_in(key, tree_root); return iterator(n.get()); } - size_t size() const - { - return tree_size; - } + size_t size() const { return tree_size; } - bool empty() const - { - return !tree_root; - } + bool empty() const { return !tree_root; } - iterator begin() - { - N* node = tree_root.get(); + iterator begin() { + N *node = tree_root.get(); if (!node) { return iterator(); @@ -324,16 +260,11 @@ public: return iterator(node); } - iterator end() - { - return iterator(); - } + iterator end() { return iterator(); } protected: - - void insert_into(const K& key, const V& value, std::unique_ptr& node, - N* parent = nullptr) - { + void insert_into(const K &key, const V &value, std::unique_ptr &node, + N *parent = nullptr) { if (!node) { node = make_node(key, value); node->parent = parent; @@ -353,8 +284,7 @@ protected: } } - void erase_node(std::unique_ptr& node) - { + void erase_node(std::unique_ptr &node) { if (!node) { return; } @@ -364,20 +294,20 @@ protected: // 1. look for the in-order successor // 2. replace the node with the in-order successor // 3. erase the in-order successor - N* n = node.get(); - std::unique_ptr& successor = find_successor(node); + N *n = node.get(); + std::unique_ptr &successor = find_successor(node); swap_successor(node, successor); // we swapped the node with successor and the // 'successor' unique pointer holds now the node if (successor.get() == n) { erase_node(successor); - }// otherwise the successor was the right child of node, + } // otherwise the successor was the right child of node, // hence node should be now the right child of 'node' // unique pointer else if (node->right.get() == n) { erase_node(node->right); - }// there are no other cases so anything else is wrong + } // there are no other cases so anything else is wrong else { throw std::logic_error("Bad rbtree swap."); } @@ -388,8 +318,8 @@ protected: // node has at most one child // in this case simply replace the node with the // single child or null if there are no children - N* parent = node->parent; - std::unique_ptr& child = node->left ? node->left : node->right; + N *parent = node->parent; + std::unique_ptr &child = node->left ? node->left : node->right; colour_t old_colour = node->colour; if (child) { @@ -413,18 +343,18 @@ protected: rb_erase_case1(leaf_node_t(parent)); } } else if (node) - // if the node was red it has to have two BLACK children - // and since at most one of those children is a non-leaf - // child actually both have to be leafs (null) in order - // to satisfy the red-black tree invariant + // if the node was red it has to have two BLACK children + // and since at most one of those children is a non-leaf + // child actually both have to be leafs (null) in order + // to satisfy the red-black tree invariant { throw rb_invariant_error(); } } - template // make it a template so it works both for constant and mutable pointers - static PTR& find_in(const K& key, PTR& node) - { + template // make it a template so it works both for constant + // and mutable pointers + static PTR &find_in(const K &key, PTR &node) { if (!node) { return null_node; } @@ -440,9 +370,9 @@ protected: } } - template // make it a template so it works both for constant and mutable pointers - static PTR& find_min(PTR& node) - { + template // make it a template so it works both for constant + // and mutable pointers + static PTR &find_min(PTR &node) { if (!node) { return null_node; } @@ -454,9 +384,9 @@ protected: return node; } - template // make it a template so it works both for constant and mutable pointers - static PTR& find_successor(PTR& node) - { + template // make it a template so it works both for constant + // and mutable pointers + static PTR &find_successor(PTR &node) { if (!node) { return null_node; } @@ -464,27 +394,22 @@ protected: return find_min(node->right); } - static bool has_two(const N* node) - { - return node->left && node->right; - } + static bool has_two(const N *node) { return node->left && node->right; } //////////////////////////////////////////////////////////////////////////////////////////////////////////// - static void replace(std::unique_ptr& ptr, N* node) - { + static void replace(std::unique_ptr &ptr, N *node) { ptr.release(); ptr.reset(node); } - virtual void right_rotation(N* node) - { + virtual void right_rotation(N *node) { if (!node) { return; } - N* parent = node->parent; - N* left_child = node->left.release(); + N *parent = node->parent; + N *left_child = node->left.release(); bool is_left = (parent && parent->left.get() == node) ? true : false; node->left.reset(left_child->right.release()); @@ -509,14 +434,13 @@ protected: } } - virtual void left_rotation(N* node) - { + virtual void left_rotation(N *node) { if (!node) { return; } - N* parent = node->parent; - N* right_child = node->right.release(); + N *parent = node->parent; + N *right_child = node->right.release(); bool is_left = (parent && parent->left.get() == node) ? true : false; node->right.reset(right_child->left.release()); @@ -543,8 +467,7 @@ protected: //////////////////////////////////////////////////////////////////////////////////////////////////////////// - N* get_grandparent(N* node) - { + N *get_grandparent(N *node) { if (!node || !node->parent) { return nullptr; } @@ -552,9 +475,8 @@ protected: return node->parent->parent; } - N* get_uncle(N* node) - { - N* grandparent = get_grandparent(node); + N *get_uncle(N *node) { + N *grandparent = get_grandparent(node); if (!grandparent) { return nullptr; @@ -567,8 +489,7 @@ protected: } } - void rb_insert_case1(N* node) - { + void rb_insert_case1(N *node) { if (node->parent == nullptr) { // it is the root node->colour = BLACK; } else { @@ -576,8 +497,7 @@ protected: } } - void rb_insert_case2(N* node) - { + void rb_insert_case2(N *node) { if (node->parent->colour == BLACK) { return; // the invariant is OK } else { @@ -585,14 +505,13 @@ protected: } } - void rb_insert_case3(N* node) - { - N* uncle = get_uncle(node); + void rb_insert_case3(N *node) { + N *uncle = get_uncle(node); if (uncle && uncle->colour == RED) { node->parent->colour = BLACK; uncle->colour = BLACK; - N* grandparent = get_grandparent(node); + N *grandparent = get_grandparent(node); grandparent->colour = RED; rb_insert_case1(grandparent); } else { @@ -600,9 +519,8 @@ protected: } } - void rb_insert_case4(N* node) - { - N* grandparent = get_grandparent(node); + void rb_insert_case4(N *node) { + N *grandparent = get_grandparent(node); if ((node == node->parent->right.get()) && (node->parent == grandparent->left.get())) { @@ -617,9 +535,8 @@ protected: rb_insert_case5(node); } - void rb_insert_case5(N* node) - { - N* grandparent = get_grandparent(node); + void rb_insert_case5(N *node) { + N *grandparent = get_grandparent(node); node->parent->colour = BLACK; grandparent->colour = RED; @@ -632,21 +549,15 @@ protected: //////////////////////////////////////////////////////////////////////////////////////////////////////////// - template - static bool is_left(NODE node) - { + template static bool is_left(NODE node) { return node == node->parent->left.get(); } - template - static bool is_right(NODE node) - { + template static bool is_right(NODE node) { return node == node->parent->right.get(); } - template - N* get_sibling(NODE node) - { + template N *get_sibling(NODE node) { if (!node || !node->parent) { return nullptr; } @@ -658,18 +569,14 @@ protected: } } - template - void rb_erase_case1(NODE node) - { + template void rb_erase_case1(NODE node) { if (node->parent != nullptr) { rb_erase_case2(node); } } - template - void rb_erase_case2(NODE node) - { - N* sibling = get_sibling(node); + template void rb_erase_case2(NODE node) { + N *sibling = get_sibling(node); if (!sibling) { throw rb_invariant_error(); @@ -689,22 +596,20 @@ protected: rb_erase_case3(node); } - template - void rb_erase_case3(NODE node) - { - N* sibling = get_sibling(node); + template void rb_erase_case3(NODE node) { + N *sibling = get_sibling(node); if (!sibling) { throw rb_invariant_error(); } - colour_t sibling_left_colour = sibling->left ? sibling->left->colour : BLACK; - colour_t sibling_right_colour = sibling->right ? sibling->right->colour : BLACK; + colour_t sibling_left_colour = + sibling->left ? sibling->left->colour : BLACK; + colour_t sibling_right_colour = + sibling->right ? sibling->right->colour : BLACK; - if (node->parent->colour == BLACK && - sibling->colour == BLACK && - sibling_left_colour == BLACK && - sibling_right_colour == BLACK) { + if (node->parent->colour == BLACK && sibling->colour == BLACK && + sibling_left_colour == BLACK && sibling_right_colour == BLACK) { sibling->colour = RED; rb_erase_case1(node->parent); } else { @@ -712,22 +617,20 @@ protected: } } - template - void rb_erase_case4(NODE node) - { - N* sibling = get_sibling(node); + template void rb_erase_case4(NODE node) { + N *sibling = get_sibling(node); if (!sibling) { throw rb_invariant_error(); } - colour_t sibling_left_colour = sibling->left ? sibling->left->colour : BLACK; - colour_t sibling_right_colour = sibling->right ? sibling->right->colour : BLACK; + colour_t sibling_left_colour = + sibling->left ? sibling->left->colour : BLACK; + colour_t sibling_right_colour = + sibling->right ? sibling->right->colour : BLACK; - if (node->parent->colour == RED && - sibling->colour == BLACK && - sibling_left_colour == BLACK && - sibling_right_colour == BLACK) { + if (node->parent->colour == RED && sibling->colour == BLACK && + sibling_left_colour == BLACK && sibling_right_colour == BLACK) { sibling->colour = RED; node->parent->colour = BLACK; } else { @@ -735,21 +638,20 @@ protected: } } - template - void rb_erase_case5(NODE node) - { - N* sibling = get_sibling(node); + template void rb_erase_case5(NODE node) { + N *sibling = get_sibling(node); if (!sibling) { throw rb_invariant_error(); } - colour_t sibling_left_colour = sibling->left ? sibling->left->colour : BLACK; - colour_t sibling_right_colour = sibling->right ? sibling->right->colour : BLACK; + colour_t sibling_left_colour = + sibling->left ? sibling->left->colour : BLACK; + colour_t sibling_right_colour = + sibling->right ? sibling->right->colour : BLACK; if (sibling->colour == BLACK) { - if (is_left(node) && - sibling_right_colour == BLACK && + if (is_left(node) && sibling_right_colour == BLACK && sibling_left_colour == RED) { sibling->colour = RED; @@ -758,8 +660,7 @@ protected: } right_rotation(sibling); - } else if (is_right(node) && - sibling_left_colour == BLACK && + } else if (is_right(node) && sibling_left_colour == BLACK && sibling_right_colour == RED) { sibling->colour = RED; @@ -774,10 +675,8 @@ protected: rb_erase_case6(node); } - template - void rb_erase_case6(NODE node) - { - N* sibling = get_sibling(node); + template void rb_erase_case6(NODE node) { + N *sibling = get_sibling(node); if (!sibling) { throw rb_invariant_error(); @@ -805,5 +704,5 @@ protected: size_t tree_size; }; -template +template std::unique_ptr rbtree::null_node; diff --git a/src/XrdApps/XrdClJCachePlugin/file/Art.hh b/src/XrdApps/XrdClJCachePlugin/file/Art.hh index e8ce0fbc760..342f08f21b5 100644 --- a/src/XrdApps/XrdClJCachePlugin/file/Art.hh +++ b/src/XrdApps/XrdClJCachePlugin/file/Art.hh @@ -24,82 +24,89 @@ #pragma once /*----------------------------------------------------------------------------*/ -#include -#include -#include #include +#include +#include #include +#include /*----------------------------------------------------------------------------*/ namespace JCache { - class Art { - public: - Art() {} - virtual ~Art() {} +class Art { +public: + Art() {} + virtual ~Art() {} - void drawCurve(const std::vector& dataPoints, double runtime) { - if (dataPoints.size() != 40) { - std::cerr << "Error: Exactly 40 data points are required." << std::endl; - return; - } - - double maxValue = *std::max_element(dataPoints.begin(), dataPoints.end()); - double minValue = *std::min_element(dataPoints.begin(), dataPoints.end()); - - const int plotHeight = 10; // Number of lines in the plot - const int plotWidth = 40; // Width of the plot in characters - const int yLegendWidth = 8; // Width of the Y legend in characters - - std::vector plot(plotHeight, std::string(plotWidth, ' ')); - - // Normalize data points to the plot height - std::vector normalizedDataPoints; - for (double point : dataPoints) { - int normalizedValue = static_cast((point - minValue) / (maxValue - minValue) * (plotHeight - 1)); - if (normalizedValue<0){ - normalizedValue=0; - } - normalizedDataPoints.push_back(normalizedValue); - } + void drawCurve(const std::vector &dataPoints, double runtime) { + if (dataPoints.size() != 40) { + std::cerr << "Error: Exactly 40 data points are required." << std::endl; + return; + } - // Draw the curve - for (size_t i = 0; i < normalizedDataPoints.size(); ++i) { - int y = plotHeight - 1 - normalizedDataPoints[i]; - plot[y][i * (plotWidth / (dataPoints.size() - 1))] = '*'; - } - - // Print the plot with Y legend - for (int i = 0; i < plotHeight; ++i) { - double yValue = minValue + (maxValue - minValue) * (plotHeight - 1 - i) / (plotHeight - 1); - if (i==0) { - std::cerr << "# " << std::setw(yLegendWidth) << std::fixed << std::setprecision(2) << yValue << " MB/s | "; - } else { - std::cerr << "# " << std::setw(yLegendWidth) << std::fixed << std::setprecision(2) << yValue << " | "; - } - std::cerr << plot[i] << std::endl; - } + double maxValue = *std::max_element(dataPoints.begin(), dataPoints.end()); + double minValue = *std::min_element(dataPoints.begin(), dataPoints.end()); + + const int plotHeight = 10; // Number of lines in the plot + const int plotWidth = 40; // Width of the plot in characters + const int yLegendWidth = 8; // Width of the Y legend in characters - // Print the X axis - std::cerr << "# " << std::string(yLegendWidth + 7, ' ') << std::string(plotWidth, '-') << std::endl; - std::cerr << "# " << std::string(yLegendWidth + 7, ' '); - for (size_t i = 0 ; i < dataPoints.size()/4; ++i) { - std::cerr << std::fixed << std::setw(4) << std::left << (i*10); + std::vector plot(plotHeight, std::string(plotWidth, ' ')); + + // Normalize data points to the plot height + std::vector normalizedDataPoints; + for (double point : dataPoints) { + int normalizedValue = static_cast( + (point - minValue) / (maxValue - minValue) * (plotHeight - 1)); + if (normalizedValue < 0) { + normalizedValue = 0; } - std::cerr << "[ " << 100 << " % = " << std::fixed << std::setprecision(2) << runtime << "s ]"<< std::endl; + normalizedDataPoints.push_back(normalizedValue); } - void drawCurve(const std::vector& data, double interval, double runtime) { - std::vector newdata; - if (interval == 0) { - interval = 0.00001; - } - if (runtime == 0) { - runtime = 0.00001; - } - for ( auto i:data ) { - newdata.push_back(i/1000000.0 / interval); + // Draw the curve + for (size_t i = 0; i < normalizedDataPoints.size(); ++i) { + int y = plotHeight - 1 - normalizedDataPoints[i]; + plot[y][i * (plotWidth / (dataPoints.size() - 1))] = '*'; + } + + // Print the plot with Y legend + for (int i = 0; i < plotHeight; ++i) { + double yValue = minValue + (maxValue - minValue) * (plotHeight - 1 - i) / + (plotHeight - 1); + if (i == 0) { + std::cerr << "# " << std::setw(yLegendWidth) << std::fixed + << std::setprecision(2) << yValue << " MB/s | "; + } else { + std::cerr << "# " << std::setw(yLegendWidth) << std::fixed + << std::setprecision(2) << yValue << " | "; } - return drawCurve(newdata, runtime); + std::cerr << plot[i] << std::endl; + } + + // Print the X axis + std::cerr << "# " << std::string(yLegendWidth + 7, ' ') + << std::string(plotWidth, '-') << std::endl; + std::cerr << "# " << std::string(yLegendWidth + 7, ' '); + for (size_t i = 0; i < dataPoints.size() / 4; ++i) { + std::cerr << std::fixed << std::setw(4) << std::left << (i * 10); + } + std::cerr << "[ " << 100 << " % = " << std::fixed << std::setprecision(2) + << runtime << "s ]" << std::endl; + } + + void drawCurve(const std::vector &data, double interval, + double runtime) { + std::vector newdata; + if (interval == 0) { + interval = 0.00001; + } + if (runtime == 0) { + runtime = 0.00001; + } + for (auto i : data) { + newdata.push_back(i / 1000000.0 / interval); } - }; + return drawCurve(newdata, runtime); + } +}; } // namespace JCache diff --git a/src/XrdApps/XrdClJCachePlugin/file/CacheStats.hh b/src/XrdApps/XrdClJCachePlugin/file/CacheStats.hh index c8f4a80c578..b1d9164890a 100644 --- a/src/XrdApps/XrdClJCachePlugin/file/CacheStats.hh +++ b/src/XrdApps/XrdClJCachePlugin/file/CacheStats.hh @@ -33,257 +33,324 @@ #include #include +#include "file/XrdClJCacheFile.hh" +#include #include +#include +#include #include #include -#include -#include -#include -#include "file/XrdClJCacheFile.hh" - +namespace JCache { +//! structure for cache hit statistics +struct CacheStats { + CacheStats(bool doe = false) + : bytesRead(0), bytesReadV(0), bytesCached(0), bytesCachedV(0), + readOps(0), readVOps(0), readVreadOps(0), nreadfiles(0), + dumponexit(doe), peakrate(0) { + // Get the current real time + struct timeval now; + gettimeofday(&now, nullptr); + startTime = now.tv_sec + now.tv_usec / 1000000.0; + } -namespace JCache -{ - //! structure for cache hit statistics - struct CacheStats { - CacheStats(bool doe=false) : - bytesRead(0), - bytesReadV(0), - bytesCached(0), - bytesCachedV(0), - readOps(0), - readVOps(0), - readVreadOps(0), - nreadfiles(0), - dumponexit(doe), - peakrate(0) - { - // Get the current real time - struct timeval now; - gettimeofday(&now, nullptr); - startTime = now.tv_sec + now.tv_usec / 1000000.0; - } - - ~CacheStats() { - if (dumponexit.load() && totaldatasize) { - using namespace std::chrono; - std::string jsonpath = XrdCl::JCacheFile::sJsonPath + "jcache."; - std::string name = getenv("XRD_APPNAME")?getenv("XRD_APPNAME"):"none"+std::string(".")+std::to_string(getpid()); - jsonpath += name; - jsonpath += ".json"; - XrdCl::JCacheFile::sStats.GetTimes(); + ~CacheStats() { + if (dumponexit.load() && totaldatasize) { + using namespace std::chrono; + std::string jsonpath = XrdCl::JCacheFile::sJsonPath + "jcache."; + std::string name = getenv("XRD_APPNAME") ? getenv("XRD_APPNAME") + : "none" + std::string(".") + + std::to_string(getpid()); + jsonpath += name; + jsonpath += ".json"; + XrdCl::JCacheFile::sStats.GetTimes(); - XrdCl::JCacheFile::sStats.bytes_per_second = XrdCl::JCacheFile::sStats.bench.GetBins((int)(realTime)); - XrdCl::JCacheFile::sStats.peakrate = *(std::max_element(XrdCl::JCacheFile::sStats.bytes_per_second.begin(), XrdCl::JCacheFile::sStats.bytes_per_second.end())); - if (realTime <1) { - XrdCl::JCacheFile::sStats.peakrate = ReadBytes() / realTime; - } - if (XrdCl::JCacheFile::sJsonPath.length()) { - XrdCl::JCacheFile::sStats.persistToJson(jsonpath, name); - } - if (XrdCl::JCacheFile::sEnableSummary) { - std::cerr << CacheStats::GlobalStats(XrdCl::JCacheFile::sStats); - } - std::vector bins = XrdCl::JCacheFile::sStats.bench.GetBins(40); - JCache::Art art; - if (XrdCl::JCacheFile::sEnableSummary) { - std::cerr << "# IO Timeprofile " << std::endl; - art.drawCurve(bins, XrdCl::JCacheFile::sStats.bench.GetTimePerBin().count() / 1000000.0, realTime); - std::cerr << "# ----------------------------------------------------------------------- #" << std::endl; - } + XrdCl::JCacheFile::sStats.bytes_per_second = + XrdCl::JCacheFile::sStats.bench.GetBins((int)(realTime)); + XrdCl::JCacheFile::sStats.peakrate = + *(std::max_element(XrdCl::JCacheFile::sStats.bytes_per_second.begin(), + XrdCl::JCacheFile::sStats.bytes_per_second.end())); + if (realTime < 1) { + XrdCl::JCacheFile::sStats.peakrate = ReadBytes() / realTime; } - } - - static std::string bytesToHumanReadable(double bytes) { - const char* suffixes[] = {"B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}; - const int numSuffixes = sizeof(suffixes) / sizeof(suffixes[0]); - - if (bytes == 0) return "0 B"; - - int exp = std::min((int)(std::log(bytes) / std::log(1000)), numSuffixes - 1); - double val = bytes / std::pow(1000, exp); - std::ostringstream oss; - oss << std::fixed << std::setprecision(2) << val << " " << suffixes[exp]; - return oss.str(); - } - - uint64_t ReadOpBytes() { - return this->bytesCached.load()+this->bytesRead.load(); - } - - uint64_t ReadVOpBytes() { - return this->bytesCachedV.load()+this->bytesReadV.load(); - } - double HitRate() { - auto n = this->bytesCached.load()+this->bytesRead.load(); - if (!n) return 100.0; - return 100.0*(this->bytesCached.load()) / n; - } - double HitRateV() { - auto n = this->bytesCachedV.load()+this->bytesReadV.load(); - if (!n) return 100.0; - return 100.0*(this->bytesCachedV.load()) / n; - } - double CombinedHitRate() { - auto n = (this->bytesCached.load()+this->bytesRead.load()+this->bytesCachedV.load()+this->bytesReadV.load()); - if (!n) return 100.0; - return 100.0*(this->bytesCached.load()+this->bytesCachedV.load()) / n; - } - void AddUrl(const std::string& url) { - std::lock_guard guard(urlMutex); - urls.insert(url); - } - bool HasUrl(const std::string& url) { - std::lock_guard guard(urlMutex); - return urls.count(url); - } - double ReadBytes() { - return (bytesRead.load()+bytesReadV.load() + bytesCached.load() + bytesCachedV.load()); - } - - double Used() { - if (totaldatasize) { - return 100.0*(bytesRead.load()+bytesReadV.load() + bytesCached.load() + bytesCachedV.load()) / totaldatasize; - } else { - return 100.0; + if (XrdCl::JCacheFile::sJsonPath.length()) { + XrdCl::JCacheFile::sStats.persistToJson(jsonpath, name); + } + if (XrdCl::JCacheFile::sEnableSummary) { + std::cerr << CacheStats::GlobalStats(XrdCl::JCacheFile::sStats); + } + std::vector bins = XrdCl::JCacheFile::sStats.bench.GetBins(40); + JCache::Art art; + if (XrdCl::JCacheFile::sEnableSummary) { + std::cerr << "# IO Timeprofile " << std::endl; + art.drawCurve(bins, + XrdCl::JCacheFile::sStats.bench.GetTimePerBin().count() / + 1000000.0, + realTime); + std::cerr << "# " + "---------------------------------------------------------" + "-------------- #" + << std::endl; } } + } - size_t UniqueUrls() { - std::lock_guard guard(urlMutex); - return urls.size(); - } + static std::string bytesToHumanReadable(double bytes) { + const char *suffixes[] = {"B", "KB", "MB", "GB", "TB", + "PB", "EB", "ZB", "YB"}; + const int numSuffixes = sizeof(suffixes) / sizeof(suffixes[0]); + + if (bytes == 0) + return "0 B"; - void GetTimes() { - struct rusage usage; - struct timeval now; + int exp = + std::min((int)(std::log(bytes) / std::log(1000)), numSuffixes - 1); + double val = bytes / std::pow(1000, exp); + std::ostringstream oss; + oss << std::fixed << std::setprecision(2) << val << " " << suffixes[exp]; + return oss.str(); + } - // Get the current real time - gettimeofday(&now, nullptr); - realTime = now.tv_sec + now.tv_usec / 1000000.0 - startTime; + uint64_t ReadOpBytes() { + return this->bytesCached.load() + this->bytesRead.load(); + } - // Get resource usage - getrusage(RUSAGE_SELF, &usage); + uint64_t ReadVOpBytes() { + return this->bytesCachedV.load() + this->bytesReadV.load(); + } + double HitRate() { + auto n = this->bytesCached.load() + this->bytesRead.load(); + if (!n) + return 100.0; + return 100.0 * (this->bytesCached.load()) / n; + } + double HitRateV() { + auto n = this->bytesCachedV.load() + this->bytesReadV.load(); + if (!n) + return 100.0; + return 100.0 * (this->bytesCachedV.load()) / n; + } + double CombinedHitRate() { + auto n = (this->bytesCached.load() + this->bytesRead.load() + + this->bytesCachedV.load() + this->bytesReadV.load()); + if (!n) + return 100.0; + return 100.0 * (this->bytesCached.load() + this->bytesCachedV.load()) / n; + } + void AddUrl(const std::string &url) { + std::lock_guard guard(urlMutex); + urls.insert(url); + } + bool HasUrl(const std::string &url) { + std::lock_guard guard(urlMutex); + return urls.count(url); + } + double ReadBytes() { + return (bytesRead.load() + bytesReadV.load() + bytesCached.load() + + bytesCachedV.load()); + } - // Get user and system time - userTime = usage.ru_utime.tv_sec + usage.ru_utime.tv_usec / 1000000.0; - sysTime = usage.ru_stime.tv_sec + usage.ru_stime.tv_usec / 1000000.0; + double Used() { + if (totaldatasize) { + return 100.0 * + (bytesRead.load() + bytesReadV.load() + bytesCached.load() + + bytesCachedV.load()) / + totaldatasize; + } else { + return 100.0; } + } - void persistToJson(const std::string& path, const std::string& name) { - std::ofstream outFile(path); - if (!outFile.is_open()) { - std::cerr << "error: failed to open JSON statistics file: " << path << std::endl; - return; - } + size_t UniqueUrls() { + std::lock_guard guard(urlMutex); + return urls.size(); + } - outFile << "{\n"; - outFile << " \"appname\": \"" << name << "\",\n"; - outFile << " \"pid\": \"" << getpid() << "\",\n"; - outFile << " \"bytesRead\": " << bytesRead.load() << ",\n"; - outFile << " \"bytesReadV\": " << bytesReadV.load() << ",\n"; - outFile << " \"bytesCached\": " << bytesCached.load() << ",\n"; - outFile << " \"bytesCachedV\": " << bytesCachedV.load() << ",\n"; - outFile << " \"readOps\": " << readOps.load() << ",\n"; - outFile << " \"readVOps\": " << readVOps.load() << ",\n"; - outFile << " \"readVreadOps\": " << readVreadOps.load() << ",\n"; - outFile << " \"nreadfiles\": " << nreadfiles.load() << ",\n"; - outFile << " \"totaldatasize\": " << totaldatasize.load() << ",\n"; + void GetTimes() { + struct rusage usage; + struct timeval now; - std::lock_guard lock(urlMutex); - outFile << " \"urls\": ["; - for (auto it = urls.begin(); it != urls.end(); ++it) { - if (it != urls.begin()) { - outFile << ", "; - } - outFile << "\"" << *it << "\""; - } - outFile << "],\n"; + // Get the current real time + gettimeofday(&now, nullptr); + realTime = now.tv_sec + now.tv_usec / 1000000.0 - startTime; - outFile << " \"bytes_per_second\": ["; - for (size_t i = 0; i < bytes_per_second.size(); ++i) { - if (i != 0) { - outFile << ", "; - } - outFile << bytes_per_second[i]; - } - outFile << "],\n"; + // Get resource usage + getrusage(RUSAGE_SELF, &usage); - outFile << std::fixed << std::setprecision(6); // Set precision for double values + // Get user and system time + userTime = usage.ru_utime.tv_sec + usage.ru_utime.tv_usec / 1000000.0; + sysTime = usage.ru_stime.tv_sec + usage.ru_stime.tv_usec / 1000000.0; + } - outFile << " \"userTime\": " << userTime.load() << ",\n"; - outFile << " \"realTime\": " << realTime.load() << ",\n"; - outFile << " \"sysTime\": " << sysTime.load() << ",\n"; - outFile << " \"startTime\": " << startTime.load() << "\n"; - outFile << "}\n"; + void persistToJson(const std::string &path, const std::string &name) { + std::ofstream outFile(path); + if (!outFile.is_open()) { + std::cerr << "error: failed to open JSON statistics file: " << path + << std::endl; + return; + } + + outFile << "{\n"; + outFile << " \"appname\": \"" << name << "\",\n"; + outFile << " \"pid\": \"" << getpid() << "\",\n"; + outFile << " \"bytesRead\": " << bytesRead.load() << ",\n"; + outFile << " \"bytesReadV\": " << bytesReadV.load() << ",\n"; + outFile << " \"bytesCached\": " << bytesCached.load() << ",\n"; + outFile << " \"bytesCachedV\": " << bytesCachedV.load() << ",\n"; + outFile << " \"readOps\": " << readOps.load() << ",\n"; + outFile << " \"readVOps\": " << readVOps.load() << ",\n"; + outFile << " \"readVreadOps\": " << readVreadOps.load() << ",\n"; + outFile << " \"nreadfiles\": " << nreadfiles.load() << ",\n"; + outFile << " \"totaldatasize\": " << totaldatasize.load() << ",\n"; - outFile.close(); + std::lock_guard lock(urlMutex); + outFile << " \"urls\": ["; + for (auto it = urls.begin(); it != urls.end(); ++it) { + if (it != urls.begin()) { + outFile << ", "; + } + outFile << "\"" << *it << "\""; } + outFile << "],\n"; - void AddToStats(CacheStats& gStats) { - gStats.readOps += readOps.load(); - gStats.readVOps += readVOps.load(); - gStats.readVreadOps += readVreadOps.load(); - gStats.bytesRead += bytesRead.load(); - gStats.bytesReadV += bytesReadV.load(); - gStats.bytesCached += bytesCached.load(); - gStats.bytesCachedV += bytesCachedV.load(); - gStats.nreadfiles += 1; + outFile << " \"bytes_per_second\": ["; + for (size_t i = 0; i < bytes_per_second.size(); ++i) { + if (i != 0) { + outFile << ", "; + } + outFile << bytes_per_second[i]; } + outFile << "],\n"; - static std::string GlobalStats(CacheStats& sStats) { - std::ostringstream oss; - oss << "# ----------------------------------------------------------------------- #" << std::endl; - oss << "# JCache : 2024 CERN.EOS - Andreas-Joachim Peters #" << std::endl; - oss << "# ----------------------------------------------------------------------- #" << std::endl; + outFile << std::fixed + << std::setprecision(6); // Set precision for double values - oss << "# JCache : cache combined hit rate : " << std::fixed << std::setprecision(2) << sStats.CombinedHitRate() << " %" << std::endl; - oss << "# JCache : cache read hit rate : " << std::fixed << std::setprecision(2) << (!sStats.ReadOpBytes()?"\033[9m":"") << sStats.HitRate() << " %" << (!sStats.ReadOpBytes()?"\033[0m":"") << std::endl; - oss << "# JCache : cache readv hit rate : " << std::fixed << std::setprecision(2) << (!sStats.ReadVOpBytes()?"\033[9m":"") < bytesRead; - std::atomic bytesReadV; - std::atomic bytesCached; - std::atomic bytesCachedV; - std::atomic readOps; - std::atomic readVOps; - std::atomic readVreadOps; - std::atomic nreadfiles; - std::atomic totaldatasize; - std::atomic dumponexit; - std::set urls; - std::mutex urlMutex; - std::atomic userTime; - std::atomic realTime; - std::atomic sysTime; - std::atomic startTime; - JCache::TimeBench bench; - std::vector bytes_per_second; - std::atomic peakrate; - }; // class CacheStats + std::atomic bytesRead; + std::atomic bytesReadV; + std::atomic bytesCached; + std::atomic bytesCachedV; + std::atomic readOps; + std::atomic readVOps; + std::atomic readVreadOps; + std::atomic nreadfiles; + std::atomic totaldatasize; + std::atomic dumponexit; + std::set urls; + std::mutex urlMutex; + std::atomic userTime; + std::atomic realTime; + std::atomic sysTime; + std::atomic startTime; + JCache::TimeBench bench; + std::vector bytes_per_second; + std::atomic peakrate; +}; // class CacheStats } // namespace JCache diff --git a/src/XrdApps/XrdClJCachePlugin/file/TimeBench.hh b/src/XrdApps/XrdClJCachePlugin/file/TimeBench.hh index 09502a10699..6007559c41a 100644 --- a/src/XrdApps/XrdClJCachePlugin/file/TimeBench.hh +++ b/src/XrdApps/XrdClJCachePlugin/file/TimeBench.hh @@ -23,72 +23,71 @@ #pragma once -#include -#include +#include #include +#include #include -#include +#include + +namespace JCache { +class TimeBench { +private: + using Clock = std::chrono::high_resolution_clock; + using TimePoint = std::chrono::time_point; + using Duration = std::chrono::microseconds; -namespace JCache -{ - class TimeBench { - private: - using Clock = std::chrono::high_resolution_clock; - using TimePoint = std::chrono::time_point; - using Duration = std::chrono::microseconds; + std::vector> measurements; + std::vector bins; + TimePoint start; + TimePoint end; + uint64_t totalBytes; + size_t nbins; + std::mutex mtx; - std::vector> measurements; - std::vector bins; - TimePoint start; - TimePoint end; - uint64_t totalBytes; - size_t nbins; - std::mutex mtx; - - public: - TimeBench() : totalBytes(0), nbins(10) {} +public: + TimeBench() : totalBytes(0), nbins(10) {} - void AddMeasurement(uint64_t bytes) { - std::lock_guard guard(mtx); - auto now = Clock::now(); - if (measurements.empty()) { - start = now; - } - measurements.push_back(std::make_pair(now,bytes)); - totalBytes += bytes; - end = now; - } + void AddMeasurement(uint64_t bytes) { + std::lock_guard guard(mtx); + auto now = Clock::now(); + if (measurements.empty()) { + start = now; + } + measurements.push_back(std::make_pair(now, bytes)); + totalBytes += bytes; + end = now; + } - std::vector GetBins(size_t bin = 10) { - std::lock_guard guard(mtx); - nbins = bin?bin:1; - Duration totalTime = std::chrono::duration_cast(end - start); - Duration binSize = totalTime / nbins; - bins.clear(); - bins.resize(nbins, 0); - std::fill(bins.begin(), bins.end(), 0); - size_t binIndex = 0; + std::vector GetBins(size_t bin = 10) { + std::lock_guard guard(mtx); + nbins = bin ? bin : 1; + Duration totalTime = std::chrono::duration_cast(end - start); + Duration binSize = totalTime / nbins; + bins.clear(); + bins.resize(nbins, 0); + std::fill(bins.begin(), bins.end(), 0); + size_t binIndex = 0; - for (auto i : measurements) { - if (binSize.count()) { - binIndex = (i.first - start)/ binSize; - } else { - binIndex = 0; - } - if (binIndex < nbins) { - bins[binIndex] += i.second; - } else { - break; // Don't process future measurements - } - } + for (auto i : measurements) { + if (binSize.count()) { + binIndex = (i.first - start) / binSize; + } else { + binIndex = 0; + } + if (binIndex < nbins) { + bins[binIndex] += i.second; + } else { + break; // Don't process future measurements + } + } - return bins; - } + return bins; + } - Duration GetTimePerBin() { - Duration totalTime = std::chrono::duration_cast(end - start); - Duration binSize = totalTime / nbins; - return binSize; - } - }; + Duration GetTimePerBin() { + Duration totalTime = std::chrono::duration_cast(end - start); + Duration binSize = totalTime / nbins; + return binSize; + } +}; } // namespace JCache diff --git a/src/XrdApps/XrdClJCachePlugin/file/XrdClJCacheFile.cc b/src/XrdApps/XrdClJCachePlugin/file/XrdClJCacheFile.cc index a1b90805c41..0d7250cd9bf 100644 --- a/src/XrdApps/XrdClJCachePlugin/file/XrdClJCacheFile.cc +++ b/src/XrdApps/XrdClJCachePlugin/file/XrdClJCacheFile.cc @@ -28,8 +28,8 @@ #include "XrdCl/XrdClMessageUtils.hh" /*----------------------------------------------------------------------------*/ -std::string XrdCl::JCacheFile::sCachePath=""; -std::string XrdCl::JCacheFile::sJsonPath="./"; +std::string XrdCl::JCacheFile::sCachePath = ""; +std::string XrdCl::JCacheFile::sJsonPath = "./"; bool XrdCl::JCacheFile::sEnableJournalCache = true; bool XrdCl::JCacheFile::sEnableVectorCache = false; bool XrdCl::JCacheFile::sEnableSummary = true; @@ -37,37 +37,28 @@ JCache::CacheStats XrdCl::JCacheFile::sStats(true); JournalManager XrdCl::JCacheFile::sJournalManager; -namespace XrdCl -{ +namespace XrdCl { //------------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------------ -JCacheFile::JCacheFile(const std::string& url): - mIsOpen(false), - pFile(0) -{ +JCacheFile::JCacheFile(const std::string &url) : mIsOpen(false), pFile(0) { mAttachedForRead = false; mLog = DefaultEnv::GetLog(); } //------------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------------ -JCacheFile::JCacheFile(): - mIsOpen(false), - pFile(0) -{ +JCacheFile::JCacheFile() : mIsOpen(false), pFile(0) { mAttachedForRead = false; mLog = DefaultEnv::GetLog(); pStats = new JCache::CacheStats(); } - //------------------------------------------------------------------------------ // Destructor //------------------------------------------------------------------------------ -JCacheFile::~JCacheFile() -{ +JCacheFile::~JCacheFile() { LogStats(); pStats->AddToStats(sStats); if (pFile) { @@ -78,23 +69,18 @@ JCacheFile::~JCacheFile() } } - //------------------------------------------------------------------------------ // Open //------------------------------------------------------------------------------ -XRootDStatus -JCacheFile::Open(const std::string& url, - OpenFlags::Flags flags, - Access::Mode mode, - ResponseHandler* handler, - uint16_t timeout) -{ +XRootDStatus JCacheFile::Open(const std::string &url, OpenFlags::Flags flags, + Access::Mode mode, ResponseHandler *handler, + uint16_t timeout) { XRootDStatus st; mFlags = flags; if (mIsOpen) { st = XRootDStatus(stError, errInvalidOp); - std::cerr << "error: file is already opened: " << pUrl << std::endl; + std::cerr << "error: file is already opened: " << pUrl << std::endl; return st; } @@ -113,33 +99,31 @@ JCacheFile::Open(const std::string& url, mIsOpen = true; if (sEnableVectorCache || sEnableJournalCache) { if ((flags & OpenFlags::Flags::Read) == OpenFlags::Flags::Read) { - std::string JournalDir = sCachePath + "/" + VectorCache::computeSHA256(pUrl); - pJournalPath = JournalDir + "/journal"; - // it can be that we cannot write the journal directory - if (!VectorCache::ensureLastSubdirectoryExists(JournalDir)) { - st = XRootDStatus(stError, errOSError); - std::cerr << "error: unable to create cache directory: " << JournalDir << std::endl; - return st; - } + std::string JournalDir = + sCachePath + "/" + VectorCache::computeSHA256(pUrl); + pJournalPath = JournalDir + "/journal"; + // it can be that we cannot write the journal directory + if (!VectorCache::ensureLastSubdirectoryExists(JournalDir)) { + st = XRootDStatus(stError, errOSError); + std::cerr << "error: unable to create cache directory: " << JournalDir + << std::endl; + return st; + } } } } return st; } - //------------------------------------------------------------------------------ // Close //------------------------------------------------------------------------------ -XRootDStatus -JCacheFile::Close(ResponseHandler* handler, - uint16_t timeout) -{ +XRootDStatus JCacheFile::Close(ResponseHandler *handler, uint16_t timeout) { XRootDStatus st; if (mIsOpen) { mIsOpen = false; - pUrl = ""; + pUrl = ""; if (pFile) { st = pFile->Close(handler, timeout); } else { @@ -155,15 +139,11 @@ JCacheFile::Close(ResponseHandler* handler, return st; } - //------------------------------------------------------------------------------ // Stat //------------------------------------------------------------------------------ -XRootDStatus -JCacheFile::Stat(bool force, - ResponseHandler* handler, - uint16_t timeout) -{ +XRootDStatus JCacheFile::Stat(bool force, ResponseHandler *handler, + uint16_t timeout) { XRootDStatus st; if (pFile) { @@ -175,17 +155,11 @@ JCacheFile::Stat(bool force, return st; } - //------------------------------------------------------------------------------ // Read //------------------------------------------------------------------------------ -XRootDStatus -JCacheFile::Read(uint64_t offset, - uint32_t size, - void* buffer, - ResponseHandler* handler, - uint16_t timeout) -{ +XRootDStatus JCacheFile::Read(uint64_t offset, uint32_t size, void *buffer, + ResponseHandler *handler, uint16_t timeout) { XRootDStatus st; if (pFile) { @@ -193,13 +167,13 @@ JCacheFile::Read(uint64_t offset, if (sEnableJournalCache && AttachForRead()) { bool eof = false; auto rb = pJournal->pread(buffer, size, offset, eof); - if ((rb == size) || (eof && rb)) { + if ((rb == size) || (eof && rb)) { pStats->bytesCached += rb; - pStats->readOps++; + pStats->readOps++; // we can only serve success full reads from the cache for now - XRootDStatus* ret_st = new XRootDStatus(st); - ChunkInfo* chunkInfo = new ChunkInfo(offset, rb, buffer); - AnyObject* obj = new AnyObject(); + XRootDStatus *ret_st = new XRootDStatus(st); + ChunkInfo *chunkInfo = new ChunkInfo(offset, rb, buffer); + AnyObject *obj = new AnyObject(); obj->Set(chunkInfo); handler->HandleResponse(ret_st, obj); st = XRootDStatus(stOK, 0); @@ -207,26 +181,23 @@ JCacheFile::Read(uint64_t offset, } } - auto jhandler = new JCacheReadHandler(handler, &pStats->bytesRead,sEnableJournalCache?pJournal.get():nullptr); + auto jhandler = + new JCacheReadHandler(handler, &pStats->bytesRead, + sEnableJournalCache ? pJournal.get() : nullptr); pStats->readOps++; - st = pFile->Read(offset, size, buffer, jhandler, timeout); + st = pFile->Read(offset, size, buffer, jhandler, timeout); } else { st = XRootDStatus(stError, errInvalidOp); } return st; } - //------------------------------------------------------------------------------ // Write //------------------------------------------------------------------------------ -XRootDStatus -JCacheFile::Write(uint64_t offset, - uint32_t size, - const void* buffer, - ResponseHandler* handler, - uint16_t timeout) -{ +XRootDStatus JCacheFile::Write(uint64_t offset, uint32_t size, + const void *buffer, ResponseHandler *handler, + uint16_t timeout) { XRootDStatus st; if (pFile) { @@ -241,26 +212,21 @@ JCacheFile::Write(uint64_t offset, //------------------------------------------------------------------------ //! PgRead //------------------------------------------------------------------------ -XRootDStatus -JCacheFile::PgRead( uint64_t offset, - uint32_t size, - void *buffer, - ResponseHandler *handler, - uint16_t timeout ) -{ +XRootDStatus JCacheFile::PgRead(uint64_t offset, uint32_t size, void *buffer, + ResponseHandler *handler, uint16_t timeout) { XRootDStatus st; if (pFile) { sStats.bench.AddMeasurement(size); if (sEnableJournalCache && AttachForRead()) { bool eof = false; auto rb = pJournal->pread(buffer, size, offset, eof); - if ((rb == size) || (eof && rb)) { + if ((rb == size) || (eof && rb)) { pStats->bytesCached += rb; - pStats->readOps++; + pStats->readOps++; // we can only serve success full reads from the cache for now - XRootDStatus* ret_st = new XRootDStatus(st); - ChunkInfo* chunkInfo = new ChunkInfo(offset, rb, buffer); - AnyObject* obj = new AnyObject(); + XRootDStatus *ret_st = new XRootDStatus(st); + ChunkInfo *chunkInfo = new ChunkInfo(offset, rb, buffer); + AnyObject *obj = new AnyObject(); obj->Set(chunkInfo); handler->HandleResponse(ret_st, obj); st = XRootDStatus(stOK, 0); @@ -268,27 +234,24 @@ JCacheFile::PgRead( uint64_t offset, } } - auto jhandler = new JCachePgReadHandler(handler, &pStats->bytesRead,sEnableJournalCache?pJournal.get():nullptr); + auto jhandler = + new JCachePgReadHandler(handler, &pStats->bytesRead, + sEnableJournalCache ? pJournal.get() : nullptr); pStats->readOps++; - st = pFile->PgRead(offset, size, buffer, jhandler, timeout); + st = pFile->PgRead(offset, size, buffer, jhandler, timeout); } else { st = XRootDStatus(stError, errInvalidOp); } return st; } - //------------------------------------------------------------------------ //! PgWrite //------------------------------------------------------------------------ -XRootDStatus -JCacheFile::PgWrite( uint64_t offset, - uint32_t nbpgs, - const void *buffer, - std::vector &cksums, - ResponseHandler *handler, - uint16_t timeout ) -{ +XRootDStatus JCacheFile::PgWrite(uint64_t offset, uint32_t nbpgs, + const void *buffer, + std::vector &cksums, + ResponseHandler *handler, uint16_t timeout) { XRootDStatus st; if (pFile) { @@ -303,10 +266,7 @@ JCacheFile::PgWrite( uint64_t offset, //------------------------------------------------------------------------------ // Sync //------------------------------------------------------------------------------ -XRootDStatus -JCacheFile::Sync(ResponseHandler* handler, - uint16_t timeout) -{ +XRootDStatus JCacheFile::Sync(ResponseHandler *handler, uint16_t timeout) { XRootDStatus st; if (pFile) { @@ -318,15 +278,11 @@ JCacheFile::Sync(ResponseHandler* handler, return st; } - //------------------------------------------------------------------------------ // Truncate //------------------------------------------------------------------------------ -XRootDStatus -JCacheFile::Truncate(uint64_t size, - ResponseHandler* handler, - uint16_t timeout) -{ +XRootDStatus JCacheFile::Truncate(uint64_t size, ResponseHandler *handler, + uint16_t timeout) { XRootDStatus st; if (pFile) { @@ -338,16 +294,12 @@ JCacheFile::Truncate(uint64_t size, return st; } - //------------------------------------------------------------------------------ // VectorRead //------------------------------------------------------------------------------ -XRootDStatus -JCacheFile::VectorRead(const ChunkList& chunks, - void* buffer, - ResponseHandler* handler, - uint16_t timeout) -{ +XRootDStatus JCacheFile::VectorRead(const ChunkList &chunks, void *buffer, + ResponseHandler *handler, + uint16_t timeout) { XRootDStatus st; if (pFile) { @@ -357,66 +309,72 @@ JCacheFile::VectorRead(const ChunkList& chunks, } sStats.bench.AddMeasurement(len); - + if (sEnableVectorCache) { - VectorCache cache(chunks, pUrl, buffer?(char*)buffer:(char*)(chunks.begin()->buffer), sCachePath); + VectorCache cache(chunks, pUrl, + buffer ? (char *)buffer + : (char *)(chunks.begin()->buffer), + sCachePath); if (cache.retrieve()) { - XRootDStatus* ret_st = new XRootDStatus(st); - *ret_st = XRootDStatus(stOK, 0); - AnyObject* obj = new AnyObject(); - VectorReadInfo* vReadInfo = new VectorReadInfo(); + XRootDStatus *ret_st = new XRootDStatus(st); + *ret_st = XRootDStatus(stOK, 0); + AnyObject *obj = new AnyObject(); + VectorReadInfo *vReadInfo = new VectorReadInfo(); vReadInfo->SetSize(len); - ChunkList& vResp = vReadInfo->GetChunks(); + ChunkList &vResp = vReadInfo->GetChunks(); vResp = chunks; obj->Set(vReadInfo); handler->HandleResponse(ret_st, obj); - pStats->readVOps++; - pStats->readVreadOps += chunks.size(); - pStats->bytesCachedV += len; + pStats->readVOps++; + pStats->readVreadOps += chunks.size(); + pStats->bytesCachedV += len; return st; } } else { if (sEnableJournalCache) { - bool inJournal = true; - size_t len = 0; - // try to get chunks from journal cache - for (auto it = chunks.begin(); it != chunks.end(); ++it) { - bool eof = false; - auto rb = pJournal->pread(it->buffer, it->length, it->offset, eof); - if (rb != it->length) { - // interrupt if we miss a piece and go remote - inJournal = false; - break; - } else { - len += it->length; - } - } - if (inJournal) { - // we found everything in the journal - pStats->readVOps++; - pStats->readVreadOps += chunks.size(); - pStats->bytesCachedV += len; - XRootDStatus* ret_st = new XRootDStatus(st); - *ret_st = XRootDStatus(stOK, 0); - AnyObject* obj = new AnyObject(); - VectorReadInfo* vReadInfo = new VectorReadInfo(); - vReadInfo->SetSize(len); - ChunkList& vResp = vReadInfo->GetChunks(); - vResp = chunks; - obj->Set(vReadInfo); - handler->HandleResponse(ret_st, obj); - return st; - } + bool inJournal = true; + size_t len = 0; + // try to get chunks from journal cache + for (auto it = chunks.begin(); it != chunks.end(); ++it) { + bool eof = false; + auto rb = pJournal->pread(it->buffer, it->length, it->offset, eof); + if (rb != it->length) { + // interrupt if we miss a piece and go remote + inJournal = false; + break; + } else { + len += it->length; + } + } + if (inJournal) { + // we found everything in the journal + pStats->readVOps++; + pStats->readVreadOps += chunks.size(); + pStats->bytesCachedV += len; + XRootDStatus *ret_st = new XRootDStatus(st); + *ret_st = XRootDStatus(stOK, 0); + AnyObject *obj = new AnyObject(); + VectorReadInfo *vReadInfo = new VectorReadInfo(); + vReadInfo->SetSize(len); + ChunkList &vResp = vReadInfo->GetChunks(); + vResp = chunks; + obj->Set(vReadInfo); + handler->HandleResponse(ret_st, obj); + return st; + } } } - - - auto jhandler = new JCacheReadVHandler(handler, &pStats->bytesReadV,sEnableJournalCache?pJournal.get():nullptr, buffer?(char*)buffer:(char*)(chunks.begin()->buffer), sEnableVectorCache?sCachePath:"", pUrl); - pStats->readVOps++; + + auto jhandler = new JCacheReadVHandler( + handler, &pStats->bytesReadV, + sEnableJournalCache ? pJournal.get() : nullptr, + buffer ? (char *)buffer : (char *)(chunks.begin()->buffer), + sEnableVectorCache ? sCachePath : "", pUrl); + pStats->readVOps++; pStats->readVreadOps += chunks.size(); - st = pFile->VectorRead(chunks, buffer, jhandler, timeout); + st = pFile->VectorRead(chunks, buffer, jhandler, timeout); } else { st = XRootDStatus(stError, errInvalidOp); @@ -425,15 +383,11 @@ JCacheFile::VectorRead(const ChunkList& chunks, return st; } - //------------------------------------------------------------------------------ // Fcntl //------------------------------------------------------------------------------ -XRootDStatus -JCacheFile::Fcntl(const XrdCl::Buffer& arg, - ResponseHandler* handler, - uint16_t timeout) -{ +XRootDStatus JCacheFile::Fcntl(const XrdCl::Buffer &arg, + ResponseHandler *handler, uint16_t timeout) { XRootDStatus st; if (pFile) { @@ -445,14 +399,10 @@ JCacheFile::Fcntl(const XrdCl::Buffer& arg, return st; } - //------------------------------------------------------------------------------ // Visa //------------------------------------------------------------------------------ -XRootDStatus -JCacheFile::Visa(ResponseHandler* handler, - uint16_t timeout) -{ +XRootDStatus JCacheFile::Visa(ResponseHandler *handler, uint16_t timeout) { XRootDStatus st; if (pFile) { @@ -464,24 +414,16 @@ JCacheFile::Visa(ResponseHandler* handler, return st; } - //------------------------------------------------------------------------------ // IsOpen //------------------------------------------------------------------------------ -bool -JCacheFile::IsOpen() const -{ - return mIsOpen; -} - +bool JCacheFile::IsOpen() const { return mIsOpen; } //------------------------------------------------------------------------------ // @see XrdCl::File::SetProperty //------------------------------------------------------------------------------ -bool -JCacheFile::SetProperty(const std::string& name, - const std::string& value) -{ +bool JCacheFile::SetProperty(const std::string &name, + const std::string &value) { if (pFile) { return pFile->SetProperty(name, value); } else { @@ -489,14 +431,11 @@ JCacheFile::SetProperty(const std::string& name, } } - //------------------------------------------------------------------------------ // @see XrdCl::File::GetProperty //------------------------------------------------------------------------------ -bool -JCacheFile::GetProperty(const std::string& name, - std::string& value) const -{ +bool JCacheFile::GetProperty(const std::string &name, + std::string &value) const { if (pFile) { return pFile->GetProperty(name, value); } else { @@ -504,9 +443,7 @@ JCacheFile::GetProperty(const std::string& name, } } -bool -JCacheFile::AttachForRead() -{ +bool JCacheFile::AttachForRead() { std::lock_guard guard(mAttachMutex); if (mAttachedForRead) { return true; @@ -514,46 +451,46 @@ JCacheFile::AttachForRead() if ((mFlags & OpenFlags::Flags::Read) == OpenFlags::Flags::Read) { // attach to a cache if (sEnableJournalCache && pFile) { - mLog->Info(1, "JCache : attaching via journalmanager to '%s'", pUrl.c_str()); + mLog->Info(1, "JCache : attaching via journalmanager to '%s'", + pUrl.c_str()); pJournal = sJournalManager.attach(pUrl); - StatInfo* sinfo = 0; + StatInfo *sinfo = 0; auto st = pFile->Stat(false, sinfo); if (sinfo) { - // only add a file if it wasn't yet added - if (!sStats.HasUrl(pUrl)) { - sStats.totaldatasize+=sinfo->GetSize(); - } - if (pJournal->attach(pJournalPath, sinfo->GetModTime(),0, sinfo->GetSize())) { - mLog->Error(1, "JCache : failed to attach to cache directory: %s", pJournalPath.c_str()); + // only add a file if it wasn't yet added + if (!sStats.HasUrl(pUrl)) { + sStats.totaldatasize += sinfo->GetSize(); + } + if (pJournal->attach(pJournalPath, sinfo->GetModTime(), 0, + sinfo->GetSize())) { + mLog->Error(1, "JCache : failed to attach to cache directory: %s", + pJournalPath.c_str()); mAttachedForRead = true; return false; } else { - mLog->Info(1, "JCache : attached to cache directory: %s", pJournalPath.c_str()); - } + mLog->Info(1, "JCache : attached to cache directory: %s", + pJournalPath.c_str()); + } } } } - sStats.AddUrl(pUrl); + sStats.AddUrl(pUrl); mAttachedForRead = true; return true; } - //---------------------------------------------------------------------------- //! @brief log cache hit statistics //---------------------------------------------------------------------------- void JCacheFile::LogStats() { - mLog->Info(1, "JCache : read:readv-ops:readv-read-ops: %lu:%lu:%lus hit-rate: total [read/readv]=%.02f%% [%.02f%%/%.02f%%] remote-bytes-read/readv: %lu / %lu cached-bytes-read/readv: %lu / %lu", - pStats->readOps.load(), - pStats->readVOps.load(), - pStats->readVreadOps.load(), - pStats->CombinedHitRate(), - pStats->HitRate(), - pStats->HitRateV(), - pStats->bytesRead.load(), - pStats->bytesReadV.load(), - pStats->bytesCached.load(), - pStats->bytesCachedV.load()); - } -} // namespace XrdCl - + mLog->Info( + 1, + "JCache : read:readv-ops:readv-read-ops: %lu:%lu:%lus hit-rate: total " + "[read/readv]=%.02f%% [%.02f%%/%.02f%%] remote-bytes-read/readv: %lu / " + "%lu cached-bytes-read/readv: %lu / %lu", + pStats->readOps.load(), pStats->readVOps.load(), + pStats->readVreadOps.load(), pStats->CombinedHitRate(), pStats->HitRate(), + pStats->HitRateV(), pStats->bytesRead.load(), pStats->bytesReadV.load(), + pStats->bytesCached.load(), pStats->bytesCachedV.load()); +} +} // namespace XrdCl diff --git a/src/XrdApps/XrdClJCachePlugin/file/XrdClJCacheFile.hh b/src/XrdApps/XrdClJCachePlugin/file/XrdClJCacheFile.hh index 3eeac533553..343a1e89063 100644 --- a/src/XrdApps/XrdClJCachePlugin/file/XrdClJCacheFile.hh +++ b/src/XrdApps/XrdClJCachePlugin/file/XrdClJCacheFile.hh @@ -24,33 +24,32 @@ #pragma once /*----------------------------------------------------------------------------*/ -#include "XrdCl/XrdClPlugInInterface.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClLog.hh" +#include "XrdCl/XrdClPlugInInterface.hh" /*----------------------------------------------------------------------------*/ +#include "cache/Journal.hh" #include "file/Art.hh" #include "file/TimeBench.hh" -#include "cache/Journal.hh" -#include "vector/XrdClVectorCache.hh" -#include "handler/XrdClJCacheReadHandler.hh" #include "handler/XrdClJCachePgReadHandler.hh" +#include "handler/XrdClJCacheReadHandler.hh" #include "handler/XrdClJCacheReadVHandler.hh" +#include "vector/XrdClVectorCache.hh" /*----------------------------------------------------------------------------*/ #include -#include #include +#include +#include #include #include -#include -#include +#include /*----------------------------------------------------------------------------*/ namespace JCache { - class CacheStats; +class CacheStats; } -namespace XrdCl -{ +namespace XrdCl { //---------------------------------------------------------------------------- //! JCache file plugin //! This XRootD Client Plugin provides a client side read cache. @@ -58,22 +57,19 @@ namespace XrdCl //! - Read Journal Cache (journalling) //! - Vector Read Cache (vector read responses are stored in binary blobs) //---------------------------------------------------------------------------- -class JCacheFile: public XrdCl::FilePlugIn -{ +class JCacheFile : public XrdCl::FilePlugIn { public: - //---------------------------------------------------------------------------- //! Constructor //---------------------------------------------------------------------------- JCacheFile(); - JCacheFile(const std::string& url); + JCacheFile(const std::string &url); //---------------------------------------------------------------------------- //! Destructor //---------------------------------------------------------------------------- virtual ~JCacheFile(); - //---------------------------------------------------------------------------- //! @brief Open a file by URL //! @param url URL of the file @@ -82,21 +78,16 @@ public: //! @param handler Response handler //! @param timeout Timeout in seconds //---------------------------------------------------------------------------- - virtual XRootDStatus Open(const std::string& url, - OpenFlags::Flags flags, - Access::Mode mode, - ResponseHandler* handler, + virtual XRootDStatus Open(const std::string &url, OpenFlags::Flags flags, + Access::Mode mode, ResponseHandler *handler, uint16_t timeout); - //---------------------------------------------------------------------------- //! @brief Close a file //! @param handler Response handler //! @param timeout Timeout in seconds //---------------------------------------------------------------------------- - virtual XRootDStatus Close(ResponseHandler* handler, - uint16_t timeout); - + virtual XRootDStatus Close(ResponseHandler *handler, uint16_t timeout); //---------------------------------------------------------------------------- //! @brief Stat a file @@ -104,11 +95,9 @@ public: //! @param handler Response handler //! @param timeout Timeout in seconds //---------------------------------------------------------------------------- - virtual XRootDStatus Stat(bool force, - ResponseHandler* handler, + virtual XRootDStatus Stat(bool force, ResponseHandler *handler, uint16_t timeout); - //---------------------------------------------------------------------------- //! @brief Read //! @param offset Offset in bytes @@ -117,12 +106,8 @@ public: //! @param handler Response handler //! @param timeout Timeout in seconds //---------------------------------------------------------------------------- - virtual XRootDStatus Read(uint64_t offset, - uint32_t size, - void* buffer, - ResponseHandler* handler, - uint16_t timeout); - + virtual XRootDStatus Read(uint64_t offset, uint32_t size, void *buffer, + ResponseHandler *handler, uint16_t timeout); //---------------------------------------------------------------------------- //! @brief Write @@ -132,21 +117,15 @@ public: //! @param handler Response handler //! @param timeout Timeout in seconds //---------------------------------------------------------------------------- - virtual XRootDStatus Write(uint64_t offset, - uint32_t size, - const void* buffer, - ResponseHandler* handler, - uint16_t timeout); - + virtual XRootDStatus Write(uint64_t offset, uint32_t size, const void *buffer, + ResponseHandler *handler, uint16_t timeout); //---------------------------------------------------------------------------- //! @brief Sync //! @param handler Response handler //! @param timeout Timeout in seconds //---------------------------------------------------------------------------- - virtual XRootDStatus Sync(ResponseHandler* handler, - uint16_t timeout); - + virtual XRootDStatus Sync(ResponseHandler *handler, uint16_t timeout); //---------------------------------------------------------------------------- //! @brief Truncate @@ -154,11 +133,9 @@ public: //! @param handler Response handler //! @param timeout Timeout in seconds //---------------------------------------------------------------------------- - virtual XRootDStatus Truncate(uint64_t size, - ResponseHandler* handler, + virtual XRootDStatus Truncate(uint64_t size, ResponseHandler *handler, uint16_t timeout); - //---------------------------------------------------------------------------- //! @brief VectorRead //! @param chunks Chunks to read @@ -166,10 +143,8 @@ public: //! @param handler Response handler //! @param timeout Timeout in seconds //---------------------------------------------------------------------------- - virtual XRootDStatus VectorRead(const ChunkList& chunks, - void* buffer, - ResponseHandler* handler, - uint16_t timeout); + virtual XRootDStatus VectorRead(const ChunkList &chunks, void *buffer, + ResponseHandler *handler, uint16_t timeout); //------------------------------------------------------------------------ //! @brief PgRead @@ -179,11 +154,9 @@ public: //! @param handler Response handler //! @param timeout Timeout in seconds //------------------------------------------------------------------------ - virtual XRootDStatus PgRead( uint64_t offset, - uint32_t size, - void *buffer, - ResponseHandler *handler, - uint16_t timeout ) override; + virtual XRootDStatus PgRead(uint64_t offset, uint32_t size, void *buffer, + ResponseHandler *handler, + uint16_t timeout) override; //------------------------------------------------------------------------ //! @brief PgWrite @@ -193,13 +166,11 @@ public: //! @param handler Response handler //! @param timeout Timeout in seconds //------------------------------------------------------------------------ - virtual XRootDStatus PgWrite( uint64_t offset, - uint32_t nbpgs, - const void *buffer, - std::vector &cksums, - ResponseHandler *handler, - uint16_t timeout ) override; - + virtual XRootDStatus PgWrite(uint64_t offset, uint32_t nbpgs, + const void *buffer, + std::vector &cksums, + ResponseHandler *handler, + uint16_t timeout) override; //------------------------------------------------------------------------ //! @brief Fcntl @@ -207,47 +178,35 @@ public: //! @param handler Response handler //! @param timeout Timeout in seconds //------------------------------------------------------------------------ - virtual XRootDStatus Fcntl(const Buffer& arg, - ResponseHandler* handler, + virtual XRootDStatus Fcntl(const Buffer &arg, ResponseHandler *handler, uint16_t timeout); - //---------------------------------------------------------------------------- //! @brief Visa //! @param handler Response handler //! @param timeout Timeout in seconds //---------------------------------------------------------------------------- - virtual XRootDStatus Visa(ResponseHandler* handler, - uint16_t timeout); - + virtual XRootDStatus Visa(ResponseHandler *handler, uint16_t timeout); //---------------------------------------------------------------------------- //! @brief check if file is open //---------------------------------------------------------------------------- virtual bool IsOpen() const; - //---------------------------------------------------------------------------- //! @see XrdCl::File::SetProperty //---------------------------------------------------------------------------- - virtual bool SetProperty(const std::string& name, - const std::string& value); - + virtual bool SetProperty(const std::string &name, const std::string &value); //---------------------------------------------------------------------------- //! @see XrdCl::File::GetProperty //---------------------------------------------------------------------------- - virtual bool GetProperty(const std::string& name, - std::string& value) const; - + virtual bool GetProperty(const std::string &name, std::string &value) const; //---------------------------------------------------------------------------- //! @brief validate the local cache //---------------------------------------------------------------------------- - inline bool IsValid() - { - return true; - } + inline bool IsValid() { return true; } //---------------------------------------------------------------------------- //! @brief set the local cache path and enable/disable journal/vector caches @@ -255,12 +214,12 @@ public: //! @param journal Enable/disable journal cache //! @param vector Enable/disable vector cache //---------------------------------------------------------------------------- - - static void SetCache(const std::string& path) { sCachePath = path; } - static void SetJournal(const bool& value) { sEnableJournalCache = value; } - static void SetVector(const bool& value) { sEnableVectorCache = value; } - static void SetJsonPath(const std::string& path) { sJsonPath = path; } - static void SetSummary(const bool& value) { sEnableSummary = value; } + + static void SetCache(const std::string &path) { sCachePath = path; } + static void SetJournal(const bool &value) { sEnableJournalCache = value; } + static void SetVector(const bool &value) { sEnableVectorCache = value; } + static void SetJsonPath(const std::string &path) { sJsonPath = path; } + static void SetSummary(const bool &value) { sEnableSummary = value; } //---------------------------------------------------------------------------- //! @brief static members pointing to cache settings @@ -271,17 +230,16 @@ public: static bool sEnableJournalCache; static bool sEnableSummary; static JournalManager sJournalManager; - + //---------------------------------------------------------------------------- //! @brief log cache hit statistics //---------------------------------------------------------------------------- void LogStats(); - + //! @brief global plugin cache hit statistics static JCache::CacheStats sStats; - -private: +private: //! @brief attach for read bool AttachForRead(); @@ -294,7 +252,7 @@ private: //! @brief boolean to track if file is open bool mIsOpen; //! @brief pointer to the remote file - XrdCl::File* pFile; + XrdCl::File *pFile; //! @brief URL of the remote file std::string pUrl; //! @brief instance of a local journal for this file @@ -302,10 +260,10 @@ private: //! @brief path to the journal of this file std::string pJournalPath; //! @brief pointer to logging object - Log* mLog; + Log *mLog; //! @brief cache hit statistics - JCache::CacheStats* pStats; + JCache::CacheStats *pStats; }; } // namespace XrdCl diff --git a/src/XrdApps/XrdClJCachePlugin/handler/XrdClJCachePgReadHandler.hh b/src/XrdApps/XrdClJCachePlugin/handler/XrdClJCachePgReadHandler.hh index a5ae639461e..dd8060e7cf9 100644 --- a/src/XrdApps/XrdClJCachePlugin/handler/XrdClJCachePgReadHandler.hh +++ b/src/XrdApps/XrdClJCachePlugin/handler/XrdClJCachePgReadHandler.hh @@ -32,41 +32,42 @@ namespace XrdCl { class JCachePgReadHandler : public XrdCl::ResponseHandler - // ---------------------------------------------------------------------- // +// ---------------------------------------------------------------------- // { public: - JCachePgReadHandler() { } + JCachePgReadHandler() {} - JCachePgReadHandler(JCacheReadHandler* other) { - rbytes = other->rbytes; - journal = other->journal; - } - - JCachePgReadHandler(XrdCl::ResponseHandler* handler, - std::atomic* rbytes, - Journal* journal) : handler(handler), rbytes(rbytes), journal(journal) {} + JCachePgReadHandler(JCacheReadHandler *other) { + rbytes = other->rbytes; + journal = other->journal; + } - virtual ~JCachePgReadHandler() {} + JCachePgReadHandler(XrdCl::ResponseHandler *handler, + std::atomic *rbytes, Journal *journal) + : handler(handler), rbytes(rbytes), journal(journal) {} - virtual void HandleResponse(XrdCl::XRootDStatus* pStatus, - XrdCl::AnyObject* pResponse) { + virtual ~JCachePgReadHandler() {} - XrdCl::PageInfo* pageInfo; - if (pStatus->IsOK()) { - if (pResponse) { - pResponse->Get(pageInfo); - // store successfull reads in the journal - if (journal) journal->pwrite(pageInfo->GetBuffer(), pageInfo->GetLength(), pageInfo->GetOffset()); - *rbytes+= pageInfo->GetLength(); - } - } - handler->HandleResponse(pStatus, pResponse); - } + virtual void HandleResponse(XrdCl::XRootDStatus *pStatus, + XrdCl::AnyObject *pResponse) { - XrdCl::ResponseHandler* handler; - std::atomic* rbytes; - Journal* journal; + XrdCl::PageInfo *pageInfo; + if (pStatus->IsOK()) { + if (pResponse) { + pResponse->Get(pageInfo); + // store successfull reads in the journal + if (journal) + journal->pwrite(pageInfo->GetBuffer(), pageInfo->GetLength(), + pageInfo->GetOffset()); + *rbytes += pageInfo->GetLength(); + } + } + handler->HandleResponse(pStatus, pResponse); + } + XrdCl::ResponseHandler *handler; + std::atomic *rbytes; + Journal *journal; }; } // namespace XrdCl diff --git a/src/XrdApps/XrdClJCachePlugin/handler/XrdClJCacheReadHandler.hh b/src/XrdApps/XrdClJCachePlugin/handler/XrdClJCacheReadHandler.hh index dd6a3814b79..baaf9178619 100644 --- a/src/XrdApps/XrdClJCachePlugin/handler/XrdClJCacheReadHandler.hh +++ b/src/XrdApps/XrdClJCachePlugin/handler/XrdClJCacheReadHandler.hh @@ -32,41 +32,42 @@ namespace XrdCl { class JCacheReadHandler : public XrdCl::ResponseHandler - // ---------------------------------------------------------------------- // +// ---------------------------------------------------------------------- // { public: - JCacheReadHandler() { } + JCacheReadHandler() {} - JCacheReadHandler(JCacheReadHandler* other) { - rbytes = other->rbytes; - journal = other->journal; - } - - JCacheReadHandler(XrdCl::ResponseHandler* handler, - std::atomic* rbytes, - Journal* journal) : handler(handler), rbytes(rbytes), journal(journal) {} + JCacheReadHandler(JCacheReadHandler *other) { + rbytes = other->rbytes; + journal = other->journal; + } - virtual ~JCacheReadHandler() {} + JCacheReadHandler(XrdCl::ResponseHandler *handler, + std::atomic *rbytes, Journal *journal) + : handler(handler), rbytes(rbytes), journal(journal) {} - virtual void HandleResponse(XrdCl::XRootDStatus* pStatus, - XrdCl::AnyObject* pResponse) { + virtual ~JCacheReadHandler() {} - XrdCl::ChunkInfo* chunkInfo; - if (pStatus->IsOK()) { - if (pResponse) { - pResponse->Get(chunkInfo); - // store successfull reads in the journal - if (journal) journal->pwrite(chunkInfo->GetBuffer(), chunkInfo->GetLength(), chunkInfo->GetOffset()); - *rbytes+= chunkInfo->GetLength(); - } - } - handler->HandleResponse(pStatus, pResponse); - } + virtual void HandleResponse(XrdCl::XRootDStatus *pStatus, + XrdCl::AnyObject *pResponse) { - XrdCl::ResponseHandler* handler; - std::atomic* rbytes; - Journal* journal; + XrdCl::ChunkInfo *chunkInfo; + if (pStatus->IsOK()) { + if (pResponse) { + pResponse->Get(chunkInfo); + // store successfull reads in the journal + if (journal) + journal->pwrite(chunkInfo->GetBuffer(), chunkInfo->GetLength(), + chunkInfo->GetOffset()); + *rbytes += chunkInfo->GetLength(); + } + } + handler->HandleResponse(pStatus, pResponse); + } + XrdCl::ResponseHandler *handler; + std::atomic *rbytes; + Journal *journal; }; } // namespace XrdCl diff --git a/src/XrdApps/XrdClJCachePlugin/handler/XrdClJCacheReadVHandler.hh b/src/XrdApps/XrdClJCachePlugin/handler/XrdClJCacheReadVHandler.hh index 59552d03bdb..f377e7671ef 100644 --- a/src/XrdApps/XrdClJCachePlugin/handler/XrdClJCacheReadVHandler.hh +++ b/src/XrdApps/XrdClJCachePlugin/handler/XrdClJCacheReadVHandler.hh @@ -33,61 +33,62 @@ namespace XrdCl { class JCacheReadVHandler : public XrdCl::ResponseHandler - // ---------------------------------------------------------------------- // +// ---------------------------------------------------------------------- // { public: - JCacheReadVHandler() { } + JCacheReadVHandler() {} - JCacheReadVHandler(JCacheReadVHandler* other) { - - journal = other->journal; - buffer = other->buffer; - rvbytes = other->rvbytes; - vcachepath = other->vcachepath; - url = other->url; - } + JCacheReadVHandler(JCacheReadVHandler *other) { + + journal = other->journal; + buffer = other->buffer; + rvbytes = other->rvbytes; + vcachepath = other->vcachepath; + url = other->url; + } - JCacheReadVHandler(XrdCl::ResponseHandler* handler, - std::atomic* rvbytes, - Journal* journal, - void* buffer, - const std::string& vcachepath, - const std::string& url) : handler(handler), rvbytes(rvbytes), journal(journal), buffer(buffer), vcachepath(vcachepath), url(url) {} + JCacheReadVHandler(XrdCl::ResponseHandler *handler, + std::atomic *rvbytes, Journal *journal, + void *buffer, const std::string &vcachepath, + const std::string &url) + : handler(handler), rvbytes(rvbytes), journal(journal), buffer(buffer), + vcachepath(vcachepath), url(url) {} - virtual ~JCacheReadVHandler() {} + virtual ~JCacheReadVHandler() {} - virtual void HandleResponse(XrdCl::XRootDStatus* pStatus, - XrdCl::AnyObject* pResponse) { - if (pStatus->IsOK()) { - if (pResponse) { - VectorReadInfo* vReadInfo; - pResponse->Get(vReadInfo); - ChunkList* chunks = &(vReadInfo->GetChunks()); - // store successfull reads in the journal if there is no vector cache - if (vcachepath.empty()) { - if (journal) { - for (auto it = chunks->begin(); it != chunks->end(); ++it) { - journal->pwrite(it->GetBuffer(), it->GetLength(), it->GetOffset()); - } - } - } else { - VectorCache cache(*chunks, url, (const char*)buffer, vcachepath); - cache.store(); - } - for (auto it = chunks->begin(); it != chunks->end(); ++it) { - *rvbytes += it->GetLength(); - } - } - } - handler->HandleResponse(pStatus, pResponse); - } + virtual void HandleResponse(XrdCl::XRootDStatus *pStatus, + XrdCl::AnyObject *pResponse) { + if (pStatus->IsOK()) { + if (pResponse) { + VectorReadInfo *vReadInfo; + pResponse->Get(vReadInfo); + ChunkList *chunks = &(vReadInfo->GetChunks()); + // store successfull reads in the journal if there is no vector cache + if (vcachepath.empty()) { + if (journal) { + for (auto it = chunks->begin(); it != chunks->end(); ++it) { + journal->pwrite(it->GetBuffer(), it->GetLength(), + it->GetOffset()); + } + } + } else { + VectorCache cache(*chunks, url, (const char *)buffer, vcachepath); + cache.store(); + } + for (auto it = chunks->begin(); it != chunks->end(); ++it) { + *rvbytes += it->GetLength(); + } + } + } + handler->HandleResponse(pStatus, pResponse); + } - XrdCl::ResponseHandler* handler; - std::atomic* rvbytes; - Journal* journal; - void* buffer; - std::string vcachepath; - std::string url; + XrdCl::ResponseHandler *handler; + std::atomic *rvbytes; + Journal *journal; + void *buffer; + std::string vcachepath; + std::string url; }; } // namespace XrdCl diff --git a/src/XrdApps/XrdClJCachePlugin/plugin/XrdClJCachePlugin.cc b/src/XrdApps/XrdClJCachePlugin/plugin/XrdClJCachePlugin.cc index a065303b5f5..1a76dda24f2 100644 --- a/src/XrdApps/XrdClJCachePlugin/plugin/XrdClJCachePlugin.cc +++ b/src/XrdApps/XrdClJCachePlugin/plugin/XrdClJCachePlugin.cc @@ -26,13 +26,10 @@ XrdVERSIONINFO(XrdClGetPlugIn, XrdClGetPlugIn) -extern "C" -{ - void* XrdClGetPlugIn( const void* arg ) - { - const std::map* config = - static_cast< const std::map* >(arg); - return static_cast(new XrdCl::JCacheFactory( config ) ); + extern "C" { + void *XrdClGetPlugIn(const void *arg) { + const std::map *config = + static_cast *>(arg); + return static_cast(new XrdCl::JCacheFactory(config)); } } - diff --git a/src/XrdApps/XrdClJCachePlugin/plugin/XrdClJCachePlugin.hh b/src/XrdApps/XrdClJCachePlugin/plugin/XrdClJCachePlugin.hh index a174e105b63..24775690f95 100644 --- a/src/XrdApps/XrdClJCachePlugin/plugin/XrdClJCachePlugin.hh +++ b/src/XrdApps/XrdClJCachePlugin/plugin/XrdClJCachePlugin.hh @@ -23,105 +23,107 @@ #pragma once /*----------------------------------------------------------------------------*/ -#include "XrdCl/XrdClPlugInInterface.hh" #include "XrdCl/XrdClDefaultEnv.hh" #include "XrdCl/XrdClLog.hh" +#include "XrdCl/XrdClPlugInInterface.hh" /*----------------------------------------------------------------------------*/ #include "../file/XrdClJCacheFile.hh" /*----------------------------------------------------------------------------*/ -#include #include +#include #include /*----------------------------------------------------------------------------*/ -namespace XrdCl -{ +namespace XrdCl { //------------------------------------------------------------------------------ //! XrdCl JCache plug-in factory //------------------------------------------------------------------------------ -class JCacheFactory : public PlugInFactory -{ - public: - //---------------------------------------------------------------------------- - //! Constructor - //! - //! @param config map containing configuration parameters - //---------------------------------------------------------------------------- - JCacheFactory( const std::map* config ) - { - if( config ) - { - auto itc = config->find( "cache" ); - JCacheFile::SetCache( itc != config->end() ? itc->second : "" ); - auto itv = config->find( "vector" ); - JCacheFile::SetVector( itv != config->end() ? itv->second == "true": false ); - auto itj = config->find( "journal" ); - JCacheFile::SetJournal( itj != config->end() ? itj->second == "true": false ); - auto itjson = config->find( "json" ); - JCacheFile::SetJsonPath( itjson != config->end() ? itjson->second : "./" ); - auto its = config->find( "summary" ); - JCacheFile::SetSummary( its != config->end() ? its->second != "false": true ); +class JCacheFactory : public PlugInFactory { +public: + //---------------------------------------------------------------------------- + //! Constructor + //! + //! @param config map containing configuration parameters + //---------------------------------------------------------------------------- + JCacheFactory(const std::map *config) { + if (config) { + auto itc = config->find("cache"); + JCacheFile::SetCache(itc != config->end() ? itc->second : ""); + auto itv = config->find("vector"); + JCacheFile::SetVector(itv != config->end() ? itv->second == "true" + : false); + auto itj = config->find("journal"); + JCacheFile::SetJournal(itj != config->end() ? itj->second == "true" + : false); + auto itjson = config->find("json"); + JCacheFile::SetJsonPath(itjson != config->end() ? itjson->second : "./"); + auto its = config->find("summary"); + JCacheFile::SetSummary(its != config->end() ? its->second != "false" + : true); - if (const char *v = getenv("XRD_JCACHE_CACHE")) { - JCacheFile::SetCache( (std::string(v).length()) ? std::string(v) : ""); - } + if (const char *v = getenv("XRD_JCACHE_CACHE")) { + JCacheFile::SetCache((std::string(v).length()) ? std::string(v) : ""); + } - if (const char *v = getenv("XRD_JCACHE_SUMMARY")) { - JCacheFile::SetSummary( (std::string(v) == "true") ? true : false); - } + if (const char *v = getenv("XRD_JCACHE_SUMMARY")) { + JCacheFile::SetSummary((std::string(v) == "true") ? true : false); + } - if (const char *v = getenv("XRD_JCACHE_JOURNAL")) { - JCacheFile::SetJournal( (std::string(v) == "true") ? true : false); - } + if (const char *v = getenv("XRD_JCACHE_JOURNAL")) { + JCacheFile::SetJournal((std::string(v) == "true") ? true : false); + } - if (const char *v = getenv("XRD_JCACHE_VECTOR")) { - JCacheFile::SetVector( (std::string(v) == "true") ? true : false); - } + if (const char *v = getenv("XRD_JCACHE_VECTOR")) { + JCacheFile::SetVector((std::string(v) == "true") ? true : false); + } - if (const char *v = getenv("XRD_JCACHE_JSON")) { - JCacheFile::SetJsonPath( (std::string(v).length()) ? std::string(v) : ""); - } + if (const char *v = getenv("XRD_JCACHE_JSON")) { + JCacheFile::SetJsonPath((std::string(v).length()) ? std::string(v) + : ""); + } - Log* log = DefaultEnv::GetLog(); - log->Info(1, "JCache : cache directory: %s", JCacheFile::sCachePath.c_str()); - log->Info(1, "JCache : caching readv in vector cache : %s", JCacheFile::sEnableVectorCache ? "true" : "false"); - log->Info(1, "JCache : caching reads in journal cache: %s", JCacheFile::sEnableJournalCache ? "true" : "false"); - log->Info(1, "JCache : summary output is: %s", JCacheFile::sEnableSummary ? "true" : "false"); - if (JCacheFile::sJsonPath.length()) { - log->Info(1, "JCache : json output to prefix: %s", JCacheFile::sJsonPath.c_str()); - } else { - log->Info(1, "JCache : json output is disabled", JCacheFile::sJsonPath.c_str()); - } + Log *log = DefaultEnv::GetLog(); + log->Info(1, "JCache : cache directory: %s", + JCacheFile::sCachePath.c_str()); + log->Info(1, "JCache : caching readv in vector cache : %s", + JCacheFile::sEnableVectorCache ? "true" : "false"); + log->Info(1, "JCache : caching reads in journal cache: %s", + JCacheFile::sEnableJournalCache ? "true" : "false"); + log->Info(1, "JCache : summary output is: %s", + JCacheFile::sEnableSummary ? "true" : "false"); + if (JCacheFile::sJsonPath.length()) { + log->Info(1, "JCache : json output to prefix: %s", + JCacheFile::sJsonPath.c_str()); + } else { + log->Info(1, "JCache : json output is disabled", + JCacheFile::sJsonPath.c_str()); } } + } - //---------------------------------------------------------------------------- - //! Destructor - //---------------------------------------------------------------------------- - virtual ~JCacheFactory() - { - } + //---------------------------------------------------------------------------- + //! Destructor + //---------------------------------------------------------------------------- + virtual ~JCacheFactory() {} - //---------------------------------------------------------------------------- - //! Create a file plug-in for the given URL - //---------------------------------------------------------------------------- - virtual FilePlugIn* CreateFile(const std::string& url) - { - std::unique_ptr ptr( new JCacheFile() ); - if( !ptr->IsValid() ) - return nullptr; - return static_cast( ptr.release() ); - } + //---------------------------------------------------------------------------- + //! Create a file plug-in for the given URL + //---------------------------------------------------------------------------- + virtual FilePlugIn *CreateFile(const std::string &url) { + std::unique_ptr ptr(new JCacheFile()); + if (!ptr->IsValid()) + return nullptr; + return static_cast(ptr.release()); + } - //---------------------------------------------------------------------------- - //! Create a file system plug-in for the given URL - //---------------------------------------------------------------------------- - virtual FileSystemPlugIn* CreateFileSystem(const std::string& url) - { - Log* log = DefaultEnv::GetLog(); - log->Error(1, "FileSystem plugin implementation not supported"); - return static_cast(0); - } + //---------------------------------------------------------------------------- + //! Create a file system plug-in for the given URL + //---------------------------------------------------------------------------- + virtual FileSystemPlugIn *CreateFileSystem(const std::string &url) { + Log *log = DefaultEnv::GetLog(); + log->Error(1, "FileSystem plugin implementation not supported"); + return static_cast(0); + } }; } // namespace XrdCl diff --git a/src/XrdApps/XrdClJCachePlugin/vector/XrdClVectorCache.cc b/src/XrdApps/XrdClJCachePlugin/vector/XrdClVectorCache.cc index 61b0180e394..d30fa3861fa 100644 --- a/src/XrdApps/XrdClJCachePlugin/vector/XrdClVectorCache.cc +++ b/src/XrdApps/XrdClJCachePlugin/vector/XrdClVectorCache.cc @@ -25,8 +25,8 @@ #include "vector/XrdClVectorCache.hh" /*----------------------------------------------------------------------------*/ #include -#include #include +#include /*----------------------------------------------------------------------------*/ namespace fs = std::filesystem; @@ -37,237 +37,244 @@ namespace XrdCl { //! serialize a vector into a buffer //---------------------------------------------------------------------------- std::vector VectorCache::serializeVector() const { - std::vector serializedData; - for (const auto& i : chunks) { - uint64_t o = i.GetOffset(); - uint64_t n = i.GetLength(); - unsigned char buffer[sizeof(uint64_t) + sizeof(size_t)]; - std::memcpy(buffer, &o, sizeof(uint64_t)); - std::memcpy(buffer + sizeof(uint64_t), &n, sizeof(size_t)); - serializedData.insert(serializedData.end(), buffer, buffer + sizeof(buffer)); - } - return serializedData; + std::vector serializedData; + for (const auto &i : chunks) { + uint64_t o = i.GetOffset(); + uint64_t n = i.GetLength(); + unsigned char buffer[sizeof(uint64_t) + sizeof(size_t)]; + std::memcpy(buffer, &o, sizeof(uint64_t)); + std::memcpy(buffer + sizeof(uint64_t), &n, sizeof(size_t)); + serializedData.insert(serializedData.end(), buffer, + buffer + sizeof(buffer)); + } + return serializedData; } //---------------------------------------------------------------------------- //! compute SHA256 signature for a given vector read //---------------------------------------------------------------------------- -std::string VectorCache::computeSHA256(const std::vector& data) { - unsigned int length = 0; +std::string VectorCache::computeSHA256(const std::vector &data) { + unsigned int length = 0; #if OPENSSL_VERSION_NUMBER < 0x30000000L - length = SHA256_DIGEST_LENGTH; - unsigned char hash[length]; - SHA256_CTX sha256; - SHA256_Init(&sha256); - SHA256_Update(&sha256, data.data(), data.size()); - SHA256_Final(hash, &sha256); + length = SHA256_DIGEST_LENGTH; + unsigned char hash[length]; + SHA256_CTX sha256; + SHA256_Init(&sha256); + SHA256_Update(&sha256, data.data(), data.size()); + SHA256_Final(hash, &sha256); #else - // things don't always get better ... we skip error handling - EVP_MD_CTX* ctx = EVP_MD_CTX_new(); - const EVP_MD* md = EVP_sha256(); - EVP_DigestInit_ex(ctx, md, NULL); - EVP_DigestUpdate(ctx, data.data(), data.size()); - length = EVP_MD_size(md); - unsigned char hash[length]; - EVP_DigestFinal_ex(ctx, hash, &length); - EVP_MD_CTX_free(ctx); + // things don't always get better ... we skip error handling + EVP_MD_CTX *ctx = EVP_MD_CTX_new(); + const EVP_MD *md = EVP_sha256(); + EVP_DigestInit_ex(ctx, md, NULL); + EVP_DigestUpdate(ctx, data.data(), data.size()); + length = EVP_MD_size(md); + unsigned char hash[length]; + EVP_DigestFinal_ex(ctx, hash, &length); + EVP_MD_CTX_free(ctx); #endif - std::stringstream ss; - for (unsigned int i = 0; i < length; ++i) { - ss << std::hex << std::setw(2) << std::setfill('0') << (int)hash[i]; - } - return ss.str(); + std::stringstream ss; + for (unsigned int i = 0; i < length; ++i) { + ss << std::hex << std::setw(2) << std::setfill('0') << (int)hash[i]; + } + return ss.str(); } //---------------------------------------------------------------------------- //! compute SHA256 signature for a string //---------------------------------------------------------------------------- -std::string VectorCache::computeSHA256(const std::string& data) { - unsigned int length = 0; +std::string VectorCache::computeSHA256(const std::string &data) { + unsigned int length = 0; #if OPENSSL_VERSION_NUMBER < 0x30000000L - length = SHA256_DIGEST_LENGTH; - unsigned char hash[length]; - SHA256_CTX sha256; - SHA256_Init(&sha256); - SHA256_Update(&sha256, data.c_str(), data.size()); - SHA256_Final(hash, &sha256); + length = SHA256_DIGEST_LENGTH; + unsigned char hash[length]; + SHA256_CTX sha256; + SHA256_Init(&sha256); + SHA256_Update(&sha256, data.c_str(), data.size()); + SHA256_Final(hash, &sha256); #else - // things don't always get better ... we skip error handling - EVP_MD_CTX* ctx = EVP_MD_CTX_new(); - const EVP_MD* md = EVP_sha256(); - EVP_DigestInit_ex(ctx, md, NULL); - EVP_DigestUpdate(ctx, data.data(), data.size()); - length = EVP_MD_size(md); - unsigned char hash[length]; - EVP_DigestFinal_ex(ctx, hash, &length); - EVP_MD_CTX_free(ctx); + // things don't always get better ... we skip error handling + EVP_MD_CTX *ctx = EVP_MD_CTX_new(); + const EVP_MD *md = EVP_sha256(); + EVP_DigestInit_ex(ctx, md, NULL); + EVP_DigestUpdate(ctx, data.data(), data.size()); + length = EVP_MD_size(md); + unsigned char hash[length]; + EVP_DigestFinal_ex(ctx, hash, &length); + EVP_MD_CTX_free(ctx); #endif - std::stringstream ss; - for (unsigned int i = 0; i < length; ++i) { - ss << std::hex << std::setw(2) << std::setfill('0') << (int)hash[i]; - } - return ss.str(); + std::stringstream ss; + for (unsigned int i = 0; i < length; ++i) { + ss << std::hex << std::setw(2) << std::setfill('0') << (int)hash[i]; + } + return ss.str(); } //---------------------------------------------------------------------------- //! compute SHA256 for vector read and name //---------------------------------------------------------------------------- std::pair VectorCache::computeHash() const { - std::vector serializedData = serializeVector(); - std::string vectorHash = computeSHA256(serializedData); - std::string nameHash = computeSHA256(name); - return {vectorHash, nameHash}; + std::vector serializedData = serializeVector(); + std::string vectorHash = computeSHA256(serializedData); + std::string nameHash = computeSHA256(name); + return {vectorHash, nameHash}; } //---------------------------------------------------------------------------- //! ensure that the last subdirectory directory exists //---------------------------------------------------------------------------- -bool VectorCache::ensureLastSubdirectoryExists(const std::string& dirName) { - fs::path dirPath(dirName); - - if (fs::exists(dirPath) && fs::is_directory(dirPath)) { - return true; - } - - // Extract the parent path - fs::path parentPath = dirPath.parent_path(); - - if (!fs::exists(parentPath)) { - std::cerr << "error: parent directory does not exist. Cannot create subdirectory.\n"; - return false; - } - - if (fs::create_directory(dirPath)) { - return true; - } else { - std::cerr << "error: failed to create subdirectory.\n"; - return false; - } +bool VectorCache::ensureLastSubdirectoryExists(const std::string &dirName) { + fs::path dirPath(dirName); + if (fs::exists(dirPath) && fs::is_directory(dirPath)) { + return true; + } + + // Extract the parent path + fs::path parentPath = dirPath.parent_path(); + + if (!fs::exists(parentPath)) { + std::cerr << "error: parent directory does not exist. Cannot create " + "subdirectory.\n"; return false; + } + + if (fs::create_directory(dirPath)) { + return true; + } else { + std::cerr << "error: failed to create subdirectory.\n"; + return false; + } + + return false; } //---------------------------------------------------------------------------- //! store a vector read in the cache //---------------------------------------------------------------------------- bool VectorCache::store() const { - // Compute hashes - auto [vectorHash, nameHash] = computeHash(); - - // Compute the total expected length from the input vector - size_t expectedLen = 0; - for (const auto& chunk : chunks) { - expectedLen += chunk.GetLength(); - } + // Compute hashes + auto [vectorHash, nameHash] = computeHash(); - // Try to have a cache toplevel directory - if (!ensureLastSubdirectoryExists(prefix)) { - return false; - } + // Compute the total expected length from the input vector + size_t expectedLen = 0; + for (const auto &chunk : chunks) { + expectedLen += chunk.GetLength(); + } - // Generate the dir name using the prefix and the hash of the name - std::string dirName = prefix + nameHash; - std::string fileName = dirName + "/" + vectorHash; - std::string tmpName = fileName + ".tmp"; + // Try to have a cache toplevel directory + if (!ensureLastSubdirectoryExists(prefix)) { + return false; + } - // Try to have a cache subdirectory for this file - if (!ensureLastSubdirectoryExists(dirName)) { - return false; - } + // Generate the dir name using the prefix and the hash of the name + std::string dirName = prefix + nameHash; + std::string fileName = dirName + "/" + vectorHash; + std::string tmpName = fileName + ".tmp"; - // Open the file for writing (binary mode - // Write specified segments of data to the file - std::ofstream outFile(tmpName, std::ios::binary); - if (outFile.is_open()) { - std::error_code ec; - outFile.write(data, expectedLen); - if (outFile.fail()) { - std::cerr << "error: failed writing to file: " << tmpName << std::endl; - outFile.close(); - fs::remove(tmpName, ec); - if (ec) { - std::cerr << "error: failed cleanup of temporary file: " << tmpName << std::endl; - } - return false; - } - outFile.close(); - fs::rename(tmpName, fileName, ec); - if (ec) { - outFile.close(); - std::cerr << "error: failed atomic rename to file: " << fileName << std::endl; - fs::remove(tmpName, ec); - if (ec) { - std::cerr << "error: failed cleanup of temporary file: " << tmpName << std::endl; - } - return false; - } - return true; - } else { - std::cerr << "error: failed to open file: " << tmpName << std::endl; - return false; + // Try to have a cache subdirectory for this file + if (!ensureLastSubdirectoryExists(dirName)) { + return false; + } + + // Open the file for writing (binary mode + // Write specified segments of data to the file + std::ofstream outFile(tmpName, std::ios::binary); + if (outFile.is_open()) { + std::error_code ec; + outFile.write(data, expectedLen); + if (outFile.fail()) { + std::cerr << "error: failed writing to file: " << tmpName << std::endl; + outFile.close(); + fs::remove(tmpName, ec); + if (ec) { + std::cerr << "error: failed cleanup of temporary file: " << tmpName + << std::endl; + } + return false; + } + outFile.close(); + fs::rename(tmpName, fileName, ec); + if (ec) { + outFile.close(); + std::cerr << "error: failed atomic rename to file: " << fileName + << std::endl; + fs::remove(tmpName, ec); + if (ec) { + std::cerr << "error: failed cleanup of temporary file: " << tmpName + << std::endl; + } + return false; } + return true; + } else { + std::cerr << "error: failed to open file: " << tmpName << std::endl; + return false; + } } //---------------------------------------------------------------------------- //! retrieve a vector read from the cache //---------------------------------------------------------------------------- bool VectorCache::retrieve() const { - // Compute the total expected length from the input vector - size_t expectedLen = 0; - for (const auto& chunk : chunks) { - expectedLen += chunk.GetLength(); - } + // Compute the total expected length from the input vector + size_t expectedLen = 0; + for (const auto &chunk : chunks) { + expectedLen += chunk.GetLength(); + } - // Compute hashes - auto [vectorHash, nameHash] = computeHash(); + // Compute hashes + auto [vectorHash, nameHash] = computeHash(); - // Generate the dir name using the prefix and the hash of the name - std::string dirName = prefix + nameHash; - std::string fileName = dirName + "/" + vectorHash; + // Generate the dir name using the prefix and the hash of the name + std::string dirName = prefix + nameHash; + std::string fileName = dirName + "/" + vectorHash; - // Check if the file exists - struct stat fileInfo; - if (stat(fileName.c_str(), &fileInfo) != 0) { - if (verbose) { - std::cerr << "error: file does not exist: " << fileName << std::endl; - } - return false; + // Check if the file exists + struct stat fileInfo; + if (stat(fileName.c_str(), &fileInfo) != 0) { + if (verbose) { + std::cerr << "error: file does not exist: " << fileName << std::endl; } + return false; + } - // Check if the file size matches the expected length - if ((size_t)fileInfo.st_size != expectedLen) { - if (verbose) { - std::cerr << "error: file size mismatch. Expected: " << expectedLen << ", Actual: " << fileInfo.st_size << std::endl; - } - return false; + // Check if the file size matches the expected length + if ((size_t)fileInfo.st_size != expectedLen) { + if (verbose) { + std::cerr << "error: file size mismatch. Expected: " << expectedLen + << ", Actual: " << fileInfo.st_size << std::endl; } + return false; + } - // Open the file for reading - std::ifstream inFile(fileName, std::ios::binary); - if (inFile.is_open()) { - inFile.read((char*)data, fileInfo.st_size); - if (inFile.fail()) { - if (verbose) { - std::cerr << "error: failed to read cached data" << std::endl; - } - } else { - std::streamsize bytesRead = inFile.gcount(); - if (bytesRead != fileInfo.st_size) { - if (verbose) { - std::cerr << "error: read only " << bytesRead << " bytes instead of " << fileInfo.st_size << " !"<< std::endl; - } - return false; - } - } - inFile.close(); - return true; + // Open the file for reading + std::ifstream inFile(fileName, std::ios::binary); + if (inFile.is_open()) { + inFile.read((char *)data, fileInfo.st_size); + if (inFile.fail()) { + if (verbose) { + std::cerr << "error: failed to read cached data" << std::endl; + } } else { + std::streamsize bytesRead = inFile.gcount(); + if (bytesRead != fileInfo.st_size) { if (verbose) { - std::cerr << "error: failed to open file: " << fileName << std::endl; + std::cerr << "error: read only " << bytesRead << " bytes instead of " + << fileInfo.st_size << " !" << std::endl; } return false; + } } + inFile.close(); + return true; + } else { + if (verbose) { + std::cerr << "error: failed to open file: " << fileName << std::endl; + } + return false; + } } } // namespace XrdCl diff --git a/src/XrdApps/XrdClJCachePlugin/vector/XrdClVectorCache.hh b/src/XrdApps/XrdClJCachePlugin/vector/XrdClVectorCache.hh index 673eee32335..cccc09f78e7 100644 --- a/src/XrdApps/XrdClJCachePlugin/vector/XrdClVectorCache.hh +++ b/src/XrdApps/XrdClJCachePlugin/vector/XrdClVectorCache.hh @@ -24,16 +24,16 @@ #pragma once /*----------------------------------------------------------------------------*/ +#include // For std::memcpy +#include // For file operations +#include #include -#include -#include -#include // For SHA-256 +#include // For SHA-256 #include -#include -#include // For std::memcpy #include -#include // For file operations -#include // For checking if file exists +#include // For checking if file exists +#include +#include /*----------------------------------------------------------------------------*/ #include "XrdCl/XrdClXRootDResponses.hh" //---------------------------------------------------------------------------- @@ -44,25 +44,27 @@ namespace XrdCl { class VectorCache { public: - VectorCache(const XrdCl::ChunkList chunks, const std::string& name, const char* data, const std::string& prefix, bool verbose=false) - : chunks(chunks), name(name), data(data), prefix(prefix), verbose(verbose) {} - - std::pair computeHash() const; - bool store() const; - bool retrieve() const; + VectorCache(const XrdCl::ChunkList chunks, const std::string &name, + const char *data, const std::string &prefix, bool verbose = false) + : chunks(chunks), name(name), data(data), prefix(prefix), + verbose(verbose) {} + + std::pair computeHash() const; + bool store() const; + bool retrieve() const; - static std::string computeSHA256(const std::vector& data); - static std::string computeSHA256(const std::string& data); - static bool ensureLastSubdirectoryExists(const std::string& dirName); + static std::string computeSHA256(const std::vector &data); + static std::string computeSHA256(const std::string &data); + static bool ensureLastSubdirectoryExists(const std::string &dirName); private: - XrdCl::ChunkList chunks; - std::string name; - const char* data; - std::string prefix; - bool verbose; + XrdCl::ChunkList chunks; + std::string name; + const char *data; + std::string prefix; + bool verbose; - std::vector serializeVector() const; + std::vector serializeVector() const; }; } // namespace XrdCl \ No newline at end of file