From 5c53f8f1bfac817cac9d32cdad10d90a25b4acfd Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Mon, 30 Oct 2023 23:29:38 -0500 Subject: [PATCH 1/2] Add bindings for buffer info in blob info --- wrapper/python/cpp/py_hermes.cpp | 11 +++++++++++ wrapper/python/py_hermes/py_hermes/py_hermes.py | 0 2 files changed, 11 insertions(+) create mode 100644 wrapper/python/py_hermes/py_hermes/py_hermes.py diff --git a/wrapper/python/cpp/py_hermes.cpp b/wrapper/python/cpp/py_hermes.cpp index 51efeabd0..433c2b41d 100644 --- a/wrapper/python/cpp/py_hermes.cpp +++ b/wrapper/python/cpp/py_hermes.cpp @@ -22,6 +22,7 @@ using hermes::BlobId; using hermes::BucketId; using hermes::TagId; using hermes::TargetId; +using hermes::BufferInfo; using hermes::BlobInfo; using hermes::TargetStats; using hermes::TagInfo; @@ -54,6 +55,15 @@ void BindUniqueId(py::module &m, const std::string &name) { .def_readonly("unique", &UniqueT::unique_); } +void BindBufferInfo(py::module &m) { + py::class_(m, "BufferInfo") + .def(py::init<>()) + .def_readwrite("tid", &BufferInfo::tid_) + .def_readwrite("t_slab", &BufferInfo::t_slab_) + .def_readwrite("t_off", &BufferInfo::t_off_) + .def_readwrite("t_size", &BufferInfo::t_size_); +} + void BindBlobInfo(py::module &m) { py::class_(m, "BlobInfo") .def(py::init<>()) @@ -118,6 +128,7 @@ PYBIND11_MODULE(py_hermes, m) { BindUniqueId(m, "BucketId"); BindUniqueId(m, "TagId"); BindUniqueId(m, "TargetId"); + BindBufferInfo(m); BindBlobInfo(m); BindTargetStats(m); BindMetadataTable(m); diff --git a/wrapper/python/py_hermes/py_hermes/py_hermes.py b/wrapper/python/py_hermes/py_hermes/py_hermes.py new file mode 100644 index 000000000..e69de29bb From 1ce183d03e4d3cbd7f83eab7ca1e39a707646446 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 31 Oct 2023 00:28:54 -0500 Subject: [PATCH 2/2] py_hermes_mdm works --- include/hermes/hermes_types.h | 10 ++++ wrapper/python/cpp/py_hermes.cpp | 5 +- .../python/py_hermes/py_hermes/py_hermes.py | 0 .../py_hermes_mdm/py_hermes_mdm.py | 59 +++++++++++++++++++ 4 files changed, 72 insertions(+), 2 deletions(-) delete mode 100644 wrapper/python/py_hermes/py_hermes/py_hermes.py create mode 100644 wrapper/python/py_hermes_mdm/py_hermes_mdm/py_hermes_mdm.py diff --git a/include/hermes/hermes_types.h b/include/hermes/hermes_types.h index 822540252..3d5381dcb 100644 --- a/include/hermes/hermes_types.h +++ b/include/hermes/hermes_types.h @@ -322,6 +322,11 @@ struct BlobInfo { last_access_.Now(); access_freq_.fetch_add(1); } + + /** Get name as std::string */ + std::string GetName() { + return name_.str(); + } }; /** Data structure used to store Bucket information */ @@ -339,6 +344,11 @@ struct TagInfo { void serialize(Ar &ar) { ar(tag_id_, name_, internal_size_, page_size_, owner_); } + + /** Get std::string of name */ + std::string GetName() { + return name_.str(); + } }; /** The types of I/O that can be performed (for IoCall RPC) */ diff --git a/wrapper/python/cpp/py_hermes.cpp b/wrapper/python/cpp/py_hermes.cpp index 433c2b41d..e6c3aebd1 100644 --- a/wrapper/python/cpp/py_hermes.cpp +++ b/wrapper/python/cpp/py_hermes.cpp @@ -71,7 +71,7 @@ void BindBlobInfo(py::module &m) { .def("UpdateReadStats", &BlobInfo::UpdateReadStats) .def_readonly("tag_id", &BlobInfo::tag_id_) .def_readonly("blob_id", &BlobInfo::blob_id_) - .def_readonly("name", &BlobInfo::name_) + .def("get_name", &BlobInfo::GetName) .def_readonly("buffers", &BlobInfo::buffers_) .def_readonly("tags", &BlobInfo::tags_) .def_readonly("blob_size", &BlobInfo::blob_size_) @@ -97,7 +97,7 @@ void BindTagInfo(py::module &m) { py::class_(m, "TagInfo") .def(py::init<>()) .def_readonly("tag_id", &TagInfo::tag_id_) - .def_readonly("name", &TagInfo::name_) + .def("get_name", &TagInfo::GetName) .def_readonly("blobs", &TagInfo::blobs_) .def_readonly("traits", &TagInfo::traits_) .def_readonly("internal_size", &TagInfo::internal_size_) @@ -131,6 +131,7 @@ PYBIND11_MODULE(py_hermes, m) { BindBufferInfo(m); BindBlobInfo(m); BindTargetStats(m); + BindTagInfo(m); BindMetadataTable(m); BindHermes(m); } diff --git a/wrapper/python/py_hermes/py_hermes/py_hermes.py b/wrapper/python/py_hermes/py_hermes/py_hermes.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/wrapper/python/py_hermes_mdm/py_hermes_mdm/py_hermes_mdm.py b/wrapper/python/py_hermes_mdm/py_hermes_mdm/py_hermes_mdm.py new file mode 100644 index 000000000..ce653f2d5 --- /dev/null +++ b/wrapper/python/py_hermes_mdm/py_hermes_mdm/py_hermes_mdm.py @@ -0,0 +1,59 @@ + +import sys +import os +print(os.getenv('PYTHONPATH')) +from py_hermes import Hermes, TRANSPARENT_HERMES +class MetadataSnapshot: + def __init__(self): + TRANSPARENT_HERMES() + self.hermes = Hermes() + self.blob_info = [] + self.target_info = [] + self.tag_info = [] + + @staticmethod + def unique(id): + return f'{id.unique}.{id.node_id}' + + def collect(self): + mdm = self.hermes.CollectMetadataSnapshot() + for blob in mdm.blob_info: + blob_info = { + 'name': str(blob.get_name()), + 'id': self.unique(blob.blob_id), + 'score': float(blob.score), + 'access_frequency': 0, + 'buffer_info': [] + } + for buf in blob.buffers: + buf_info = { + 'target_id': self.unique(buf.tid), + 'size': int(buf.t_size) + } + blob_info['buffer_info'].append(buf_info) + self.blob_info.append(blob_info) + for tag in mdm.bkt_info: + tag_info = { + 'id': self.unique(tag.tag_id), + 'name': str(tag.get_name()), + 'blobs': [self.unique(blob.blob_id) for blob in tag.blobs] + } + self.tag_info.append(tag_info) + for target in mdm.target_info: + target_info = { + 'name': None, + 'id': self.unique(target.tgt_id), + 'rem_cap': target.rem_cap, + 'max_cap': target.max_cap, + 'bandwidth': target.bandwidth, + 'latency': target.latency, + 'score': target.score, + } + self.target_info.append(target_info) + self.target_info.sort(reverse=True, key=lambda x: x['bandwidth']) + for i, target in enumerate(self.target_info): + target['name'] = f'Tier {i}' + +# mdm = MetadataSnapshot() +# mdm.collect() +# print('Done')