Merge pull request #642 from lukemartinlogan/dev

Python binding converter for Metadata Snapshot

include/hermes/hermes_types.h

@@ -322,6 +322,11 @@ struct BlobInfo {
     last_access_.Now();
     access_freq_.fetch_add(1);
   }
+
+  /** Get name as std::string */
+  std::string GetName() {
+    return name_.str();
+  }
 };
 
 /** Data structure used to store Bucket information */
@@ -339,6 +344,11 @@ struct TagInfo {
   void serialize(Ar &ar) {
     ar(tag_id_, name_, internal_size_, page_size_, owner_);
   }
+
+  /** Get std::string of name */
+  std::string GetName() {
+    return name_.str();
+  }
 };
 
 /** The types of I/O that can be performed (for IoCall RPC) */

wrapper/python/cpp/py_hermes.cpp

@@ -22,6 +22,7 @@ using hermes::BlobId;
 using hermes::BucketId;
 using hermes::TagId;
 using hermes::TargetId;
+using hermes::BufferInfo;
 using hermes::BlobInfo;
 using hermes::TargetStats;
 using hermes::TagInfo;
@@ -54,14 +55,23 @@ void BindUniqueId(py::module &m, const std::string &name) {
       .def_readonly("unique", &UniqueT::unique_);
 }
 
+void BindBufferInfo(py::module &m) {
+  py::class_<BufferInfo>(m, "BufferInfo")
+      .def(py::init<>())
+      .def_readwrite("tid", &BufferInfo::tid_)
+      .def_readwrite("t_slab", &BufferInfo::t_slab_)
+      .def_readwrite("t_off", &BufferInfo::t_off_)
+      .def_readwrite("t_size", &BufferInfo::t_size_);
+}
+
 void BindBlobInfo(py::module &m) {
   py::class_<BlobInfo>(m, "BlobInfo")
       .def(py::init<>())
       .def("UpdateWriteStats", &BlobInfo::UpdateWriteStats)
       .def("UpdateReadStats", &BlobInfo::UpdateReadStats)
       .def_readonly("tag_id", &BlobInfo::tag_id_)
       .def_readonly("blob_id", &BlobInfo::blob_id_)
-      .def_readonly("name", &BlobInfo::name_)
+      .def("get_name", &BlobInfo::GetName)
       .def_readonly("buffers", &BlobInfo::buffers_)
       .def_readonly("tags", &BlobInfo::tags_)
       .def_readonly("blob_size", &BlobInfo::blob_size_)
@@ -87,7 +97,7 @@ void BindTagInfo(py::module &m) {
   py::class_<TagInfo>(m, "TagInfo")
       .def(py::init<>())
       .def_readonly("tag_id", &TagInfo::tag_id_)
-      .def_readonly("name", &TagInfo::name_)
+      .def("get_name", &TagInfo::GetName)
       .def_readonly("blobs", &TagInfo::blobs_)
       .def_readonly("traits", &TagInfo::traits_)
       .def_readonly("internal_size", &TagInfo::internal_size_)
@@ -118,8 +128,10 @@ PYBIND11_MODULE(py_hermes, m) {
   BindUniqueId<BucketId>(m, "BucketId");
   BindUniqueId<TagId>(m, "TagId");
   BindUniqueId<TargetId>(m, "TargetId");
+  BindBufferInfo(m);
   BindBlobInfo(m);
   BindTargetStats(m);
+  BindTagInfo(m);
   BindMetadataTable(m);
   BindHermes(m);
 }

wrapper/python/py_hermes_mdm/py_hermes_mdm/py_hermes_mdm.py

@@ -0,0 +1,59 @@
+
+import sys
+import os
+print(os.getenv('PYTHONPATH'))
+from py_hermes import Hermes, TRANSPARENT_HERMES
+class MetadataSnapshot:
+    def __init__(self):
+        TRANSPARENT_HERMES()
+        self.hermes = Hermes()
+        self.blob_info = []
+        self.target_info = []
+        self.tag_info = []
+
+    @staticmethod
+    def unique(id):
+        return f'{id.unique}.{id.node_id}'
+
+    def collect(self):
+        mdm = self.hermes.CollectMetadataSnapshot()
+        for blob in mdm.blob_info:
+            blob_info = {
+                'name': str(blob.get_name()),
+                'id': self.unique(blob.blob_id),
+                'score': float(blob.score),
+                'access_frequency': 0,
+                'buffer_info': []
+            }
+            for buf in blob.buffers:
+                buf_info = {
+                    'target_id': self.unique(buf.tid),
+                    'size': int(buf.t_size)
+                }
+                blob_info['buffer_info'].append(buf_info)
+            self.blob_info.append(blob_info)
+        for tag in mdm.bkt_info:
+            tag_info = {
+                'id': self.unique(tag.tag_id),
+                'name': str(tag.get_name()),
+                'blobs': [self.unique(blob.blob_id) for blob in tag.blobs]
+            }
+            self.tag_info.append(tag_info)
+        for target in mdm.target_info:
+            target_info = {
+                'name': None,
+                'id':  self.unique(target.tgt_id),
+                'rem_cap': target.rem_cap,
+                'max_cap': target.max_cap,
+                'bandwidth': target.bandwidth,
+                'latency': target.latency,
+                'score': target.score,
+            }
+            self.target_info.append(target_info)
+        self.target_info.sort(reverse=True, key=lambda x: x['bandwidth'])
+        for i, target in enumerate(self.target_info):
+            target['name'] = f'Tier {i}'
+
+# mdm = MetadataSnapshot()
+# mdm.collect()
+# print('Done')