add objgraph ref count; also dumps a few dict size to view growth

quarkchain/cluster/slave.py

@@ -788,7 +788,8 @@ async def connect_to_slave(self, slave_info: SlaveInfo) -> str:
         return ""
 
 
-async def tracemalloc_snapshot(slave_id):
+async def tracemalloc_snapshot(slave):
+    slave_id = slave.id.decode('ascii')
     import tracemalloc
     import time
     tracemalloc.start(25)
@@ -803,6 +804,8 @@ async def tracemalloc_snapshot(slave_id):
     import pathlib
     pathlib.Path("./mem{}".format(slave_id)).mkdir(parents=True, exist_ok=True)
 
+    import objgraph
+
     while True:
         # tracemalloc ***
         ts = int(time.time())
@@ -817,6 +820,21 @@ async def tracemalloc_snapshot(slave_id):
             for stat in diff[:10]:
                 Logger.warning(stat)
         snapshot_old = snapshot_new
+        Logger.warning(objgraph.growth())
+        Logger.warning(objgraph.most_common_types(limit=20))
+        Logger.warning("slave.add_block_futures={}".format(len(slave.add_block_futures)))
+        for branch, shard in slave.shards.items():
+            Logger.warning("shard.state.new_block_pool={}".format(len(shard.state.new_block_pool)))
+
+            Logger.warning("shard.add_block_futures={}".format(len(shard.add_block_futures)))
+
+            Logger.warning("shard.state.db.m_header_pool={}".format(len(shard.state.db.m_header_pool)))
+            Logger.warning("shard.state.db.m_meta_pool={}".format(len(shard.state.db.m_meta_pool)))
+            Logger.warning("shard.state.db.x_shard_set={}".format(len(shard.state.db.x_shard_set)))
+            Logger.warning("shard.state.db.r_header_pool={}".format(len(shard.state.db.r_header_pool)))
+            Logger.warning("shard.state.db.r_minor_header_pool={}".format(len(shard.state.db.r_minor_header_pool)))
+            Logger.warning("shard.state.db.height_to_minor_block_hashes={}".format(len(shard.state.db.height_to_minor_block_hashes)))
+
         await asyncio.sleep(60)
 
 
@@ -918,7 +936,7 @@ async def __start_server(self):
         )
 
     def start(self):
-        self.loop.create_task(tracemalloc_snapshot(self.id.decode('ascii')))
+        self.loop.create_task(tracemalloc_snapshot(self))
         self.loop.create_task(self.__start_server())
 
     def start_and_loop(self):

requirements.txt

@@ -19,6 +19,7 @@ pyethash>=0.1.27,<1.0.0
 py_ecc==1.4.3
 eth-hash[pycryptodome]==0.1.4
 vmprof==0.4.12
+objgraph==3.4.0
 
 # p2p
 pytest>=3.6,<3.7