diff --git a/quarkchain/cluster/master.py b/quarkchain/cluster/master.py index a98675219..73af7497e 100644 --- a/quarkchain/cluster/master.py +++ b/quarkchain/cluster/master.py @@ -1493,6 +1493,33 @@ def parse_args(): return env +async def tracemalloc_snapshot(env): + import tracemalloc + tracemalloc.start(25) + snapshot_old = None + + # vmprof *** + import vmprof + import os + f=os.open("./vmprofout", os.O_RDWR|os.O_CREAT) + vmprof.enable(f, period=0.5, memory=True) # 2 Hz, see https://github.com/blue-yonder/vmprof-viewer-client + + while True: + # tracemalloc *** + ts = int(time.time()) + snapshot_new = tracemalloc.take_snapshot() + fn = "./mem/{}".format(ts) + snapshot_new.dump(fn) + Logger.warning("dumped mem snapshot to {}".format(fn)) + + if snapshot_old: + diff = snapshot_new.compare_to(snapshot_old, 'lineno') + Logger.warning("[ Top 10 differences ]") + for stat in diff[:10]: + Logger.warning(stat) + snapshot_old = snapshot_new + await asyncio.sleep(60) + def main(): from quarkchain.cluster.jsonrpc import JSONRPCServer @@ -1522,6 +1549,8 @@ def main(): public_json_rpc_server = JSONRPCServer.start_public_server(env, master) private_json_rpc_server = JSONRPCServer.start_private_server(env, master) + asyncio.ensure_future(tracemalloc_snapshot(env)) + try: loop.run_until_complete(master.shutdown_future) except KeyboardInterrupt: diff --git a/requirements.txt b/requirements.txt index ab249def0..d3ff96377 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,7 @@ eth-bloom==1.0.0 pyethash>=0.1.27,<1.0.0 py_ecc==1.4.3 eth-hash[pycryptodome]==0.1.4 +vmprof==0.4.12 # p2p pytest>=3.6,<3.7