From a948957140e572e38af3669a83f65160328e6bcb Mon Sep 17 00:00:00 2001 From: kudkudak Date: Wed, 14 May 2014 20:47:56 +0200 Subject: [PATCH] Passing 1000 test, #146 --- graph_workers/spidercrab.py | 14 ++++++++++---- tests/spidercrab_1000_test.py | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/graph_workers/spidercrab.py b/graph_workers/spidercrab.py index be2ed31..264ce59 100644 --- a/graph_workers/spidercrab.py +++ b/graph_workers/spidercrab.py @@ -106,6 +106,10 @@ def __init__( @param master: master Spidercrab object @type master: Spidercrab """ + + # Clean logs. TODO: do not use unix command + os.system("rm "+self.STANDARD_STATS_EXPORT_FILE) + self.logger = logger # Config used to be stored inside the database (only values from file) self.given_config = dict() @@ -279,7 +283,7 @@ def _end_run(self): info_level, self.fullname + ' Finished!\nStats:\n' + str(self.stats)) self._export_stats_to(self.STANDARD_STATS_EXPORT_FILE) - if not self.export_stats_to is None: + if not self.export_stats_to is None and self.export_stats_to != self.STANDARD_STATS_EXPORT_FILE: self._export_stats_to(self.export_stats_to) def _export_stats_to(self, file_name): @@ -288,12 +292,14 @@ def _export_stats_to(self, file_name): 'runtime_id': self.runtime_id, 'stats': self.stats } + data = None if not os.path.isfile(file_name): with open(file_name, 'w') as json_file: json_file.write(json.dumps({})) try: - with open(file_name, 'r') as json_file: - data = json.load(json_file) + with open(file_name, 'r') as json_file: + x= json_file.read() + data = json.loads(x) try: graph_worker_id = self.config[self.C_GRAPH_WORKER_ID] if graph_worker_id not in data: @@ -303,7 +309,7 @@ def _export_stats_to(self, file_name): data[graph_worker_id][self.level].append(json_export) finally: with open(file_name, 'w') as json_file: - json_file.write(json.dumps(data, indent=4, sort_keys=True)) + json_file.write(json.dumps(data)) except IOError as e: print e pass diff --git a/tests/spidercrab_1000_test.py b/tests/spidercrab_1000_test.py index 156a26d..558e8ef 100755 --- a/tests/spidercrab_1000_test.py +++ b/tests/spidercrab_1000_test.py @@ -53,7 +53,7 @@ def run_slave(): ] NUMBER_OF_SLAVES = 10 - NEWS_TO_BE_FETCHED = 25 # for each slave + NEWS_TO_BE_FETCHED = 50 # for each slave print 'Running', __file__ print '\nNOTE: You need following file to run this test ' \