Improve Rosie with year filter

rosie/.gitignore

@@ -4,3 +4,4 @@
 .python-version
 __pycache__/
 htmlcov/
+output/

rosie/Dockerfile

@@ -1,13 +1,15 @@
-FROM conda/miniconda3
+FROM continuumio/conda-ci-linux-64-python3.7
+
+ENTRYPOINT ["/opt/conda/bin/python"]
 
 ARG AMAZON_BUCKET=serenata-de-amor-data
 ARG AMAZON_ENDPOINT=https://nyc3.digitaloceanspaces.com
 ARG AMAZON_REGION=nyc3
 
 WORKDIR /code
 COPY requirements.txt ./
-COPY rosie.py ./
 
 RUN pip install -r requirements.txt
 
+COPY rosie.py ./
 COPY rosie ./rosie

rosie/README.md

@@ -9,10 +9,15 @@ A Python application reading receipts from the Quota for Exercising Parliamentar
 #### Running
 
 ```console
-$ docker run --rm -v /tmp/serenata-data:/tmp/serenata-data serenata/rosie python rosie.py run <module-name>
+$ docker build . -t serenata/rosie
+$ docker run --rm \
+             -v /tmp/serenata-data:/tmp/serenata-data \
+             -v rosie_output_backup:/home/test_user:rw \
+             serenata/rosie \
+             rosie.py run <module-name> --output /home/test_user [--last_years=10]
 ```
 
-`<module-name>` might be either `chamber_of_deputies` or `federal_senate`. After running it, check your `/tmp/serenata-data/` directory in you host machine for `suspicions.xz`. It's a compacted CSV with all the irregularities Rosie was able to find.
+`<module-name>` might be either `chamber_of_deputies` or `federal_senate`. After running it, check your `/home/test_user/` directory in you host machine for `suspicions.xz`. It's a compacted CSV with all the irregularities Rosie was able to find.
 
 #### Testing
 

rosie/rosie.py

@@ -5,22 +5,26 @@
 control of public administration.
 
 Usage:
-  rosie.py run (chamber_of_deputies|federal_senate) [--output=<directory>]
+  rosie.py run (chamber_of_deputies|federal_senate) [--output=<directory>] [--last_years=<x>]
   rosie.py test [chamber_of_deputies|federal_senate|core]
 
 Options:
-  --help                Show this screen
-  --output=<directory>  Output directory [default: /tmp/serenata-data]
+  --help                 Show this screen
+  --output=<directory>   Output directory  [default: /tmp/serenata-data]
+  --last_years=<x>       Only last X years
 """
 import os
 import unittest
+import logging
+from datetime import date
 
 from docopt import docopt
 
 import rosie
 import rosie.chamber_of_deputies
 import rosie.federal_senate
 
+log = logging.getLogger('rosie')
 
 def get_module(arguments):
     modules = ('chamber_of_deputies', 'federal_senate', 'core')
@@ -29,9 +33,9 @@ def get_module(arguments):
             return module
 
 
-def run(module, directory):
+def run(module, directory, starting_year):
     module = getattr(rosie, module)
-    module.main(directory)
+    module.main(starting_year=starting_year, target_directory=directory)
 
 
 def test(module=None):
@@ -47,6 +51,11 @@ def test(module=None):
     if not result.wasSuccessful():
         exit(1)
 
+def get_starting_year(last_years):
+    if last_years:
+        return date.today().year - int(last_years) + 1
+    else:
+        return 2009
 
 def main():
     arguments = docopt(__doc__)
@@ -57,8 +66,11 @@ def main():
 
     if arguments['run']:
         module = module if module != 'core' else None
-        run(module, arguments['--output'])
+        starting_year = get_starting_year(arguments['--last_years'])
+        log.info(f'Running from {starting_year}')
+        run(module, arguments['--output'], starting_year)
 
+    log.info('done')
 
 if __name__ == '__main__':
     main()

rosie/rosie/chamber_of_deputies/__init__.py

@@ -3,7 +3,7 @@
 from rosie.core import Core
 
 
-def main(target_directory='/tmp/serenata-data'):
-    adapter = Adapter(target_directory)
+def main(starting_year, target_directory='/tmp/serenata-data'):
+    adapter = Adapter(target_directory, starting_year)
     core = Core(settings, adapter)
     core()

rosie/rosie/chamber_of_deputies/adapter.py

@@ -2,7 +2,7 @@
 import os
 from datetime import date
 from pathlib import Path
-from re import match
+import re
 
 import numpy as np
 import pandas as pd
@@ -14,9 +14,7 @@
 
 class Adapter:
 
-    STARTING_YEAR = 2009
     COMPANIES_DATASET = '2016-09-03-companies.xz'
-    REIMBURSEMENTS_PATTERN = r'reimbursements-\d{4}\.csv'
     RENAME_COLUMNS = {
         'subquota_description': 'category',
         'total_net_value': 'net_value',
@@ -30,9 +28,12 @@ class Adapter:
         'subquota_number': np.str
     }
 
-    def __init__(self, path):
+    def __init__(self, path, starting_year):
         self.path = path
+        self.starting_year = starting_year
         self.log = logging.getLogger(__name__)
+        next_year = date.today().year + 1
+        self.years = range(self.starting_year, next_year)
 
     @property
     def dataset(self):
@@ -57,11 +58,11 @@ def companies(self):
 
     @property
     def reimbursements(self):
-        df = pd.DataFrame()
+        years_pattern = ",".join(map(str, list(self.years)))
         paths = (
-            str(path) for path in Path(self.path).glob('*.csv')
-            if match(self.REIMBURSEMENTS_PATTERN, path.name)
+            str(path) for path in Path(self.path).glob(f"*reimbursements-[{years_pattern}]*.csv")
         )
+        df = pd.DataFrame()
 
         for path in paths:
             self.log.info(f'Loading reimbursements from {path}')
@@ -77,14 +78,12 @@ def update_datasets(self):
     def update_companies(self):
         self.log.info('Updating companies')
         os.makedirs(self.path, exist_ok=True)
-        fetch(self.COMPANIES_DATASET, self.path)
-
-    def update_reimbursements(self, years=None):
-        if not years:
-            next_year = date.today().year + 1
-            years = range(self.STARTING_YEAR, next_year)
+        file_name = f"{self.path}/{self.COMPANIES_DATASET}"
+        if not os.path.isfile(file_name):
+            fetch(self.COMPANIES_DATASET, self.path)
 
-        for year in years:
+    def update_reimbursements(self):
+        for year in self.years:
             self.log.info(f'Updating reimbursements from {year}')
             try:
                 Reimbursements(year, self.path)()

rosie/rosie/federal_senate/__init__.py

@@ -3,7 +3,7 @@
 from rosie.core import Core
 
 
-def main(target_directory='/tmp/serenata-data'):
-    adapter = Adapter(target_directory)
+def main(starting_year, target_directory='/tmp/serenata-data'):
+    adapter = Adapter(target_directory, starting_year)
     core = Core(settings, adapter)
-    core()
+    core()

rosie/rosie/federal_senate/adapter.py

@@ -14,8 +14,9 @@
 
 class Adapter:
 
-    def __init__(self, path):
+    def __init__(self, path, starting_year):
         self.path = path
+        self.starting_year = starting_year
 
     @property
     def dataset(self):