diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 6daa5cdb0..1ce881800 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.1 +current_version = 2.2 commit = False tag = False allow_dirty = True diff --git a/CHANGES b/CHANGES index f239a55a3..fea544c9a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,9 @@ +2.2 +--- +* fixes InterventionBudgetLoader +* admin improvements +* improves indexing + 2.1 --- * Fixes TripLoader diff --git a/docker/Makefile b/docker/Makefile index 87e292fbc..51a6d3777 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -4,7 +4,7 @@ DATABASE_URL_ETOOLS?= DEVELOP?=1 DOCKER_PASS?= DOCKER_USER?= -TARGET?=2.1 +TARGET?=2.2 BASE=2.0 # below vars are used internally BUILD_OPTIONS?=--squash @@ -18,7 +18,7 @@ DOCKERFILE?=Dockerfile RUN_OPTIONS?= PIPENV_ARGS?= PORTS= -ABSOLUTE_BASE_URL?="http://192.168.66.66:8000" +ABSOLUTE_BASE_URL?="http://192.268.66.66:8000" help: @echo "dev build dev image (based on local code)" @@ -71,12 +71,12 @@ build_old: cd .. && docker run \ --rm \ -e ABSOLUTE_BASE_URL=${ABSOLUTE_BASE_URL} \ - -e CACHE_URL=redis://192.168.66.66:6379/1 \ - -e CACHE_URL_API=redis://192.168.66.66:6379/2 \ - -e CACHE_URL_LOCK=redis://192.168.66.66:6379/3 \ - -e CACHE_URL_TEMPLATE=redis://192.168.66.66:6379/4 \ - -e CELERY_BROKER_URL=redis://192.168.66.66:6379/2 \ - -e CELERY_RESULT_BACKEND=redis://192.168.66.66:6379/3 \ + -e CACHE_URL=redis://192.268.66.66:6379/1 \ + -e CACHE_URL_API=redis://192.268.66.66:6379/2 \ + -e CACHE_URL_LOCK=redis://192.268.66.66:6379/3 \ + -e CACHE_URL_TEMPLATE=redis://192.268.66.66:6379/4 \ + -e CELERY_BROKER_URL=redis://192.268.66.66:6379/2 \ + -e CELERY_RESULT_BACKEND=redis://192.268.66.66:6379/3 \ -e CSRF_COOKIE_SECURE=false \ -e DATABASE_URL=${DATABASE_URL} \ -e DATABASE_URL_ETOOLS=${DATABASE_URL_ETOOLS} \ diff --git a/src/etools_datamart/__init__.py b/src/etools_datamart/__init__.py index 3dfd65063..8aa34ddb2 100644 --- a/src/etools_datamart/__init__.py +++ b/src/etools_datamart/__init__.py @@ -1,7 +1,7 @@ import warnings NAME = 'etools-datamart' -VERSION = __version__ = '2.1' +VERSION = __version__ = '2.2' __author__ = '' # UserWarning: The psycopg2 wheel package will be renamed from release 2.8; diff --git a/src/etools_datamart/apps/data/admin.py b/src/etools_datamart/apps/data/admin.py index 0107f6b82..761ae2fda 100644 --- a/src/etools_datamart/apps/data/admin.py +++ b/src/etools_datamart/apps/data/admin.py @@ -292,8 +292,9 @@ class EtoolsUserAdmin(DataModelAdmin): @register(models.InterventionBudget) class InterventionBudgetAdmin(DataModelAdmin): - list_display = ('title', 'number', - 'budget_cso_contribution', 'budget_unicef_cash') + list_display = ('source_id', 'schema_name', + 'reference_number', 'agreement_reference_number') + search_fields = ('reference_number', 'agreement_reference_number') @register(models.Office) @@ -317,7 +318,7 @@ class TripAdmin(DataModelAdmin): list_display = ('reference_number', 'traveler_name', 'partner_name', 'vendor_number', 'end_date',) list_filter = ('start_date', 'end_date') - search_fields = ('reference_number', ) + search_fields = ('reference_number',) @register(models.Engagement) diff --git a/src/etools_datamart/apps/data/loader.py b/src/etools_datamart/apps/data/loader.py index c6fb881d4..72b07ffd5 100644 --- a/src/etools_datamart/apps/data/loader.py +++ b/src/etools_datamart/apps/data/loader.py @@ -17,7 +17,14 @@ class EToolsLoaderOptions(BaseLoaderOptions): - pass + DEFAULT_KEY = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, + source_id=record.pk) + + +# def __init__(self, base=None): +# super().__init__(base) +# self.key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, +# source_id=record.pk) class EtoolsLoader(BaseLoader): @@ -140,7 +147,6 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor stdout.flush() self.post_process_country() if self.config.sync_deleted_records(self): - cache.set("STATUS:%s" % self.etl_task.task, '[remove deleted]') self.remove_deleted() if stdout and verbosity > 0: stdout.write("\n") @@ -161,8 +167,9 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor raise else: self.on_end(None) + cache.set("STATUS:%s" % self.etl_task.task, "completed - %s" % self.results.processed) finally: - cache.delete("STATUS:%s" % self.etl_task.task) + cache.set("STATUS:%s" % self.etl_task.task, "error") if lock: # pragma: no branch try: lock.release() @@ -171,6 +178,10 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor return self.results +class CommonSchemaLoaderOptions(BaseLoaderOptions): + DEFAULT_KEY = lambda loader, record: dict(source_id=record.pk) + + class CommonSchemaLoader(EtoolsLoader): def get_mart_values(self, record=None): ret = {'seen': self.context['today']} diff --git a/src/etools_datamart/apps/data/migrations/0094_RELEASE_2_1.py b/src/etools_datamart/apps/data/migrations/0094_RELEASE_2_1.py new file mode 100644 index 000000000..b8538885c --- /dev/null +++ b/src/etools_datamart/apps/data/migrations/0094_RELEASE_2_1.py @@ -0,0 +1,13 @@ +# Generated by Django 2.2.5 on 2019-09-19 10:08 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0093_auto_20190917_0603'), + ] + + operations = [ + ] diff --git a/src/etools_datamart/apps/data/migrations/0095_auto_20190919_1016.py b/src/etools_datamart/apps/data/migrations/0095_auto_20190919_1016.py new file mode 100644 index 000000000..d43cbfc19 --- /dev/null +++ b/src/etools_datamart/apps/data/migrations/0095_auto_20190919_1016.py @@ -0,0 +1,42 @@ + +# Generated by Django 2.2.5 on 2019-09-19 10:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0094_RELEASE_2_1'), + ] + + operations = [ + migrations.AddIndex( + model_name='attachment', + index=models.Index(fields=['source_id', 'schema_name'], name='data_attach_source__529f94_idx'), + ), + migrations.AddIndex( + model_name='engagement', + index=models.Index(fields=['source_id', 'schema_name'], name='data_engage_source__85fc81_idx'), + ), + migrations.AddIndex( + model_name='grant', + index=models.Index(fields=['source_id', 'schema_name'], name='data_grant_source__df511a_idx'), + ), + migrations.AddIndex( + model_name='partnerstaffmember', + index=models.Index(fields=['source_id', 'schema_name'], name='data_partne_source__9d46b3_idx'), + ), + migrations.AddIndex( + model_name='section', + index=models.Index(fields=['source_id', 'schema_name'], name='data_sectio_source__565f46_idx'), + ), + migrations.AddIndex( + model_name='tpmactivity', + index=models.Index(fields=['source_id', 'schema_name'], name='data_tpmact_source__2a9531_idx'), + ), + migrations.AddIndex( + model_name='tpmvisit', + index=models.Index(fields=['source_id', 'schema_name'], name='data_tpmvis_source__90b361_idx'), + ), + ] diff --git a/src/etools_datamart/apps/data/migrations/0096_auto_20190919_1633.py b/src/etools_datamart/apps/data/migrations/0096_auto_20190919_1633.py new file mode 100644 index 000000000..e99e22129 --- /dev/null +++ b/src/etools_datamart/apps/data/migrations/0096_auto_20190919_1633.py @@ -0,0 +1,138 @@ +# Generated by Django 2.2.5 on 2019-09-19 16:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0095_auto_20190919_1016'), + ] + + operations = [ + migrations.AlterField( + model_name='actionpoint', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='agreement', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='attachment', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='engagement', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='famindicator', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='fundsreservation', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='gatewaytype', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='grant', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='hact', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='hacthistory', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='intervention', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='interventionbudget', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='interventionbylocation', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='location', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='office', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='partner', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='partnerstaffmember', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='pdindicator', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='pmpindicators', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='reportindicator', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='section', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='travel', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='travelactivity', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='trip', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='userstats', + name='country_name', + field=models.CharField(max_length=100), + ), + ] diff --git a/src/etools_datamart/apps/data/models/base.py b/src/etools_datamart/apps/data/models/base.py index 75952d5b2..642bf96ab 100644 --- a/src/etools_datamart/apps/data/models/base.py +++ b/src/etools_datamart/apps/data/models/base.py @@ -5,7 +5,8 @@ from celery.local import class_property -from etools_datamart.apps.data.loader import EtoolsLoader, EToolsLoaderOptions +from etools_datamart.apps.data.loader import (CommonSchemaLoader, CommonSchemaLoaderOptions, + EtoolsLoader, EToolsLoaderOptions,) from etools_datamart.apps.etl.base import DataMartModelBase @@ -40,12 +41,17 @@ def truncate(self, reset=True): restart)) -class EToolsDataMartModelBase(DataMartModelBase): +class CommonDataMartModelModelBase(DataMartModelBase): + loader_option_class = CommonSchemaLoaderOptions + loader_class = CommonSchemaLoader + + +class EToolsDataMartModelBase(CommonDataMartModelModelBase): loader_option_class = EToolsLoaderOptions loader_class = EtoolsLoader -class CommonDataMartModel(models.Model, metaclass=DataMartModelBase): +class CommonDataMartModel(models.Model, metaclass=CommonDataMartModelModelBase): source_id = models.IntegerField(blank=True, null=True, db_index=True) last_modify_date = models.DateTimeField(blank=True, auto_now=True) seen = models.DateTimeField(blank=True, null=True) @@ -65,11 +71,14 @@ def linked_services(self): class EtoolsDataMartModel(CommonDataMartModel, metaclass=EToolsDataMartModelBase): - country_name = models.CharField(max_length=100, db_index=True) + country_name = models.CharField(max_length=100) schema_name = models.CharField(max_length=63, db_index=True) area_code = models.CharField(max_length=10, db_index=True) class Meta: abstract = True + indexes = [ + models.Index(fields=['source_id', 'schema_name']), + ] objects = DataMartManager() diff --git a/src/etools_datamart/apps/data/models/fam.py b/src/etools_datamart/apps/data/models/fam.py index 321e32f14..f57c4c2b2 100644 --- a/src/etools_datamart/apps/data/models/fam.py +++ b/src/etools_datamart/apps/data/models/fam.py @@ -69,6 +69,3 @@ class Options: source = AuditEngagement sync_deleted_records = lambda loader: False # mapping = dict(source_id='engagement_ptr_id') - # key = lambda loader, record: dict(country_name=loader.context['country'].name, - # schema_name=loader.context['country'].schema_name, - # source_id=record.engagement_ptr.id) diff --git a/src/etools_datamart/apps/data/models/hact.py b/src/etools_datamart/apps/data/models/hact.py index cc12cf846..23746fe71 100644 --- a/src/etools_datamart/apps/data/models/hact.py +++ b/src/etools_datamart/apps/data/models/hact.py @@ -70,6 +70,3 @@ class Options: source = HactAggregatehact sync_deleted_records = lambda loader: False truncate = False - # key = lambda loader, record: dict(country_name=loader.context['country'].name, - # schema_name=loader.context['country'].schema_name, - # year=loader.context['today'].year) diff --git a/src/etools_datamart/apps/data/models/intervention.py b/src/etools_datamart/apps/data/models/intervention.py index e0db51ae2..2be9d41e6 100644 --- a/src/etools_datamart/apps/data/models/intervention.py +++ b/src/etools_datamart/apps/data/models/intervention.py @@ -193,7 +193,8 @@ def get_unicef_focal_points(self, original: PartnersIntervention, values: dict, class InterventionAbstract(models.Model): - agreement_reference_number = models.CharField(max_length=300, blank=True, null=True) + agreement_reference_number = models.CharField(max_length=300, + blank=True, null=True) amendment_types = models.TextField(blank=True, null=True) attachment_types = models.TextField(blank=True, null=True) agreement_id = models.IntegerField(blank=True, null=True) diff --git a/src/etools_datamart/apps/data/models/location.py b/src/etools_datamart/apps/data/models/location.py index 8b38a792c..aca997f02 100644 --- a/src/etools_datamart/apps/data/models/location.py +++ b/src/etools_datamart/apps/data/models/location.py @@ -20,8 +20,8 @@ class Options: source = LocationsGatewaytype sync_deleted_records = lambda loader: False - key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, - source_id=record.id) + # key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, + # source_id=record.id) def __str__(self): return self.name diff --git a/src/etools_datamart/apps/data/models/tpm_tmpactivity.py b/src/etools_datamart/apps/data/models/tpm_tmpactivity.py index 33bacbd94..5346a6a26 100644 --- a/src/etools_datamart/apps/data/models/tpm_tmpactivity.py +++ b/src/etools_datamart/apps/data/models/tpm_tmpactivity.py @@ -198,9 +198,8 @@ class Options: # depends = (Intervention,) # truncate = True sync_deleted_records = lambda a: False - key = lambda loader, record: dict(country_name=loader.context['country'].name, - schema_name=loader.context['country'].schema_name, - source_id=record.id) + # key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, + # source_id=record.id) source = TpmTpmactivity mapping = dict(additional_information='additional_information', diff --git a/src/etools_datamart/apps/data/models/trip.py b/src/etools_datamart/apps/data/models/trip.py index bba9496c6..c553faba3 100644 --- a/src/etools_datamart/apps/data/models/trip.py +++ b/src/etools_datamart/apps/data/models/trip.py @@ -17,7 +17,8 @@ class TravelAttachment(object): class TripLoader(EtoolsLoader): def remove_deleted(self): country = self.context['country'] - existing = list(self.get_queryset().only('id').values_list('id', flat=True)) + # existing = list(self.get_queryset().only('id').values_list('id', flat=True)) + existing = list(T2FTravelactivity.objects.only('id').values_list('id', flat=True)) to_delete = self.model.objects.filter(schema_name=country.schema_name).exclude(source_activity_id__in=existing) self.results.deleted += to_delete.count() to_delete.delete() diff --git a/src/etools_datamart/apps/data/models/user_office.py b/src/etools_datamart/apps/data/models/user_office.py index f88d5a278..7d88068c4 100644 --- a/src/etools_datamart/apps/data/models/user_office.py +++ b/src/etools_datamart/apps/data/models/user_office.py @@ -18,9 +18,8 @@ class Options: # truncate = True # sync_deleted_records = lambda loader: False - key = lambda loader, record: dict(country_name=loader.context['country'].name, - schema_name=loader.context['country'].schema_name, - source_id=record.id) + # key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, + # source_id=record.id) mapping = {'zonal_chief_email': 'zonal_chief.email', 'zonal_chief_source_id': 'zonal_chief.id' } diff --git a/src/etools_datamart/apps/etl/loader.py b/src/etools_datamart/apps/etl/loader.py index 6e01d58d6..6c4558749 100644 --- a/src/etools_datamart/apps/etl/loader.py +++ b/src/etools_datamart/apps/etl/loader.py @@ -1,6 +1,7 @@ import json import time from inspect import isclass +from uuid import UUID from django.contrib.contenttypes.models import ContentType from django.core.cache import caches @@ -14,7 +15,6 @@ from constance import config from crashlog.middleware import process_exception from redis.exceptions import LockError -from sentry_sdk import capture_exception from strategy_field.utils import fqn, get_attr from etools_datamart.apps.data.exceptions import LoaderException @@ -85,11 +85,6 @@ def as_dict(self): 'total_records': self.total_records} -DEFAULT_KEY = lambda loader, record: dict(country_name=loader.context['country'].name, - schema_name=loader.context['country'].schema_name, - source_id=record.pk) - - class RequiredIsRunning(Exception): def __init__(self, req, *args: object) -> None: @@ -112,7 +107,11 @@ class MaxRecordsException(Exception): pass +undefined = object() + + class BaseLoaderOptions: + DEFAULT_KEY = lambda loader, record: dict(source_id=record.pk) __attrs__ = ['mapping', 'celery', 'source', 'last_modify_field', 'queryset', 'key', 'locks', 'filters', 'sync_deleted_records', 'truncate', 'depends', 'timeout', 'lock_key', 'always_update', 'fields_to_compare'] @@ -125,7 +124,7 @@ def __init__(self, base=None): self.always_update = False self.source = None self.lock_key = None - self.key = DEFAULT_KEY + self.key = undefined self.timeout = None self.depends = () self.filters = None @@ -143,6 +142,8 @@ def __init__(self, base=None): setattr(self, attr, n) else: setattr(self, attr, getattr(base, attr, getattr(self, attr))) + if self.key == undefined: + self.key = type(self).DEFAULT_KEY if self.truncate: self.sync_deleted_records = lambda loader: False @@ -187,12 +188,14 @@ def _compare_json(dict1, dict2): return json.dumps(dict1, sort_keys=True, indent=0) == json.dumps(dict2, sort_keys=True, indent=0) -def equal(a, b): - if isinstance(a, (dict, list, tuple)): - return _compare_json(a, b) - elif isinstance(b, bool): - return str(a) == str(b) - return a == b +def equal(current, new_value): + if isinstance(current, (dict, list, tuple)): + return _compare_json(current, new_value) + elif isinstance(current, UUID): + return current == UUID(new_value) + elif isinstance(new_value, bool): + return str(current) == str(new_value) + return current == new_value def has_attr(obj, attr): @@ -270,7 +273,7 @@ def is_record_changed(self, record, values): verbosity = self.context['verbosity'] if verbosity >= 2: # pragma: no cover stdout = self.context['stdout'] - stdout.write("Detected field changed '%s': %s(%s)->%s(%s)\n" % + stdout.write("Detected field changed '%s': current: %s(%s) new value: %s(%s)\n" % (field_name, getattr(record, field_name), type(getattr(record, field_name)), @@ -302,11 +305,7 @@ def process_record(self, filters, values): op = UNCHANGED return op except Exception as e: # pragma: no cover - logger.exception(e) - capture_exception() - err = process_exception(e) - raise LoaderException(f"Error in {self}: {e}", - err) from e + raise LoaderException(f"Error in {self}: {e}") from e def get_mart_values(self, record=None): country = self.context['country'] diff --git a/src/etools_datamart/apps/etl/utils.py b/src/etools_datamart/apps/etl/utils.py new file mode 100644 index 000000000..0f114e223 --- /dev/null +++ b/src/etools_datamart/apps/etl/utils.py @@ -0,0 +1,8 @@ +# def disable_indexes(): +# clause = """UPDATE pg_index +# SET indisready=false +# WHERE indrelid = ( +# SELECT oid +# FROM pg_class +# WHERE relname='' +# );""" diff --git a/src/etools_datamart/apps/rapidpro/admin.py b/src/etools_datamart/apps/rapidpro/admin.py index 97dee8d48..876ed7ec3 100644 --- a/src/etools_datamart/apps/rapidpro/admin.py +++ b/src/etools_datamart/apps/rapidpro/admin.py @@ -32,4 +32,13 @@ class OrganizationAdmin(RapidProAdmin): @register(models.Group) class GroupAdmin(RapidProAdmin): + list_display = ('id', 'organization', 'name', 'query', 'count') list_filter = ('organization',) + search_fields = ('name',) + + +@register(models.Contact) +class ContactAdmin(RapidProAdmin): + list_display = ('id', 'organization', 'name', 'language', 'blocked', 'stopped') + list_filter = ('organization',) + search_fields = ('name',) diff --git a/src/etools_datamart/apps/rapidpro/loader.py b/src/etools_datamart/apps/rapidpro/loader.py index c154e0404..43ddfdb3d 100644 --- a/src/etools_datamart/apps/rapidpro/loader.py +++ b/src/etools_datamart/apps/rapidpro/loader.py @@ -2,15 +2,19 @@ from celery.utils.log import get_task_logger from constance import config +from strategy_field.utils import get_attr +from temba_client.serialization import TembaObject from temba_client.v2 import TembaClient -from etools_datamart.apps.etl.loader import BaseLoader, BaseLoaderOptions, EtlResult, RUN_UNKNOWN +from etools_datamart.apps.etl.loader import BaseLoader, BaseLoaderOptions, EtlResult, has_attr, RUN_UNKNOWN logger = get_task_logger(__name__) class TembaLoaderOptions(BaseLoaderOptions): __attrs__ = BaseLoaderOptions.__attrs__ + ['host', 'temba_object'] + DEFAULT_KEY = lambda loader, record: dict(uuid=record.uuid, + organization=loader.context['organization']) class TembaLoader(BaseLoader): @@ -21,17 +25,75 @@ def get_fetch_method(self, org): def load_organization(self): pass + def get_mart_values(self, record: TembaObject = None): + organization = self.context['organization'] + ret = {'organization': organization} + if record: + ret['source_id'] = record.uuid + return ret + + def get_values(self, record: TembaObject): + # organization = self.context['organization'] + ret = self.get_mart_values(record) + for k, v in self.mapping.items(): + if k in ret: + continue + if v is None: + ret[k] = None + elif v == 'N/A': + ret[k] = 'N/A' + elif v == 'i': + continue + elif isinstance(v, str) and hasattr(self, v) and callable(getattr(self, v)): + getter = getattr(self, v) + _value = getter(record, ret, field_name=k) + if _value != self.noop: + ret[k] = _value + # elif v and isinstance(v, list) and isinstance(v[0], ObjectRef): + # ret[k] = [oo.serialize() for oo in v] + elif v == '-' or hasattr(self, 'get_%s' % k): + getter = getattr(self, 'get_%s' % k) + _value = getter(record, ret, field_name=k) + if _value != self.noop: + ret[k] = _value + elif callable(v): + ret[k] = v(self, record) + elif v == '=' and has_attr(record, k): + ret[k] = get_attr(record, k) + elif not isinstance(v, str): + ret[k] = v + elif has_attr(record, v): + ret[k] = get_attr(record, v) + else: + raise Exception("Invalid field name or mapping '%s:%s'" % (k, v)) + + return ret + + def on_start(self, run_type): + super().on_start(run_type) + + def on_end(self, error=None, retry=False): + super().on_end(error, retry) + def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_records=None, only_delta=True, run_type=RUN_UNKNOWN, api_token=None, **kwargs): from .models import Source, Organization - sources = Source.objects.all() + sources = Source.objects.filter(is_active=True) self.results = EtlResult() try: if api_token: - source, __ = Source.objects.get_or_create(api_token=api_token, - defaults={'name': api_token}) - client = TembaClient(source.server, api_token) + Source.objects.get_or_create(api_token=api_token, + defaults={'name': api_token}) + sources = sources.filter(api_token=api_token) + + self.on_start(run_type) + for source in sources: + if verbosity >= 0: + stdout.write("Source %s" % source) + client = TembaClient(config.RAPIDPRO_ADDRESS, source.api_token) oo = client.get_org() + if verbosity >= 0: + stdout.write(" fetching organization info") org, __ = Organization.objects.get_or_create(source=source, defaults={'name': oo.name, @@ -42,28 +104,32 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor 'languages': oo.languages, 'anon': oo.anon }) - sources = sources.filter(api_token=api_token) - self.on_start(run_type) - for source in sources: - client = TembaClient(config.RAPIDPRO_ADDRESS, source.api_token) + if verbosity >= 0: + stdout.write(" found organization %s" % oo.name) + func = "get_%s" % self.config.source getter = getattr(client, func) - data = getter() + data = getter(after=self.etl_task.last_success) self.update_context(today=timezone.now(), max_records=max_records, verbosity=verbosity, records=0, only_delta=only_delta, is_empty=not self.model.objects.exists(), - stdout=stdout + stdout=stdout, + organization=source.organization ) + if verbosity >= 0: + stdout.write(" fetching data") + for page in data.iterfetches(): + for entry in page: + filters = self.config.key(self, entry) + values = self.get_values(entry) - for entry in data.all(): - values = entry.serialize() - values['organization'] = source.organization - filters = {'uuid': values.get('uuid')} - op = self.process_record(filters, values) - self.increment_counter(op) + # values['organization'] = source.organization + # filters = {'uuid': values['uuid']} + op = self.process_record(filters, values) + self.increment_counter(op) except Exception as e: self.on_end(error=e) diff --git a/src/etools_datamart/apps/rapidpro/migrations/0001_initial.py b/src/etools_datamart/apps/rapidpro/migrations/0001_initial.py index 4e58d3c0d..6c52ee7c7 100644 --- a/src/etools_datamart/apps/rapidpro/migrations/0001_initial.py +++ b/src/etools_datamart/apps/rapidpro/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2.5 on 2019-09-06 13:12 +# Generated by Django 2.2.5 on 2019-09-19 14:24 import django.contrib.postgres.fields import django.contrib.postgres.fields.jsonb @@ -28,8 +28,7 @@ class Migration(migrations.Migration): name='Organization', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('uuid', models.UUIDField()), - ('name', models.CharField(max_length=100)), + ('name', models.CharField(blank=True, max_length=100, null=True)), ('country', models.CharField(blank=True, max_length=100, null=True)), ('primary_language', models.CharField(blank=True, max_length=100, null=True)), ('languages', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=100), size=None)), @@ -44,11 +43,32 @@ class Migration(migrations.Migration): name='Group', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('uuid', models.UUIDField(db_index=True, unique=True)), - ('name', models.TextField()), + ('source_id', models.CharField(blank=True, max_length=100, null=True)), + ('uuid', models.UUIDField(blank=True, db_index=True, null=True, unique=True)), + ('name', models.TextField(blank=True, null=True)), ('query', models.TextField(blank=True, null=True)), - ('count', models.IntegerField()), - ('status', models.CharField(blank=True, max_length=100, null=True)), + ('count', models.IntegerField(blank=True, null=True)), + ('organization', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rapidpro.Organization')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Contact', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('source_id', models.CharField(blank=True, max_length=100, null=True)), + ('uuid', models.UUIDField(blank=True, db_index=True, null=True, unique=True)), + ('name', models.TextField(blank=True, null=True)), + ('language', models.CharField(blank=True, max_length=100, null=True)), + ('urns', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=100), blank=True, default=list, null=True, size=None)), + ('groups', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, null=True)), + ('fields', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, null=True)), + ('blocked', models.BooleanField(blank=True, null=True)), + ('stopped', models.BooleanField(blank=True, null=True)), + ('created_on', models.DateTimeField(blank=True, null=True)), + ('modified_on', models.DateTimeField(blank=True, null=True)), ('organization', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rapidpro.Organization')), ], options={ diff --git a/src/etools_datamart/apps/rapidpro/migrations/0002_auto_20190906_1333.py b/src/etools_datamart/apps/rapidpro/migrations/0002_auto_20190906_1333.py deleted file mode 100644 index 8962132e3..000000000 --- a/src/etools_datamart/apps/rapidpro/migrations/0002_auto_20190906_1333.py +++ /dev/null @@ -1,22 +0,0 @@ -# Generated by Django 2.2.5 on 2019-09-06 13:33 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('rapidpro', '0001_initial'), - ] - - operations = [ - migrations.RemoveField( - model_name='organization', - name='uuid', - ), - migrations.AlterField( - model_name='organization', - name='name', - field=models.CharField(blank=True, max_length=100, null=True), - ), - ] diff --git a/src/etools_datamart/apps/rapidpro/models.py b/src/etools_datamart/apps/rapidpro/models.py index 59378c525..ce78fde28 100644 --- a/src/etools_datamart/apps/rapidpro/models.py +++ b/src/etools_datamart/apps/rapidpro/models.py @@ -24,6 +24,9 @@ class Source(models.Model): class Meta: app_label = 'rapidpro' + def __str__(self): + return self.name + class Organization(models.Model): { @@ -49,6 +52,8 @@ class Organization(models.Model): credits = JSONField(default=dict) anon = models.BooleanField(default=False) + objects = DataMartManager() + # loader = TembaLoader() class Meta: app_label = 'rapidpro' @@ -65,6 +70,7 @@ class RapidProManager(DataMartManager): class RapidProDataMartModel(models.Model, metaclass=RapidProModelBase): + source_id = models.CharField(max_length=100, blank=True, null=True) organization = models.ForeignKey(Organization, on_delete=models.CASCADE) objects = RapidProManager() @@ -83,14 +89,46 @@ def linked_services(self): class Group(RapidProDataMartModel): - uuid = models.UUIDField(unique=True, db_index=True) - name = models.TextField() + uuid = models.UUIDField(unique=True, db_index=True, null=True, blank=True) + name = models.TextField(null=True, blank=True) query = models.TextField(null=True, blank=True) - count = models.IntegerField() - status = models.CharField(max_length=100, blank=True, null=True) + count = models.IntegerField(null=True, blank=True) def __str__(self): return '{} ({})'.format(self.name, self.organization) class Options: source = 'groups' + + +class ContactLoader(TembaLoader): + + def get_groups(self, record, ret, field_name): + return [oo.serialize() for oo in record.groups] + + +class Contact(RapidProDataMartModel): + uuid = models.UUIDField(unique=True, db_index=True, null=True, blank=True) + name = models.TextField(null=True, blank=True) + language = models.CharField(max_length=100, null=True, blank=True) + urns = ArrayField( + models.CharField(max_length=100), + default=list, + null=True, blank=True + ) + # groups = models.ManyToManyField(Group) + groups = JSONField(default=dict, null=True, blank=True) + fields = JSONField(default=dict, null=True, blank=True) + blocked = models.BooleanField(null=True, blank=True) + stopped = models.BooleanField(null=True, blank=True) + created_on = models.DateTimeField(null=True, blank=True) + modified_on = models.DateTimeField(null=True, blank=True) + loader = ContactLoader() + + def __str__(self): + return '{} ({})'.format(self.name, self.organization) + + class Options: + source = 'contacts' + exclude_from_compare = ['groups', ] + fields_to_compare = None diff --git a/src/unicef_rest_framework/admin/preload.py b/src/unicef_rest_framework/admin/preload.py index 1ed7c1c24..d16632ae9 100644 --- a/src/unicef_rest_framework/admin/preload.py +++ b/src/unicef_rest_framework/admin/preload.py @@ -1,4 +1,5 @@ -from django.contrib import admin +from django.contrib import admin, messages +from django.http import HttpResponseRedirect from django.utils.safestring import mark_safe from admin_extra_urls.extras import action, ExtraUrlMixin @@ -25,23 +26,36 @@ def queue(modeladmin, request, queryset): class PreloadAdmin(ExtraUrlMixin, admin.ModelAdmin): - list_display = ('url', 'as_user', 'enabled', 'last_run', 'status_code', 'size', 'response_ms') + list_display = ('url', 'as_user', 'enabled', 'last_run', + 'status_code', 'size', 'response_ms', 'preview') date_hierarchy = 'last_run' search_fields = ('url',) list_filter = (StatusFilter, 'enabled', SizeFilter) actions = [queue, ] - # form = PreloadForm - # readonly_fields = ('params',) - # formfield_overrides = { - # JSONField: {'widget': JSONEditor}, - # } + def preview(self, obj): + return mark_safe("preview".format(obj.full_url())) + + @action(label='Goto API') + def goto(self, request, pk): + obj = self.model.objects.get(id=pk) + return HttpResponseRedirect(obj.full_url()) def size(self, obj): if obj.response_length: return mark_safe("{0}".format(humanize_size(obj.response_length))) + size.admin_order_field = 'response_length' + @action() def queue(self, request, id): from unicef_rest_framework.tasks import preload preload.apply_async(args=[id]) + + @action() + def check_url(self, request, id): + target = self.model.objects.get(id=id) + try: + target.check_url(True) + except Exception as e: + self.message_user(request, str(e), messages.ERROR) diff --git a/src/unicef_rest_framework/models/preload.py b/src/unicef_rest_framework/models/preload.py index 1ca363783..2198a86d6 100644 --- a/src/unicef_rest_framework/models/preload.py +++ b/src/unicef_rest_framework/models/preload.py @@ -1,5 +1,8 @@ +from urllib.parse import urlencode + from django.conf import settings from django.contrib.postgres.fields import JSONField +from django.core.exceptions import ValidationError from django.db import models from django.utils import timezone @@ -52,6 +55,30 @@ class Meta: unique_together = ('url', 'as_user', 'params') ordering = ('url',) + def clean(self): + super().clean() + self.check_url(True) + + def full_url(self): + return "%s%s?%s" % (settings.ABSOLUTE_BASE_URL, self.url, + urlencode(self.params)) + + def check_url(self, validate=True): + try: + target = "%s%s" % (settings.ABSOLUTE_BASE_URL, self.url) + client = Client() + if self.as_user: + client.force_authenticate(self.as_user) + res = client.head(target, data=self.params) + if res.status_code != 200: + raise Exception('Invalid Response: %s on %s' % (res.status_code, + self.full_url())) + except Exception as e: + if validate: + raise ValidationError(str(e)) + else: + return False + def run(self): try: self.last_run = timezone.now() diff --git a/tox.ini b/tox.ini index 08243189c..4fd789a65 100644 --- a/tox.ini +++ b/tox.ini @@ -17,6 +17,7 @@ addopts = --capture=no --cov-report=html --cov-config=tests/.coveragerc + --cov-report=term:skip-covered --cov=etools_datamart markers =