From 98951e97697bf75a61251bd994f446faddaf140a Mon Sep 17 00:00:00 2001 From: saxix Date: Tue, 17 Sep 2019 16:45:37 +0200 Subject: [PATCH 01/10] update version --- .bumpversion.cfg | 2 +- CHANGES | 4 ++++ docker/Makefile | 16 ++++++++-------- src/etools_datamart/__init__.py | 2 +- src/etools_datamart/apps/data/loader.py | 1 - src/etools_datamart/apps/data/models/trip.py | 3 ++- 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 6daa5cdb0..6185324a2 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.1 +current_version = 2.2.0a commit = False tag = False allow_dirty = True diff --git a/CHANGES b/CHANGES index f239a55a3..9dfff33fc 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,7 @@ +2.2 (dev) +--------- +* + 2.1 --- * Fixes TripLoader diff --git a/docker/Makefile b/docker/Makefile index 87e292fbc..70a431fba 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -4,7 +4,7 @@ DATABASE_URL_ETOOLS?= DEVELOP?=1 DOCKER_PASS?= DOCKER_USER?= -TARGET?=2.1 +TARGET?=2.2.0a BASE=2.0 # below vars are used internally BUILD_OPTIONS?=--squash @@ -18,7 +18,7 @@ DOCKERFILE?=Dockerfile RUN_OPTIONS?= PIPENV_ARGS?= PORTS= -ABSOLUTE_BASE_URL?="http://192.168.66.66:8000" +ABSOLUTE_BASE_URL?="http://192.2.0a68.66.66:8000" help: @echo "dev build dev image (based on local code)" @@ -71,12 +71,12 @@ build_old: cd .. && docker run \ --rm \ -e ABSOLUTE_BASE_URL=${ABSOLUTE_BASE_URL} \ - -e CACHE_URL=redis://192.168.66.66:6379/1 \ - -e CACHE_URL_API=redis://192.168.66.66:6379/2 \ - -e CACHE_URL_LOCK=redis://192.168.66.66:6379/3 \ - -e CACHE_URL_TEMPLATE=redis://192.168.66.66:6379/4 \ - -e CELERY_BROKER_URL=redis://192.168.66.66:6379/2 \ - -e CELERY_RESULT_BACKEND=redis://192.168.66.66:6379/3 \ + -e CACHE_URL=redis://192.2.0a68.66.66:6379/1 \ + -e CACHE_URL_API=redis://192.2.0a68.66.66:6379/2 \ + -e CACHE_URL_LOCK=redis://192.2.0a68.66.66:6379/3 \ + -e CACHE_URL_TEMPLATE=redis://192.2.0a68.66.66:6379/4 \ + -e CELERY_BROKER_URL=redis://192.2.0a68.66.66:6379/2 \ + -e CELERY_RESULT_BACKEND=redis://192.2.0a68.66.66:6379/3 \ -e CSRF_COOKIE_SECURE=false \ -e DATABASE_URL=${DATABASE_URL} \ -e DATABASE_URL_ETOOLS=${DATABASE_URL_ETOOLS} \ diff --git a/src/etools_datamart/__init__.py b/src/etools_datamart/__init__.py index 3dfd65063..205ac4050 100644 --- a/src/etools_datamart/__init__.py +++ b/src/etools_datamart/__init__.py @@ -1,7 +1,7 @@ import warnings NAME = 'etools-datamart' -VERSION = __version__ = '2.1' +VERSION = __version__ = '2.2.0a' __author__ = '' # UserWarning: The psycopg2 wheel package will be renamed from release 2.8; diff --git a/src/etools_datamart/apps/data/loader.py b/src/etools_datamart/apps/data/loader.py index c6fb881d4..85ba86b48 100644 --- a/src/etools_datamart/apps/data/loader.py +++ b/src/etools_datamart/apps/data/loader.py @@ -140,7 +140,6 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor stdout.flush() self.post_process_country() if self.config.sync_deleted_records(self): - cache.set("STATUS:%s" % self.etl_task.task, '[remove deleted]') self.remove_deleted() if stdout and verbosity > 0: stdout.write("\n") diff --git a/src/etools_datamart/apps/data/models/trip.py b/src/etools_datamart/apps/data/models/trip.py index bba9496c6..c553faba3 100644 --- a/src/etools_datamart/apps/data/models/trip.py +++ b/src/etools_datamart/apps/data/models/trip.py @@ -17,7 +17,8 @@ class TravelAttachment(object): class TripLoader(EtoolsLoader): def remove_deleted(self): country = self.context['country'] - existing = list(self.get_queryset().only('id').values_list('id', flat=True)) + # existing = list(self.get_queryset().only('id').values_list('id', flat=True)) + existing = list(T2FTravelactivity.objects.only('id').values_list('id', flat=True)) to_delete = self.model.objects.filter(schema_name=country.schema_name).exclude(source_activity_id__in=existing) self.results.deleted += to_delete.count() to_delete.delete() From 28e799c1c4e1afe6cdbd9312405bbddfd730da5e Mon Sep 17 00:00:00 2001 From: saxix Date: Tue, 17 Sep 2019 17:41:11 +0200 Subject: [PATCH 02/10] rapidpro groups --- src/etools_datamart/apps/data/loader.py | 3 ++- src/etools_datamart/apps/rapidpro/loader.py | 14 +++++++++++++- src/etools_datamart/apps/rapidpro/models.py | 2 ++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/etools_datamart/apps/data/loader.py b/src/etools_datamart/apps/data/loader.py index 85ba86b48..780741a94 100644 --- a/src/etools_datamart/apps/data/loader.py +++ b/src/etools_datamart/apps/data/loader.py @@ -160,8 +160,9 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor raise else: self.on_end(None) + cache.set("STATUS:%s" % self.etl_task.task, "completed - %s" % self.results.processed) finally: - cache.delete("STATUS:%s" % self.etl_task.task) + cache.set("STATUS:%s" % self.etl_task.task, "error") if lock: # pragma: no branch try: lock.release() diff --git a/src/etools_datamart/apps/rapidpro/loader.py b/src/etools_datamart/apps/rapidpro/loader.py index c154e0404..4f415a47e 100644 --- a/src/etools_datamart/apps/rapidpro/loader.py +++ b/src/etools_datamart/apps/rapidpro/loader.py @@ -24,7 +24,7 @@ def load_organization(self): def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_records=None, only_delta=True, run_type=RUN_UNKNOWN, api_token=None, **kwargs): from .models import Source, Organization - sources = Source.objects.all() + sources = Source.objects.filter(is_active=True) self.results = EtlResult() try: if api_token: @@ -46,6 +46,18 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor self.on_start(run_type) for source in sources: client = TembaClient(config.RAPIDPRO_ADDRESS, source.api_token) + oo = client.get_org() + + org, __ = Organization.objects.get_or_create(source=source, + defaults={'name': oo.name, + 'country': oo.country, + 'primary_language': oo.primary_language, + 'timezone': oo.timezone, + 'date_style': oo.date_style, + 'languages': oo.languages, + 'anon': oo.anon + }) + func = "get_%s" % self.config.source getter = getattr(client, func) data = getter() diff --git a/src/etools_datamart/apps/rapidpro/models.py b/src/etools_datamart/apps/rapidpro/models.py index 59378c525..7505360fe 100644 --- a/src/etools_datamart/apps/rapidpro/models.py +++ b/src/etools_datamart/apps/rapidpro/models.py @@ -49,6 +49,8 @@ class Organization(models.Model): credits = JSONField(default=dict) anon = models.BooleanField(default=False) + objects = DataMartManager() + # loader = TembaLoader() class Meta: app_label = 'rapidpro' From 6669e570d9c2c2360f7bf1807f6756aa6cc04d34 Mon Sep 17 00:00:00 2001 From: saxix Date: Thu, 19 Sep 2019 08:49:22 +0200 Subject: [PATCH 03/10] updates migrations --- src/etools_datamart/apps/data/admin.py | 7 ++-- src/etools_datamart/apps/data/loader.py | 3 +- .../migrations/0094_auto_20190919_0647.py | 41 +++++++++++++++++++ src/etools_datamart/apps/data/models/base.py | 4 ++ src/etools_datamart/apps/data/models/fam.py | 3 -- src/etools_datamart/apps/data/models/hact.py | 3 -- .../apps/data/models/intervention.py | 3 +- .../apps/data/models/location.py | 4 +- .../apps/data/models/tpm_tmpactivity.py | 5 +-- .../apps/data/models/user_office.py | 5 +-- src/etools_datamart/apps/etl/loader.py | 16 ++------ src/etools_datamart/apps/rapidpro/loader.py | 16 ++++++-- .../migrations/0003_group_source_id.py | 18 ++++++++ src/etools_datamart/apps/rapidpro/models.py | 15 +++++++ src/unicef_rest_framework/admin/preload.py | 28 +++++++++---- src/unicef_rest_framework/models/preload.py | 27 ++++++++++++ 16 files changed, 156 insertions(+), 42 deletions(-) create mode 100644 src/etools_datamart/apps/data/migrations/0094_auto_20190919_0647.py create mode 100644 src/etools_datamart/apps/rapidpro/migrations/0003_group_source_id.py diff --git a/src/etools_datamart/apps/data/admin.py b/src/etools_datamart/apps/data/admin.py index 0107f6b82..761ae2fda 100644 --- a/src/etools_datamart/apps/data/admin.py +++ b/src/etools_datamart/apps/data/admin.py @@ -292,8 +292,9 @@ class EtoolsUserAdmin(DataModelAdmin): @register(models.InterventionBudget) class InterventionBudgetAdmin(DataModelAdmin): - list_display = ('title', 'number', - 'budget_cso_contribution', 'budget_unicef_cash') + list_display = ('source_id', 'schema_name', + 'reference_number', 'agreement_reference_number') + search_fields = ('reference_number', 'agreement_reference_number') @register(models.Office) @@ -317,7 +318,7 @@ class TripAdmin(DataModelAdmin): list_display = ('reference_number', 'traveler_name', 'partner_name', 'vendor_number', 'end_date',) list_filter = ('start_date', 'end_date') - search_fields = ('reference_number', ) + search_fields = ('reference_number',) @register(models.Engagement) diff --git a/src/etools_datamart/apps/data/loader.py b/src/etools_datamart/apps/data/loader.py index 780741a94..d527cc3d3 100644 --- a/src/etools_datamart/apps/data/loader.py +++ b/src/etools_datamart/apps/data/loader.py @@ -17,7 +17,8 @@ class EToolsLoaderOptions(BaseLoaderOptions): - pass + DEFAULT_KEY = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, + source_id=record.pk) class EtoolsLoader(BaseLoader): diff --git a/src/etools_datamart/apps/data/migrations/0094_auto_20190919_0647.py b/src/etools_datamart/apps/data/migrations/0094_auto_20190919_0647.py new file mode 100644 index 000000000..898cd70e8 --- /dev/null +++ b/src/etools_datamart/apps/data/migrations/0094_auto_20190919_0647.py @@ -0,0 +1,41 @@ +# Generated by Django 2.2.5 on 2019-09-19 06:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0093_auto_20190917_0603'), + ] + + operations = [ + migrations.AddIndex( + model_name='attachment', + index=models.Index(fields=['source_id', 'schema_name'], name='source'), + ), + migrations.AddIndex( + model_name='engagement', + index=models.Index(fields=['source_id', 'schema_name'], name='source'), + ), + migrations.AddIndex( + model_name='grant', + index=models.Index(fields=['source_id', 'schema_name'], name='source'), + ), + migrations.AddIndex( + model_name='partnerstaffmember', + index=models.Index(fields=['source_id', 'schema_name'], name='source'), + ), + migrations.AddIndex( + model_name='section', + index=models.Index(fields=['source_id', 'schema_name'], name='source'), + ), + migrations.AddIndex( + model_name='tpmactivity', + index=models.Index(fields=['source_id', 'schema_name'], name='source'), + ), + migrations.AddIndex( + model_name='tpmvisit', + index=models.Index(fields=['source_id', 'schema_name'], name='source'), + ), + ] diff --git a/src/etools_datamart/apps/data/models/base.py b/src/etools_datamart/apps/data/models/base.py index 75952d5b2..15dc36e68 100644 --- a/src/etools_datamart/apps/data/models/base.py +++ b/src/etools_datamart/apps/data/models/base.py @@ -71,5 +71,9 @@ class EtoolsDataMartModel(CommonDataMartModel, metaclass=EToolsDataMartModelBase class Meta: abstract = True + indexes = [ + models.Index(name='source', + fields=['source_id', 'schema_name']), + ] objects = DataMartManager() diff --git a/src/etools_datamart/apps/data/models/fam.py b/src/etools_datamart/apps/data/models/fam.py index 321e32f14..f57c4c2b2 100644 --- a/src/etools_datamart/apps/data/models/fam.py +++ b/src/etools_datamart/apps/data/models/fam.py @@ -69,6 +69,3 @@ class Options: source = AuditEngagement sync_deleted_records = lambda loader: False # mapping = dict(source_id='engagement_ptr_id') - # key = lambda loader, record: dict(country_name=loader.context['country'].name, - # schema_name=loader.context['country'].schema_name, - # source_id=record.engagement_ptr.id) diff --git a/src/etools_datamart/apps/data/models/hact.py b/src/etools_datamart/apps/data/models/hact.py index cc12cf846..23746fe71 100644 --- a/src/etools_datamart/apps/data/models/hact.py +++ b/src/etools_datamart/apps/data/models/hact.py @@ -70,6 +70,3 @@ class Options: source = HactAggregatehact sync_deleted_records = lambda loader: False truncate = False - # key = lambda loader, record: dict(country_name=loader.context['country'].name, - # schema_name=loader.context['country'].schema_name, - # year=loader.context['today'].year) diff --git a/src/etools_datamart/apps/data/models/intervention.py b/src/etools_datamart/apps/data/models/intervention.py index e0db51ae2..2be9d41e6 100644 --- a/src/etools_datamart/apps/data/models/intervention.py +++ b/src/etools_datamart/apps/data/models/intervention.py @@ -193,7 +193,8 @@ def get_unicef_focal_points(self, original: PartnersIntervention, values: dict, class InterventionAbstract(models.Model): - agreement_reference_number = models.CharField(max_length=300, blank=True, null=True) + agreement_reference_number = models.CharField(max_length=300, + blank=True, null=True) amendment_types = models.TextField(blank=True, null=True) attachment_types = models.TextField(blank=True, null=True) agreement_id = models.IntegerField(blank=True, null=True) diff --git a/src/etools_datamart/apps/data/models/location.py b/src/etools_datamart/apps/data/models/location.py index 8b38a792c..aca997f02 100644 --- a/src/etools_datamart/apps/data/models/location.py +++ b/src/etools_datamart/apps/data/models/location.py @@ -20,8 +20,8 @@ class Options: source = LocationsGatewaytype sync_deleted_records = lambda loader: False - key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, - source_id=record.id) + # key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, + # source_id=record.id) def __str__(self): return self.name diff --git a/src/etools_datamart/apps/data/models/tpm_tmpactivity.py b/src/etools_datamart/apps/data/models/tpm_tmpactivity.py index 33bacbd94..5346a6a26 100644 --- a/src/etools_datamart/apps/data/models/tpm_tmpactivity.py +++ b/src/etools_datamart/apps/data/models/tpm_tmpactivity.py @@ -198,9 +198,8 @@ class Options: # depends = (Intervention,) # truncate = True sync_deleted_records = lambda a: False - key = lambda loader, record: dict(country_name=loader.context['country'].name, - schema_name=loader.context['country'].schema_name, - source_id=record.id) + # key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, + # source_id=record.id) source = TpmTpmactivity mapping = dict(additional_information='additional_information', diff --git a/src/etools_datamart/apps/data/models/user_office.py b/src/etools_datamart/apps/data/models/user_office.py index f88d5a278..7d88068c4 100644 --- a/src/etools_datamart/apps/data/models/user_office.py +++ b/src/etools_datamart/apps/data/models/user_office.py @@ -18,9 +18,8 @@ class Options: # truncate = True # sync_deleted_records = lambda loader: False - key = lambda loader, record: dict(country_name=loader.context['country'].name, - schema_name=loader.context['country'].schema_name, - source_id=record.id) + # key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, + # source_id=record.id) mapping = {'zonal_chief_email': 'zonal_chief.email', 'zonal_chief_source_id': 'zonal_chief.id' } diff --git a/src/etools_datamart/apps/etl/loader.py b/src/etools_datamart/apps/etl/loader.py index 6e01d58d6..979ea6735 100644 --- a/src/etools_datamart/apps/etl/loader.py +++ b/src/etools_datamart/apps/etl/loader.py @@ -14,7 +14,6 @@ from constance import config from crashlog.middleware import process_exception from redis.exceptions import LockError -from sentry_sdk import capture_exception from strategy_field.utils import fqn, get_attr from etools_datamart.apps.data.exceptions import LoaderException @@ -85,11 +84,6 @@ def as_dict(self): 'total_records': self.total_records} -DEFAULT_KEY = lambda loader, record: dict(country_name=loader.context['country'].name, - schema_name=loader.context['country'].schema_name, - source_id=record.pk) - - class RequiredIsRunning(Exception): def __init__(self, req, *args: object) -> None: @@ -113,6 +107,8 @@ class MaxRecordsException(Exception): class BaseLoaderOptions: + DEFAULT_KEY = lambda loader, record: dict(source_id=record.pk) + __attrs__ = ['mapping', 'celery', 'source', 'last_modify_field', 'queryset', 'key', 'locks', 'filters', 'sync_deleted_records', 'truncate', 'depends', 'timeout', 'lock_key', 'always_update', 'fields_to_compare'] @@ -125,7 +121,7 @@ def __init__(self, base=None): self.always_update = False self.source = None self.lock_key = None - self.key = DEFAULT_KEY + self.key = self.DEFAULT_KEY self.timeout = None self.depends = () self.filters = None @@ -302,11 +298,7 @@ def process_record(self, filters, values): op = UNCHANGED return op except Exception as e: # pragma: no cover - logger.exception(e) - capture_exception() - err = process_exception(e) - raise LoaderException(f"Error in {self}: {e}", - err) from e + raise LoaderException(f"Error in {self}: {e}") from e def get_mart_values(self, record=None): country = self.context['country'] diff --git a/src/etools_datamart/apps/rapidpro/loader.py b/src/etools_datamart/apps/rapidpro/loader.py index 4f415a47e..0a7383a4f 100644 --- a/src/etools_datamart/apps/rapidpro/loader.py +++ b/src/etools_datamart/apps/rapidpro/loader.py @@ -11,6 +11,8 @@ class TembaLoaderOptions(BaseLoaderOptions): __attrs__ = BaseLoaderOptions.__attrs__ + ['host', 'temba_object'] + DEFAULT_KEY = lambda loader, record: dict(uuid=record['uuid'], + organization=loader.context['organization']) class TembaLoader(BaseLoader): @@ -21,6 +23,9 @@ def get_fetch_method(self, org): def load_organization(self): pass + def get_values(self, record): + return record.serialize() + def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_records=None, only_delta=True, run_type=RUN_UNKNOWN, api_token=None, **kwargs): from .models import Source, Organization @@ -67,13 +72,16 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor records=0, only_delta=only_delta, is_empty=not self.model.objects.exists(), - stdout=stdout + stdout=stdout, + organization=source.organization ) for entry in data.all(): - values = entry.serialize() - values['organization'] = source.organization - filters = {'uuid': values.get('uuid')} + filters = self.config.key(self, entry) + values = self.get_values(entry) + + # values['organization'] = source.organization + # filters = {'uuid': values['uuid']} op = self.process_record(filters, values) self.increment_counter(op) diff --git a/src/etools_datamart/apps/rapidpro/migrations/0003_group_source_id.py b/src/etools_datamart/apps/rapidpro/migrations/0003_group_source_id.py new file mode 100644 index 000000000..74fc8a31c --- /dev/null +++ b/src/etools_datamart/apps/rapidpro/migrations/0003_group_source_id.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.5 on 2019-09-19 06:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('rapidpro', '0002_auto_20190906_1333'), + ] + + operations = [ + migrations.AddField( + model_name='group', + name='source_id', + field=models.CharField(blank=True, max_length=100, null=True), + ), + ] diff --git a/src/etools_datamart/apps/rapidpro/models.py b/src/etools_datamart/apps/rapidpro/models.py index 7505360fe..6a1ff151c 100644 --- a/src/etools_datamart/apps/rapidpro/models.py +++ b/src/etools_datamart/apps/rapidpro/models.py @@ -67,6 +67,7 @@ class RapidProManager(DataMartManager): class RapidProDataMartModel(models.Model, metaclass=RapidProModelBase): + source_id = models.CharField(max_length=100, blank=True, null=True) organization = models.ForeignKey(Organization, on_delete=models.CASCADE) objects = RapidProManager() @@ -96,3 +97,17 @@ def __str__(self): class Options: source = 'groups' + + +# class Group(RapidProDataMartModel): +# uuid = models.UUIDField(unique=True, db_index=True) +# name = models.TextField() +# query = models.TextField(null=True, blank=True) +# count = models.IntegerField() +# status = models.CharField(max_length=100, blank=True, null=True) +# +# def __str__(self): +# return '{} ({})'.format(self.name, self.organization) +# +# class Options: +# source = 'groups' diff --git a/src/unicef_rest_framework/admin/preload.py b/src/unicef_rest_framework/admin/preload.py index 1ed7c1c24..d16632ae9 100644 --- a/src/unicef_rest_framework/admin/preload.py +++ b/src/unicef_rest_framework/admin/preload.py @@ -1,4 +1,5 @@ -from django.contrib import admin +from django.contrib import admin, messages +from django.http import HttpResponseRedirect from django.utils.safestring import mark_safe from admin_extra_urls.extras import action, ExtraUrlMixin @@ -25,23 +26,36 @@ def queue(modeladmin, request, queryset): class PreloadAdmin(ExtraUrlMixin, admin.ModelAdmin): - list_display = ('url', 'as_user', 'enabled', 'last_run', 'status_code', 'size', 'response_ms') + list_display = ('url', 'as_user', 'enabled', 'last_run', + 'status_code', 'size', 'response_ms', 'preview') date_hierarchy = 'last_run' search_fields = ('url',) list_filter = (StatusFilter, 'enabled', SizeFilter) actions = [queue, ] - # form = PreloadForm - # readonly_fields = ('params',) - # formfield_overrides = { - # JSONField: {'widget': JSONEditor}, - # } + def preview(self, obj): + return mark_safe("preview".format(obj.full_url())) + + @action(label='Goto API') + def goto(self, request, pk): + obj = self.model.objects.get(id=pk) + return HttpResponseRedirect(obj.full_url()) def size(self, obj): if obj.response_length: return mark_safe("{0}".format(humanize_size(obj.response_length))) + size.admin_order_field = 'response_length' + @action() def queue(self, request, id): from unicef_rest_framework.tasks import preload preload.apply_async(args=[id]) + + @action() + def check_url(self, request, id): + target = self.model.objects.get(id=id) + try: + target.check_url(True) + except Exception as e: + self.message_user(request, str(e), messages.ERROR) diff --git a/src/unicef_rest_framework/models/preload.py b/src/unicef_rest_framework/models/preload.py index 1ca363783..2198a86d6 100644 --- a/src/unicef_rest_framework/models/preload.py +++ b/src/unicef_rest_framework/models/preload.py @@ -1,5 +1,8 @@ +from urllib.parse import urlencode + from django.conf import settings from django.contrib.postgres.fields import JSONField +from django.core.exceptions import ValidationError from django.db import models from django.utils import timezone @@ -52,6 +55,30 @@ class Meta: unique_together = ('url', 'as_user', 'params') ordering = ('url',) + def clean(self): + super().clean() + self.check_url(True) + + def full_url(self): + return "%s%s?%s" % (settings.ABSOLUTE_BASE_URL, self.url, + urlencode(self.params)) + + def check_url(self, validate=True): + try: + target = "%s%s" % (settings.ABSOLUTE_BASE_URL, self.url) + client = Client() + if self.as_user: + client.force_authenticate(self.as_user) + res = client.head(target, data=self.params) + if res.status_code != 200: + raise Exception('Invalid Response: %s on %s' % (res.status_code, + self.full_url())) + except Exception as e: + if validate: + raise ValidationError(str(e)) + else: + return False + def run(self): try: self.last_run = timezone.now() From 66eb5572a5b3df6f259b239ac36aaf5eaf8a37c8 Mon Sep 17 00:00:00 2001 From: saxix Date: Thu, 19 Sep 2019 14:06:40 +0200 Subject: [PATCH 04/10] fixes InterventionBudgetLoader --- src/etools_datamart/apps/data/loader.py | 7 ++-- .../apps/data/migrations/0094_RELEASE_2_1.py | 13 ++++++++ ...919_0647.py => 0095_auto_20190919_1016.py} | 18 +++++------ src/etools_datamart/apps/data/models/base.py | 5 ++- src/etools_datamart/apps/etl/loader.py | 7 ++-- src/etools_datamart/apps/etl/utils.py | 8 +++++ src/etools_datamart/apps/rapidpro/loader.py | 9 ++++-- src/etools_datamart/apps/rapidpro/models.py | 32 ++++++++++++------- 8 files changed, 68 insertions(+), 31 deletions(-) create mode 100644 src/etools_datamart/apps/data/migrations/0094_RELEASE_2_1.py rename src/etools_datamart/apps/data/migrations/{0094_auto_20190919_0647.py => 0095_auto_20190919_1016.py} (74%) create mode 100644 src/etools_datamart/apps/etl/utils.py diff --git a/src/etools_datamart/apps/data/loader.py b/src/etools_datamart/apps/data/loader.py index d527cc3d3..ce4c0c9f0 100644 --- a/src/etools_datamart/apps/data/loader.py +++ b/src/etools_datamart/apps/data/loader.py @@ -17,8 +17,11 @@ class EToolsLoaderOptions(BaseLoaderOptions): - DEFAULT_KEY = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, - source_id=record.pk) + + def __init__(self, base=None): + super().__init__(base) + self.key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, + source_id=record.pk) class EtoolsLoader(BaseLoader): diff --git a/src/etools_datamart/apps/data/migrations/0094_RELEASE_2_1.py b/src/etools_datamart/apps/data/migrations/0094_RELEASE_2_1.py new file mode 100644 index 000000000..b8538885c --- /dev/null +++ b/src/etools_datamart/apps/data/migrations/0094_RELEASE_2_1.py @@ -0,0 +1,13 @@ +# Generated by Django 2.2.5 on 2019-09-19 10:08 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0093_auto_20190917_0603'), + ] + + operations = [ + ] diff --git a/src/etools_datamart/apps/data/migrations/0094_auto_20190919_0647.py b/src/etools_datamart/apps/data/migrations/0095_auto_20190919_1016.py similarity index 74% rename from src/etools_datamart/apps/data/migrations/0094_auto_20190919_0647.py rename to src/etools_datamart/apps/data/migrations/0095_auto_20190919_1016.py index 898cd70e8..5166f18a9 100644 --- a/src/etools_datamart/apps/data/migrations/0094_auto_20190919_0647.py +++ b/src/etools_datamart/apps/data/migrations/0095_auto_20190919_1016.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2.5 on 2019-09-19 06:47 +# Generated by Django 2.2.5 on 2019-09-19 10:16 from django.db import migrations, models @@ -6,36 +6,36 @@ class Migration(migrations.Migration): dependencies = [ - ('data', '0093_auto_20190917_0603'), + ('data', '0094_RELEASE_2_1'), ] operations = [ migrations.AddIndex( model_name='attachment', - index=models.Index(fields=['source_id', 'schema_name'], name='source'), + index=models.Index(fields=['source_id', 'schema_name'], name='data_attach_source__529f94_idx'), ), migrations.AddIndex( model_name='engagement', - index=models.Index(fields=['source_id', 'schema_name'], name='source'), + index=models.Index(fields=['source_id', 'schema_name'], name='data_engage_source__85fc81_idx'), ), migrations.AddIndex( model_name='grant', - index=models.Index(fields=['source_id', 'schema_name'], name='source'), + index=models.Index(fields=['source_id', 'schema_name'], name='data_grant_source__df511a_idx'), ), migrations.AddIndex( model_name='partnerstaffmember', - index=models.Index(fields=['source_id', 'schema_name'], name='source'), + index=models.Index(fields=['source_id', 'schema_name'], name='data_partne_source__9d46b3_idx'), ), migrations.AddIndex( model_name='section', - index=models.Index(fields=['source_id', 'schema_name'], name='source'), + index=models.Index(fields=['source_id', 'schema_name'], name='data_sectio_source__565f46_idx'), ), migrations.AddIndex( model_name='tpmactivity', - index=models.Index(fields=['source_id', 'schema_name'], name='source'), + index=models.Index(fields=['source_id', 'schema_name'], name='data_tpmact_source__2a9531_idx'), ), migrations.AddIndex( model_name='tpmvisit', - index=models.Index(fields=['source_id', 'schema_name'], name='source'), + index=models.Index(fields=['source_id', 'schema_name'], name='data_tpmvis_source__90b361_idx'), ), ] diff --git a/src/etools_datamart/apps/data/models/base.py b/src/etools_datamart/apps/data/models/base.py index 15dc36e68..65dba9074 100644 --- a/src/etools_datamart/apps/data/models/base.py +++ b/src/etools_datamart/apps/data/models/base.py @@ -65,15 +65,14 @@ def linked_services(self): class EtoolsDataMartModel(CommonDataMartModel, metaclass=EToolsDataMartModelBase): - country_name = models.CharField(max_length=100, db_index=True) + country_name = models.CharField(max_length=100) schema_name = models.CharField(max_length=63, db_index=True) area_code = models.CharField(max_length=10, db_index=True) class Meta: abstract = True indexes = [ - models.Index(name='source', - fields=['source_id', 'schema_name']), + models.Index(fields=['source_id', 'schema_name']), ] objects = DataMartManager() diff --git a/src/etools_datamart/apps/etl/loader.py b/src/etools_datamart/apps/etl/loader.py index 979ea6735..b4d46ddce 100644 --- a/src/etools_datamart/apps/etl/loader.py +++ b/src/etools_datamart/apps/etl/loader.py @@ -106,9 +106,10 @@ class MaxRecordsException(Exception): pass -class BaseLoaderOptions: - DEFAULT_KEY = lambda loader, record: dict(source_id=record.pk) +DEFAULT_KEY = lambda loader, record: dict(source_id=record.pk) + +class BaseLoaderOptions: __attrs__ = ['mapping', 'celery', 'source', 'last_modify_field', 'queryset', 'key', 'locks', 'filters', 'sync_deleted_records', 'truncate', 'depends', 'timeout', 'lock_key', 'always_update', 'fields_to_compare'] @@ -121,7 +122,7 @@ def __init__(self, base=None): self.always_update = False self.source = None self.lock_key = None - self.key = self.DEFAULT_KEY + self.key = DEFAULT_KEY self.timeout = None self.depends = () self.filters = None diff --git a/src/etools_datamart/apps/etl/utils.py b/src/etools_datamart/apps/etl/utils.py new file mode 100644 index 000000000..0f114e223 --- /dev/null +++ b/src/etools_datamart/apps/etl/utils.py @@ -0,0 +1,8 @@ +# def disable_indexes(): +# clause = """UPDATE pg_index +# SET indisready=false +# WHERE indrelid = ( +# SELECT oid +# FROM pg_class +# WHERE relname='' +# );""" diff --git a/src/etools_datamart/apps/rapidpro/loader.py b/src/etools_datamart/apps/rapidpro/loader.py index 0a7383a4f..5fd98455c 100644 --- a/src/etools_datamart/apps/rapidpro/loader.py +++ b/src/etools_datamart/apps/rapidpro/loader.py @@ -8,11 +8,16 @@ logger = get_task_logger(__name__) +DEFAULT_KEY = lambda loader, record: dict(uuid=record['uuid'], + organization=loader.context['organization']) + class TembaLoaderOptions(BaseLoaderOptions): __attrs__ = BaseLoaderOptions.__attrs__ + ['host', 'temba_object'] - DEFAULT_KEY = lambda loader, record: dict(uuid=record['uuid'], - organization=loader.context['organization']) + + def __init__(self, base=None): + super().__init__(base) + self.key = DEFAULT_KEY class TembaLoader(BaseLoader): diff --git a/src/etools_datamart/apps/rapidpro/models.py b/src/etools_datamart/apps/rapidpro/models.py index 6a1ff151c..f4e166b50 100644 --- a/src/etools_datamart/apps/rapidpro/models.py +++ b/src/etools_datamart/apps/rapidpro/models.py @@ -99,15 +99,23 @@ class Options: source = 'groups' -# class Group(RapidProDataMartModel): -# uuid = models.UUIDField(unique=True, db_index=True) -# name = models.TextField() -# query = models.TextField(null=True, blank=True) -# count = models.IntegerField() -# status = models.CharField(max_length=100, blank=True, null=True) -# -# def __str__(self): -# return '{} ({})'.format(self.name, self.organization) -# -# class Options: -# source = 'groups' +class Contact(RapidProDataMartModel): + uuid = models.UUIDField(unique=True, db_index=True) + name = models.TextField(null=True, blank=True) + language = models.CharField(max_length=100, null=True, blank=True) + urns = ArrayField( + models.CharField(max_length=100), + default=list + ) + groups = models.ManyToManyField(Group) + fields = JSONField(default=dict) + blocked = models.NullBooleanField() + stopped = models.NullBooleanField() + created_on = models.DateTimeField(null=True, blank=True) + modified_on = models.DateTimeField(null=True, blank=True) + + def __str__(self): + return '{} ({})'.format(self.name, self.organization) + + class Options: + source = 'contacts' From c6bfc9c0d0d3021a8ac9647159d819d5c70664f9 Mon Sep 17 00:00:00 2001 From: saxix Date: Thu, 19 Sep 2019 14:45:12 +0200 Subject: [PATCH 05/10] fixes --- .../migrations/0095_auto_20190919_1016.py | 1 + src/etools_datamart/apps/rapidpro/loader.py | 79 ++++++++++++++++--- src/etools_datamart/apps/rapidpro/models.py | 42 +++++----- 3 files changed, 89 insertions(+), 33 deletions(-) diff --git a/src/etools_datamart/apps/data/migrations/0095_auto_20190919_1016.py b/src/etools_datamart/apps/data/migrations/0095_auto_20190919_1016.py index 5166f18a9..d43cbfc19 100644 --- a/src/etools_datamart/apps/data/migrations/0095_auto_20190919_1016.py +++ b/src/etools_datamart/apps/data/migrations/0095_auto_20190919_1016.py @@ -1,3 +1,4 @@ + # Generated by Django 2.2.5 on 2019-09-19 10:16 from django.db import migrations, models diff --git a/src/etools_datamart/apps/rapidpro/loader.py b/src/etools_datamart/apps/rapidpro/loader.py index 5fd98455c..31cdffd44 100644 --- a/src/etools_datamart/apps/rapidpro/loader.py +++ b/src/etools_datamart/apps/rapidpro/loader.py @@ -28,8 +28,73 @@ def get_fetch_method(self, org): def load_organization(self): pass + def get_mart_values(self, record=None): + organization = self.context['organization'] + ret = {'organization': organization, + 'seen': self.context['today'] + } + if record: + ret['source_id'] = record['uuid'] + return ret + def get_values(self, record): - return record.serialize() + organization = self.context['organization'] + ret = self.get_mart_values(record) + + for k, v in self.mapping.items(): + if k in ret: + continue + if v is None: + ret[k] = None + elif v == 'N/A': + ret[k] = 'N/A' + elif v == 'i': + continue + elif isinstance(v, str) and hasattr(self, v) and callable(getattr(self, v)): + getter = getattr(self, v) + _value = getter(record, ret, field_name=k) + if _value != self.noop: + ret[k] = _value + elif v == '-' or hasattr(self, 'get_%s' % k): + getter = getattr(self, 'get_%s' % k) + _value = getter(record, ret, field_name=k) + if _value != self.noop: + ret[k] = _value + elif v == '__self__': + try: + ret[k] = self.model.objects.get(schema_name=country.schema_name, + source_id=getattr(record, k).id) + except AttributeError: + ret[k] = None + except self.model.DoesNotExist: + ret[k] = None + self.tree_parents.append((record.id, getattr(record, k).id)) + + elif isclass(v) and issubclass(v, models.Model): + try: + ret[k] = v.objects.get(schema_name=country.schema_name, + source_id=getattr(record, k).id) + except ObjectDoesNotExist: # pragma: no cover + ret[k] = None + except AttributeError: # pragma: no cover + pass + elif callable(v): + ret[k] = v(self, record) + elif v == '=' and has_attr(record, k): + ret[k] = get_attr(record, k) + # elif has_attr(record, k): + # ret[k] = get_attr(record, k) + elif not isinstance(v, str): + ret[k] = v + elif has_attr(record, v): + ret[k] = get_attr(record, v) + else: + raise Exception("Invalid field name or mapping '%s:%s'" % (k, v)) + + return ret + # + # def get_values(self, record): + # return record.serialize() def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_records=None, only_delta=True, run_type=RUN_UNKNOWN, api_token=None, **kwargs): @@ -41,18 +106,8 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor source, __ = Source.objects.get_or_create(api_token=api_token, defaults={'name': api_token}) client = TembaClient(source.server, api_token) - oo = client.get_org() - - org, __ = Organization.objects.get_or_create(source=source, - defaults={'name': oo.name, - 'country': oo.country, - 'primary_language': oo.primary_language, - 'timezone': oo.timezone, - 'date_style': oo.date_style, - 'languages': oo.languages, - 'anon': oo.anon - }) sources = sources.filter(api_token=api_token) + self.on_start(run_type) for source in sources: client = TembaClient(config.RAPIDPRO_ADDRESS, source.api_token) diff --git a/src/etools_datamart/apps/rapidpro/models.py b/src/etools_datamart/apps/rapidpro/models.py index f4e166b50..1dad78655 100644 --- a/src/etools_datamart/apps/rapidpro/models.py +++ b/src/etools_datamart/apps/rapidpro/models.py @@ -98,24 +98,24 @@ def __str__(self): class Options: source = 'groups' - -class Contact(RapidProDataMartModel): - uuid = models.UUIDField(unique=True, db_index=True) - name = models.TextField(null=True, blank=True) - language = models.CharField(max_length=100, null=True, blank=True) - urns = ArrayField( - models.CharField(max_length=100), - default=list - ) - groups = models.ManyToManyField(Group) - fields = JSONField(default=dict) - blocked = models.NullBooleanField() - stopped = models.NullBooleanField() - created_on = models.DateTimeField(null=True, blank=True) - modified_on = models.DateTimeField(null=True, blank=True) - - def __str__(self): - return '{} ({})'.format(self.name, self.organization) - - class Options: - source = 'contacts' +# +# class Contact(RapidProDataMartModel): +# uuid = models.UUIDField(unique=True, db_index=True) +# name = models.TextField(null=True, blank=True) +# language = models.CharField(max_length=100, null=True, blank=True) +# urns = ArrayField( +# models.CharField(max_length=100), +# default=list +# ) +# groups = models.ManyToManyField(Group) +# fields = JSONField(default=dict) +# blocked = models.NullBooleanField() +# stopped = models.NullBooleanField() +# created_on = models.DateTimeField(null=True, blank=True) +# modified_on = models.DateTimeField(null=True, blank=True) +# +# def __str__(self): +# return '{} ({})'.format(self.name, self.organization) +# +# class Options: +# source = 'contacts' From 6577fa28f66e3c755a63d7ffd4cbf6abdadab566 Mon Sep 17 00:00:00 2001 From: saxix Date: Thu, 19 Sep 2019 18:02:34 +0200 Subject: [PATCH 06/10] updates rp --- src/etools_datamart/apps/etl/loader.py | 17 ++-- src/etools_datamart/apps/rapidpro/admin.py | 9 ++ src/etools_datamart/apps/rapidpro/loader.py | 83 +++++++++---------- .../apps/rapidpro/migrations/0001_initial.py | 34 ++++++-- .../migrations/0002_auto_20190906_1333.py | 22 ----- .../migrations/0003_group_source_id.py | 18 ---- src/etools_datamart/apps/rapidpro/models.py | 77 +++++++++++------ 7 files changed, 135 insertions(+), 125 deletions(-) delete mode 100644 src/etools_datamart/apps/rapidpro/migrations/0002_auto_20190906_1333.py delete mode 100644 src/etools_datamart/apps/rapidpro/migrations/0003_group_source_id.py diff --git a/src/etools_datamart/apps/etl/loader.py b/src/etools_datamart/apps/etl/loader.py index b4d46ddce..201079dcb 100644 --- a/src/etools_datamart/apps/etl/loader.py +++ b/src/etools_datamart/apps/etl/loader.py @@ -1,6 +1,7 @@ import json import time from inspect import isclass +from uuid import UUID from django.contrib.contenttypes.models import ContentType from django.core.cache import caches @@ -184,12 +185,14 @@ def _compare_json(dict1, dict2): return json.dumps(dict1, sort_keys=True, indent=0) == json.dumps(dict2, sort_keys=True, indent=0) -def equal(a, b): - if isinstance(a, (dict, list, tuple)): - return _compare_json(a, b) - elif isinstance(b, bool): - return str(a) == str(b) - return a == b +def equal(current, new_value): + if isinstance(current, (dict, list, tuple)): + return _compare_json(current, new_value) + elif isinstance(current, UUID): + return current == UUID(new_value) + elif isinstance(new_value, bool): + return str(current) == str(new_value) + return current == new_value def has_attr(obj, attr): @@ -267,7 +270,7 @@ def is_record_changed(self, record, values): verbosity = self.context['verbosity'] if verbosity >= 2: # pragma: no cover stdout = self.context['stdout'] - stdout.write("Detected field changed '%s': %s(%s)->%s(%s)\n" % + stdout.write("Detected field changed '%s': current: %s(%s) new value: %s(%s)\n" % (field_name, getattr(record, field_name), type(getattr(record, field_name)), diff --git a/src/etools_datamart/apps/rapidpro/admin.py b/src/etools_datamart/apps/rapidpro/admin.py index 97dee8d48..876ed7ec3 100644 --- a/src/etools_datamart/apps/rapidpro/admin.py +++ b/src/etools_datamart/apps/rapidpro/admin.py @@ -32,4 +32,13 @@ class OrganizationAdmin(RapidProAdmin): @register(models.Group) class GroupAdmin(RapidProAdmin): + list_display = ('id', 'organization', 'name', 'query', 'count') list_filter = ('organization',) + search_fields = ('name',) + + +@register(models.Contact) +class ContactAdmin(RapidProAdmin): + list_display = ('id', 'organization', 'name', 'language', 'blocked', 'stopped') + list_filter = ('organization',) + search_fields = ('name',) diff --git a/src/etools_datamart/apps/rapidpro/loader.py b/src/etools_datamart/apps/rapidpro/loader.py index 31cdffd44..aebe106f5 100644 --- a/src/etools_datamart/apps/rapidpro/loader.py +++ b/src/etools_datamart/apps/rapidpro/loader.py @@ -2,13 +2,15 @@ from celery.utils.log import get_task_logger from constance import config +from strategy_field.utils import get_attr +from temba_client.serialization import TembaObject from temba_client.v2 import TembaClient -from etools_datamart.apps.etl.loader import BaseLoader, BaseLoaderOptions, EtlResult, RUN_UNKNOWN +from etools_datamart.apps.etl.loader import BaseLoader, BaseLoaderOptions, EtlResult, has_attr, RUN_UNKNOWN logger = get_task_logger(__name__) -DEFAULT_KEY = lambda loader, record: dict(uuid=record['uuid'], +DEFAULT_KEY = lambda loader, record: dict(uuid=record.uuid, organization=loader.context['organization']) @@ -28,19 +30,16 @@ def get_fetch_method(self, org): def load_organization(self): pass - def get_mart_values(self, record=None): + def get_mart_values(self, record: TembaObject = None): organization = self.context['organization'] - ret = {'organization': organization, - 'seen': self.context['today'] - } + ret = {'organization': organization} if record: - ret['source_id'] = record['uuid'] + ret['source_id'] = record.uuid return ret - def get_values(self, record): - organization = self.context['organization'] + def get_values(self, record: TembaObject): + # organization = self.context['organization'] ret = self.get_mart_values(record) - for k, v in self.mapping.items(): if k in ret: continue @@ -55,35 +54,17 @@ def get_values(self, record): _value = getter(record, ret, field_name=k) if _value != self.noop: ret[k] = _value + # elif v and isinstance(v, list) and isinstance(v[0], ObjectRef): + # ret[k] = [oo.serialize() for oo in v] elif v == '-' or hasattr(self, 'get_%s' % k): getter = getattr(self, 'get_%s' % k) _value = getter(record, ret, field_name=k) if _value != self.noop: ret[k] = _value - elif v == '__self__': - try: - ret[k] = self.model.objects.get(schema_name=country.schema_name, - source_id=getattr(record, k).id) - except AttributeError: - ret[k] = None - except self.model.DoesNotExist: - ret[k] = None - self.tree_parents.append((record.id, getattr(record, k).id)) - - elif isclass(v) and issubclass(v, models.Model): - try: - ret[k] = v.objects.get(schema_name=country.schema_name, - source_id=getattr(record, k).id) - except ObjectDoesNotExist: # pragma: no cover - ret[k] = None - except AttributeError: # pragma: no cover - pass elif callable(v): ret[k] = v(self, record) elif v == '=' and has_attr(record, k): ret[k] = get_attr(record, k) - # elif has_attr(record, k): - # ret[k] = get_attr(record, k) elif not isinstance(v, str): ret[k] = v elif has_attr(record, v): @@ -92,9 +73,12 @@ def get_values(self, record): raise Exception("Invalid field name or mapping '%s:%s'" % (k, v)) return ret - # - # def get_values(self, record): - # return record.serialize() + + def on_start(self, run_type): + super().on_start(run_type) + + def on_end(self, error=None, retry=False): + super().on_end(error, retry) def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_records=None, only_delta=True, run_type=RUN_UNKNOWN, api_token=None, **kwargs): @@ -103,15 +87,18 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor self.results = EtlResult() try: if api_token: - source, __ = Source.objects.get_or_create(api_token=api_token, - defaults={'name': api_token}) - client = TembaClient(source.server, api_token) + Source.objects.get_or_create(api_token=api_token, + defaults={'name': api_token}) sources = sources.filter(api_token=api_token) self.on_start(run_type) for source in sources: + if verbosity >= 0: + stdout.write("Source %s" % source) client = TembaClient(config.RAPIDPRO_ADDRESS, source.api_token) oo = client.get_org() + if verbosity >= 0: + stdout.write(" fetching organization info") org, __ = Organization.objects.get_or_create(source=source, defaults={'name': oo.name, @@ -122,10 +109,12 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor 'languages': oo.languages, 'anon': oo.anon }) + if verbosity >= 0: + stdout.write(" found organization %s" % oo.name) func = "get_%s" % self.config.source getter = getattr(client, func) - data = getter() + data = getter(after=self.etl_task.last_success) self.update_context(today=timezone.now(), max_records=max_records, verbosity=verbosity, @@ -135,15 +124,17 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor stdout=stdout, organization=source.organization ) - - for entry in data.all(): - filters = self.config.key(self, entry) - values = self.get_values(entry) - - # values['organization'] = source.organization - # filters = {'uuid': values['uuid']} - op = self.process_record(filters, values) - self.increment_counter(op) + if verbosity >= 0: + stdout.write(" fetching data") + for page in data.iterfetches(): + for entry in page: + filters = self.config.key(self, entry) + values = self.get_values(entry) + + # values['organization'] = source.organization + # filters = {'uuid': values['uuid']} + op = self.process_record(filters, values) + self.increment_counter(op) except Exception as e: self.on_end(error=e) diff --git a/src/etools_datamart/apps/rapidpro/migrations/0001_initial.py b/src/etools_datamart/apps/rapidpro/migrations/0001_initial.py index 4e58d3c0d..6c52ee7c7 100644 --- a/src/etools_datamart/apps/rapidpro/migrations/0001_initial.py +++ b/src/etools_datamart/apps/rapidpro/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2.5 on 2019-09-06 13:12 +# Generated by Django 2.2.5 on 2019-09-19 14:24 import django.contrib.postgres.fields import django.contrib.postgres.fields.jsonb @@ -28,8 +28,7 @@ class Migration(migrations.Migration): name='Organization', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('uuid', models.UUIDField()), - ('name', models.CharField(max_length=100)), + ('name', models.CharField(blank=True, max_length=100, null=True)), ('country', models.CharField(blank=True, max_length=100, null=True)), ('primary_language', models.CharField(blank=True, max_length=100, null=True)), ('languages', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=100), size=None)), @@ -44,11 +43,32 @@ class Migration(migrations.Migration): name='Group', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('uuid', models.UUIDField(db_index=True, unique=True)), - ('name', models.TextField()), + ('source_id', models.CharField(blank=True, max_length=100, null=True)), + ('uuid', models.UUIDField(blank=True, db_index=True, null=True, unique=True)), + ('name', models.TextField(blank=True, null=True)), ('query', models.TextField(blank=True, null=True)), - ('count', models.IntegerField()), - ('status', models.CharField(blank=True, max_length=100, null=True)), + ('count', models.IntegerField(blank=True, null=True)), + ('organization', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rapidpro.Organization')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Contact', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('source_id', models.CharField(blank=True, max_length=100, null=True)), + ('uuid', models.UUIDField(blank=True, db_index=True, null=True, unique=True)), + ('name', models.TextField(blank=True, null=True)), + ('language', models.CharField(blank=True, max_length=100, null=True)), + ('urns', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=100), blank=True, default=list, null=True, size=None)), + ('groups', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, null=True)), + ('fields', django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, null=True)), + ('blocked', models.BooleanField(blank=True, null=True)), + ('stopped', models.BooleanField(blank=True, null=True)), + ('created_on', models.DateTimeField(blank=True, null=True)), + ('modified_on', models.DateTimeField(blank=True, null=True)), ('organization', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rapidpro.Organization')), ], options={ diff --git a/src/etools_datamart/apps/rapidpro/migrations/0002_auto_20190906_1333.py b/src/etools_datamart/apps/rapidpro/migrations/0002_auto_20190906_1333.py deleted file mode 100644 index 8962132e3..000000000 --- a/src/etools_datamart/apps/rapidpro/migrations/0002_auto_20190906_1333.py +++ /dev/null @@ -1,22 +0,0 @@ -# Generated by Django 2.2.5 on 2019-09-06 13:33 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('rapidpro', '0001_initial'), - ] - - operations = [ - migrations.RemoveField( - model_name='organization', - name='uuid', - ), - migrations.AlterField( - model_name='organization', - name='name', - field=models.CharField(blank=True, max_length=100, null=True), - ), - ] diff --git a/src/etools_datamart/apps/rapidpro/migrations/0003_group_source_id.py b/src/etools_datamart/apps/rapidpro/migrations/0003_group_source_id.py deleted file mode 100644 index 74fc8a31c..000000000 --- a/src/etools_datamart/apps/rapidpro/migrations/0003_group_source_id.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.5 on 2019-09-19 06:47 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('rapidpro', '0002_auto_20190906_1333'), - ] - - operations = [ - migrations.AddField( - model_name='group', - name='source_id', - field=models.CharField(blank=True, max_length=100, null=True), - ), - ] diff --git a/src/etools_datamart/apps/rapidpro/models.py b/src/etools_datamart/apps/rapidpro/models.py index 1dad78655..102b4fb57 100644 --- a/src/etools_datamart/apps/rapidpro/models.py +++ b/src/etools_datamart/apps/rapidpro/models.py @@ -24,6 +24,9 @@ class Source(models.Model): class Meta: app_label = 'rapidpro' + def __str__(self): + return self.name + class Organization(models.Model): { @@ -85,12 +88,25 @@ def linked_services(self): return [s for s in Service.objects.all() if s.managed_model == self] +class SyncCheckpoint(models.Model): + organization = models.ForeignKey(Organization, db_index=True, on_delete=models.CASCADE) + collection_name = models.CharField(max_length=100) + subcollection_name = models.CharField(max_length=100, null=True, blank=True) + last_started = models.DateTimeField() + last_saved = models.DateTimeField(null=True, blank=True) + + class Meta: + unique_together = ('organization', 'collection_name', 'subcollection_name') + + def __str__(self): + return '{}: {} {}'.format(self.organization, self.collection_name, self.subcollection_name or '').strip() + + class Group(RapidProDataMartModel): - uuid = models.UUIDField(unique=True, db_index=True) - name = models.TextField() + uuid = models.UUIDField(unique=True, db_index=True, null=True, blank=True) + name = models.TextField(null=True, blank=True) query = models.TextField(null=True, blank=True) - count = models.IntegerField() - status = models.CharField(max_length=100, blank=True, null=True) + count = models.IntegerField(null=True, blank=True) def __str__(self): return '{} ({})'.format(self.name, self.organization) @@ -98,24 +114,35 @@ def __str__(self): class Options: source = 'groups' -# -# class Contact(RapidProDataMartModel): -# uuid = models.UUIDField(unique=True, db_index=True) -# name = models.TextField(null=True, blank=True) -# language = models.CharField(max_length=100, null=True, blank=True) -# urns = ArrayField( -# models.CharField(max_length=100), -# default=list -# ) -# groups = models.ManyToManyField(Group) -# fields = JSONField(default=dict) -# blocked = models.NullBooleanField() -# stopped = models.NullBooleanField() -# created_on = models.DateTimeField(null=True, blank=True) -# modified_on = models.DateTimeField(null=True, blank=True) -# -# def __str__(self): -# return '{} ({})'.format(self.name, self.organization) -# -# class Options: -# source = 'contacts' + +class ContactLoader(TembaLoader): + + def get_groups(self, record, ret, field_name): + return [oo.serialize() for oo in record.groups] + + +class Contact(RapidProDataMartModel): + uuid = models.UUIDField(unique=True, db_index=True, null=True, blank=True) + name = models.TextField(null=True, blank=True) + language = models.CharField(max_length=100, null=True, blank=True) + urns = ArrayField( + models.CharField(max_length=100), + default=list, + null=True, blank=True + ) + # groups = models.ManyToManyField(Group) + groups = JSONField(default=dict, null=True, blank=True) + fields = JSONField(default=dict, null=True, blank=True) + blocked = models.BooleanField(null=True, blank=True) + stopped = models.BooleanField(null=True, blank=True) + created_on = models.DateTimeField(null=True, blank=True) + modified_on = models.DateTimeField(null=True, blank=True) + loader = ContactLoader() + + def __str__(self): + return '{} ({})'.format(self.name, self.organization) + + class Options: + source = 'contacts' + exclude_from_compare = ['groups', ] + fields_to_compare = None From 012f4b8e912fd1c5d26932c8ac0795801ff142e7 Mon Sep 17 00:00:00 2001 From: saxix Date: Thu, 19 Sep 2019 18:41:00 +0200 Subject: [PATCH 07/10] removes wrong model --- .../migrations/0096_auto_20190919_1633.py | 138 ++++++++++++++++++ src/etools_datamart/apps/rapidpro/models.py | 14 -- tox.ini | 1 + 3 files changed, 139 insertions(+), 14 deletions(-) create mode 100644 src/etools_datamart/apps/data/migrations/0096_auto_20190919_1633.py diff --git a/src/etools_datamart/apps/data/migrations/0096_auto_20190919_1633.py b/src/etools_datamart/apps/data/migrations/0096_auto_20190919_1633.py new file mode 100644 index 000000000..e99e22129 --- /dev/null +++ b/src/etools_datamart/apps/data/migrations/0096_auto_20190919_1633.py @@ -0,0 +1,138 @@ +# Generated by Django 2.2.5 on 2019-09-19 16:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('data', '0095_auto_20190919_1016'), + ] + + operations = [ + migrations.AlterField( + model_name='actionpoint', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='agreement', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='attachment', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='engagement', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='famindicator', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='fundsreservation', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='gatewaytype', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='grant', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='hact', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='hacthistory', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='intervention', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='interventionbudget', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='interventionbylocation', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='location', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='office', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='partner', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='partnerstaffmember', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='pdindicator', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='pmpindicators', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='reportindicator', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='section', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='travel', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='travelactivity', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='trip', + name='country_name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='userstats', + name='country_name', + field=models.CharField(max_length=100), + ), + ] diff --git a/src/etools_datamart/apps/rapidpro/models.py b/src/etools_datamart/apps/rapidpro/models.py index 102b4fb57..ce78fde28 100644 --- a/src/etools_datamart/apps/rapidpro/models.py +++ b/src/etools_datamart/apps/rapidpro/models.py @@ -88,20 +88,6 @@ def linked_services(self): return [s for s in Service.objects.all() if s.managed_model == self] -class SyncCheckpoint(models.Model): - organization = models.ForeignKey(Organization, db_index=True, on_delete=models.CASCADE) - collection_name = models.CharField(max_length=100) - subcollection_name = models.CharField(max_length=100, null=True, blank=True) - last_started = models.DateTimeField() - last_saved = models.DateTimeField(null=True, blank=True) - - class Meta: - unique_together = ('organization', 'collection_name', 'subcollection_name') - - def __str__(self): - return '{}: {} {}'.format(self.organization, self.collection_name, self.subcollection_name or '').strip() - - class Group(RapidProDataMartModel): uuid = models.UUIDField(unique=True, db_index=True, null=True, blank=True) name = models.TextField(null=True, blank=True) diff --git a/tox.ini b/tox.ini index 08243189c..4fd789a65 100644 --- a/tox.ini +++ b/tox.ini @@ -17,6 +17,7 @@ addopts = --capture=no --cov-report=html --cov-config=tests/.coveragerc + --cov-report=term:skip-covered --cov=etools_datamart markers = From 1ef7c95d86f13b7d05874b3134b955a4d0aeb7f1 Mon Sep 17 00:00:00 2001 From: saxix Date: Thu, 19 Sep 2019 20:12:18 +0200 Subject: [PATCH 08/10] fixes key loader --- src/etools_datamart/apps/data/loader.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/etools_datamart/apps/data/loader.py b/src/etools_datamart/apps/data/loader.py index ce4c0c9f0..bde4bd9f2 100644 --- a/src/etools_datamart/apps/data/loader.py +++ b/src/etools_datamart/apps/data/loader.py @@ -17,11 +17,11 @@ class EToolsLoaderOptions(BaseLoaderOptions): - - def __init__(self, base=None): - super().__init__(base) - self.key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, - source_id=record.pk) + pass +# def __init__(self, base=None): +# super().__init__(base) +# self.key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, +# source_id=record.pk) class EtoolsLoader(BaseLoader): From 7f64bacc92bcc144d43db4cb400150c83b1cece2 Mon Sep 17 00:00:00 2001 From: saxix Date: Thu, 19 Sep 2019 20:12:18 +0200 Subject: [PATCH 09/10] fixes key loader --- src/etools_datamart/apps/data/loader.py | 9 ++++++++- src/etools_datamart/apps/data/models/base.py | 12 +++++++++--- src/etools_datamart/apps/etl/loader.py | 7 +++++-- src/etools_datamart/apps/rapidpro/loader.py | 9 ++------- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/etools_datamart/apps/data/loader.py b/src/etools_datamart/apps/data/loader.py index bde4bd9f2..72b07ffd5 100644 --- a/src/etools_datamart/apps/data/loader.py +++ b/src/etools_datamart/apps/data/loader.py @@ -17,7 +17,10 @@ class EToolsLoaderOptions(BaseLoaderOptions): - pass + DEFAULT_KEY = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, + source_id=record.pk) + + # def __init__(self, base=None): # super().__init__(base) # self.key = lambda loader, record: dict(schema_name=loader.context['country'].schema_name, @@ -175,6 +178,10 @@ def load(self, *, verbosity=0, stdout=None, ignore_dependencies=False, max_recor return self.results +class CommonSchemaLoaderOptions(BaseLoaderOptions): + DEFAULT_KEY = lambda loader, record: dict(source_id=record.pk) + + class CommonSchemaLoader(EtoolsLoader): def get_mart_values(self, record=None): ret = {'seen': self.context['today']} diff --git a/src/etools_datamart/apps/data/models/base.py b/src/etools_datamart/apps/data/models/base.py index 65dba9074..642bf96ab 100644 --- a/src/etools_datamart/apps/data/models/base.py +++ b/src/etools_datamart/apps/data/models/base.py @@ -5,7 +5,8 @@ from celery.local import class_property -from etools_datamart.apps.data.loader import EtoolsLoader, EToolsLoaderOptions +from etools_datamart.apps.data.loader import (CommonSchemaLoader, CommonSchemaLoaderOptions, + EtoolsLoader, EToolsLoaderOptions,) from etools_datamart.apps.etl.base import DataMartModelBase @@ -40,12 +41,17 @@ def truncate(self, reset=True): restart)) -class EToolsDataMartModelBase(DataMartModelBase): +class CommonDataMartModelModelBase(DataMartModelBase): + loader_option_class = CommonSchemaLoaderOptions + loader_class = CommonSchemaLoader + + +class EToolsDataMartModelBase(CommonDataMartModelModelBase): loader_option_class = EToolsLoaderOptions loader_class = EtoolsLoader -class CommonDataMartModel(models.Model, metaclass=DataMartModelBase): +class CommonDataMartModel(models.Model, metaclass=CommonDataMartModelModelBase): source_id = models.IntegerField(blank=True, null=True, db_index=True) last_modify_date = models.DateTimeField(blank=True, auto_now=True) seen = models.DateTimeField(blank=True, null=True) diff --git a/src/etools_datamart/apps/etl/loader.py b/src/etools_datamart/apps/etl/loader.py index 201079dcb..6c4558749 100644 --- a/src/etools_datamart/apps/etl/loader.py +++ b/src/etools_datamart/apps/etl/loader.py @@ -107,10 +107,11 @@ class MaxRecordsException(Exception): pass -DEFAULT_KEY = lambda loader, record: dict(source_id=record.pk) +undefined = object() class BaseLoaderOptions: + DEFAULT_KEY = lambda loader, record: dict(source_id=record.pk) __attrs__ = ['mapping', 'celery', 'source', 'last_modify_field', 'queryset', 'key', 'locks', 'filters', 'sync_deleted_records', 'truncate', 'depends', 'timeout', 'lock_key', 'always_update', 'fields_to_compare'] @@ -123,7 +124,7 @@ def __init__(self, base=None): self.always_update = False self.source = None self.lock_key = None - self.key = DEFAULT_KEY + self.key = undefined self.timeout = None self.depends = () self.filters = None @@ -141,6 +142,8 @@ def __init__(self, base=None): setattr(self, attr, n) else: setattr(self, attr, getattr(base, attr, getattr(self, attr))) + if self.key == undefined: + self.key = type(self).DEFAULT_KEY if self.truncate: self.sync_deleted_records = lambda loader: False diff --git a/src/etools_datamart/apps/rapidpro/loader.py b/src/etools_datamart/apps/rapidpro/loader.py index aebe106f5..43ddfdb3d 100644 --- a/src/etools_datamart/apps/rapidpro/loader.py +++ b/src/etools_datamart/apps/rapidpro/loader.py @@ -10,16 +10,11 @@ logger = get_task_logger(__name__) -DEFAULT_KEY = lambda loader, record: dict(uuid=record.uuid, - organization=loader.context['organization']) - class TembaLoaderOptions(BaseLoaderOptions): __attrs__ = BaseLoaderOptions.__attrs__ + ['host', 'temba_object'] - - def __init__(self, base=None): - super().__init__(base) - self.key = DEFAULT_KEY + DEFAULT_KEY = lambda loader, record: dict(uuid=record.uuid, + organization=loader.context['organization']) class TembaLoader(BaseLoader): From 742f2864355c5fa78d74682108d1ce5437a0c03d Mon Sep 17 00:00:00 2001 From: saxix Date: Thu, 19 Sep 2019 21:00:33 +0200 Subject: [PATCH 10/10] bump version --- .bumpversion.cfg | 2 +- CHANGES | 8 +++++--- docker/Makefile | 16 ++++++++-------- src/etools_datamart/__init__.py | 2 +- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 6185324a2..1ce881800 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.2.0a +current_version = 2.2 commit = False tag = False allow_dirty = True diff --git a/CHANGES b/CHANGES index 9dfff33fc..fea544c9a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,8 @@ -2.2 (dev) ---------- -* +2.2 +--- +* fixes InterventionBudgetLoader +* admin improvements +* improves indexing 2.1 --- diff --git a/docker/Makefile b/docker/Makefile index 70a431fba..51a6d3777 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -4,7 +4,7 @@ DATABASE_URL_ETOOLS?= DEVELOP?=1 DOCKER_PASS?= DOCKER_USER?= -TARGET?=2.2.0a +TARGET?=2.2 BASE=2.0 # below vars are used internally BUILD_OPTIONS?=--squash @@ -18,7 +18,7 @@ DOCKERFILE?=Dockerfile RUN_OPTIONS?= PIPENV_ARGS?= PORTS= -ABSOLUTE_BASE_URL?="http://192.2.0a68.66.66:8000" +ABSOLUTE_BASE_URL?="http://192.268.66.66:8000" help: @echo "dev build dev image (based on local code)" @@ -71,12 +71,12 @@ build_old: cd .. && docker run \ --rm \ -e ABSOLUTE_BASE_URL=${ABSOLUTE_BASE_URL} \ - -e CACHE_URL=redis://192.2.0a68.66.66:6379/1 \ - -e CACHE_URL_API=redis://192.2.0a68.66.66:6379/2 \ - -e CACHE_URL_LOCK=redis://192.2.0a68.66.66:6379/3 \ - -e CACHE_URL_TEMPLATE=redis://192.2.0a68.66.66:6379/4 \ - -e CELERY_BROKER_URL=redis://192.2.0a68.66.66:6379/2 \ - -e CELERY_RESULT_BACKEND=redis://192.2.0a68.66.66:6379/3 \ + -e CACHE_URL=redis://192.268.66.66:6379/1 \ + -e CACHE_URL_API=redis://192.268.66.66:6379/2 \ + -e CACHE_URL_LOCK=redis://192.268.66.66:6379/3 \ + -e CACHE_URL_TEMPLATE=redis://192.268.66.66:6379/4 \ + -e CELERY_BROKER_URL=redis://192.268.66.66:6379/2 \ + -e CELERY_RESULT_BACKEND=redis://192.268.66.66:6379/3 \ -e CSRF_COOKIE_SECURE=false \ -e DATABASE_URL=${DATABASE_URL} \ -e DATABASE_URL_ETOOLS=${DATABASE_URL_ETOOLS} \ diff --git a/src/etools_datamart/__init__.py b/src/etools_datamart/__init__.py index 205ac4050..8aa34ddb2 100644 --- a/src/etools_datamart/__init__.py +++ b/src/etools_datamart/__init__.py @@ -1,7 +1,7 @@ import warnings NAME = 'etools-datamart' -VERSION = __version__ = '2.2.0a' +VERSION = __version__ = '2.2' __author__ = '' # UserWarning: The psycopg2 wheel package will be renamed from release 2.8;