From 11f85d66fb1fef8ce2f9d217dd95b1a2db01065f Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 1 Feb 2019 20:32:44 +0100 Subject: [PATCH 1/9] fixes init-setup PeriodicTask names --- .../apps/init/management/commands/init-setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/etools_datamart/apps/init/management/commands/init-setup.py b/src/etools_datamart/apps/init/management/commands/init-setup.py index c3139f0be..8dc6a5601 100644 --- a/src/etools_datamart/apps/init/management/commands/init-setup.py +++ b/src/etools_datamart/apps/init/management/commands/init-setup.py @@ -262,8 +262,8 @@ def handle(self, *args, **options): for loadeable in loadeables: model = apps.get_model(loadeable) loaders.append(loadeable) - __, is_new = PeriodicTask.objects.get_or_create(task=f"ETL {model.loader.task.name}", - defaults={'name': loadeable, + __, is_new = PeriodicTask.objects.get_or_create(name=f"ETL {model.loader.task.name}", + defaults={'task': model.loader.task.name, 'service': Service.objects.get_for_model(model), 'crontab': midnight}) if is_new: From 2af19172240c32519de8e168c0f158d1c4de4429 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 1 Feb 2019 20:39:33 +0100 Subject: [PATCH 2/9] fixex init-setup --- .../apps/init/management/commands/init-setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/etools_datamart/apps/init/management/commands/init-setup.py b/src/etools_datamart/apps/init/management/commands/init-setup.py index 8dc6a5601..3256d40eb 100644 --- a/src/etools_datamart/apps/init/management/commands/init-setup.py +++ b/src/etools_datamart/apps/init/management/commands/init-setup.py @@ -261,7 +261,7 @@ def handle(self, *args, **options): loaders = [] for loadeable in loadeables: model = apps.get_model(loadeable) - loaders.append(loadeable) + loaders.append(model.loader.task.name) __, is_new = PeriodicTask.objects.get_or_create(name=f"ETL {model.loader.task.name}", defaults={'task': model.loader.task.name, 'service': Service.objects.get_for_model(model), @@ -280,7 +280,7 @@ def handle(self, *args, **options): if is_new: self.stdout.write(f"NEW preload task for '{url}'") - ret = PeriodicTask.objects.filter(name__startswith='data.').exclude(name__in=loaders).delete() + ret = PeriodicTask.objects.filter(task__startswith='load_').exclude(task__in=loaders).delete() counters[False] = ret[0] EtlTask.objects.inspect() From 94f4b6a4f3e8cab0b5ed57e47445f83d084d8bc1 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 1 Feb 2019 20:44:48 +0100 Subject: [PATCH 3/9] use TRUNCATE..CASCADE --- src/etools_datamart/apps/data/models/base.py | 2 +- src/unicef_rest_framework/admin/base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/etools_datamart/apps/data/models/base.py b/src/etools_datamart/apps/data/models/base.py index 8bb4a841f..cda6e37e9 100644 --- a/src/etools_datamart/apps/data/models/base.py +++ b/src/etools_datamart/apps/data/models/base.py @@ -31,7 +31,7 @@ def filter_schemas(self, *schemas): class DataMartManager(BaseManager.from_queryset(DataMartQuerySet)): def truncate(self): - self.raw('TRUNCATE TABLE {0}'.format(self.model._meta.db_table)) + self.raw('TRUNCATE TABLE {0} CASCADE'.format(self.model._meta.db_table)) class DataMartModelBase(ModelBase): diff --git a/src/unicef_rest_framework/admin/base.py b/src/unicef_rest_framework/admin/base.py index 71d814eef..c007411cd 100644 --- a/src/unicef_rest_framework/admin/base.py +++ b/src/unicef_rest_framework/admin/base.py @@ -24,7 +24,7 @@ class TruncateTableMixin(ExtraUrlMixin): def _truncate(self, request): from django.db import connection cursor = connection.cursor() - cursor.execute('TRUNCATE TABLE {0}'.format(self.model._meta.db_table)) + cursor.execute('TRUNCATE TABLE "{0}" CASCADE '.format(self.model._meta.db_table)) @link(label='Truncate', permission=lambda request, obj: request.user.is_superuser) def truncate(self, request): From f1dd379dc56872e8ccc73ddc4a7a2bc8bbfedf6a Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 1 Feb 2019 20:47:54 +0100 Subject: [PATCH 4/9] add abulity to truncate tables in DataModelAdmin --- src/etools_datamart/apps/data/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etools_datamart/apps/data/admin.py b/src/etools_datamart/apps/data/admin.py index c33675266..c304e3de1 100644 --- a/src/etools_datamart/apps/data/admin.py +++ b/src/etools_datamart/apps/data/admin.py @@ -30,7 +30,7 @@ class DatamartChangeList(ChangeList): pass -class DataModelAdmin(ExtraUrlMixin, ModelAdmin): +class DataModelAdmin(ExtraUrlMixin, TruncateTableMixin, ModelAdmin): actions = [mass_update, ] def get_list_filter(self, request): From 53b7f599e6d2d48dbcb6290d43dd394ab94a15f2 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 1 Feb 2019 20:51:40 +0100 Subject: [PATCH 5/9] fixes DataModelAdmin MRO --- src/etools_datamart/apps/data/admin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/etools_datamart/apps/data/admin.py b/src/etools_datamart/apps/data/admin.py index c304e3de1..0a552b33e 100644 --- a/src/etools_datamart/apps/data/admin.py +++ b/src/etools_datamart/apps/data/admin.py @@ -9,7 +9,7 @@ from django.http import HttpResponseRedirect from django.urls import reverse -from admin_extra_urls.extras import ExtraUrlMixin, link +from admin_extra_urls.extras import link from adminactions.mass_update import mass_update from adminfilters.filters import AllValuesComboFilter from crashlog.middleware import process_exception @@ -30,7 +30,7 @@ class DatamartChangeList(ChangeList): pass -class DataModelAdmin(ExtraUrlMixin, TruncateTableMixin, ModelAdmin): +class DataModelAdmin(TruncateTableMixin, ModelAdmin): actions = [mass_update, ] def get_list_filter(self, request): From d224d5c1b500ef6adf7efce032544d01a80d06c1 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 1 Feb 2019 20:56:22 +0100 Subject: [PATCH 6/9] updates CHANGES --- CHANGES | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES b/CHANGES index 929b7fd01..beff6ef45 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,8 @@ +1.15 +---- +* add ability to truncate DataMart tables +* fixes init-setup PeriodicTask creation + 1.14 ---- * Fix loading Intervention From e8b93db02707bd123338d1884dbda85650554b25 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 1 Feb 2019 21:16:11 +0100 Subject: [PATCH 7/9] =?UTF-8?q?Bump=20version:=201.14=20=E2=86=92=201.15?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 4 ++-- CHANGES | 6 +++--- docker/Makefile | 2 +- src/etools_datamart/__init__.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0fdbd0ceb..aeefd700e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.14 +current_version = 1.15 commit = True tag = False allow_dirty = True @@ -21,7 +21,7 @@ serialize = {major}.{minor} [bumpversion:file:docker/Makefile] -serialize = +serialize = {major}.{minor}{release}{num} [bumpversion:file:src/etools_datamart/__init__.py] diff --git a/CHANGES b/CHANGES index beff6ef45..f6584325d 100644 --- a/CHANGES +++ b/CHANGES @@ -3,18 +3,18 @@ * add ability to truncate DataMart tables * fixes init-setup PeriodicTask creation -1.14 +1.15 ---- * Fix loading Intervention * migrations reset -1.14 +1.15 --------- * new endpoint DPIndicator * preload helpers -1.14 +1.15 --------- * add impersonation * fixes schema permission check diff --git a/docker/Makefile b/docker/Makefile index ba6a82310..009b0a149 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -4,7 +4,7 @@ DATABASE_URL_ETOOLS?= DEVELOP?=0 DOCKER_PASS?= DOCKER_USER?= -TARGET?=1.14 +TARGET?=1.15a0 # below vars are used internally BUILD_OPTIONS?=--squash CMD?=datamart diff --git a/src/etools_datamart/__init__.py b/src/etools_datamart/__init__.py index 909cb376b..31249cc06 100644 --- a/src/etools_datamart/__init__.py +++ b/src/etools_datamart/__init__.py @@ -1,7 +1,7 @@ import warnings NAME = 'etools-datamart' -VERSION = __version__ = '1.14' +VERSION = __version__ = '1.15' __author__ = '' # UserWarning: The psycopg2 wheel package will be renamed from release 2.8; From c28a3c439648c5627c092820bdf79d97787861a6 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 1 Feb 2019 21:20:50 +0100 Subject: [PATCH 8/9] fixes CHANGES --- CHANGES | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index f6584325d..e1ad1ed59 100644 --- a/CHANGES +++ b/CHANGES @@ -3,18 +3,18 @@ * add ability to truncate DataMart tables * fixes init-setup PeriodicTask creation -1.15 +1.14 ---- * Fix loading Intervention * migrations reset -1.15 +1.12 --------- * new endpoint DPIndicator * preload helpers -1.15 +1.11 --------- * add impersonation * fixes schema permission check From 099d2d1693dd3e83af987faa38b63695145d19c7 Mon Sep 17 00:00:00 2001 From: sax Date: Fri, 1 Feb 2019 21:22:48 +0100 Subject: [PATCH 9/9] update bumpversion config --- .bumpversion.cfg | 4 ++-- docker/Makefile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index aeefd700e..533085955 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -3,7 +3,7 @@ current_version = 1.15 commit = True tag = False allow_dirty = True -parse = (?P\d+)\.(?P\d+)((?P[a-z]+)(?P\d+))? +parse = (?P\d+)\.(?P\d+)\.(?P[a-z]+) serialize = {major}.{minor}{release}{num} {major}.{minor} @@ -22,7 +22,7 @@ serialize = [bumpversion:file:docker/Makefile] serialize = - {major}.{minor}{release}{num} + {major}.{minor}{release} [bumpversion:file:src/etools_datamart/__init__.py] serialize = diff --git a/docker/Makefile b/docker/Makefile index 009b0a149..611eed563 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -4,7 +4,7 @@ DATABASE_URL_ETOOLS?= DEVELOP?=0 DOCKER_PASS?= DOCKER_USER?= -TARGET?=1.15a0 +TARGET?=1.15 # below vars are used internally BUILD_OPTIONS?=--squash CMD?=datamart