refactoring versioning

.github/workflows/test.yml

@@ -117,6 +117,11 @@ jobs:
             type=ref,event=tag
             type=semver,pattern={{version}}
             type=semver,pattern={{raw}}
+      - name: DockerHub login
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       - id: last_commit
         uses: ./.github/actions/last_commit
       - name: Build and push
@@ -129,13 +134,13 @@ jobs:
           tags: unicef/hope-country-workspace:test-${{env.BRANCH}}
           platforms: linux/amd64
           target: test
-          push: true
-          cache-from: type=registry,ref=unicef/hope-country-workspace:test-${{env.BRANCH}},ref=unicef/hope-country-workspace:${{env.BRANCH}}
-          cache-to: type=registry,ref=unicef/hope-country-workspace:${{env.BRANCH}}-cache,mode=max,image-manifest=true
           build-args: |
               GITHUB_SERVER_URL=${{ github.server_url }}
               GITHUB_REPOSITORY=${{ github.repository }}
               SOURCE_COMMIT=${{ steps.last_commit.outputs.last_commit_short_sha }}
+#          push: true
+#          cache-from: type=registry,ref=unicef/hope-country-workspace:test-${{env.BRANCH}},ref=unicef/hope-country-workspace:${{env.BRANCH}}
+#          cache-to: type=registry,ref=unicef/hope-country-workspace:${{env.BRANCH}}-cache,mode=max,image-manifest=true
 
 #              BUILD_DATE=${{ env.BUILD_DATE }}
 #          labels: "${{ steps.meta.outputs.labels }}\nchecksum=${{ inputs.code_checksum }}\ndistro=${{ inputs.target }}"

pyproject.toml

@@ -16,6 +16,7 @@ dependencies = [
     "django-adminactions>=2.3.0",
     "django-adminfilters>=2.5.0",
     "django-celery-beat>=2.6.0",
+    "django-celery-boost>=0.2.0",
     "django-celery-results>=2.5.1",
     "django-constance>=3.1.0",
     "django-csp",

src/country_workspace/config/__init__.py

@@ -31,7 +31,7 @@ class Group(Enum):
         list,
         [],
         [],
-        True,
+        False,
         "list of emails that will automatically created as superusers",
     ),
     "ADMIN_EMAIL": (str, "", "admin", True, "Initial user created at first deploy"),

src/country_workspace/config/settings.py

@@ -38,6 +38,7 @@
     "jsoneditor",
     "django_celery_beat",
     "django_celery_results",
+    "django_celery_boost",
     "hope_flex_fields",
     "hope_smart_import",
     "hope_smart_export",

src/country_workspace/config/urls.py

@@ -16,3 +16,7 @@
     path(r"__debug__/", include(debug_toolbar.urls)),
     path(r"", workspace.urls),
 ]
+
+admin.site.site_header = "Workspace Admin"
+admin.site.site_title = "Workspace Admin Portal"
+admin.site.index_title = "Welcome to HOPE Workspace"

src/country_workspace/management/commands/upgrade.py

@@ -166,6 +166,7 @@ def handle(self, *args: Any, **options: Any) -> None:  # noqa
                 ),
                 name=settings.TENANT_HQ,
             )
+            call_command("upgradescripts", ["apply"])
             echo("Upgrade completed", style_func=self.style.SUCCESS)
         except ValidationError as e:
             self.halt(Exception("\n- ".join(["Wrong argument(s):", *e.messages])))

src/country_workspace/versioning/management/commands/upgradescripts.py

@@ -0,0 +1,80 @@
+from pathlib import Path
+
+from django.core.management.base import BaseCommand, CommandError, no_translations
+from django.utils.timezone import now
+
+import country_workspace
+from country_workspace.versioning.management.manager import Manager
+
+VERSION_TEMPLATE = """# Generated by HCW %(version)s on %(today)s
+from packaging.version import Version
+
+_script_for_version = Version("%(version)s")
+
+
+def forward():
+    pass
+
+
+def backward():
+    pass
+
+
+class Scripts:
+    operations = [(forward, backward)]
+"""
+
+
+class Command(BaseCommand):
+    help = "Creates new version for apps."
+
+    def add_arguments(self, parser):
+        subparsers = parser.add_subparsers(dest="command")
+        subparsers.add_parser("list", help="Show applied/available scripts")
+
+        parser_create = subparsers.add_parser("create", help="Create a new empty script")
+        parser_create.add_argument("--label", help="name of the new script", metavar="LABEL")
+
+        parser_apply = subparsers.add_parser("apply", help="Apply scripts")
+        parser_apply.add_argument(
+            "num",
+            nargs="?",
+            help='Scripts will be applied to the one after that selected. Use the name "zero" to unapply all scripts.',
+        )
+        parser_apply.add_argument(
+            "--fake", action="store_true", help="Mark script as run without actually running them."
+        )
+
+    @no_translations
+    def handle(self, command, label=None, num=None, fake=None, zero=False, **options):
+        m = Manager()
+        if command == "list":
+            for entry in m.existing:
+                stem = Path(entry).stem
+                x = "x" if m.is_processed(entry) else " "
+                self.stdout.write("[{x}] {name}".format(x=x, name=stem))
+        elif command == "create":
+            new_ver = m.max_version + 1
+            ts = now().strftime("%Y_%m_%d_%H%M%S")
+            today = now().strftime("%Y %m %d %H:%M:%S")
+
+            dest_file = m.folder / "{:>04}_{}.py".format(new_ver, label or ts)
+            with dest_file.open("w") as f:
+                f.write(VERSION_TEMPLATE % {"timestamp": ts, "version": country_workspace.VERSION, "today": today})
+            if options["verbosity"] > 0:
+                self.stdout.write(f"Created script {dest_file.name}")
+
+        elif command == "apply":
+            if num == "zero":
+                m.zero(out=self.stdout)
+            else:
+                if not num:
+                    num = m.max_version
+                else:
+                    num = int(num)
+                if num >= m.max_applied_version:
+                    m.forward(num, fake, self.stdout)
+                else:
+                    m.backward(num, self.stdout)
+        else:
+            raise CommandError("Invalid command")

src/country_workspace/versioning/management/manager.py

@@ -1,13 +1,13 @@
 import importlib.util
 import re
+import sys
 from pathlib import Path
 from typing import Callable
 
 from country_workspace import VERSION
 from country_workspace.versioning.models import Version
 
 regex = re.compile(r"(\d+).*")
-default_folder = Path(__file__).parent.parent / "scripts"
 
 
 def get_version(filename):
@@ -19,11 +19,11 @@ def get_version(filename):
 def get_funcs(filename: Path, direction: str = "forward"):
     if not filename.exists():  # pragma: no cover
         raise FileNotFoundError(filename)
-    spec = importlib.util.spec_from_file_location("version", filename.absolute())
+    spec = importlib.util.spec_from_file_location(filename.stem, filename.absolute())
     module = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(module)
     funcs = []
-    for op in module.Version.operations:
+    for op in module.Scripts.operations:
         if isinstance(op, (list, tuple)):
             if direction == "forward":
                 funcs.append(op[0])
@@ -39,8 +39,10 @@ def get_funcs(filename: Path, direction: str = "forward"):
 
 
 class Manager:
-    def __init__(self, folder: Path = default_folder):
-        self.folder = folder
+    default_folder = Path(__file__).parent.parent / "scripts"
+
+    def __init__(self, folder: Path = ""):
+        self.folder = folder or self.default_folder
         self.existing = []
         self.applied = list(Version.objects.order_by("name").values_list("name", flat=True))
         self.max_version = 0
@@ -53,34 +55,44 @@ def __init__(self, folder: Path = default_folder):
                 self.existing.append(filename)
                 self.max_version = max(self.max_version, v)
 
-    def zero(self):
-        self.backward(0)
+    def is_processed(self, entry):
+        return Path(entry).name in self.applied
+
+    def zero(self, out=sys.stdout):
+        self.backward(0, out=out)
 
-    def forward(self, to_num) -> list[tuple[Path, list[Callable[[None], None]]]]:
-        print("Upgrading...")
+    def forward(
+        self, to_num=None, fake: bool = False, out=sys.stdout
+    ) -> list[tuple[Path, list[Callable[[None], None]]]]:
+        out.write("Upgrading...\n")
+        if not to_num:
+            to_num = self.max_version
         processed = []
         for entry in self.existing:
             if get_version(entry.stem) > to_num:
                 break
             if entry.name not in self.applied:
                 funcs = get_funcs(entry, direction="forward")
-                print(f"    Applying {entry.stem}")
-                for func in funcs:
-                    func()
+                if fake:
+                    out.write(f"   Applying {entry.stem} (fake)\n")
+                else:
+                    out.write(f"   Applying {entry.stem}\n")
+                    for func in funcs:
+                        func()
                 Version.objects.create(name=entry.name, version=VERSION)
                 processed.append((entry, funcs))
         self.applied = list(Version.objects.order_by("name").values_list("name", flat=True))
         return processed
 
-    def backward(self, to_num) -> list[tuple[Path, list[Callable[[None], None]]]]:
-        print("Downgrading...")
+    def backward(self, to_num, out=sys.stdout) -> list[tuple[Path, list[Callable[[None], None]]]]:
+        out.write("Downgrading...\n")
         processed = []
         for entry in reversed(self.applied):
             if get_version(entry) <= to_num:
                 break
             file_path = Path(self.folder) / entry
             funcs = get_funcs(file_path, direction="backward")
-            print(f"  Discharging {file_path.stem}")
+            out.write(f"   Discharging {file_path.stem}\n")
             for func in funcs:
                 func()
             Version.objects.get(name=file_path.name).delete()

src/country_workspace/versioning/scripts/0001_initial.py

@@ -17,7 +17,7 @@ def removes_hope_core_fieldset():
     DataChecker.objects.filter(name=HOUSEHOLD_CHECKER_NAME).delete()
 
 
-class Version:
+class Scripts:
     operations = [
         (create_hope_field_definitions, removes_hope_field_definitions),
         (create_hope_core_fieldset, removes_hope_core_fieldset),

src/country_workspace/versioning/scripts/0002_synclog.py

@@ -1,5 +1,5 @@
 from country_workspace.versioning.synclog import create_default_synclog, removes_default_synclog
 
 
-class Version:
+class Scripts:
     operations = [(create_default_synclog, removes_default_synclog)]