updates tests and CI

.dockerignore

@@ -1,8 +1,19 @@
 *
+
 !docker/bin
 !docker/conf
+!src/country_workspace/**/*
+
 !pyproject.toml
 !uv.lock
-!src
+
 !LICENSE
 !README.md
+!AUTHORS
+!CHANGES.md
+!MANIFEST.in
+
+*.egg-info
+.*
+~*
+**/~*

.github/workflows/docs.yml

@@ -23,33 +23,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-
-#      - name: Restore cached venv
-#        id: cache-venv-restore
-#        uses: actions/cache/restore@v4
-#        with:
-#          path: |
-#            .cache-uv/
-#            .venv/
-#          key: ${{ matrix.python-version }}-${{matrix.django-version}}-venv
-#
-#      - uses: yezz123/setup-uv@v4
       - name: Build Doc
         run: |
             pip install "mkdocs>=1.6.1" "markdown-include" "mkdocs-material>=9.5.36" \
                           "mkdocs-awesome-pages-plugin>=2.9.3" "mkdocstrings-python" "mdx-gh-links>=0.4"
             mkdocs build -d ./docs-output
 
-#      - name: Cache venv
-#        if: steps.cache-venv-restore.outputs.cache-hit != 'true'
-#        id: cache-venv-save
-#        uses: actions/cache/save@v4
-#        with:
-#          path: |
-#            .cache-uv/
-#            .venv/
-#          key: ${{ matrix.python-version }}-${{matrix.django-version}}-venv
-
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3
         with:

.github/workflows/test.yml

@@ -92,13 +92,12 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
+      - uses: yezz123/setup-uv@v4
       - name: Run tests
         run: |          
           apt-get update
           apt-get install python3-dev
-          pip install uv
-          uv venv
-          uv sync
+          uv sync --frozen
           uv run pytest tests/
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4

MANIFEST.in

@@ -1,10 +1,21 @@
+exclude *
+exclude requirements.txt
+
 include README.md
 include MANIFEST.in
 include AUTHORS
 include CHANGES.md
 include LICENSE
-
+include pyproject.toml
+include *.py
+include uv.lock
 
 recursive-include src/country_workspace *
+recursive-include src/country_workspace *.html
+recursive-exclude tests *
 
 prune **/~*
+global-exclude .*
+global-exclude *~
+global-exclude ~*
+exclude manage.py

compose.yaml

@@ -2,7 +2,7 @@ x-common: &common
   build:
     context: .
     dockerfile: docker/Dockerfile
-    target: python_dev_deps
+    target: builder
   platform: linux/amd64
   environment:
     - DEBUG=true
@@ -18,7 +18,7 @@ x-common: &common
     - SECRET_KEY=sensitive-secret-key
     - STATIC_ROOT=/var/country_workspace/static
   volumes:
-    - .:/code
+    - .:/app
     - /var/run/docker.sock:/var/run/docker.sock
   restart: always
   depends_on:

docker/Dockerfile

@@ -1,6 +1,28 @@
 FROM python:3.12-slim-bookworm as python_base
+ARG GOSU_VERSION=1.17
+ARG GOSU_SHA256=bbc4136d03ab138b1ad66fa4fc051bafc6cc7ffae632b069a53657279a450de3
+
+
+RUN set -x \
+  && runtimeDeps=" \
+    libxml2 \
+  " \
+  && buildDeps=" \
+  wget \
+  " \
+  && apt-get update && apt-get install -y --no-install-recommends ${buildDeps} ${runtimeDeps} \
+  && rm -rf /var/lib/apt/lists/* \
+  && wget --quiet -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" \
+  && echo "$GOSU_SHA256 /usr/local/bin/gosu" | sha256sum --check --status \
+  && chmod +x /usr/local/bin/gosu \
+  && apt-get purge -y --auto-remove $buildDeps
+
+RUN groupadd --gid 1024 app \
+    && adduser --disabled-login --disabled-password --no-create-home --ingroup app -q user
+
 
 FROM python_base AS builder
+
 RUN set -x \
     && buildDeps="build-essential \
         cmake \
@@ -16,39 +38,40 @@ RUN set -x \
     && apt-get install -y --no-install-recommends $buildDeps \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
-RUN pip install uwsgi uv
+RUN pip install uv
 
-
-COPY docker/conf /conf/
-COPY docker/bin /usr/local/bin/
+COPY pyproject.toml uv.lock AUTHORS LICENSE README.md MANIFEST.in /app/
+COPY src /app/src
 
 WORKDIR /app
-COPY . /app/
-
-RUN uv venv \
-    && uv sync --no-dev --no-editable --frozen \
-    && uv pip install . \
-    && ls -al /app/.venv/
-
-FROM python_base
+RUN uv sync --no-dev --no-editable --frozen
+#RUN ls -al .venv/lib/python3.12/site-packages/country_workspace/workspaces/
 
+#RUN uv export --no-dev --no-editable --frozen --no-sources --no-hashes -o requirement.txt \
+#    && python -m venv .venv \
+#    && .venv/bin/pip install -r requirement.txt
 
+#
 FROM python_base
-ENV PATH=/app/.venv/bin:$PATH
+ENV PATH=/app/.venv/bin:$PATH \
+    DJANGO_SETTINGS_MODULE=country_workspace.config.settings \
+    UWSGI_PROCESSES=4
 
 COPY docker/conf /conf/
-COPY docker/bin /usr/local/bin/
+COPY docker/bin/* /usr/local/bin/
 COPY --chown=user:app --from=builder /app/.venv /app/.venv
+#COPY --chown=user:app --from=builder /usr/local/bin/uwsgi /usr/local/bin/uwsgi
 
 
 EXPOSE 8000
 ENTRYPOINT exec docker-entrypoint.sh "$0" "$@"
 CMD ["run"]
 
+
 LABEL distro="final"
 LABEL maintainer="[email protected]"
 LABEL org.opencontainers.image.authors="[email protected]"
-LABEL org.opencontainers.image.created="$BUILD_DATE"
+#LABEL org.opencontainers.image.created="$BUILD_DATE"
 LABEL org.opencontainers.image.description="Hope Country Workspace"
 LABEL org.opencontainers.image.documentation="https://github.com/unicef/hope-country-workspace/"
 LABEL org.opencontainers.image.licenses="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${SOURCE_COMMIT:-master}/LICENSE"

docker/bin/docker-entrypoint.sh

@@ -4,7 +4,7 @@
 export MEDIA_ROOT="${MEDIA_ROOT:-/var/run/app/media}"
 export STATIC_ROOT="${STATIC_ROOT:-/var/run/app/static}"
 export UWSGI_PROCESSES="${UWSGI_PROCESSES:-"4"}"
-export DJANGO_SETTINGS_MODULE="${DJANGO_SETTINGS_MODULE:-"country_workspace.config.settings"}"
+#export DJANGO_SETTINGS_MODULE="${DJANGO_SETTINGS_MODULE:-"country_workspace.config.settings"}"
 
 ls -al /app/
 

docker/conf/uwsgi.ini

@@ -1,19 +1,23 @@
 [uwsgi]
 http=0.0.0.0:8000
-enable-threads=0
-honour-range=1
-master=1
+;enable-threads=0
+;honour-range=1
+;master=1
 module=country_workspace.config.wsgi
-processes=$(UWSGI_PROCESSES)
-;virtualenv=/code/.venv/
+;processes=$(UWSGI_PROCESSES)
+;virtualenv=/app/.venv/
+;pythonpath=/app/.venv/lib/python3.12/site-packages
 ;virtualenv=%(_)
 ;venv=%(_)
 ;chdir=code/
-username = user
-gropuname = app
+uid = user
+gid = app
+;username = user
+;gropuname = app
 ;offload-threads=%k
 ;static-gzip-all=true
-route = /static/(.*) static:$(STATIC_ROOT)/$1
-http-keepalive = 1
-collect-header=Content-Type RESPONSE_CONTENT_TYPE
-mimefile=/etc/mime.types
+;route = /static/(.*) static:$(STATIC_ROOT)/$1
+;http-keepalive = 1
+;collect-header=Content-Type RESPONSE_CONTENT_TYPE
+mimefile=/conf/mime.types
+;honour-stdin = true

pyproject.toml

@@ -41,9 +41,11 @@ dependencies = [
     "sentry-sdk>=2.7.1",
     "social-auth-app-django",
     "unicef-security>=1.5.1",
+    "uwsgi>=2.0.27",
 ]
 
 [tool.uv]
+package = true
 dev-dependencies = [
     "black>=24.4.2",
     "bump2version>=1.0.1",
@@ -70,10 +72,6 @@ dev-dependencies = [
     "vcrpy>=6.0.2",
 ]
 
-#[tool.uv.sources]
-#hope-smart-export = { git = "https://github.com/unicef/hope-smart-export/tree/0.3" }
-#hope-flex-fields = { git = "https://github.com/unicef/hope-flex-fields/tree/0.3" }
-
 [tool.black]
 line-length = 120
 include = '\.pyi?$'
@@ -103,3 +101,30 @@ skip = ["migrations", "snapshots", ".venv"]
 
 [tool.django-stubs]
 django_settings_module = "country_workspace.config.settings"
+
+#[build-system]
+#requires = ["setuptools"]
+#build-backend = "setuptools.build_meta"
+#
+#[build-system]
+#requires = ["hatchling"]
+#build-backend = "hatchling.build"
+#
+#[tool.hatch.build.targets.sdist]
+#include = [
+#  "/src",
+#]
+#
+#[tool.hatch.build.targets.wheel]
+#packages = ["src/"]
+#[tool.hatch.build]
+#include = [
+#  "README.md",
+#  "src/**/*.csv",
+#  "src/**/*.html",
+#  "src/**/*.js",
+#  "src/**/*.py",
+#]
+#exclude = [
+#  "tests/**",
+#]

src/country_workspace/config/settings.py

@@ -41,6 +41,7 @@
     "hope_flex_fields",
     "hope_smart_import",
     "hope_smart_export",
+    "smart_env",
     "country_workspace.security",
     "country_workspace.apps.Config",
     "country_workspace.workspaces.apps.Config",

src/country_workspace/workspaces/admin/actions/regex.py

@@ -0,0 +1,52 @@
+from typing import TYPE_CHECKING
+
+from django import forms
+from django.db import transaction
+from django.db.models import QuerySet
+from django.shortcuts import render
+from django.utils.translation import gettext as _
+
+if TYPE_CHECKING:
+    from hope_flex_fields.models import DataChecker
+
+    from country_workspace.types import Beneficiary
+    from country_workspace.workspaces.admin.hh_ind import CountryHouseholdIndividualBaseAdmin
+
+    RegexRule = tuple[str, str]
+    RegexRules = list(RegexRule)
+
+
+class RegexUpdateForm(forms.Form):
+    action = forms.CharField(widget=forms.HiddenInput)
+    select_across = forms.BooleanField(widget=forms.HiddenInput)
+    _selected_action = forms.CharField(widget=forms.HiddenInput)
+    field = forms.ChoiceField(choices=[])
+
+    def __init__(self, *args, **kwargs):
+        checker: "DataChecker" = kwargs.pop("checker")
+        super().__init__(*args, **kwargs)
+        field_names = checker.get_form()().fields.keys()
+        self.fields["field"].choices = zip(field_names, field_names)
+
+
+def regex_update_impl(records: "QuerySet[Beneficiary]", config: "RegexRules") -> None:
+    with transaction.atomic():
+        for record in records:
+            pass
+    # for field_name, attrs in config.items():
+    #     op, new_value = attrs
+    #     old_value = record.flex_fields[field_name]
+    #     func = operations.get_function_by_id(op)
+    #     record.flex_fields[field_name] = func(old_value, new_value)
+    # record.save()
+
+
+def regex_update(model_admin: "CountryHouseholdIndividualBaseAdmin", request, queryset):
+    ctx = model_admin.get_common_context(request, title=_("Mass update"))
+    ctx["checker"] = checker = model_admin.get_checker(request)
+    form = RegexUpdateForm(request.POST, checker=checker)
+    ctx["form"] = form
+    if "_apply" in request.POST:
+        if form.is_valid():
+            regex_update_impl(queryset.all(), form.get_selected())
+    return render(request, "actions/mass_update.html", ctx)

tests/conftest.py

@@ -122,20 +122,20 @@ def active_marks(request):
 
 @pytest.fixture()
 def force_migrated_records(request, active_marks):
-    if request.config.option.enable_selenium or "selenium" in active_marks:
-        from django.apps import apps
+    from django.apps import apps
 
-        from hope_flex_fields.apps import sync_content_types
-        from hope_flex_fields.utils import create_default_fields
+    from hope_flex_fields.apps import sync_content_types
+    from hope_flex_fields.utils import create_default_fields
 
-        from country_workspace.versioning.api import run_scripts
-        from country_workspace.versioning.checkers import create_hope_core_fieldset, create_hope_field_definitions
-        from country_workspace.versioning.synclog import create_default_synclog
+    from country_workspace.versioning.api import run_scripts
+    from country_workspace.versioning.checkers import create_hope_core_fieldset, create_hope_field_definitions
+    from country_workspace.versioning.synclog import create_default_synclog
 
+    if request.config.option.enable_selenium or "selenium" in active_marks:
         # we need to recreate these records because with selenium they are not available
         create_default_fields(apps, None)
         sync_content_types(None)
-        create_hope_field_definitions()
-        create_hope_core_fieldset()
-        create_default_synclog()
-        run_scripts()
+    create_hope_field_definitions()
+    create_hope_core_fieldset()
+    create_default_synclog()
+    run_scripts()

tests/workspace/test_ws_import.py

@@ -45,7 +45,7 @@ def app(django_app_factory: "MixinWithInstanceVariables") -> "DjangoTestApp":
     yield django_app
 
 
-def test_import_rdi(app, program):
+def test_import_rdi(force_migrated_records, app, program):
     res = app.get("/").follow()
     res.forms["select-tenant"]["tenant"] = program.country_office.pk
     res.forms["select-tenant"].submit()