From 134ab5e46eb98197946fda669765973a5ff6d3d7 Mon Sep 17 00:00:00 2001 From: Stefano Apostolico Date: Wed, 16 Oct 2024 22:04:41 +0200 Subject: [PATCH] updates tests and CI --- .dockerignore | 13 ++++- .github/workflows/docs.yml | 21 -------- .github/workflows/test.yml | 5 +- MANIFEST.in | 13 ++++- compose.yaml | 4 +- docker/Dockerfile | 53 +++++++++++++------ docker/bin/docker-entrypoint.sh | 2 +- docker/conf/uwsgi.ini | 26 +++++---- pyproject.toml | 33 ++++++++++-- src/country_workspace/config/settings.py | 1 + .../workspaces/admin/actions/regex.py | 52 ++++++++++++++++++ tests/conftest.py | 22 ++++---- tests/workspace/test_ws_import.py | 2 +- 13 files changed, 176 insertions(+), 71 deletions(-) create mode 100644 src/country_workspace/workspaces/admin/actions/regex.py diff --git a/.dockerignore b/.dockerignore index 8161e3a..2928301 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,19 @@ * + !docker/bin !docker/conf +!src/country_workspace/**/* + !pyproject.toml !uv.lock -!src + !LICENSE !README.md +!AUTHORS +!CHANGES.md +!MANIFEST.in + +*.egg-info +.* +~* +**/~* diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3e3693b..28c31af 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,33 +23,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - -# - name: Restore cached venv -# id: cache-venv-restore -# uses: actions/cache/restore@v4 -# with: -# path: | -# .cache-uv/ -# .venv/ -# key: ${{ matrix.python-version }}-${{matrix.django-version}}-venv -# -# - uses: yezz123/setup-uv@v4 - name: Build Doc run: | pip install "mkdocs>=1.6.1" "markdown-include" "mkdocs-material>=9.5.36" \ "mkdocs-awesome-pages-plugin>=2.9.3" "mkdocstrings-python" "mdx-gh-links>=0.4" mkdocs build -d ./docs-output -# - name: Cache venv -# if: steps.cache-venv-restore.outputs.cache-hit != 'true' -# id: cache-venv-save -# uses: actions/cache/save@v4 -# with: -# path: | -# .cache-uv/ -# .venv/ -# key: ${{ matrix.python-version }}-${{matrix.django-version}}-venv - - name: Upload artifact uses: actions/upload-pages-artifact@v3 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 218645d..e3ec949 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -92,13 +92,12 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + - uses: yezz123/setup-uv@v4 - name: Run tests run: | apt-get update apt-get install python3-dev - pip install uv - uv venv - uv sync + uv sync --frozen uv run pytest tests/ - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 diff --git a/MANIFEST.in b/MANIFEST.in index 80fffd7..203855c 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,10 +1,21 @@ +exclude * +exclude requirements.txt + include README.md include MANIFEST.in include AUTHORS include CHANGES.md include LICENSE - +include pyproject.toml +include *.py +include uv.lock recursive-include src/country_workspace * +recursive-include src/country_workspace *.html +recursive-exclude tests * prune **/~* +global-exclude .* +global-exclude *~ +global-exclude ~* +exclude manage.py diff --git a/compose.yaml b/compose.yaml index d1453e9..516138c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,7 +2,7 @@ x-common: &common build: context: . dockerfile: docker/Dockerfile - target: python_dev_deps + target: builder platform: linux/amd64 environment: - DEBUG=true @@ -18,7 +18,7 @@ x-common: &common - SECRET_KEY=sensitive-secret-key - STATIC_ROOT=/var/country_workspace/static volumes: - - .:/code + - .:/app - /var/run/docker.sock:/var/run/docker.sock restart: always depends_on: diff --git a/docker/Dockerfile b/docker/Dockerfile index 1f1360a..7aae89b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,28 @@ FROM python:3.12-slim-bookworm as python_base +ARG GOSU_VERSION=1.17 +ARG GOSU_SHA256=bbc4136d03ab138b1ad66fa4fc051bafc6cc7ffae632b069a53657279a450de3 + + +RUN set -x \ + && runtimeDeps=" \ + libxml2 \ + " \ + && buildDeps=" \ + wget \ + " \ + && apt-get update && apt-get install -y --no-install-recommends ${buildDeps} ${runtimeDeps} \ + && rm -rf /var/lib/apt/lists/* \ + && wget --quiet -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" \ + && echo "$GOSU_SHA256 /usr/local/bin/gosu" | sha256sum --check --status \ + && chmod +x /usr/local/bin/gosu \ + && apt-get purge -y --auto-remove $buildDeps + +RUN groupadd --gid 1024 app \ + && adduser --disabled-login --disabled-password --no-create-home --ingroup app -q user + FROM python_base AS builder + RUN set -x \ && buildDeps="build-essential \ cmake \ @@ -16,39 +38,40 @@ RUN set -x \ && apt-get install -y --no-install-recommends $buildDeps \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -RUN pip install uwsgi uv +RUN pip install uv - -COPY docker/conf /conf/ -COPY docker/bin /usr/local/bin/ +COPY pyproject.toml uv.lock AUTHORS LICENSE README.md MANIFEST.in /app/ +COPY src /app/src WORKDIR /app -COPY . /app/ - -RUN uv venv \ - && uv sync --no-dev --no-editable --frozen \ - && uv pip install . \ - && ls -al /app/.venv/ - -FROM python_base +RUN uv sync --no-dev --no-editable --frozen +#RUN ls -al .venv/lib/python3.12/site-packages/country_workspace/workspaces/ +#RUN uv export --no-dev --no-editable --frozen --no-sources --no-hashes -o requirement.txt \ +# && python -m venv .venv \ +# && .venv/bin/pip install -r requirement.txt +# FROM python_base -ENV PATH=/app/.venv/bin:$PATH +ENV PATH=/app/.venv/bin:$PATH \ + DJANGO_SETTINGS_MODULE=country_workspace.config.settings \ + UWSGI_PROCESSES=4 COPY docker/conf /conf/ -COPY docker/bin /usr/local/bin/ +COPY docker/bin/* /usr/local/bin/ COPY --chown=user:app --from=builder /app/.venv /app/.venv +#COPY --chown=user:app --from=builder /usr/local/bin/uwsgi /usr/local/bin/uwsgi EXPOSE 8000 ENTRYPOINT exec docker-entrypoint.sh "$0" "$@" CMD ["run"] + LABEL distro="final" LABEL maintainer="hope@unicef.org" LABEL org.opencontainers.image.authors="hope@unicef.org" -LABEL org.opencontainers.image.created="$BUILD_DATE" +#LABEL org.opencontainers.image.created="$BUILD_DATE" LABEL org.opencontainers.image.description="Hope Country Workspace" LABEL org.opencontainers.image.documentation="https://github.com/unicef/hope-country-workspace/" LABEL org.opencontainers.image.licenses="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${SOURCE_COMMIT:-master}/LICENSE" diff --git a/docker/bin/docker-entrypoint.sh b/docker/bin/docker-entrypoint.sh index cc12ca7..ed7c128 100755 --- a/docker/bin/docker-entrypoint.sh +++ b/docker/bin/docker-entrypoint.sh @@ -4,7 +4,7 @@ export MEDIA_ROOT="${MEDIA_ROOT:-/var/run/app/media}" export STATIC_ROOT="${STATIC_ROOT:-/var/run/app/static}" export UWSGI_PROCESSES="${UWSGI_PROCESSES:-"4"}" -export DJANGO_SETTINGS_MODULE="${DJANGO_SETTINGS_MODULE:-"country_workspace.config.settings"}" +#export DJANGO_SETTINGS_MODULE="${DJANGO_SETTINGS_MODULE:-"country_workspace.config.settings"}" ls -al /app/ diff --git a/docker/conf/uwsgi.ini b/docker/conf/uwsgi.ini index 9d7c46f..b41debe 100644 --- a/docker/conf/uwsgi.ini +++ b/docker/conf/uwsgi.ini @@ -1,19 +1,23 @@ [uwsgi] http=0.0.0.0:8000 -enable-threads=0 -honour-range=1 -master=1 +;enable-threads=0 +;honour-range=1 +;master=1 module=country_workspace.config.wsgi -processes=$(UWSGI_PROCESSES) -;virtualenv=/code/.venv/ +;processes=$(UWSGI_PROCESSES) +;virtualenv=/app/.venv/ +;pythonpath=/app/.venv/lib/python3.12/site-packages ;virtualenv=%(_) ;venv=%(_) ;chdir=code/ -username = user -gropuname = app +uid = user +gid = app +;username = user +;gropuname = app ;offload-threads=%k ;static-gzip-all=true -route = /static/(.*) static:$(STATIC_ROOT)/$1 -http-keepalive = 1 -collect-header=Content-Type RESPONSE_CONTENT_TYPE -mimefile=/etc/mime.types +;route = /static/(.*) static:$(STATIC_ROOT)/$1 +;http-keepalive = 1 +;collect-header=Content-Type RESPONSE_CONTENT_TYPE +mimefile=/conf/mime.types +;honour-stdin = true diff --git a/pyproject.toml b/pyproject.toml index 9f5fea1..72d9ee2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,9 +41,11 @@ dependencies = [ "sentry-sdk>=2.7.1", "social-auth-app-django", "unicef-security>=1.5.1", + "uwsgi>=2.0.27", ] [tool.uv] +package = true dev-dependencies = [ "black>=24.4.2", "bump2version>=1.0.1", @@ -70,10 +72,6 @@ dev-dependencies = [ "vcrpy>=6.0.2", ] -#[tool.uv.sources] -#hope-smart-export = { git = "https://github.com/unicef/hope-smart-export/tree/0.3" } -#hope-flex-fields = { git = "https://github.com/unicef/hope-flex-fields/tree/0.3" } - [tool.black] line-length = 120 include = '\.pyi?$' @@ -103,3 +101,30 @@ skip = ["migrations", "snapshots", ".venv"] [tool.django-stubs] django_settings_module = "country_workspace.config.settings" + +#[build-system] +#requires = ["setuptools"] +#build-backend = "setuptools.build_meta" +# +#[build-system] +#requires = ["hatchling"] +#build-backend = "hatchling.build" +# +#[tool.hatch.build.targets.sdist] +#include = [ +# "/src", +#] +# +#[tool.hatch.build.targets.wheel] +#packages = ["src/"] +#[tool.hatch.build] +#include = [ +# "README.md", +# "src/**/*.csv", +# "src/**/*.html", +# "src/**/*.js", +# "src/**/*.py", +#] +#exclude = [ +# "tests/**", +#] diff --git a/src/country_workspace/config/settings.py b/src/country_workspace/config/settings.py index 489cf54..918bb6b 100644 --- a/src/country_workspace/config/settings.py +++ b/src/country_workspace/config/settings.py @@ -41,6 +41,7 @@ "hope_flex_fields", "hope_smart_import", "hope_smart_export", + "smart_env", "country_workspace.security", "country_workspace.apps.Config", "country_workspace.workspaces.apps.Config", diff --git a/src/country_workspace/workspaces/admin/actions/regex.py b/src/country_workspace/workspaces/admin/actions/regex.py new file mode 100644 index 0000000..dc758ab --- /dev/null +++ b/src/country_workspace/workspaces/admin/actions/regex.py @@ -0,0 +1,52 @@ +from typing import TYPE_CHECKING + +from django import forms +from django.db import transaction +from django.db.models import QuerySet +from django.shortcuts import render +from django.utils.translation import gettext as _ + +if TYPE_CHECKING: + from hope_flex_fields.models import DataChecker + + from country_workspace.types import Beneficiary + from country_workspace.workspaces.admin.hh_ind import CountryHouseholdIndividualBaseAdmin + + RegexRule = tuple[str, str] + RegexRules = list(RegexRule) + + +class RegexUpdateForm(forms.Form): + action = forms.CharField(widget=forms.HiddenInput) + select_across = forms.BooleanField(widget=forms.HiddenInput) + _selected_action = forms.CharField(widget=forms.HiddenInput) + field = forms.ChoiceField(choices=[]) + + def __init__(self, *args, **kwargs): + checker: "DataChecker" = kwargs.pop("checker") + super().__init__(*args, **kwargs) + field_names = checker.get_form()().fields.keys() + self.fields["field"].choices = zip(field_names, field_names) + + +def regex_update_impl(records: "QuerySet[Beneficiary]", config: "RegexRules") -> None: + with transaction.atomic(): + for record in records: + pass + # for field_name, attrs in config.items(): + # op, new_value = attrs + # old_value = record.flex_fields[field_name] + # func = operations.get_function_by_id(op) + # record.flex_fields[field_name] = func(old_value, new_value) + # record.save() + + +def regex_update(model_admin: "CountryHouseholdIndividualBaseAdmin", request, queryset): + ctx = model_admin.get_common_context(request, title=_("Mass update")) + ctx["checker"] = checker = model_admin.get_checker(request) + form = RegexUpdateForm(request.POST, checker=checker) + ctx["form"] = form + if "_apply" in request.POST: + if form.is_valid(): + regex_update_impl(queryset.all(), form.get_selected()) + return render(request, "actions/mass_update.html", ctx) diff --git a/tests/conftest.py b/tests/conftest.py index 19afe88..d80b615 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -122,20 +122,20 @@ def active_marks(request): @pytest.fixture() def force_migrated_records(request, active_marks): - if request.config.option.enable_selenium or "selenium" in active_marks: - from django.apps import apps + from django.apps import apps - from hope_flex_fields.apps import sync_content_types - from hope_flex_fields.utils import create_default_fields + from hope_flex_fields.apps import sync_content_types + from hope_flex_fields.utils import create_default_fields - from country_workspace.versioning.api import run_scripts - from country_workspace.versioning.checkers import create_hope_core_fieldset, create_hope_field_definitions - from country_workspace.versioning.synclog import create_default_synclog + from country_workspace.versioning.api import run_scripts + from country_workspace.versioning.checkers import create_hope_core_fieldset, create_hope_field_definitions + from country_workspace.versioning.synclog import create_default_synclog + if request.config.option.enable_selenium or "selenium" in active_marks: # we need to recreate these records because with selenium they are not available create_default_fields(apps, None) sync_content_types(None) - create_hope_field_definitions() - create_hope_core_fieldset() - create_default_synclog() - run_scripts() + create_hope_field_definitions() + create_hope_core_fieldset() + create_default_synclog() + run_scripts() diff --git a/tests/workspace/test_ws_import.py b/tests/workspace/test_ws_import.py index db9be5e..aea59b9 100644 --- a/tests/workspace/test_ws_import.py +++ b/tests/workspace/test_ws_import.py @@ -45,7 +45,7 @@ def app(django_app_factory: "MixinWithInstanceVariables") -> "DjangoTestApp": yield django_app -def test_import_rdi(app, program): +def test_import_rdi(force_migrated_records, app, program): res = app.get("/").follow() res.forms["select-tenant"]["tenant"] = program.country_office.pk res.forms["select-tenant"].submit()