diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..757d0a79 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,201 @@ +name: CI + +on: + push: + branches: + - develop + - master + pull_request: + branches: + - develop + - master + +jobs: + build_and_push_dev: + runs-on: ubuntu-latest + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Checkout code + uses: actions/checkout@v2 + - name: DockerHub login + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push dev + run: | + docker buildx create --use + docker buildx build \ + --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-${{ github.sha }}-dev \ + --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-latest-dev \ + --cache-to ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-${{ github.sha }}-dev \ + --cache-to ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-latest-dev \ + -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:country-report-${{ github.sha }}-dev \ + -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:country-report-latest-dev \ + -f ./docker/Dockerfile \ + --target dev \ + --push \ + ./ + + black: + runs-on: ubuntu-latest + needs: [build_and_push_dev] + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: DockerHub login + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Check + run: | + docker run --rm -i \ + ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:country-report-${{ github.sha }}-dev \ + black . --check + + flake8: + runs-on: ubuntu-latest + needs: [build_and_push_dev] + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: DockerHub login + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Check + run: | + docker run --rm -i \ + ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:country-report-${{ github.sha }}-dev \ + flake8 . + + unit_tests: + runs-on: ubuntu-latest + needs: [build_and_push_dev] + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: DockerHub login + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Unit tests + run: | + backend_image=${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:country-report-${{ github.sha }}-dev docker compose \ + -f ./ops/compose.ci-test.yml \ + up --exit-code-from backend + + build_and_push_prd: + needs: [build_and_push_dev] + runs-on: ubuntu-latest + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - name: Checkout code + uses: actions/checkout@v2 + - name: DockerHub login + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Push prd + run: | + docker buildx create --use + + # Base part of the command + build_command="docker buildx build \ + --progress=plain \ + --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-${{ github.sha }}-dev \ + --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-latest-dev \ + --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-${{ github.sha }}-prd \ + --cache-from ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-latest-prd \ + --cache-to ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-${{ github.sha }}-prd \ + --cache-to ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:cache-country-report-latest-prd \ + -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:country-report-${{ github.sha }}-prd \ + -t ${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:country-report-${{ github.sha }} \ + -f ./docker/Dockerfile \ + --target prd \ + --push ./" + + if [ "${{ github.ref }}" = "refs/heads/master" ]; then + version=$(python3 -c "import sys; version=None; [version:=line.split('=')[1].strip().strip('\"') for line in open('pyproject.toml', 'r') if line.strip().startswith('version =')]; print(version if version else sys.exit(1))") + tagged_image=${{ vars.DOCKERHUB_ORGANIZATION }}/hope:country-report-$version + build_command="$build_command -t $tagged_image" + fi + + eval $build_command + + trivy: + runs-on: ubuntu-latest + needs: [build_and_push_prd] + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: DockerHub login + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: '${{ vars.DOCKERHUB_ORGANIZATION }}/hope-support-images:country-report-${{ github.sha }}' + format: 'table' + exit-code: '0' + ignore-unfixed: true + vuln-type: 'os,library' + severity: 'CRITICAL,HIGH' + + deploy: + runs-on: ubuntu-latest + needs: [unit_tests, black, flake8, build_and_push_prd] + if: | + github.event_name == 'push' && + ( + github.ref == 'refs/heads/develop' || + github.ref == 'refs/heads/master' + ) + steps: + - name: Trigger deploy + run: | + if [ ${{ github.ref }} == 'refs/heads/develop' ]; then + pipelineId=1149 + elif [ ${{ github.ref }} == 'refs/heads/staging' ]; then + pipelineId=1286 + elif [ ${{ github.ref }} == 'refs/heads/master' ]; then + pipelineId=1233 + else + echo "No pipeline to trigger for ref ${{ github.ref }}" + exit 0 + fi + + IFS=',' read -ra pipelines <<< "$pipelineId" + for pipeline in "${pipelines[@]}"; do + jsonBody='{"variables": {"sha": {"isSecret": false, "value": "${{ github.sha }}"}, "tag": {"isSecret": false, "value": "country-report-${{ github.sha }}"}}}' + contentLength=$(echo -n $jsonBody | wc -c) + project=ICTD-HCT-MIS + organization=unicef + + echo Triggering deploy for pipeline $pipeline + echo JSON body: $jsonBody + + curl -f -v -L \ + -u ":${{ secrets.AZURE_PAT }}" \ + -H "Content-Type: application/json" \ + -H "Content-Length: $contentLength" \ + -d "$jsonBody" \ + https://dev.azure.com/$organization/$project/_apis/pipelines/$pipeline/runs?api-version=7.1-preview.1 + if [ $? -ne 0 ]; then + echo "Failed to trigger deploy for pipeline $pipeline" + exit 1 + fi + done \ No newline at end of file diff --git a/.gitignore b/.gitignore index e0969fe4..54740acc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ -.* +.vscode +.pdm_python ~* node_modules *.pyc diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 22ff44b8..00000000 --- a/Dockerfile +++ /dev/null @@ -1,52 +0,0 @@ -FROM python:3.11-slim-bookworm as base - -RUN apt-get update \ - && apt-get install -y \ - gcc curl libgdal-dev wkhtmltopdf chromium-driver chromium \ - && apt-get clean && rm -rf /var/lib/apt/lists/* \ - && addgroup --system --gid 82 hcr \ - && adduser \ - --system --uid 82 \ - --disabled-password --home /home/hcr \ - --shell /sbin.nologin --group hcr --gecos hcr \ - && mkdir -p /code /tmp /data /static \ - && chown -R hcr:hcr /code /tmp /data /static - -ENV PACKAGES_DIR=/packages -ENV PYPACKAGES=$PACKAGES_DIR/__pypackages__/3.11 -ENV LIB_DIR=$PYPACKAGES/lib -ENV PYTHONPATH=$PYTHONPATH:$LIB_DIR:/code/src -ENV PATH=$PATH:$PYPACKAGES/bin - -WORKDIR /code - -FROM base as builder - -WORKDIR $PACKAGES_DIR -RUN pip install pdm==2.10.4 -ADD pyproject.toml ./ -ADD pdm.toml.template ./pdm.toml -ADD pdm.lock ./ -RUN pdm sync --prod --no-editable --no-self - -FROM builder AS dev - -RUN pdm sync --no-editable --no-self - -WORKDIR /code -COPY ./ ./ - -ADD entrypoint.sh /usr/local/bin/entrypoint.sh -ENTRYPOINT ["entrypoint.sh"] - - -FROM base AS prd - -ENV PATH=$PATH:/code/.venv/bin/ - -COPY --chown=hcr:hcr ./ ./ -COPY --chown=hcr:hcr --from=builder $PACKAGES_DIR $PACKAGES_DIR -USER hcr - -ADD entrypoint.sh /usr/local/bin/entrypoint.sh -ENTRYPOINT ["entrypoint.sh"] diff --git a/action.yml b/action.yml deleted file mode 100644 index 782c8302..00000000 --- a/action.yml +++ /dev/null @@ -1,16 +0,0 @@ -# action.yml -name: 'Hello World' -description: 'Greet someone and record the time' -inputs: - who-to-greet: # id of input - description: 'Who to greet' - required: true - default: 'World' -outputs: - time: # id of output - description: 'The time we greeted you' -runs: - using: 'docker' - image: 'Dockerfile' - args: - - ${{ inputs.who-to-greet }} diff --git a/compose.prd.yml b/compose.prd.yml index a2c06271..5855c426 100644 --- a/compose.prd.yml +++ b/compose.prd.yml @@ -35,9 +35,9 @@ services: backend: build: context: . - dockerfile: Dockerfile + dockerfile: docker/Dockerfile target: prd - command: sh -c "entrypoint.sh ./wait-for-it.sh db:5432 && entrypoint.sh ./wait-for-it.sh hopedb:5432 && entrypoint.sh prd" + command: sh -c "entrypoint.sh ./docker/wait-for-it.sh db:5432 && entrypoint.sh ./docker/wait-for-it.sh hopedb:5432 && entrypoint.sh prd" environment: *django-env ports: - 8000:8000 @@ -102,7 +102,7 @@ services: celery_worker: build: context: . - dockerfile: Dockerfile + dockerfile: docker/Dockerfile target: prd command: celery_worker environment: *django-env @@ -119,7 +119,7 @@ services: celery_beat: build: context: . - dockerfile: Dockerfile + dockerfile: docker/Dockerfile target: prd command: celery_beat environment: *django-env diff --git a/compose.test.yml b/compose.test.yml index f79f4cd0..f74c5cf2 100644 --- a/compose.test.yml +++ b/compose.test.yml @@ -4,7 +4,7 @@ services: backend: build: context: . - dockerfile: Dockerfile + dockerfile: docker/Dockerfile target: dev command: tests environment: diff --git a/compose.yml b/compose.yml index e9e2423e..309dde46 100644 --- a/compose.yml +++ b/compose.yml @@ -30,7 +30,7 @@ services: backend: build: context: . - dockerfile: Dockerfile + dockerfile: docker/Dockerfile target: dev command: dev environment: *django-env @@ -97,7 +97,7 @@ services: celery_worker: build: context: . - dockerfile: Dockerfile + dockerfile: docker/Dockerfile target: dev command: celery_worker environment: *django-env @@ -113,7 +113,7 @@ services: celery_beat: build: context: . - dockerfile: Dockerfile + dockerfile: docker/Dockerfile target: dev command: celery_beat environment: *django-env diff --git a/docker/Dockerfile b/docker/Dockerfile index 5d5f5d90..d44baf9e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,85 +1,53 @@ -# syntax=docker/dockerfile:1.3 -ARG PYTHON_VER=3.11 -ARG PKG_DIR=/code/__pypackages__/$PYTHON_VER/lib -ARG BUILD_DATE -ARG CHECKSUM - -FROM python:${PYTHON_VER}-slim-bookworm AS python - -FROM python AS base -ARG PKG_DIR -ENV ADMINS="" \ - UWSGI_PROCESSES=4 \ - VERSION=${VERSION} - -ENV PYTHONPATH=$PKG_DIR:/code/src/ \ - PATH=${PATH}:$PKG_DIR/../bin/ - -RUN groupadd --gid 1024 hcr \ - && adduser --system --disabled-login --disabled-password --no-create-home --group hcr -q --gecos www - - -FROM python AS cache -ARG PKG_DIR -ENV BUILD_DATE=$BUILD_DATE \ - VERSION=$VERSION \ - PYTHONDONTWRITEBYTECODE=1 -ENV buildDeps="build-essential gcc libjpeg-dev zlib1g-dev libffi-dev libssl-dev libpq-dev " -ENV runtimeDeps="postgresql-client gettext wkhtmltopdf" -RUN rm -f /etc/apt/apt.conf.d/docker-clean \ - && echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache - -RUN apt-get update - -RUN apt-get install -y --no-install-recommends $buildDeps - -RUN apt-get install -y --no-install-recommends $runtimeDeps - -RUN rm -rf /var/lib/apt/lists/* -RUN pip install -U pip setuptools && pip install pdm - -FROM cache AS builder -ARG PKG_DIR -ARG CHECKSUM -WORKDIR /code/ -COPY pyproject.toml pdm.lock README.md /code/ -RUN mkdir __pypackages__ \ - && pdm sync --prod --no-editable --no-self - -RUN echo $CHECKSUM > /CHECKSUM - - -FROM builder AS builder-test -ARG PKG_DIR -WORKDIR /code/ -ENV PYTHONPATH=$PKG_DIR:/code/src/ \ - PATH=${PATH}:$PKG_DIR/../bin/ +FROM python:3.11-slim-bookworm as base + +RUN apt update \ + && apt install --no-install-recommends -y \ + gcc curl libgdal-dev wkhtmltopdf chromium-driver chromium \ + && apt clean && rm -rf /var/lib/apt/lists/* \ + && addgroup --system --gid 82 hcr \ + && adduser \ + --system --uid 82 \ + --disabled-password --home /home/hcr \ + --shell /sbin.nologin --group hcr --gecos hcr \ + && mkdir -p /code /tmp /data /static \ + && chown -R hcr:hcr /code /tmp /data /static + +ENV PACKAGES_DIR=/packages +ENV PYPACKAGES=$PACKAGES_DIR/__pypackages__/3.11 +ENV LIB_DIR=$PYPACKAGES/lib +ENV PYTHONPATH=$PYTHONPATH:$LIB_DIR:/code/src +ENV PATH=$PATH:$PYPACKAGES/bin + +WORKDIR /code + +FROM base as builder + +WORKDIR $PACKAGES_DIR +RUN pip install pdm +COPY ../pyproject.toml ./ +COPY ../pdm.lock ./ +RUN pdm config python.use_venv false +RUN pdm config venv.in_project true +RUN pdm sync --prod --no-editable --no-self + +FROM builder AS dev RUN pdm sync --no-editable --no-self +WORKDIR /code +COPY .. ./ + +COPY docker/entrypoint.sh /usr/local/bin/entrypoint.sh +ENTRYPOINT ["entrypoint.sh"] -FROM base AS dist -ARG PKG_DIR -COPY docker/bin/* /usr/local/bin/ -COPY docker/conf/* /conf/ -WORKDIR /code/ -COPY --from=cache /usr/bin/envsubst /usr/bin/ -COPY --from=builder /code/__pypackages__ /code/__pypackages__ -COPY --from=builder /CHECKSUM /CHECKSUM -COPY ./src /code/src -ENTRYPOINT ["docker-entrypoint.sh"] -CMD ["run"] +FROM base AS prd +ENV PATH=$PATH:/code/.venv/bin/ -FROM base AS test -ARG PKG_DIR -COPY docker/bin/* /usr/local/bin/ -COPY docker/conf/* /conf/ -WORKDIR /code/ -COPY --from=builder-test /code/__pypackages__ /code/__pypackages__ -COPY --from=builder /CHECKSUM /CHECKSUM -COPY . /code/ +COPY --chown=hcr:hcr .. ./ +COPY --chown=hcr:hcr --from=builder $PACKAGES_DIR $PACKAGES_DIR +USER hcr -ENTRYPOINT ["docker-entrypoint.sh"] -CMD ["run"] +COPY docker/entrypoint.sh /usr/local/bin/entrypoint.sh +ENTRYPOINT ["entrypoint.sh"] diff --git a/entrypoint.sh b/docker/entrypoint.sh similarity index 90% rename from entrypoint.sh rename to docker/entrypoint.sh index c3bae67e..7960d4cd 100755 --- a/entrypoint.sh +++ b/docker/entrypoint.sh @@ -17,12 +17,12 @@ fi case "$1" in dev) - ./wait-for-it.sh db:5432 + ./docker/wait-for-it.sh db:5432 python3 manage.py migrate python3 manage.py runserver 0.0.0.0:8000 ;; tests) - ./wait-for-it.sh db:5432 + ./docker/wait-for-it.sh db:5432 pytest tests/ --create-db --cov-report term --maxfail 5 --with-selenium ;; prd) diff --git a/wait-for-it.sh b/docker/wait-for-it.sh similarity index 100% rename from wait-for-it.sh rename to docker/wait-for-it.sh diff --git a/ops/ci.yaml b/ops/ci.yaml deleted file mode 100644 index 88f11e24..00000000 --- a/ops/ci.yaml +++ /dev/null @@ -1,221 +0,0 @@ -trigger: - batch: true - branches: - include: - - develop - - staging - - master -pr: -- develop -- staging -- master -resources: -- repo: self - -variables: - Docker.backend.repository: "hope-country-report" - tag: $(Build.SourceVersion) - additionalTag: $(Build.SourceBranchName) - - ${{ if eq(variables['Build.SourceBranchName'], 'develop') }}: - Docker.registryConnection: "ICTD-HOPE-DEV-ACR" - Docker.url: "uniappsakshopedev.azurecr.io" - pipelineId: 1149 - ${{ elseif eq(variables['Build.SourceBranchName'], 'staging') }}: - Docker.registryConnection: "ICTD-HOPE-DEV-ACR" - Docker.url: "uniappsakshopedev.azurecr.io" - pipelineId: 1286 - ${{ elseif eq(variables['Build.SourceBranchName'], 'master') }}: - Docker.registryConnection: "ICTD-HOPE-PRD-ACR" - Docker.url: "uniappsakshope.azurecr.io" - pipelineId: 1233 - ${{ else }}: - Docker.registryConnection: "ICTD-HOPE-DEV-ACR" - Docker.url: "uniappsakshopedev.azurecr.io" - pipelineId: null - -stages: -- stage: build_and_push_backend_dev - dependsOn: [] - displayName: BUILD and PUSH DEV - jobs: - - job: build_push_backend - pool: - vmImage: ubuntu-latest - displayName: "[BACKEND_BUILD]" - steps: - - task: Docker@2 - inputs: - containerRegistry: '$(Docker.registryConnection)' - command: 'login' - - script: | - docker buildx create --use - docker buildx build \ - --cache-from=$(Docker.url)/$(Docker.backend.repository)-cache:dev-$(additionalTag) \ - --cache-from=$(Docker.url)/$(Docker.backend.repository)-cache:dev-latest \ - --cache-to=$(Docker.url)/$(Docker.backend.repository)-cache:dev-$(additionalTag) \ - --cache-to=$(Docker.url)/$(Docker.backend.repository)-cache:dev-latest \ - -t $(Docker.url)/$(Docker.backend.repository):dev-$(tag) \ - --target dev \ - --push \ - ./ - displayName: Docker build dev - -- stage: check_format - displayName: FORMAT - dependsOn: build_and_push_backend_dev - jobs: - - job: tests - pool: - vmImage: ubuntu-latest - displayName: "[DJANGO FORMAT]" - steps: - - task: Docker@2 - displayName: "[ACR] Login" - inputs: - command: login - containerRegistry: $(Docker.registryConnection) - - task: DockerCompose@0 - displayName: "Check format" - inputs: - containerregistrytype: 'Azure Container Registry' - azureContainerRegistry: $(Docker.registryConnection) - dockerComposeFile: 'ops/compose.test.yml' - action: 'Run a Docker Compose command' - dockerComposeCommand: 'run --no-deps backend black --check .' - dockerComposeFileArgs: | - backend_image=$(Docker.url)/$(Docker.backend.repository):dev-$(tag) - -- stage: lint_backend - displayName: LINT - dependsOn: build_and_push_backend_dev - jobs: - - job: lint - pool: - vmImage: ubuntu-latest - displayName: "[DJANGO LINT]" - steps: - - task: Docker@2 - displayName: "[ACR] Login" - inputs: - command: login - containerRegistry: $(Docker.registryConnection) - - task: DockerCompose@0 - displayName: "Run lint" - inputs: - containerregistrytype: 'Azure Container Registry' - azureContainerRegistry: $(Docker.registryConnection) - dockerComposeFile: 'ops/compose.test.yml' - action: 'Run a Docker Compose command' - dockerComposeCommand: 'run --no-deps backend flake8 .' - dockerComposeFileArgs: | - backend_image=$(Docker.url)/$(Docker.backend.repository):dev-$(tag) - -- stage: test_dev - displayName: TEST - dependsOn: [check_format, lint_backend] - jobs: - - job: tests - pool: - vmImage: ubuntu-latest - displayName: "[DJANGO TEST]" - steps: - - task: Docker@2 - displayName: "[ACR] Login" - inputs: - command: login - containerRegistry: $(Docker.registryConnection) - - task: DockerCompose@0 - displayName: "Run tests" - inputs: - containerregistrytype: 'Azure Container Registry' - azureContainerRegistry: $(Docker.registryConnection) - dockerComposeFile: 'ops/compose.test.yml' - action: 'Run a Docker Compose command' - dockerComposeCommand: 'up --exit-code-from backend' - dockerComposeFileArgs: | - backend_image=$(Docker.url)/$(Docker.backend.repository):dev-$(tag) - -- stage: build_and_push_backend_prd - displayName: BUILD AND PUSH PRD - dependsOn: [build_and_push_backend_dev] - jobs: - - job: build_push_backend - pool: - vmImage: ubuntu-latest - displayName: "[BACKEND_BUILD]" - steps: - - task: Docker@2 - inputs: - containerRegistry: '$(Docker.registryConnection)' - command: 'login' - - script: | - docker buildx create --use - docker buildx build \ - --cache-from=$(Docker.url)/$(Docker.backend.repository)-cache:dev-$(additionalTag) \ - --cache-from=$(Docker.url)/$(Docker.backend.repository)-cache:dev-latest \ - --cache-from=$(Docker.url)/$(Docker.backend.repository)-cache:prd-$(additionalTag) \ - --cache-from=$(Docker.url)/$(Docker.backend.repository)-cache:prd-latest \ - --cache-to=$(Docker.url)/$(Docker.backend.repository)-cache:prd-$(additionalTag) \ - --cache-to=$(Docker.url)/$(Docker.backend.repository)-cache:prd-latest \ - -t $(Docker.url)/$(Docker.backend.repository):prd-$(tag) \ - --target prd \ - --push \ - ./ - displayName: Docker build prd - -- stage: trivy - dependsOn: [build_and_push_backend_prd] - displayName: Trivy - jobs: - - job: run_trivy - pool: - vmImage: ubuntu-latest - displayName: "[TRIVY]" - steps: - - task: Docker@2 - displayName: "[ACR] Login" - inputs: - command: login - containerRegistry: $(Docker.registryConnection) - - script: | - docker pull $(Docker.url)/$(Docker.backend.repository):prd-$(tag) - docker run -v /var/run/docker.sock:/var/run/docker.sock -e TRIVY_EXIT_CODE=2 -e TRIVY_SEVERITY=HIGH,CRITICAL aquasec/trivy:0.47.0 image $(Docker.url)/$(Docker.backend.repository):prd-$(tag) - continueOnError: true - - -- stage: trigger_deploy - displayName: Trigger deploy - dependsOn: [build_and_push_backend_prd, test_dev] - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) - jobs: - - job: trigger_deploy - pool: - vmImage: ubuntu-latest - displayName: "[TRIGGER_DEPLOY]" - steps: - - script: | - pipelineId=$(pipelineId) - IFS=',' read -ra pipelines <<< "$pipelineId" - for pipeline in "${pipelines[@]}"; do - jsonBody='{"variables": {"tag": {"isSecret": false, "value": "$(tag)"}}}' - contentLength=$(echo -n $jsonBody | wc -c) - project=ICTD-HCT-MIS - organization=unicef - - echo Triggering deploy for pipeline $pipeline - echo JSON body: $jsonBody - - curl -v -L \ - -u ":$(AZURE_PAT)" \ - -H "Content-Type: application/json" \ - -H "Content-Length: $contentLength" \ - -d "$jsonBody" \ - https://dev.azure.com/$organization/$project/_apis/pipelines/$pipeline/runs?api-version=7.1-preview.1 - if [ $? -ne 0 ]; then - echo "Failed to trigger deploy for pipeline $pipeline" - exit 1 - fi - done - displayName: "Trigger deploy" - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) diff --git a/ops/compose.test.yml b/ops/compose.ci-test.yml similarity index 100% rename from ops/compose.test.yml rename to ops/compose.ci-test.yml diff --git a/pdm.lock b/pdm.lock index 72025396..30d11080 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["cross_platform"] lock_version = "4.4.1" -content_hash = "sha256:cd4016c06aa6a4654695f74bd1af66c10f34358f6ef36f8b3a765aaaf53b6820" +content_hash = "sha256:33665906566c6f1306d1331efc5d9c15ce262fb79dfc76df2fb5e3b4a33f4fb0" [[package]] name = "amqp" @@ -166,7 +166,7 @@ files = [ [[package]] name = "black" -version = "24.4.1" +version = "24.4.2" requires_python = ">=3.8" summary = "The uncompromising code formatter." dependencies = [ @@ -177,16 +177,16 @@ dependencies = [ "platformdirs>=2", ] files = [ - {file = "black-24.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9dae3ae59d6f2dc93700fd5034a3115434686e66fd6e63d4dcaa48d19880f2b0"}, - {file = "black-24.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5f8698974a81af83283eb47644f2711b5261138d6d9180c863fce673cbe04b13"}, - {file = "black-24.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f404b6e77043b23d0321fb7772522b876b6de737ad3cb97d6b156638d68ce81"}, - {file = "black-24.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:c94e52b766477bdcd010b872ba0714d5458536dc9d0734eff6583ba7266ffd89"}, - {file = "black-24.4.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:962d9e953872cdb83b97bb737ad47244ce2938054dc946685a4cad98520dab38"}, - {file = "black-24.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b1d8e3b2486b7dd522b1ab2ba1ec4907f0aa8f5e10a33c4271fb331d1d10b70c"}, - {file = "black-24.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed77e214b785148f57e43ca425b6e0850165144aa727d66ac604e56a70bb7825"}, - {file = "black-24.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:4ef4764437d7eba8386689cd06e1fb5341ee0ae2e9e22582b21178782de7ed94"}, - {file = "black-24.4.1-py3-none-any.whl", hash = "sha256:ecbab810604fe02c70b3a08afd39beb599f7cc9afd13e81f5336014133b4fe35"}, - {file = "black-24.4.1.tar.gz", hash = "sha256:5241612dc8cad5b6fd47432b8bd04db80e07cfbc53bb69e9ae18985063bcb8dd"}, + {file = "black-24.4.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:257d724c2c9b1660f353b36c802ccece186a30accc7742c176d29c146df6e474"}, + {file = "black-24.4.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bdde6f877a18f24844e381d45e9947a49e97933573ac9d4345399be37621e26c"}, + {file = "black-24.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e151054aa00bad1f4e1f04919542885f89f5f7d086b8a59e5000e6c616896ffb"}, + {file = "black-24.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:7e122b1c4fb252fd85df3ca93578732b4749d9be076593076ef4d07a0233c3e1"}, + {file = "black-24.4.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:accf49e151c8ed2c0cdc528691838afd217c50412534e876a19270fea1e28e2d"}, + {file = "black-24.4.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:88c57dc656038f1ab9f92b3eb5335ee9b021412feaa46330d5eba4e51fe49b04"}, + {file = "black-24.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be8bef99eb46d5021bf053114442914baeb3649a89dc5f3a555c88737e5e98fc"}, + {file = "black-24.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:415e686e87dbbe6f4cd5ef0fbf764af7b89f9057b97c908742b6008cc554b9c0"}, + {file = "black-24.4.2-py3-none-any.whl", hash = "sha256:d36ed1124bb81b32f8614555b34cc4259c3fbc7eec17870e8ff8ded335b58d8c"}, + {file = "black-24.4.2.tar.gz", hash = "sha256:c872b53057f000085da66a19c55d68f6f8ddcac2642392ad3a355878406fbd4d"}, ] [[package]] @@ -611,10 +611,10 @@ files = [ [[package]] name = "django-admin-extra-buttons" -version = "1.5.7" +version = "1.5.8" summary = "Django mixin to easily add buttons to any ModelAdmin" files = [ - {file = "django-admin-extra-buttons-1.5.7.tar.gz", hash = "sha256:08dd070cbd46dfe1cb46d7676495a943860e68f99bac85ab5b164cbeb5e73dae"}, + {file = "django_admin_extra_buttons-1.5.8.tar.gz", hash = "sha256:48dc9d470ade3f5f29c80753af4d367c08947d02c9a5757aa7a9223eda6812c5"}, ] [[package]] @@ -915,17 +915,17 @@ files = [ [[package]] name = "django-import-export" -version = "3.3.8" +version = "4.0.0" requires_python = ">=3.8" summary = "Django application and library for importing and exporting data with included admin integration." dependencies = [ "Django>=3.2", "diff-match-patch", - "tablib[html,ods,xls,xlsx,yaml]==3.5.0", + "tablib==3.5.0", ] files = [ - {file = "django-import-export-3.3.8.tar.gz", hash = "sha256:4deabc557801d368093608c86fd0f4831bc9540e2ea41ca2f023e2efb3eb6f48"}, - {file = "django_import_export-3.3.8-py3-none-any.whl", hash = "sha256:2eac09e8cec8670f36e24314760448011ad23c51e8fb930d55f50d0c3c926da0"}, + {file = "django_import_export-4.0.0-py3-none-any.whl", hash = "sha256:6e8c881ad8ef93ca1f302bbcc39f2b7fc0bb5b3acbdaf4563a6cf271a56c8086"}, + {file = "django_import_export-4.0.0.tar.gz", hash = "sha256:888bcd4385f92c27ce544fad7ca4cc4f9cf9d84b51cb89e402999a0988afd6f8"}, ] [[package]] @@ -993,15 +993,14 @@ files = [ [[package]] name = "django-push-notifications" -version = "3.0.2" -requires_python = ">=3.6" +version = "3.1.0" +requires_python = ">=3.7" summary = "Send push notifications to mobile devices through GCM, APNS or WNS and to WebPush (Chrome, Firefox and Opera) in Django" dependencies = [ "Django>=2.2", ] files = [ - {file = "django-push-notifications-3.0.2.tar.gz", hash = "sha256:9429f6611662ed5a71085e604e689a05e4ef0804d7d28855429972bf2d211bd8"}, - {file = "django_push_notifications-3.0.2-py3-none-any.whl", hash = "sha256:2059bcd47b565ec72827630ae023e8c070f7ef10f87a0e6f523e2b764ef3a52d"}, + {file = "django-push-notifications-3.1.0.tar.gz", hash = "sha256:be14b4b8043a5a9d1d2f6b36bedcb954a6e30f51c917ba2a70e3d501db626217"}, ] [[package]] @@ -1076,31 +1075,31 @@ files = [ [[package]] name = "django-storages" -version = "1.14.2" +version = "1.14.3" requires_python = ">=3.7" summary = "Support for many storage backends in Django" dependencies = [ "Django>=3.2", ] files = [ - {file = "django-storages-1.14.2.tar.gz", hash = "sha256:51b36af28cc5813b98d5f3dfe7459af638d84428c8df4a03990c7d74d1bea4e5"}, - {file = "django_storages-1.14.2-py3-none-any.whl", hash = "sha256:1db759346b52ada6c2efd9f23d8241ecf518813eb31db9e2589207174f58f6ad"}, + {file = "django-storages-1.14.3.tar.gz", hash = "sha256:95a12836cd998d4c7a4512347322331c662d9114c4344f932f5e9c0fce000608"}, + {file = "django_storages-1.14.3-py3-none-any.whl", hash = "sha256:31f263389e95ce3a1b902fb5f739a7ed32895f7d8b80179fe7453ecc0dfe102e"}, ] [[package]] name = "django-storages" -version = "1.14.2" +version = "1.14.3" extras = ["azure"] requires_python = ">=3.7" summary = "Support for many storage backends in Django" dependencies = [ "azure-core>=1.13", "azure-storage-blob>=12", - "django-storages==1.14.2", + "django-storages==1.14.3", ] files = [ - {file = "django-storages-1.14.2.tar.gz", hash = "sha256:51b36af28cc5813b98d5f3dfe7459af638d84428c8df4a03990c7d74d1bea4e5"}, - {file = "django_storages-1.14.2-py3-none-any.whl", hash = "sha256:1db759346b52ada6c2efd9f23d8241ecf518813eb31db9e2589207174f58f6ad"}, + {file = "django-storages-1.14.3.tar.gz", hash = "sha256:95a12836cd998d4c7a4512347322331c662d9114c4344f932f5e9c0fce000608"}, + {file = "django_storages-1.14.3-py3-none-any.whl", hash = "sha256:31f263389e95ce3a1b902fb5f739a7ed32895f7d8b80179fe7453ecc0dfe102e"}, ] [[package]] @@ -1116,24 +1115,24 @@ files = [ [[package]] name = "django-stubs" -version = "4.2.7" +version = "5.0.0" requires_python = ">=3.8" summary = "Mypy stubs for Django" dependencies = [ + "asgiref", "django", - "django-stubs-ext>=4.2.7", + "django-stubs-ext>=5.0.0", "types-PyYAML", - "types-pytz", "typing-extensions", ] files = [ - {file = "django-stubs-4.2.7.tar.gz", hash = "sha256:8ccd2ff4ee5adf22b9e3b7b1a516d2e1c2191e9d94e672c35cc2bc3dd61e0f6b"}, - {file = "django_stubs-4.2.7-py3-none-any.whl", hash = "sha256:4cf4de258fa71adc6f2799e983091b9d46cfc67c6eebc68fe111218c9a62b3b8"}, + {file = "django_stubs-5.0.0-py3-none-any.whl", hash = "sha256:084484cbe16a6d388e80ec687e46f529d67a232f3befaf55c936b3b476be289d"}, + {file = "django_stubs-5.0.0.tar.gz", hash = "sha256:b8a792bee526d6cab31e197cb414ee7fa218abd931a50948c66a80b3a2548621"}, ] [[package]] name = "django-stubs-ext" -version = "4.2.7" +version = "5.0.0" requires_python = ">=3.8" summary = "Monkey-patching and extensions for django-stubs" dependencies = [ @@ -1141,8 +1140,8 @@ dependencies = [ "typing-extensions", ] files = [ - {file = "django-stubs-ext-4.2.7.tar.gz", hash = "sha256:519342ac0849cda1559746c9a563f03ff99f636b0ebe7c14b75e816a00dfddc3"}, - {file = "django_stubs_ext-4.2.7-py3-none-any.whl", hash = "sha256:45a5d102417a412e3606e3c358adb4744988a92b7b58ccf3fd64bddd5d04d14c"}, + {file = "django_stubs_ext-5.0.0-py3-none-any.whl", hash = "sha256:8e1334fdf0c8bff87e25d593b33d4247487338aaed943037826244ff788b56a8"}, + {file = "django_stubs_ext-5.0.0.tar.gz", hash = "sha256:5bacfbb498a206d5938454222b843d81da79ea8b6fcd1a59003f529e775bc115"}, ] [[package]] @@ -1319,18 +1318,18 @@ files = [ [[package]] name = "docxtpl" -version = "0.16.8" +version = "0.17.0" summary = "Python docx template engine" dependencies = [ "docxcompose", "jinja2", "lxml", - "python-docx", + "python-docx>=1.1.1", "six", ] files = [ - {file = "docxtpl-0.16.8-py2.py3-none-any.whl", hash = "sha256:dffd549bc707a63ff1b5ed00a4dec613d6b6a242b66ee467a6e010c43a324d48"}, - {file = "docxtpl-0.16.8.tar.gz", hash = "sha256:461d3d4e261f670d31de4eee2c8c3e00af2686e7afdb5c5ccb465bd0650da6eb"}, + {file = "docxtpl-0.17.0-py2.py3-none-any.whl", hash = "sha256:475e6e922ca4c3f4c26e8941f2170826c4d80a72c5219369070554719bc8a1a4"}, + {file = "docxtpl-0.17.0.tar.gz", hash = "sha256:19d719efa46017653451f978442395ceeb5a45da4f4b879e6bf9f49df14fe465"}, ] [[package]] @@ -1710,14 +1709,6 @@ files = [ {file = "Markdown-3.6.tar.gz", hash = "sha256:ed4f41f6daecbeeb96e576ce414c41d2d876daa9a16cb35fa8ed8c2ddfad0224"}, ] -[[package]] -name = "markuppy" -version = "1.14" -summary = "An HTML/XML generator" -files = [ - {file = "MarkupPy-1.14.tar.gz", hash = "sha256:1adee2c0a542af378fe84548ff6f6b0168f3cb7f426b46961038a2bcfaad0d5f"}, -] - [[package]] name = "markupsafe" version = "2.1.5" @@ -1922,17 +1913,6 @@ files = [ {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, ] -[[package]] -name = "odfpy" -version = "1.4.1" -summary = "Python API and tools to manipulate OpenDocument files" -dependencies = [ - "defusedxml", -] -files = [ - {file = "odfpy-1.4.1.tar.gz", hash = "sha256:db766a6e59c5103212f3cc92ec8dd50a0f3a02790233ed0b52148b70d3c438ec"}, -] - [[package]] name = "openpyxl" version = "3.1.2" @@ -2069,12 +2049,12 @@ files = [ [[package]] name = "pluggy" -version = "1.4.0" +version = "1.5.0" requires_python = ">=3.8" summary = "plugin and hook calling mechanisms for python" files = [ - {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, - {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] [[package]] @@ -2369,18 +2349,18 @@ files = [ [[package]] name = "pytest" -version = "8.1.1" +version = "8.2.0" requires_python = ">=3.8" summary = "pytest: simple powerful testing with Python" dependencies = [ "colorama; sys_platform == \"win32\"", "iniconfig", "packaging", - "pluggy<2.0,>=1.4", + "pluggy<2.0,>=1.5", ] files = [ - {file = "pytest-8.1.1-py3-none-any.whl", hash = "sha256:2a8386cfc11fa9d2c50ee7b2a57e7d898ef90470a7a34c4b949ff59662bb78b7"}, - {file = "pytest-8.1.1.tar.gz", hash = "sha256:ac978141a75948948817d360297b7aae0fcb9d6ff6bc9ec6d514b85d5a65c044"}, + {file = "pytest-8.2.0-py3-none-any.whl", hash = "sha256:1733f0620f6cda4095bbf0d9ff8022486e91892245bb9e7d5542c018f612f233"}, + {file = "pytest-8.2.0.tar.gz", hash = "sha256:d507d4482197eac0ba2bae2e9babf0672eb333017bcedaa5fb1a3d42c1174b3f"}, ] [[package]] @@ -2557,16 +2537,16 @@ files = [ [[package]] name = "python-docx" -version = "1.1.0" +version = "1.1.2" requires_python = ">=3.7" summary = "Create, read, and update Microsoft Word .docx files." dependencies = [ "lxml>=3.1.0", - "typing-extensions", + "typing-extensions>=4.9.0", ] files = [ - {file = "python-docx-1.1.0.tar.gz", hash = "sha256:5829b722141cf1ab79aedf0c34d9fe9924b29764584c0f2164eb2b02dcdf17c9"}, - {file = "python_docx-1.1.0-py3-none-any.whl", hash = "sha256:bac9773278098a1ddc43a52d84e22f5909c4a3080a624530b3ecb3771b07c6cd"}, + {file = "python_docx-1.1.2-py3-none-any.whl", hash = "sha256:08c20d6058916fb19853fcf080f7f42b6270d89eac9fa5f8c15f691c0017fabe"}, + {file = "python_docx-1.1.2.tar.gz", hash = "sha256:0cf1f22e95b9002addca7948e16f2cd7acdfd498047f1941ca5d293db7762efd"}, ] [[package]] @@ -2981,17 +2961,12 @@ files = [ [[package]] name = "tablib" version = "3.5.0" -extras = ["html", "ods", "xls", "xlsx", "yaml"] +extras = ["xlsx"] requires_python = ">=3.8" summary = "Format agnostic tabular data library (XLS, JSON, YAML, CSV, etc.)" dependencies = [ - "markuppy", - "odfpy", "openpyxl>=2.6.0", - "pyyaml", "tablib==3.5.0", - "xlrd", - "xlwt", ] files = [ {file = "tablib-3.5.0-py3-none-any.whl", hash = "sha256:9821caa9eca6062ff7299fa645e737aecff982e6b2b42046928a6413c8dabfd9"}, @@ -3039,7 +3014,7 @@ files = [ [[package]] name = "tox" -version = "4.14.2" +version = "4.15.0" requires_python = ">=3.8" summary = "tox is a generic virtualenv management and test command line tool" dependencies = [ @@ -3054,8 +3029,8 @@ dependencies = [ "virtualenv>=20.25", ] files = [ - {file = "tox-4.14.2-py3-none-any.whl", hash = "sha256:2900c4eb7b716af4a928a7fdc2ed248ad6575294ed7cfae2ea41203937422847"}, - {file = "tox-4.14.2.tar.gz", hash = "sha256:0defb44f6dafd911b61788325741cc6b2e12ea71f987ac025ad4d649f1f1a104"}, + {file = "tox-4.15.0-py3-none-any.whl", hash = "sha256:300055f335d855b2ab1b12c5802de7f62a36d4fd53f30bd2835f6a201dda46ea"}, + {file = "tox-4.15.0.tar.gz", hash = "sha256:7a0beeef166fbe566f54f795b4906c31b428eddafc0102ac00d20998dd1933f6"}, ] [[package]] diff --git a/pdm.toml.template b/pdm.toml.template deleted file mode 100644 index dfe8d052..00000000 --- a/pdm.toml.template +++ /dev/null @@ -1,5 +0,0 @@ -[python] -use_venv = false - -[venv] -in_project = true diff --git a/pyproject.toml b/pyproject.toml index e3fe87b0..1c68c678 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ dependencies = [ "social-auth-app-django", "sqlparse", "swapper", - "tablib", + "tablib[xlsx]", "tzdata", "unicef-security", "pathvalidate", diff --git a/src/hope_country_report/apps/core/management/commands/inspect_hope.py b/src/hope_country_report/apps/core/management/commands/inspect_hope.py index 5e715ba5..c509755c 100644 --- a/src/hope_country_report/apps/core/management/commands/inspect_hope.py +++ b/src/hope_country_report/apps/core/management/commands/inspect_hope.py @@ -348,7 +348,7 @@ def strip_prefix(s): yield f" return str(self.{attr})" break - def normalize_col_name(self, col_name: str, used_column_names: List[str], is_relation: bool): + def normalize_col_name(self, col_name: str, used_column_names: List[str], is_relation: bool): # noqa """ Modify the column name to make it Python-compatible as a field name """ diff --git a/src/hope_country_report/apps/core/management/commands/upgrade.py b/src/hope_country_report/apps/core/management/commands/upgrade.py index 7374ccb3..71be1460 100644 --- a/src/hope_country_report/apps/core/management/commands/upgrade.py +++ b/src/hope_country_report/apps/core/management/commands/upgrade.py @@ -107,7 +107,7 @@ def halt(self, e: Exception) -> None: # pragma: no cover sys.exit(1) - def handle(self, *args: Any, **options: Any) -> None: + def handle(self, *args: Any, **options: Any) -> None: # noqa from hope_country_report.apps.core.models import CountryOffice, User self.get_options(options) diff --git a/src/hope_country_report/apps/hope/admin/__init__.py b/src/hope_country_report/apps/hope/admin/__init__.py index 410f5bd4..38d8c277 100644 --- a/src/hope_country_report/apps/hope/admin/__init__.py +++ b/src/hope_country_report/apps/hope/admin/__init__.py @@ -22,7 +22,7 @@ def register_all_app_models() -> None: if not state.inspecting: # pragma: no branch - from .generic import * - from .utils import modeladmin_factory + from .generic import * # noqa + from .utils import modeladmin_factory # noqa register_all_app_models() diff --git a/src/hope_country_report/apps/hope/models/__init__.py b/src/hope_country_report/apps/hope/models/__init__.py index d8757448..5d843c5f 100644 --- a/src/hope_country_report/apps/hope/models/__init__.py +++ b/src/hope_country_report/apps/hope/models/__init__.py @@ -1 +1 @@ -from ._inspect import * +from ._inspect import * # noqa diff --git a/src/hope_country_report/apps/hope/patcher/__init__.py b/src/hope_country_report/apps/hope/patcher/__init__.py index f2c7e658..08733851 100644 --- a/src/hope_country_report/apps/hope/patcher/__init__.py +++ b/src/hope_country_report/apps/hope/patcher/__init__.py @@ -6,7 +6,6 @@ from django.apps import AppConfig, apps from django.db import models from django.db.models import Model -from django.utils.functional import cached_property from hope_country_report.apps.hope import models as hope_models from hope_country_report.apps.hope.models import HopeModel diff --git a/src/hope_country_report/apps/power_query/models/__init__.py b/src/hope_country_report/apps/power_query/models/__init__.py index 43427e3a..726cd76c 100644 --- a/src/hope_country_report/apps/power_query/models/__init__.py +++ b/src/hope_country_report/apps/power_query/models/__init__.py @@ -1,9 +1,9 @@ -from ._base import CeleryEnabled -from .arguments import Parametrizer -from .chart import ChartPage -from .dataset import Dataset -from .formatter import Formatter -from .query import Query -from .report import ReportConfiguration -from .report_document import ReportDocument -from .report_template import ReportTemplate +from ._base import CeleryEnabled # noqa +from .arguments import Parametrizer # noqa +from .chart import ChartPage # noqa +from .dataset import Dataset # noqa +from .formatter import Formatter # noqa +from .query import Query # noqa +from .report import ReportConfiguration # noqa +from .report_document import ReportDocument # noqa +from .report_template import ReportTemplate # noqa diff --git a/src/hope_country_report/apps/power_query/models/report_document.py b/src/hope_country_report/apps/power_query/models/report_document.py index c4f017bf..091d3a83 100644 --- a/src/hope_country_report/apps/power_query/models/report_document.py +++ b/src/hope_country_report/apps/power_query/models/report_document.py @@ -70,7 +70,7 @@ def filename(self): @classmethod def process( self, report: "ReportConfiguration", dataset: "Dataset", formatter: "Formatter", notify: bool = True - ) -> "Tuple[int|None, Exception|str]": + ) -> "Tuple[int|None, Exception|str]": # noqa try: args_desc = "_".join([str(value) for value in dataset.arguments.values()]) context = { diff --git a/src/hope_country_report/apps/power_query/processors.py b/src/hope_country_report/apps/power_query/processors.py index 9c1ead97..75c7a1fd 100644 --- a/src/hope_country_report/apps/power_query/processors.py +++ b/src/hope_country_report/apps/power_query/processors.py @@ -23,7 +23,7 @@ from strategy_field.registry import Registry from strategy_field.utils import fqn -from hope_country_report.apps.power_query.storage import DataSetStorage, HopeStorage +from hope_country_report.apps.power_query.storage import HopeStorage from .utils import to_dataset diff --git a/src/hope_country_report/apps/power_query/storage.py b/src/hope_country_report/apps/power_query/storage.py index be84e48f..882981a7 100644 --- a/src/hope_country_report/apps/power_query/storage.py +++ b/src/hope_country_report/apps/power_query/storage.py @@ -12,6 +12,22 @@ def get_available_name(self, name: str, max_length: int | None = None) -> str: return name +class ReadOnlyStorageMixin: + def delete(self, name): + raise RuntimeError("This storage cannot delete files") + + def save(self, name, content, max_length=None): + raise RuntimeError("This storage cannot save files") + + def open(self, name, mode="rb"): + if "w" in mode: + raise RuntimeError("This storage cannot open files in write mode") + return super().open(name, mode="rb") + + def listdir(self, path=""): + return [] + + class HCRAzureStorage(AzureStorage): prefix = "" @@ -23,7 +39,7 @@ def get_default_settings(self): return base -class HopeStorage(HCRAzureStorage): +class HopeStorage(ReadOnlyStorageMixin, HCRAzureStorage): prefix = "HOPE" diff --git a/src/hope_country_report/apps/power_query/utils.py b/src/hope_country_report/apps/power_query/utils.py index 96da1ac4..9a6cf57f 100644 --- a/src/hope_country_report/apps/power_query/utils.py +++ b/src/hope_country_report/apps/power_query/utils.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, TYPE_CHECKING, Union +from typing import Any, Dict, TYPE_CHECKING import base64 import binascii @@ -16,7 +16,6 @@ from django.http import HttpRequest, HttpResponse from django.utils.safestring import mark_safe -import pytz import tablib from constance import config from sentry_sdk import configure_scope @@ -44,7 +43,7 @@ def make_naive(value: datetime.datetime) -> datetime.datetime: return value -def to_dataset(result: "QuerySet[AnyModel]|Iterable[Any]|tablib.Dataset|Dict[str,Any]") -> tablib.Dataset: +def to_dataset(result: "QuerySet[AnyModel]|Iterable[Any]|tablib.Dataset|Dict[str,Any]") -> tablib.Dataset: # noqa if isinstance(result, QuerySet): data = tablib.Dataset() fields = result.__dict__["_fields"] diff --git a/src/hope_country_report/config/__init__.py b/src/hope_country_report/config/__init__.py index cf49ee54..ed8a58af 100644 --- a/src/hope_country_report/config/__init__.py +++ b/src/hope_country_report/config/__init__.py @@ -1,11 +1,8 @@ -from typing import Any, Dict, reveal_type, Tuple - -from collections.abc import Mapping from enum import Enum from environ import Env -DJANGO_HELP_BASE = "https://docs.djangoproject.com/en/4.2/ref/settings" +DJANGO_HELP_BASE = "https://docs.djangoproject.com/en/5.0/ref/settings" def setting(anchor): diff --git a/src/hope_country_report/web/views/__init__.py b/src/hope_country_report/web/views/__init__.py index 8240b886..ee9283d7 100644 --- a/src/hope_country_report/web/views/__init__.py +++ b/src/hope_country_report/web/views/__init__.py @@ -1,12 +1,12 @@ -from .charts import ChartDetailView, ChartListView -from .document import ( +from .charts import ChartDetailView, ChartListView # noqa +from .document import ( # noqa OfficeDocumentDisplayView, OfficeDocumentDownloadView, OfficeReportDocumentDetailView, OfficeReportDocumentListView, RequestAccessView, ) -from .generic import ( +from .generic import ( # noqa download, index, OfficeHomeView, @@ -15,5 +15,5 @@ OfficePreferencesView, select_tenant, ) -from .report import OfficeConfigurationDetailView, OfficeConfigurationListView -from .user import UserProfileView +from .report import OfficeConfigurationDetailView, OfficeConfigurationListView # noqa +from .user import UserProfileView # noqa diff --git a/tests/power_query/test_pq_report.py b/tests/power_query/test_pq_report.py index 4bac6759..456ea085 100644 --- a/tests/power_query/test_pq_report.py +++ b/tests/power_query/test_pq_report.py @@ -65,7 +65,7 @@ def query(data: "_DATA"): @pytest.fixture() def query_impl(data: "_DATA", query): - from testutils.factories import ContentTypeFactory, QueryFactory + from testutils.factories import QueryFactory return QueryFactory( target=None,