From 91da27dbd7b266df2864db41e4fae67c68e06105 Mon Sep 17 00:00:00 2001 From: saxix Date: Fri, 5 Jul 2024 16:16:08 +0200 Subject: [PATCH 01/16] CI Initial commit --- .github/.gitignore | 4 + .github/actions/checksum/action.yml | 22 +++ .github/actions/docker_build/action.yml | 193 +++++++++++++++++++++ .github/actions/last_commit/action.yml | 31 ++++ .github/codeql/codeql-config.yml | 4 + .github/file-filters.yml | 39 +++++ .github/workflows/delete_image.yml | 49 ++++++ .github/workflows/dump.yml | 87 ++++++++++ .github/workflows/label-pullrequest.yml | 38 +++++ .github/workflows/lint.yml | 91 ++++++++++ .github/workflows/security.yml | 77 +++++++++ .github/workflows/test.yml | 215 ++++++++++++++++++++++++ .gitignore | 7 + .pdm-python | 1 + docker/Dockerfile | 65 ++++--- docker/Makefile | 184 -------------------- docker/bin/docker-entrypoint.sh | 3 + docker/conf/circus.conf | 74 -------- docker/conf/uwsgi.ini | 2 +- 19 files changed, 902 insertions(+), 284 deletions(-) create mode 100644 .github/.gitignore create mode 100644 .github/actions/checksum/action.yml create mode 100644 .github/actions/docker_build/action.yml create mode 100644 .github/actions/last_commit/action.yml create mode 100644 .github/codeql/codeql-config.yml create mode 100644 .github/file-filters.yml create mode 100644 .github/workflows/delete_image.yml create mode 100644 .github/workflows/dump.yml create mode 100644 .github/workflows/label-pullrequest.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/security.yml create mode 100644 .github/workflows/test.yml create mode 100644 .gitignore create mode 100644 .pdm-python delete mode 100644 docker/Makefile delete mode 100644 docker/conf/circus.conf diff --git a/.github/.gitignore b/.github/.gitignore new file mode 100644 index 0000000..db11503 --- /dev/null +++ b/.github/.gitignore @@ -0,0 +1,4 @@ +_workflows/* +_actions/* +_* +icons.sh diff --git a/.github/actions/checksum/action.yml b/.github/actions/checksum/action.yml new file mode 100644 index 0000000..b3bdd51 --- /dev/null +++ b/.github/actions/checksum/action.yml @@ -0,0 +1,22 @@ +# ref: https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action +name: 'Calculate Release Hash' +description: 'Calculate deps and os hash' +inputs: + files: + description: 'Files to use to calculate the hash' + required: true + default: "pdm.lock docker/bin/* docker/conf/* docker/Dockerfile" +outputs: + checksum: # id of output + description: 'The time we greeted you' + value: ${{ steps.calc.outputs.checksum }} + +runs: + using: 'composite' + steps: + - id: calc + shell: bash + run: | + set -x + LOCK_SHA=$(sha1sum ${{ inputs.files }} | sha1sum | awk '{print $1}' | cut -c 1-8) + echo "checksum=$LOCK_SHA" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/docker_build/action.yml b/.github/actions/docker_build/action.yml new file mode 100644 index 0000000..a37dc4a --- /dev/null +++ b/.github/actions/docker_build/action.yml @@ -0,0 +1,193 @@ +# ref: https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action +name: 'Docker build' +description: 'Calculate deps and os hash' +inputs: + image: + description: "Image name to build and push" + required: true + type: string + default: ${{ github.repository }} + target: + description: "Target Dockerfile Stage" + type: string + username: + description: 'DockerHub username ' + required: false + password: + description: 'DockerHub password ' + required: false + code_checksum: + description: 'Current codde checksum' + required: false + rebuild: + description: 'Always rebuild image, Ignore checksum' + required: false + default: false + dryrun: + description: 'Dryrun build' + required: false + default: 'false' + + +defaults: + run: + shell: bash + + +outputs: + image: + description: 'Built image name' + value: ${{ steps.image_name.outputs.name }} + version: + description: 'Built image version' + value: ${{ steps.meta.outputs.version }} + created: + description: 'True if new image has been created' + value: ${{ steps.status.outputs.created }} + digest: + description: 'Built image digest' + value: ${{ steps.build_push.outputs.digest }} + imageId: + description: 'Built image ID' + value: ${{ steps.build_push.outputs.imageId }} + + +runs: + using: 'composite' + steps: + + - name: Restore cached regclient + id: cache-regclient-restore + uses: actions/cache/restore@v4 + with: + path: $HOME/.regctl + key: ${{ runner.os }}-regclient + - name: Install regctl + if: steps.cache-regclient.outputs.cache-hit != 'true' + uses: regclient/actions/regctl-installer@main + - name: Cache regclient + id: cache-regclient-save + uses: actions/cache/save@v4 + with: + path: $HOME/.regctl + key: ${{ runner.os }}-regclient + + - name: DockerHub login + uses: docker/login-action@v3 + with: + username: ${{ inputs.username }} + password: ${{ inputs.password }} + - id: checksum + uses: ./.github/actions/checksum + + - shell: bash + run: | + build_date=$(date +"%Y-%m-%d %H:%M") + echo "BUILD_DATE=$build_date" >> $GITHUB_ENV + + if [[ "${{inputs.target}}" == "dist" ]]; then + echo "TAG_PREFIX=" >> $GITHUB_ENV + else + echo "TAG_PREFIX=test-" >> $GITHUB_ENV + fi + - id: last_commit + uses: ./.github/actions/last_commit + - name: Docker meta + id: meta + uses: docker/metadata-action@v5.5.1 + with: + images: ${{ inputs.image }} + flavor: | + prefix=${{env.TAG_PREFIX}} + tags: | + type=ref,event=branch + type=ref,event=pr,prefix=${{env.TAG_PREFIX}}pr + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index + - name: "Image Name" + shell: bash + id: image_name + run: | + echo "name=${{ inputs.image }}:${{ steps.meta.outputs.version }}" >> $GITHUB_OUTPUT + echo "test_name=${{ inputs.image }}:test-${{ steps.meta.outputs.version }}" >> $GITHUB_OUTPUT + - name: "Check Image" + id: image_status + shell: bash + run: | + set +e + echo "::notice::ℹ Checking checksum for ${{ steps.image_name.outputs.name }}" + image_checksum=$(regctl image inspect \ + -p linux/amd64 \ + --format '{{index .Config.Labels "checksum"}}' \ + -v fatal \ + ${{ steps.image_name.outputs.name }} 2>/dev/null) + code_checksum="${{ inputs.code_checksum }}" + + if [[ -z "$image_checksum" ]]; then + echo "::warning::🤔 No image checksum found" + echo "updated=false" >> $GITHUB_OUTPUT + elif [[ $image_checksum == $code_checksum ]]; then + echo "::notice::😀 Image is updated" + echo "updated=true" >> $GITHUB_OUTPUT + else + echo "::warning::🤬 Checksum: found '${image_checksum}' expected '${code_checksum}'" + echo "updated=false" >> $GITHUB_OUTPUT + fi + if [[ "${{inputs.rebuild}}" == "true" ]]; then + echo "::warning::⚠ Forced build due input parameter" + fi + - name: Set up Docker BuildX + if: steps.image_status.outputs.updated != 'true' || inputs.rebuild == 'true' + uses: docker/setup-buildx-action@v3.3.0 + with: + platforms: linux/amd64 + driver: docker-container + driver-opts: | + image=moby/buildkit:v0.13.2 + network=host + - name: Build and push + if: (steps.image_status.outputs.updated != 'true' || inputs.rebuild == 'true') && inputs.dryrun == 'true' + shell: bash + run: | + echo "::notice:: Dryrun build of ${{ steps.meta.outputs.tags }}" + echo "Building image ${{ steps.meta.outputs.tags }}" + echo " TAG_PREFIX ${{ env.TAG_PREFIX }}" + echo " target ${{ inputs.target }}" + echo " checksum ${{ inputs.code_checksum }}" + echo " version ${{ steps.meta.outputs.version }}" + echo " commit ${{ steps.last_commit.outputs.last_commit_short_sha }}" + + - name: Build and push + if: (steps.image_status.outputs.updated != 'true' || inputs.rebuild == 'true') && inputs.dryrun != 'true' + id: build_push + uses: docker/build-push-action@v6 + with: + context: . + tags: ${{ steps.meta.outputs.tags }} + labels: "${{ steps.meta.outputs.labels }}\nchecksum=${{ inputs.code_checksum }}\ndistro=${{ inputs.target }}" + annotations: "${{ steps.meta.outputs.annotations }}\nchecksum=${{ inputs.code_checksum }}\ndistro=${{ inputs.target }}" + target: ${{ inputs.target }} + file: ./docker/Dockerfile + platforms: linux/amd64 + push: true + sbom: true + provenance: true + cache-from: type=registry,ref=${{ steps.image_name.outputs.name }}-cache,ref=${{ steps.image_name.outputs.test_name }}-cache + cache-to: type=registry,ref=${{ steps.image_name.outputs.name }}-cache,mode=max,image-manifest=true + build-args: | + GITHUB_SERVER_URL=${{ github.server_url }} + GITHUB_REPOSITORY=${{ github.repository }} + BUILD_DATE=${{ env.BUILD_DATE }} + DISTRO=${{ inputs.target }} + CHECKSUM=${{ inputs.code_checksum }} + VERSION=${{ steps.meta.outputs.version }} + SOURCE_COMMIT=${{ steps.last_commit.outputs.last_commit_short_sha }} + - name: Status + id: status + if: (steps.image_status.outputs.updated != 'true' || inputs.rebuild == 'true') && inputs.dryrun != 'true' + shell: bash + run: | + echo "${{ toJSON(steps.build_push.outputs) }}" + regctl image inspect -p linux/amd64 ${{ steps.image_name.outputs.name }} + echo "::notice:: Image ${{ steps.meta.outputs.tags }} successfully built and pushed" + echo "created=true" >> $GITHUB_OUTPUT diff --git a/.github/actions/last_commit/action.yml b/.github/actions/last_commit/action.yml new file mode 100644 index 0000000..e42013e --- /dev/null +++ b/.github/actions/last_commit/action.yml @@ -0,0 +1,31 @@ +name: 'Get Last commit' +description: '' + + +outputs: + last_commit_sha: + description: 'last_commit_sha' + value: ${{ steps.result.outputs.last_commit_sha }} + last_commit_short_sha: + description: 'last_commit_short_sha' + value: ${{ steps.result.outputs.last_commit_short_sha }} + +runs: + using: "composite" + steps: + - name: Setup Environment (PR) + if: ${{ github.event_name == 'pull_request' }} + shell: bash + run: | + echo "LAST_COMMIT_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV + - name: Setup Environment (Push) + if: ${{ github.event_name == 'push' }} + shell: bash + run: | + echo "LAST_COMMIT_SHA=${GITHUB_SHA}" >> $GITHUB_ENV + - id: result + shell: bash + run: | + raw=${{env.LAST_COMMIT_SHA}} + echo "last_commit_sha=$raw" >> $GITHUB_OUTPUT + echo "last_commit_short_sha=${raw::8}" >> $GITHUB_OUTPUT diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml new file mode 100644 index 0000000..979cf37 --- /dev/null +++ b/.github/codeql/codeql-config.yml @@ -0,0 +1,4 @@ +name: 'App CodeQL Config' + +paths-ignore: + - '**/tests/**' \ No newline at end of file diff --git a/.github/file-filters.yml b/.github/file-filters.yml new file mode 100644 index 0000000..c960202 --- /dev/null +++ b/.github/file-filters.yml @@ -0,0 +1,39 @@ +# This is used by the action https://github.com/dorny/paths-filter +docker: &docker + - added|modified: './docker/**/*' + - added|modified: './docker/*' + +dependencies: &dependencies + - 'pdm.lock' + - 'pyproject.toml' + +python: &python + - added|modified: 'src/**' + - added|modified: 'tests/**' + - 'manage.py' + +changelog: + - added|modified: 'changes/**' + - 'CHANGELOG.md' + +mypy: + - *python + - 'mypy.ini' + +docker_base: + - *docker + - *dependencies + +run_tests: + - *python + - *docker + - *dependencies + - 'pytest.ini' + +migrations: + - added|modified: 'src/**/migrations/*' + +lint: + - *python + - '.flake8' + - 'pyproject.toml' diff --git a/.github/workflows/delete_image.yml b/.github/workflows/delete_image.yml new file mode 100644 index 0000000..7d51ec6 --- /dev/null +++ b/.github/workflows/delete_image.yml @@ -0,0 +1,49 @@ +name: Branch Deleted +on: delete +jobs: + delete: + if: github.event.ref_type == 'branch' + runs-on: ubuntu-latest + steps: + - name: Install regctl + uses: regclient/actions/regctl-installer@main + - name: regctl login + uses: regclient/actions/regctl-login@main + with: + registry: docker.io + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - shell: bash + run: | + ref="${{github.event.ref}}" + tag=$(echo $ref | sed -e "s#refs/heads/##g" | sed -e s#/#-##g) + name="${{vars.DOCKER_IMAGE}}:test-${{github.event.ref}}" + echo "Delete $name" +# - name: Delete Test Docker Image +# shell: bash +# run: | +# name="${{vars.DOCKER_IMAGE}}:test-${{github.event.ref}}" +# registry="https://registry-1.docker.io" +# curl -v -sSL -X DELETE "http://${registry}/v2/${name}/manifests/$( +# curl -sSL -I \ +# -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ +# "http://${registry}/v2/${name}/manifests/$( +# curl -sSL "http://${registry}/v2/${name}/tags/list" | jq -r '.tags[0]' +# )" \ +# | awk '$1 == "Docker-Content-Digest:" { print $2 }' \ +# | tr -d $'\r' \ +# )" +# - name: Delete linked Docker Image +# shell: bash +# run: | +# name="${{vars.DOCKER_IMAGE}}:${{github.event.ref}}" +# registry="https://registry-1.docker.io" +# curl -v -sSL -X DELETE "http://${registry}/v2/${name}/manifests/$( +# curl -sSL -I \ +# -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ +# "http://${registry}/v2/${name}/manifests/$( +# curl -sSL "http://${registry}/v2/${name}/tags/list" | jq -r '.tags[0]' +# )" \ +# | awk '$1 == "Docker-Content-Digest:" { print $2 }' \ +# | tr -d $'\r' \ +# )" diff --git a/.github/workflows/dump.yml b/.github/workflows/dump.yml new file mode 100644 index 0000000..461d80c --- /dev/null +++ b/.github/workflows/dump.yml @@ -0,0 +1,87 @@ +name: "[DEBUG] Dump" + +on: + check_run: + create: + delete: + discussion: + discussion_comment: + fork: + issues: + issue_comment: + milestone: + pull_request: + pull_request_review_comment: + pull_request_review: + push: + release: + workflow_dispatch: + + +jobs: + dump: + name: "[DEBUG] Echo Full Context" + if: ${{ contains(github.event.head_commit.message, 'ci:debug') }} + runs-on: ubuntu-latest + steps: + - name: Dump Env vars + run: | + echo "====== ENVIRONMENT =================" + env | sort + echo "====================================" + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJSON(github) }} + run: | + echo "====== GITHUB_CONTEXT ==============" + echo "$GITHUB_CONTEXT" + echo "====================================" + - name: Dump job context + env: + JOB_CONTEXT: ${{ toJSON(job) }} + run: | + echo "====== JOB_CONTEXT ==============" + echo "$JOB_CONTEXT" + echo "====================================" + - name: Dump steps context + env: + STEPS_CONTEXT: ${{ toJSON(steps) }} + run: | + echo "====== STEPS_CONTEXT ==============" + echo "$STEPS_CONTEXT" + echo "====================================" + - name: Dump runner context + env: + RUNNER_CONTEXT: ${{ toJSON(runner) }} + run: | + echo "====== RUNNER_CONTEXT ==============" + echo "$RUNNER_CONTEXT" + echo "====================================" + - name: Dump strategy context + env: + STRATEGY_CONTEXT: ${{ toJSON(strategy) }} + run: | + echo "====== STRATEGY_CONTEXT ==============" + echo "$STRATEGY_CONTEXT" + echo "====================================" + - name: Dump matrix context + env: + MATRIX_CONTEXT: ${{ toJSON(matrix) }} + run: | + echo "====== MATRIX_CONTEXT ==============" + echo "$MATRIX_CONTEXT" + echo "====================================" + - name: Dump vars context + env: + VARS_CONTEXT: ${{ toJSON(vars) }} + run: | + echo "====== VARS ==============" + echo "$VARS_CONTEXT" + echo "====================================" + - name: Dump env context + env: + ENV_CONTEXT: ${{ toJSON(env) }} + run: | + echo "====== ENV ==============" + echo "$ENV_CONTEXT" + echo "====================================" diff --git a/.github/workflows/label-pullrequest.yml b/.github/workflows/label-pullrequest.yml new file mode 100644 index 0000000..5f9c071 --- /dev/null +++ b/.github/workflows/label-pullrequest.yml @@ -0,0 +1,38 @@ +# Adds labels to pull requests for the type of change the PR makes +name: Adds labels + +on: + pull_request: + types: [opened, synchronize, edited, ready_for_review] + +jobs: + label-pullrequest: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name + permissions: + contents: read + pull-requests: write + name: labels pull requests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + persist-credentials: false + + - name: Check for file changes + uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0 + id: changes + with: + token: ${{ github.token }} + filters: .github/file-filters.yml + + - name: Add Migration label + uses: actions-ecosystem/action-add-labels@v1 + if: steps.changes.outputs.migrations == 'true' + with: + labels: 'Contains new migration(s)' + + - name: Add Dependencies label + uses: actions-ecosystem/action-add-labels@v1 + if: steps.changes.outputs.dependencies == 'true' + with: + labels: 'Add/Change dependencies' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..2ba7924 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,91 @@ +name: Lint +on: + push: + branches: + - develop + - master + - staging + - release/* + - feature/* + - bugfix/* + - hotfix/* +# pull_request: +# branches: [develop, master] +# types: [synchronize, opened, reopened, ready_for_review] + +defaults: + run: + shell: bash + + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true + + +permissions: + contents: read + +jobs: + changes: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name + name: check files + runs-on: ubuntu-latest + timeout-minutes: 3 + outputs: + lint: ${{ steps.changes.outputs.lint }} + docker: ${{ steps.changes.outputs.docker_base }} + steps: + - run: git config --global --add safe.directory $(realpath .) + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - id: changes + name: Check for backend file changes + uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0 + with: + base: ${{ github.ref }} + token: ${{ github.token }} + filters: .github/file-filters.yml + + flake8: + needs: changes + runs-on: ubuntu-latest + if: github.event.pull_request.draft == false && needs.changes.outputs.lint + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install requirements + run: pip install flake8 pycodestyle + - name: Check syntax + # Stop the build if there are Python syntax errors or undefined names + run: flake8 src/ --count --statistics --max-line-length=127 + + - name: Warnings + run: flake8 src --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --extend-exclude="" + isort: + needs: changes + runs-on: ubuntu-latest + if: github.event.pull_request.draft == false && needs.changes.outputs.lint + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install requirements + run: pip install isort + - name: iSort + run: isort src/ --check-only + black: + needs: changes + runs-on: ubuntu-latest + if: github.event.pull_request.draft == false && needs.changes.outputs.lint + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install requirements + run: pip install black + - name: Black + run: black src/ --check diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml new file mode 100644 index 0000000..1efe142 --- /dev/null +++ b/.github/workflows/security.yml @@ -0,0 +1,77 @@ +name: Security +on: + push: + branches: + - develop + - master + - staging + - release/* + - feature/* + - bugfix/* + - hotfix/* +# pull_request: +# branches: [develop, master] +# types: [synchronize, opened, reopened, ready_for_review] + +defaults: + run: + shell: bash + + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true + + +permissions: + contents: read + +jobs: + changes: + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name + name: check files + runs-on: ubuntu-latest + timeout-minutes: 3 + outputs: + lint: ${{ steps.changes.outputs.lint }} + docker: ${{ steps.changes.outputs.docker_base }} + steps: + - run: git config --global --add safe.directory $(realpath .) + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - id: changes + name: Check for backend file changes + uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0 + with: + base: ${{ github.ref }} + token: ${{ github.token }} + filters: .github/file-filters.yml + + bandit: + needs: changes + runs-on: ubuntu-latest + if: github.event.pull_request.draft == false && needs.changes.outputs.lint + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + steps: + - uses: actions/checkout@v4 + - name: Bandit Scan + uses: shundor/python-bandit-scan@9cc5aa4a006482b8a7f91134412df6772dbda22c + with: # optional arguments + # exit with 0, even with results found + exit_zero: true # optional, default is DEFAULT + # Github token of the repository (automatically created by Github) + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information. + # File or directory to run bandit on + path: src # optional, default is . + # Report only issues of a given severity level or higher. Can be LOW, MEDIUM or HIGH. Default is UNDEFINED (everything) + # level: # optional, default is UNDEFINED + # Report only issues of a given confidence level or higher. Can be LOW, MEDIUM or HIGH. Default is UNDEFINED (everything) + # confidence: # optional, default is UNDEFINED + # comma-separated list of paths (glob patterns supported) to exclude from scan (note that these are in addition to the excluded paths provided in the config file) (default: .svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.eggs,*.egg) + # excluded_paths: # optional, default is DEFAULT + # comma-separated list of test IDs to skip + # skips: # optional, default is DEFAULT + # path to a .bandit file that supplies command line arguments + # ini_path: # optional, default is DEFAULT diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..1f896f8 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,215 @@ +name: Test + +on: + create: + branches: + - releases/* + push: + branches: + - develop + - master + - staging + - release/* + - feature/* + - bugfix/* + - hotfix/* +# pull_request: +# branches: [ develop, master ] +# types: [ synchronize, opened, reopened, ready_for_review ] + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true + +defaults: + run: + shell: bash + +permissions: + id-token: write + attestations: write + + +jobs: + changes: + if: (github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name) + || github.event_name == 'create' + runs-on: ubuntu-latest + timeout-minutes: 1 + defaults: + run: + shell: bash + outputs: + run_tests: ${{ steps.changes.outputs.run_tests }} + steps: + - name: Checkout code + uses: actions/checkout@v4.1.7 + - id: changes + name: Check for file changes + uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0 + with: + base: ${{ github.ref }} + token: ${{ github.token }} + filters: .github/file-filters.yml + - name: info + shell: bash + run: | + force_build="${{ contains(github.event.head_commit.message, 'ci:build') || contains(github.event.head_commit.message, 'ci:release')}}" + force_scan="${{ contains(github.event.head_commit.message, 'ci:scan') }}" + force_test="${{ contains(github.event.head_commit.message, 'ci:test') }}" + + if [[ $force_build == "true" ]]; then + echo "::notice:: Forced build docker due to commit message" + elif [[ $force_test == "true" ]]; then + echo "::notice:: Forced python tests due to commit message" + elif [[ $force_scan == "true" ]]; then + echo "::notice:: Forced trivy scan due to commit message" + fi + if [[ $force_build == "true" || "${{needs.changes.outputs.run_tests}}" == "true" ]]; then + echo "BUILD=true" >> $GITHUB_ENV + fi + + build: + needs: [ changes ] + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + shell: bash + outputs: + image: ${{ steps.build.outputs.image }} + version: ${{ steps.build.outputs.version }} + created: ${{ steps.build.outputs.created }} + steps: + - name: Checkout code + uses: actions/checkout@v4.1.7 + - id: checksum + uses: ./.github/actions/checksum + - name: Build Image + id: build + uses: ./.github/actions/docker_build + with: + dryrun: ${{ env.ACT || 'false' }} + rebuild: ${{ env.BUILD == 'true'}} + image: ${{ vars.DOCKER_IMAGE }} + target: 'python_dev_deps' + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + code_checksum: ${{ steps.checksum.outputs.checksum }} + + test: + name: Run Test Suite + needs: [ changes,build ] + if: (needs.changes.outputs.run_tests == 'true' + || contains(github.event.head_commit.message, 'ci:test') + || contains(github.event.head_commit.message, 'ci:all') + || github.event_name == 'create') + runs-on: ubuntu-latest + services: + redis: + image: redis + db: + image: postgres:14 + env: + POSTGRES_DATABASE: dedupe + POSTGRES_PASSWORD: postgres + POSTGRES_USERNAME: postgres + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + DOCKER_DEFAULT_PLATFORM: linux/amd64 + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Run tests + run: | + docker run --rm \ + -e DATABASE_URL=postgres://postgres:postgres@localhost:5432/dedupe \ + -e SECRET_KEY=secret_key \ + -e CACHE_URL=redis://redis:6379/0 \ + -e CELERY_BROKER_URL=redis://redis:6379/0 \ + --network host \ + -v $PWD:/code/app \ + -w /code/app \ + -t ${{needs.build.outputs.image}} \ + pytest tests -v --create-db -v --maxfail=10 + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + env_vars: OS,PYTHON + fail_ci_if_error: true + files: coverage.xml + token: ${{ secrets.CODECOV_TOKEN }} + verbose: false + name: codecov-${{env.GITHUB_REF_NAME}} + + deployable: + if: + contains(fromJSON('["refs/heads/develop", "refs/heads/staging", "refs/heads/master", "refs/heads/release"]'), github.ref) + || contains(github.event.head_commit.message, 'ci:release') + || contains(github.event.head_commit.message, 'ci:all') + + name: "Build deployable Docker" + needs: [ test ] + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + shell: bash + outputs: + image: ${{ steps.build.outputs.image }} + version: ${{ steps.build.outputs.version }} + created: ${{ steps.build.outputs.created }} + steps: + - name: Checkout code + uses: actions/checkout@v4.1.7 + - id: checksum + uses: ./.github/actions/checksum + - name: Build + id: build + uses: ./.github/actions/docker_build + with: + dryrun: ${{ env.ACT || 'false' }} + rebuild: ${{ env.BUILD == 'true'}} + image: ${{ vars.DOCKER_IMAGE }} + target: 'dist' + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + code_checksum: ${{ contains(github.event.head_commit.message, 'ci:build') && steps.checksum.outputs.checksum || '' }} + - shell: bash + run: | + echo "${{ toJSON(steps.build.outputs) }}" + + trivy: + name: Check Image with Trivy + runs-on: ubuntu-latest + needs: [ deployable ] + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + if: needs.release.outputs.created == 'true' + || contains(github.event.head_commit.message, 'ci:scan') + || contains(github.event.head_commit.message, 'ci:all') + || github.event_name == 'create' + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@master + with: + image-ref: ${{needs.deployable.outputs.image}} + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH' + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: 'trivy-results.sarif' diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bbbc9d4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.* +~* +!.dockerignore +!.gitignore +!.pdm-python +!.github +Makefile diff --git a/.pdm-python b/.pdm-python new file mode 100644 index 0000000..a9ff23a --- /dev/null +++ b/.pdm-python @@ -0,0 +1 @@ +/Users/sax/Documents/data/PROGETTI/UNICEF/hope-country-workspace/.venv/bin/python \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 9c0a544..12de2cc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -121,6 +121,8 @@ ARG VERSION ENV VERSION=$VERSION ARG BUILD_DATE ENV BUILD_DATE=$BUILD_DATE +ARG DISTRO +ENV DISTRO=$DISTRO ARG SOURCE_COMMIT ENV SOURCE_COMMIT=$SOURCE_COMMIT ARG GITHUB_SERVER_URL @@ -132,26 +134,29 @@ ENV GITHUB_REPOSITORY=$GITHUB_REPOSITORY LABEL date=$BUILD_DATE LABEL version=$VERSION LABEL checksum=$CHECKSUM +LABEL distro="test" +#COPY pyproject.toml pdm.lock ./ +#COPY docker/conf/config.toml /etc/xdg/pdm/config.toml +COPY . /code WORKDIR /code -COPY pyproject.toml pdm.lock ./ -COPY docker/conf/config.toml /etc/xdg/pdm/config.toml + +RUN set -x \ + && pip install -U pip pdm \ + && mkdir -p $PKG_DIR \ + && pdm sync --no-editable -v --no-self + RUN < /RELEASE {"version": "$VERSION", "commit": "$SOURCE_COMMIT", "date": "$BUILD_DATE", + "distro": "test", "checksum": "$CHECKSUM", - "source": "${GITHUB_SERVER_URL}/$${GITHUB_REPOSITORY}/tree/${SOURCE_COMMIT:-master}/" + "source": "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/tree/${SOURCE_COMMIT:-master}/" } EOF -RUN set -x \ - && pip install -U pip pdm \ - && mkdir -p $PKG_DIR \ - && pdm sync --no-editable -v --no-self - - -FROM python_dev_deps AS python_prod_deps +FROM build_deps AS python_prod_deps ARG PKG_DIR ARG CHECKSUM ENV CHECKSUM=$CHECKSUM @@ -169,15 +174,18 @@ ENV GITHUB_REPOSITORY=$GITHUB_REPOSITORY LABEL date=$BUILD_DATE LABEL version=$VERSION LABEL checksum=$CHECKSUM +LABEL distro="builder-prod" -WORKDIR /code COPY docker/conf/config.toml /etc/xdg/pdm/config.toml -COPY pyproject.toml pdm.lock ./ -COPY ./src /code/src +#COPY pyproject.toml pdm.lock /README.md /LICENSE ./ +#COPY ./src /code/src +COPY . /code +WORKDIR /code -RUN mkdir -p $PKG_DIR \ - && pip install -U pdm \ +RUN set -x \ + && pip install -U pip pdm \ + && mkdir -p $PKG_DIR \ && pdm sync --no-editable -v --prod @@ -197,34 +205,41 @@ ENV GITHUB_SERVER_URL=$GITHUB_SERVER_URL ARG GITHUB_REPOSITORY ENV GITHUB_REPOSITORY=$GITHUB_REPOSITORY + +WORKDIR /code +COPY --chown=user:app --from=python_prod_deps /code/__pypackages__ /code/__pypackages__ +COPY --chown=user:app --from=python_prod_deps /code/README.md /code/LICENSE / + +ENV PATH=${APATH}:${PATH} \ + PYTHONPATH=${APYTHONPATH} \ + PYTHONDBUFFERED=1 \ + PYTHONDONTWRITEBYTCODE=1 + RUN < /RELEASE {"version": "$VERSION", "commit": "$SOURCE_COMMIT", "date": "$BUILD_DATE", + "distro": "dist", "checksum": "$CHECKSUM", "source": "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/tree/${SOURCE_COMMIT:-master}/" } EOF -WORKDIR /code -COPY --chown=user:app --from=python_prod_deps /code /code -COPY --chown=user:app --from=python_prod_deps /RELEASE /RELEASE - VOLUME /var/run/app/ EXPOSE 8000 ENTRYPOINT exec docker-entrypoint.sh "$0" "$@" CMD ["run"] -LABEL maintainer="mnt@app.io" -LABEL org.opencontainers.image.authors="author@app.io" +LABEL distro="final" +LABEL maintainer="hope@unicef.org" +LABEL org.opencontainers.image.authors="hope@unicef.org" LABEL org.opencontainers.image.created="$BUILD_DATE" LABEL org.opencontainers.image.description="App runtime image" LABEL org.opencontainers.image.documentation="https://github.com/saxix/trash" LABEL org.opencontainers.image.licenses="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${SOURCE_COMMIT:-master}/LICENSE" LABEL org.opencontainers.image.revision=$SOURCE_COMMIT LABEL org.opencontainers.image.source="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/tree/${SOURCE_COMMIT:-master}/" -LABEL org.opencontainers.image.title="App" -LABEL org.opencontainers.image.url="https://app.io/" -LABEL org.opencontainers.image.vendor="App ltd" +LABEL org.opencontainers.image.title="Hope Deduplication Engine" LABEL org.opencontainers.image.version="$VERSION" - +#LABEL org.opencontainers.image.url="https://app.io/" +#LABEL org.opencontainers.image.vendor="App ltd" diff --git a/docker/Makefile b/docker/Makefile deleted file mode 100644 index d013ae6..0000000 --- a/docker/Makefile +++ /dev/null @@ -1,184 +0,0 @@ -BUILD_DATE:=$(shell date +"%Y-%m-%d %H:%M") -CHECKSUM?=$(shell sha1sum ../pdm.lock | awk '{print $$1}') - -VERSION?=0.1.0 -HASH_SEEDS=pdm.lock docker/bin/* docker/conf/* docker/Dockerfile -AA=$(cd .. && sha1sum ${HASH_SEEDS}) -LOCK_SHA?=$(shell cd .. && sha1sum ${HASH_SEEDS} | sha1sum | awk '{print $1}' | cut -c 1-8) - -COMMIT_SHA?=$(shell git rev-parse --short HEAD) -CI_REGISTRY_IMAGE?="" -CONTAINER_NAME?=hde - - -define PRINT_HELP_PYSCRIPT -import re, sys - -for line in sys.stdin: - match = re.match(r'^([a-zA-Z0-9_-]+):.*?## (.*)$$', line) - if match: - target, help = match.groups() - print("%-20s %s" % (target, help)) -endef -export PRINT_HELP_PYSCRIPT - -help: - @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) - - -# --cache-from "${CI_REGISTRY_IMAGE}/base:${LOCK_SHA}" \ -# --cache-from "${CI_REGISTRY_IMAGE}/deps:${LOCK_SHA}" \ -# --cache-from "${CI_REGISTRY_IMAGE}/builder:${LOCK_SHA}" \ - -sha: -# @cd .. && sha1sum ${HASH_SEEDS} | sha1sum | awk '{print $1}' | cut -c 1-8 - @echo ${LOCK_SHA} -xxx: - docker run -it --rm -t python:3.12-slim-bookworm /bin/bash - - -.build: - @echo ${DOCKERHUB_TOKEN} | docker login -u saxix --password-stdin - cd .. && docker buildx build \ - --progress=plain \ - --build-arg BUILDKIT_INLINE_CACHE=1 \ - --build-arg VERSION=${VERSION} \ - --build-arg BUILD_DATE="${BUILD_DATE}" \ - --build-arg CHECKSUM="${LOCK_SHA}" \ - --build-arg COMMIT="${COMMIT_SHA}" \ - ${EXTRA} \ - -t "${DOCKER_IMAGE}:${TAG}" \ - --push \ - -f docker/Dockerfile . - docker image ls "${DOCKER_IMAGE}:${TAG}" - @docker inspect -f "{{json .Config.Labels}}" ${DOCKER_IMAGE}:${TAG} - -base: ## build 'builder' image - STAGE="python_base" \ - EXTRA='--target python_base' \ - DOCKER_IMAGE="${CI_REGISTRY_IMAGE}" \ - TAG="base-${LOCK_SHA}" \ - $(MAKE) .build - -buildDeps: ## build 'builder' image - STAGE="build_deps" \ - EXTRA='--cache-from "${CI_REGISTRY_IMAGE}:base-${LOCK_SHA}" --target build_deps' \ - DOCKER_IMAGE="${CI_REGISTRY_IMAGE}" \ - TAG="deps-${LOCK_SHA}" \ - $(MAKE) .build - -pythonDevDeps: ## build 'builder' image - STAGE="python_dev_deps" \ - EXTRA='--cache-from "${CI_REGISTRY_IMAGE}:dev-${LOCK_SHA}" --target python_dev_deps' \ - DOCKER_IMAGE="${CI_REGISTRY_IMAGE}" \ - TAG="dev-${LOCK_SHA}" \ - $(MAKE) .build - -pythonProdDeps: ## build 'builder' image - STAGE="python_prod_deps" \ - EXTRA='--cache-from "${CI_REGISTRY_IMAGE}:deps-${LOCK_SHA}" --target python_prod_deps' \ - DOCKER_IMAGE="${CI_REGISTRY_IMAGE}" \ - TAG="python_prod_deps-${LOCK_SHA}" \ - $(MAKE) .build - -#dev: ## build dev image -# STAGE='dev' \ -# EXTRA='--cache-from "${CI_REGISTRY_IMAGE}:python_dev_deps-${LOCK_SHA}" --target dev' \ -# DOCKER_IMAGE="${CI_REGISTRY_IMAGE}" \ -# TAG="dev" \ -# $(MAKE) .build - - -#test: ## build test image -# EXTRA='--target test ' \ -# DOCKER_IMAGE="${CI_REGISTRY_IMAGE}/test" \ -# TAG="${COMMIT_SHA}" \ -# $(MAKE) .build - -dist: ## build prod image - STAGE='dist' \ - EXTRA='--cache-from "${CI_REGISTRY_IMAGE}:base-${LOCK_SHA}" \ - --cache-from "${CI_REGISTRY_IMAGE}:deps-${LOCK_SHA}" --target dist' \ - DOCKER_IMAGE="${CI_REGISTRY_IMAGE}" \ - TAG="${VERSION}" \ - $(MAKE) .build - docker tag ${CI_REGISTRY_IMAGE}:${VERSION} ${CI_REGISTRY_IMAGE}:latest - -push: ## build prod image - - echo ${IMAGE} - -all: base buildDeps pythonDevDeps pythonProdDeps dev info - -info: - docker image ls "${CI_REGISTRY_IMAGE}" - -check: - DOCKER_IMAGE=${CI_REGISTRY_IMAGE}:${VERSION} \ - RUN_OPTIONS=-it \ - CMD="django-admin check" \ - $(MAKE) .run - -# DOCKER_IMAGE="${CI_REGISTRY_IMAGE}" \ -# TAG="${VERSION}-${COMMIT}" \ -# $(MAKE) .build - -.run: - cd .. && docker run \ - --rm \ - --name=${CONTAINER_NAME} \ - -p 8000:8000 \ - -e ADMINS="${ADMINS}" \ - -e ADMIN_EMAIL="${ADMIN_EMAIL}" \ - -e ADMIN_PASSWORD="${ADMIN_PASSWORD}" \ - -e ALLOWED_HOSTS="*" \ - -e CACHE_URL="${CACHE_URL}" \ - -e CELERY_BROKER_URL="${CELERY_BROKER_URL}" \ - -e CSRF_COOKIE_SECURE="${CSRF_COOKIE_SECURE}" \ - -e DATABASE_URL="${DATABASE_URL}" \ - -e DEBUG="0" \ - -e INIT_RUN_UPGRADE=1 \ - -e MEDIA_ROOT="/tmp/media" \ - -e SECRET_KEY="${SECRET_KEY}" \ - -e SENTRY_DSN="${SENTRY_DSN}" \ - -e STATIC_ROOT="/tmp/static" \ - ${RUN_OPTIONS} \ - -t "${DOCKER_IMAGE}" \ - ${CMD} - -# -v ${PWD}/docker/bin/docker-entrypoint.sh:/usr/local/bin/docker-entrypoint.sh \ -# -v ${PWD}/docker/conf/uwsgi.ini:/conf/uwsgi.ini \ - -shell: ## run production image - DOCKER_IMAGE=${CI_REGISTRY_IMAGE}:${VERSION} \ - RUN_OPTIONS=-it \ - CMD=/bin/bash \ - $(MAKE) .run - - -shell-target: ## run production image - DOCKER_IMAGE="${CI_REGISTRY_IMAGE}/$S:latest" \ - RUN_OPTIONS=-it \ - CMD=/bin/bash \ - $(MAKE) .run - -run: ## run production image - DOCKER_IMAGE=${CI_REGISTRY_IMAGE}:${VERSION} \ - CMD=run \ - $(MAKE) .run - -test: - DOCKER_IMAGE=${CI_REGISTRY_IMAGE}/python_dev_deps:${LOCK_SHA} \ - RUN_OPTIONS="-v .:/hde/code/ -w /hde/code/ -it" \ - CMD="pytest tests/ --create-db -v; ls -al " \ - $(MAKE) .run -# -#run-target: ## run production image -# DOCKER_IMAGE="${CI_REGISTRY_IMAGE}/$S:latest" \ -# $(MAKE) .run -# -#test: ## run production image -# DOCKER_IMAGE="${CI_REGISTRY_IMAGE}/tests:${COMMIT}" \ -# RUN_OPTIONS=-it \ -# CMD="/bin/bash" \ -# $(MAKE) .run diff --git a/docker/bin/docker-entrypoint.sh b/docker/bin/docker-entrypoint.sh index 7c67a60..455425b 100755 --- a/docker/bin/docker-entrypoint.sh +++ b/docker/bin/docker-entrypoint.sh @@ -4,10 +4,13 @@ export MEDIA_ROOT="${MEDIA_ROOT:-/var/run/app/media}" export STATIC_ROOT="${STATIC_ROOT:-/var/run/app/static}" export UWSGI_PROCESSES="${UWSGI_PROCESSES:-"4"}" +export DJANGO_SETTINGS_MODULE="${DJANGO_SETTINGS_MODULE:-"hope_dedup_engine.config.settings"}" mkdir -p "${MEDIA_ROOT}" "${STATIC_ROOT}" || echo "Cannot create dirs ${MEDIA_ROOT} ${STATIC_ROOT}" case "$1" in run) + django-admin check --deploy + django-admin upgrade set -- tini -- "$@" set -- gosu user:app uwsgi --ini /conf/uwsgi.ini ;; diff --git a/docker/conf/circus.conf b/docker/conf/circus.conf deleted file mode 100644 index 8d59544..0000000 --- a/docker/conf/circus.conf +++ /dev/null @@ -1,74 +0,0 @@ -[circus] -check_delay = 5 -endpoint = tcp://127.0.0.1:5555 -pubsub_endpoint = tcp://127.0.0.1:5556 -umask = 002 -working_dir = $(CIRCUS.ENV.PWD) -debug = false -stdout_stream.class = StdoutStream -stderr_stream.class = StdoutStream - -# [watcher:web] -# cmd = nginx -# args = -c /conf/nginx.conf -# user = www -# group = sos -# use_sockets = True -# copy_env = true -# autostart = $(CIRCUS.ENV.INIT_START_WEB) - -[watcher:app] -cmd = uwsgi -args = --ini /conf/uwsgi.ini -user = www -group = bitcaster -use_sockets = True -copy_env = true -autostart = $(CIRCUS.ENV.INIT_START_BOB) -numprocesses = 1 -send_hup = True -stop_signal = QUIT -warmup_delay = 0 - -# [watcher:clearly] -# cmd = clearly -# args = server $(CIRCUS.ENV.CELERY_BROKER_URL) -p $(CIRCUS.ENV.CLEARLY_PORT) -# user = www -# group = sos -# use_sockets = True -# copy_env = true -# autostart = $(CIRCUS.ENV.INIT_START_CLEARLY) - - -[watcher:daphne] -cmd = daphne -args = -b 0.0.0.0 -p 8001 bitcaster.config.asgi:application -user = www -group = bitcaster -copy_env = true -autostart = $(CIRCUS.ENV.INIT_START_DAPHNE) - -[watcher:celery-worker] -cmd = celery -args = -A bitcaster.config.celery worker -E --loglevel=ERROR --concurrency=4 --uid www --gid sos -user = www -group = bitcaster -copy_env = true -autostart = $(CIRCUS.ENV.INIT_START_CELERY) -warmup_delay = 0 - -[watcher:celery-beat] -cmd = celery -args = -A bitcaster.config.celery beat --loglevel=ERROR --scheduler django_celery_beat.schedulers:DatabaseScheduler -user = www -group = bitcaster -copy_env = true -autostart = $(CIRCUS.ENV.INIT_START_BEAT) - -[watcher:flower] -cmd = celery -args = -A bitcaster.config.celery -b ${CIRCUS.ENV.CELERY_BROKER_URL} flower --loglevel=ERROR --auth='.*@os4d\.org' --url-prefix=flower --purge-offline-workers=3600 -user = www -group = bitcaster -copy_env = true -autostart = $(CIRCUS.ENV.INIT_START_FLOWER) diff --git a/docker/conf/uwsgi.ini b/docker/conf/uwsgi.ini index 919740d..2492291 100644 --- a/docker/conf/uwsgi.ini +++ b/docker/conf/uwsgi.ini @@ -3,7 +3,7 @@ http=0.0.0.0:8000 enable-threads=0 honour-range=1 master=1 -module=trash.wsgi +module=hope_dedup_engine.config.wsgi processes=$(UWSGI_PROCESSES) ;virtualenv=/code/.venv/ ;virtualenv=%(_) From 00265df7e5e883e62e8a684eca00e03506824417 Mon Sep 17 00:00:00 2001 From: saxix Date: Fri, 5 Jul 2024 17:18:14 +0200 Subject: [PATCH 02/16] updates --- .github/workflows/test.yml | 4 +- pdm.lock | 229 +++++++++++++++++- pyproject.toml | 9 +- pytest.ini | 2 +- src/hope_country_workspace/admin_site/site.py | 12 +- .../config/fragments/constance.py | 1 - src/hope_country_workspace/config/settings.py | 18 +- .../management/commands/demo.py | 16 +- .../management/commands/env.py | 26 +- .../management/commands/upgrade.py | 23 +- .../middleware/exception.py | 22 +- .../middleware/state.py | 7 +- .../models/household.py | 1 + src/hope_country_workspace/security/models.py | 3 + src/hope_country_workspace/state.py | 12 +- src/hope_country_workspace/tenant/sites.py | 30 +-- src/hope_country_workspace/tenant/utils.py | 4 +- src/hope_country_workspace/types/__init__.py | 0 src/hope_country_workspace/types/django.pyi | 9 + src/hope_country_workspace/types/hope.pyi | 0 src/hope_country_workspace/types/http.pyi | 26 ++ src/hope_country_workspace/utils/__init__.py | 13 + src/hope_country_workspace/utils/flags.py | 20 +- .../web/templatetags/filters.py | 5 +- tests/conftest.py | 89 +++++++ tests/extras/testutils/__init__.py | 0 tests/extras/testutils/decorators.py | 15 ++ tests/extras/testutils/factories/__init__.py | 49 ++++ tests/extras/testutils/factories/base.py | 23 ++ .../testutils/factories/contenttypes.py | 12 + .../extras/testutils/factories/django_auth.py | 32 +++ .../testutils/factories/django_celery_beat.py | 47 ++++ tests/extras/testutils/factories/household.py | 11 + tests/extras/testutils/factories/log.py | 11 + tests/extras/testutils/factories/social.py | 12 + tests/extras/testutils/factories/user.py | 49 ++++ tests/extras/testutils/factories/userrole.py | 23 ++ tests/extras/testutils/perms.py | 169 +++++++++++++ tests/tenant/test_tenant_backend.py | 127 ++++++++++ tests/tenant/test_tenant_filtering.py | 38 +++ tests/tenant/test_tenant_manager.py | 84 +++++++ tests/tenant/test_tenant_middleware.py | 21 ++ tests/tenant/test_tenant_permissions.py | 91 +++++++ 43 files changed, 1329 insertions(+), 66 deletions(-) create mode 100644 src/hope_country_workspace/types/__init__.py create mode 100644 src/hope_country_workspace/types/django.pyi create mode 100644 src/hope_country_workspace/types/hope.pyi create mode 100644 src/hope_country_workspace/types/http.pyi create mode 100644 tests/conftest.py create mode 100644 tests/extras/testutils/__init__.py create mode 100644 tests/extras/testutils/decorators.py create mode 100644 tests/extras/testutils/factories/__init__.py create mode 100644 tests/extras/testutils/factories/base.py create mode 100644 tests/extras/testutils/factories/contenttypes.py create mode 100644 tests/extras/testutils/factories/django_auth.py create mode 100644 tests/extras/testutils/factories/django_celery_beat.py create mode 100644 tests/extras/testutils/factories/household.py create mode 100644 tests/extras/testutils/factories/log.py create mode 100644 tests/extras/testutils/factories/social.py create mode 100644 tests/extras/testutils/factories/user.py create mode 100644 tests/extras/testutils/factories/userrole.py create mode 100644 tests/extras/testutils/perms.py create mode 100644 tests/tenant/test_tenant_backend.py create mode 100644 tests/tenant/test_tenant_filtering.py create mode 100644 tests/tenant/test_tenant_manager.py create mode 100644 tests/tenant/test_tenant_middleware.py create mode 100644 tests/tenant/test_tenant_permissions.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1f896f8..2a64f03 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -112,7 +112,7 @@ jobs: db: image: postgres:14 env: - POSTGRES_DATABASE: dedupe + POSTGRES_DATABASE: country_workspace POSTGRES_PASSWORD: postgres POSTGRES_USERNAME: postgres ports: @@ -130,7 +130,7 @@ jobs: - name: Run tests run: | docker run --rm \ - -e DATABASE_URL=postgres://postgres:postgres@localhost:5432/dedupe \ + -e DATABASE_URL=postgres://postgres:postgres@localhost:5432/country_workspace \ -e SECRET_KEY=secret_key \ -e CACHE_URL=redis://redis:6379/0 \ -e CELERY_BROKER_URL=redis://redis:6379/0 \ diff --git a/pdm.lock b/pdm.lock index 833f47b..a3f16e9 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.1" -content_hash = "sha256:b014daf25ac65c1ce6ee1edf205f03ea0676acfccc0a37b8920b027df8de2dbf" +content_hash = "sha256:8744d30ad35acc7ffb7119daa600cc8b2557e66f380d27f20ecc3104ba587ae6" [[package]] name = "amqp" @@ -43,6 +43,20 @@ files = [ {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, ] +[[package]] +name = "beautifulsoup4" +version = "4.12.3" +requires_python = ">=3.6.0" +summary = "Screen-scraping library" +groups = ["dev"] +dependencies = [ + "soupsieve>1.2", +] +files = [ + {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"}, + {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"}, +] + [[package]] name = "billiard" version = "4.2.0" @@ -227,6 +241,52 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "coverage" +version = "7.5.4" +requires_python = ">=3.8" +summary = "Code coverage measurement for Python" +groups = ["dev"] +files = [ + {file = "coverage-7.5.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:54317c2b806354cbb2dc7ac27e2b93f97096912cc16b18289c5d4e44fc663233"}, + {file = "coverage-7.5.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:042183de01f8b6d531e10c197f7f0315a61e8d805ab29c5f7b51a01d62782747"}, + {file = "coverage-7.5.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6bb74ed465d5fb204b2ec41d79bcd28afccf817de721e8a807d5141c3426638"}, + {file = "coverage-7.5.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3d45ff86efb129c599a3b287ae2e44c1e281ae0f9a9bad0edc202179bcc3a2e"}, + {file = "coverage-7.5.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5013ed890dc917cef2c9f765c4c6a8ae9df983cd60dbb635df8ed9f4ebc9f555"}, + {file = "coverage-7.5.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1014fbf665fef86cdfd6cb5b7371496ce35e4d2a00cda501cf9f5b9e6fced69f"}, + {file = "coverage-7.5.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3684bc2ff328f935981847082ba4fdc950d58906a40eafa93510d1b54c08a66c"}, + {file = "coverage-7.5.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:581ea96f92bf71a5ec0974001f900db495488434a6928a2ca7f01eee20c23805"}, + {file = "coverage-7.5.4-cp312-cp312-win32.whl", hash = "sha256:73ca8fbc5bc622e54627314c1a6f1dfdd8db69788f3443e752c215f29fa87a0b"}, + {file = "coverage-7.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:cef4649ec906ea7ea5e9e796e68b987f83fa9a718514fe147f538cfeda76d7a7"}, + {file = "coverage-7.5.4-pp38.pp39.pp310-none-any.whl", hash = "sha256:79b356f3dd5b26f3ad23b35c75dbdaf1f9e2450b6bcefc6d0825ea0aa3f86ca5"}, + {file = "coverage-7.5.4.tar.gz", hash = "sha256:a44963520b069e12789d0faea4e9fdb1e410cdc4aab89d94f7f55cbb7fef0353"}, +] + +[[package]] +name = "coverage" +version = "7.5.4" +extras = ["toml"] +requires_python = ">=3.8" +summary = "Code coverage measurement for Python" +groups = ["dev"] +dependencies = [ + "coverage==7.5.4", +] +files = [ + {file = "coverage-7.5.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:54317c2b806354cbb2dc7ac27e2b93f97096912cc16b18289c5d4e44fc663233"}, + {file = "coverage-7.5.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:042183de01f8b6d531e10c197f7f0315a61e8d805ab29c5f7b51a01d62782747"}, + {file = "coverage-7.5.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6bb74ed465d5fb204b2ec41d79bcd28afccf817de721e8a807d5141c3426638"}, + {file = "coverage-7.5.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3d45ff86efb129c599a3b287ae2e44c1e281ae0f9a9bad0edc202179bcc3a2e"}, + {file = "coverage-7.5.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5013ed890dc917cef2c9f765c4c6a8ae9df983cd60dbb635df8ed9f4ebc9f555"}, + {file = "coverage-7.5.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1014fbf665fef86cdfd6cb5b7371496ce35e4d2a00cda501cf9f5b9e6fced69f"}, + {file = "coverage-7.5.4-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:3684bc2ff328f935981847082ba4fdc950d58906a40eafa93510d1b54c08a66c"}, + {file = "coverage-7.5.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:581ea96f92bf71a5ec0974001f900db495488434a6928a2ca7f01eee20c23805"}, + {file = "coverage-7.5.4-cp312-cp312-win32.whl", hash = "sha256:73ca8fbc5bc622e54627314c1a6f1dfdd8db69788f3443e752c215f29fa87a0b"}, + {file = "coverage-7.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:cef4649ec906ea7ea5e9e796e68b987f83fa9a718514fe147f538cfeda76d7a7"}, + {file = "coverage-7.5.4-pp38.pp39.pp310-none-any.whl", hash = "sha256:79b356f3dd5b26f3ad23b35c75dbdaf1f9e2450b6bcefc6d0825ea0aa3f86ca5"}, + {file = "coverage-7.5.4.tar.gz", hash = "sha256:a44963520b069e12789d0faea4e9fdb1e410cdc4aab89d94f7f55cbb7fef0353"}, +] + [[package]] name = "cron-descriptor" version = "1.4.3" @@ -591,6 +651,19 @@ files = [ {file = "django_timezone_field-6.1.0.tar.gz", hash = "sha256:d40f7059d7bae4075725d04a9dae601af9fe3c7f0119a69b0e2c6194a782f797"}, ] +[[package]] +name = "django-webtest" +version = "1.9.11" +summary = "Instant integration of Ian Bicking's WebTest (http://docs.pylonsproject.org/projects/webtest/) with Django's testing framework." +groups = ["dev"] +dependencies = [ + "webtest>=1.3.3", +] +files = [ + {file = "django-webtest-1.9.11.tar.gz", hash = "sha256:9597d26ced599bc5d4d9366bb451469fc9707b4779f79543cdf401ae6c5aeb35"}, + {file = "django_webtest-1.9.11-py3-none-any.whl", hash = "sha256:e29baf8337e7fe7db41ce63ca6661f7b5c77fe56f506f48b305e09313f5475b4"}, +] + [[package]] name = "djangorestframework" version = "3.15.2" @@ -704,6 +777,17 @@ files = [ {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] +[[package]] +name = "inflection" +version = "0.5.1" +requires_python = ">=3.5" +summary = "A port of Ruby on Rails inflector to Python" +groups = ["dev"] +files = [ + {file = "inflection-0.5.1-py2.py3-none-any.whl", hash = "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2"}, + {file = "inflection-0.5.1.tar.gz", hash = "sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417"}, +] + [[package]] name = "iniconfig" version = "2.0.0" @@ -1009,6 +1093,47 @@ files = [ {file = "pytest_celery-1.0.0.tar.gz", hash = "sha256:17a066b1554d4fa8797d4928e8b8cda1bfb441dae4688ca29fdbde28ffa49ff7"}, ] +[[package]] +name = "pytest-cov" +version = "5.0.0" +requires_python = ">=3.8" +summary = "Pytest plugin for measuring coverage." +groups = ["dev"] +dependencies = [ + "coverage[toml]>=5.2.1", + "pytest>=4.6", +] +files = [ + {file = "pytest-cov-5.0.0.tar.gz", hash = "sha256:5837b58e9f6ebd335b0f8060eecce69b662415b16dc503883a02f45dfeb14857"}, + {file = "pytest_cov-5.0.0-py3-none-any.whl", hash = "sha256:4f0764a1219df53214206bf1feea4633c3b558a2925c8b59f144f682861ce652"}, +] + +[[package]] +name = "pytest-cover" +version = "3.0.0" +summary = "Pytest plugin for measuring coverage. Forked from `pytest-cov`." +groups = ["dev"] +dependencies = [ + "pytest-cov>=2.0", +] +files = [ + {file = "pytest-cover-3.0.0.tar.gz", hash = "sha256:5bdb6c1cc3dd75583bb7bc2c57f5e1034a1bfcb79d27c71aceb0b16af981dbf4"}, + {file = "pytest_cover-3.0.0-py2.py3-none-any.whl", hash = "sha256:578249955eb3b5f3991209df6e532bb770b647743b7392d3d97698dc02f39ebb"}, +] + +[[package]] +name = "pytest-coverage" +version = "0.0" +summary = "Pytest plugin for measuring coverage. Forked from `pytest-cov`." +groups = ["dev"] +dependencies = [ + "pytest-cover", +] +files = [ + {file = "pytest-coverage-0.0.tar.gz", hash = "sha256:db6af2cbd7e458c7c9fd2b4207cee75258243c8a81cad31a7ee8cfad5be93c05"}, + {file = "pytest_coverage-0.0-py2.py3-none-any.whl", hash = "sha256:dedd084c5e74d8e669355325916dc011539b190355021b037242514dee546368"}, +] + [[package]] name = "pytest-django" version = "4.8.0" @@ -1052,6 +1177,24 @@ files = [ {file = "pytest_echo-1.7.3-py2.py3-none-any.whl", hash = "sha256:683f4d2fef8dd701aeaf47db834ccc114d43f580abcfea53f3ce2ffe8166c3c0"}, ] +[[package]] +name = "pytest-factoryboy" +version = "2.7.0" +requires_python = ">=3.8" +summary = "Factory Boy support for pytest." +groups = ["dev"] +dependencies = [ + "factory-boy>=2.10.0", + "inflection", + "packaging", + "pytest>=6.2", + "typing-extensions", +] +files = [ + {file = "pytest_factoryboy-2.7.0-py3-none-any.whl", hash = "sha256:bf3222db22d954fbf46f4bff902a0a8d82f3fc3594a47c04bbdc0546ff4c59a6"}, + {file = "pytest_factoryboy-2.7.0.tar.gz", hash = "sha256:67fc54ec8669a3feb8ac60094dd57cd71eb0b20b2c319d2957873674c776a77b"}, +] + [[package]] name = "python-crontab" version = "3.2.0" @@ -1114,6 +1257,23 @@ files = [ {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, ] +[[package]] +name = "pyyaml" +version = "6.0.1" +requires_python = ">=3.6" +summary = "YAML parser and emitter for Python" +groups = ["dev"] +files = [ + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + [[package]] name = "redis" version = "5.0.7" @@ -1157,6 +1317,22 @@ files = [ {file = "requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36"}, ] +[[package]] +name = "responses" +version = "0.25.3" +requires_python = ">=3.8" +summary = "A utility library for mocking out the `requests` Python library." +groups = ["dev"] +dependencies = [ + "pyyaml", + "requests<3.0,>=2.30.0", + "urllib3<3.0,>=1.25.10", +] +files = [ + {file = "responses-0.25.3-py3-none-any.whl", hash = "sha256:521efcbc82081ab8daa588e08f7e8a64ce79b91c39f6e62199b19159bea7dbcb"}, + {file = "responses-0.25.3.tar.gz", hash = "sha256:617b9247abd9ae28313d57a75880422d55ec63c29d33d629697590a034358dba"}, +] + [[package]] name = "retry" version = "0.9.2" @@ -1243,6 +1419,17 @@ files = [ {file = "social_auth_core-4.5.4-py3-none-any.whl", hash = "sha256:33cf970a623c442376f9d4a86fb187579e4438649daa5b5be993d05e74d7b2db"}, ] +[[package]] +name = "soupsieve" +version = "2.5" +requires_python = ">=3.8" +summary = "A modern CSS selector implementation for Beautiful Soup." +groups = ["dev"] +files = [ + {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, + {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, +] + [[package]] name = "sqlparse" version = "0.5.0" @@ -1259,7 +1446,7 @@ name = "typing-extensions" version = "4.12.2" requires_python = ">=3.8" summary = "Backported and Experimental Type Hints for Python 3.8+" -groups = ["default"] +groups = ["default", "dev"] files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, @@ -1329,6 +1516,17 @@ files = [ {file = "vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0"}, ] +[[package]] +name = "waitress" +version = "3.0.0" +requires_python = ">=3.8.0" +summary = "Waitress WSGI server" +groups = ["dev"] +files = [ + {file = "waitress-3.0.0-py3-none-any.whl", hash = "sha256:2a06f242f4ba0cc563444ca3d1998959447477363a2d7e9b8b4d75d35cfd1669"}, + {file = "waitress-3.0.0.tar.gz", hash = "sha256:005da479b04134cdd9dd602d1ee7c49d79de0537610d653674cc6cbde222b8a1"}, +] + [[package]] name = "wcwidth" version = "0.2.13" @@ -1339,6 +1537,33 @@ files = [ {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, ] +[[package]] +name = "webob" +version = "1.8.7" +requires_python = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*" +summary = "WSGI request and response object" +groups = ["dev"] +files = [ + {file = "WebOb-1.8.7-py2.py3-none-any.whl", hash = "sha256:73aae30359291c14fa3b956f8b5ca31960e420c28c1bec002547fb04928cf89b"}, + {file = "WebOb-1.8.7.tar.gz", hash = "sha256:b64ef5141be559cfade448f044fa45c2260351edcb6a8ef6b7e00c7dcef0c323"}, +] + +[[package]] +name = "webtest" +version = "3.0.0" +requires_python = ">=3.6, <4" +summary = "Helper to test WSGI applications" +groups = ["dev"] +dependencies = [ + "WebOb>=1.2", + "beautifulsoup4", + "waitress>=0.8.5", +] +files = [ + {file = "WebTest-3.0.0-py3-none-any.whl", hash = "sha256:2a001a9efa40d2a7e5d9cd8d1527c75f41814eb6afce2c3d207402547b1e5ead"}, + {file = "WebTest-3.0.0.tar.gz", hash = "sha256:54bd969725838d9861a9fa27f8d971f79d275d94ae255f5c501f53bb6d9929eb"}, +] + [[package]] name = "wmctrl" version = "0.5" diff --git a/pyproject.toml b/pyproject.toml index 61d583e..1f4ecab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,10 @@ dev = [ "isort>=5.13.2", "flake8>=7.1.0", "pdbpp>=0.10.3", + "pytest-coverage>=0.0", + "responses>=0.25.3", + "pytest-factoryboy>=2.7.0", + "django-webtest>=1.9.11", ] [tool.black] @@ -69,7 +73,6 @@ exclude = ''' )/ ''' - [tool.isort] profile = "black" line_length = 88 @@ -79,3 +82,7 @@ known_django = "django" sections = ["FUTURE","STDLIB","DJANGO","THIRDPARTY","FIRSTPARTY","LOCALFOLDER"] include_trailing_comma = true skip = ["migrations", "snapshots", ".venv"] + + +[tool.django-stubs] +django_settings_module = "hope_country_workspace.config.settings" diff --git a/pytest.ini b/pytest.ini index a4982ca..b1ade59 100644 --- a/pytest.ini +++ b/pytest.ini @@ -14,7 +14,7 @@ tmp_path_retention_count=0 addopts = --tb=short --capture=sys - --cov hope_flex_fields + --cov hope_country_workspace --cov-config=tests/.coveragerc --cov-report html --cov-report xml:coverage.xml diff --git a/src/hope_country_workspace/admin_site/site.py b/src/hope_country_workspace/admin_site/site.py index 1fab79f..13a1fcd 100644 --- a/src/hope_country_workspace/admin_site/site.py +++ b/src/hope_country_workspace/admin_site/site.py @@ -1,14 +1,12 @@ import logging from typing import Any -from django.conf import settings from django.http import HttpRequest -from django.utils.text import slugify from django.utils.translation import gettext_lazy from hope_country_workspace.tenant.forms import TenantAuthenticationForm from hope_country_workspace.tenant.sites import TenantAdminSite -from hope_country_workspace.tenant.utils import get_selected_tenant, must_tenant, is_hq_active +from hope_country_workspace.tenant.utils import get_selected_tenant, is_hq_active logger = logging.getLogger(__name__) @@ -27,9 +25,13 @@ def _build_app_dict(self, request: "HttpRequest", label=None) -> dict[str, Any]: app_dict = {} for app_label, data in original_app_dict.items(): if is_hq_active(): - data["models"] = [m for m in data["models"] if not hasattr(m["model"], "Tenant")] + data["models"] = [ + m for m in data["models"] if not hasattr(m["model"], "Tenant") + ] else: - data["models"] = [m for m in data["models"] if hasattr(m["model"], "Tenant")] + data["models"] = [ + m for m in data["models"] if hasattr(m["model"], "Tenant") + ] if data["models"]: app_dict[app_label] = data return app_dict diff --git a/src/hope_country_workspace/config/fragments/constance.py b/src/hope_country_workspace/config/fragments/constance.py index 3934979..c317e06 100644 --- a/src/hope_country_workspace/config/fragments/constance.py +++ b/src/hope_country_workspace/config/fragments/constance.py @@ -7,7 +7,6 @@ "Group to assign to any new user", str, ), - } diff --git a/src/hope_country_workspace/config/settings.py b/src/hope_country_workspace/config/settings.py index 1c5f5cd..089e6f1 100644 --- a/src/hope_country_workspace/config/settings.py +++ b/src/hope_country_workspace/config/settings.py @@ -53,7 +53,6 @@ "django.middleware.clickjacking.XFrameOptionsMiddleware", # "unicef_security.middleware.UNICEFSocialAuthExceptionMiddleware", "hope_country_workspace.middleware.state.StateClearMiddleware", - ) AUTHENTICATION_BACKENDS = ( @@ -169,6 +168,21 @@ "level": "DEBUG", "propagate": True, }, + "celery": { + "handlers": ["console"], + "level": "ERROR", + "propagate": False, + }, + "faker": { + "handlers": ["console"], + "level": "ERROR", + "propagate": False, + }, + "factory": { + "handlers": ["console"], + "level": "ERROR", + "propagate": False, + }, }, } @@ -194,7 +208,7 @@ from .fragments.rest_framework import * # noqa from .fragments.root import * # noqa from .fragments.sentry import * # noqa +from .fragments.smart_admin import * # noqa from .fragments.social_auth import * # noqa from .fragments.spectacular import * # noqa from .fragments.storages import * # noqa -from .fragments.smart_admin import * # noqa diff --git a/src/hope_country_workspace/management/commands/demo.py b/src/hope_country_workspace/management/commands/demo.py index 88abcfc..490529e 100644 --- a/src/hope_country_workspace/management/commands/demo.py +++ b/src/hope_country_workspace/management/commands/demo.py @@ -1,6 +1,5 @@ -from typing import Any - import logging +from typing import Any from django.conf import settings from django.contrib.auth.models import Group @@ -31,9 +30,16 @@ def handle(self, *args: Any, **options: Any) -> None: Site.objects.clear_cache() for flag in settings.FLAGS.keys(): - FlagState.objects.get_or_create(name=flag, condition="hostname", value="127.0.0.1,localhost") - - CountryOffice.objects.get_or_create(slug=slugify(settings.TENANT_HQ, ), name=settings.TENANT_HQ) + FlagState.objects.get_or_create( + name=flag, condition="hostname", value="127.0.0.1,localhost" + ) + + CountryOffice.objects.get_or_create( + slug=slugify( + settings.TENANT_HQ, + ), + name=settings.TENANT_HQ, + ) for co in ["afghanistan", "ukraine", "sudan", "haiti"]: CountryOffice.objects.get_or_create(slug=co, name=co.capitalize()) diff --git a/src/hope_country_workspace/management/commands/env.py b/src/hope_country_workspace/management/commands/env.py index ebf5e20..4045341 100644 --- a/src/hope_country_workspace/management/commands/env.py +++ b/src/hope_country_workspace/management/commands/env.py @@ -25,10 +25,14 @@ def add_arguments(self, parser: "CommandParser") -> None: help="Only dumps keys, without values", ) parser.add_argument( - "--develop", action="store_true", help="Get values from teh code not from the current environment" + "--develop", + action="store_true", + help="Get values from teh code not from the current environment", ) parser.add_argument( - "--changed", action="store_true", help="Get values from teh code not from the current environment" + "--changed", + action="store_true", + help="Get values from teh code not from the current environment", ) parser.add_argument( @@ -39,14 +43,22 @@ def add_arguments(self, parser: "CommandParser") -> None: help="Check env for variable availability", ) parser.add_argument( - "--check", action="store_true", dest="check", default=False, help="Check env for variable availability" + "--check", + action="store_true", + dest="check", + default=False, + help="Check env for variable availability", ) parser.add_argument( - "--ignore-errors", action="store_true", dest="ignore_errors", default=False, help="Do not fail" + "--ignore-errors", + action="store_true", + dest="ignore_errors", + default=False, + help="Do not fail", ) def handle(self, *args: "Any", **options: "Any") -> None: - from hope_country_workspace.config import CONFIG, env, EXPLICIT_SET + from hope_country_workspace.config import CONFIG, EXPLICIT_SET, env check_failure = False pattern = options["pattern"] @@ -61,7 +73,9 @@ def handle(self, *args: "Any", **options: "Any") -> None: else: value: Any = env.get_value(k) if not options["changed"] or (value != default): - self.stdout.write(pattern.format(key=k, value=value, help=help, default=default)) + self.stdout.write( + pattern.format(key=k, value=value, help=help, default=default) + ) if check_failure and not options["ignore_errors"]: raise CommandError("Env check command failure!") diff --git a/src/hope_country_workspace/management/commands/upgrade.py b/src/hope_country_workspace/management/commands/upgrade.py index dda79c8..ef431c5 100644 --- a/src/hope_country_workspace/management/commands/upgrade.py +++ b/src/hope_country_workspace/management/commands/upgrade.py @@ -1,13 +1,11 @@ -from typing import Any, TYPE_CHECKING - import logging import os import sys from pathlib import Path +from typing import TYPE_CHECKING, Any from django.conf import settings from django.contrib.auth.models import Group -from django.contrib.gis.utils import LayerMapping from django.core.exceptions import ValidationError from django.core.management import BaseCommand, call_command from django.core.management.base import CommandError, SystemCheckError @@ -15,6 +13,7 @@ from django.utils.text import slugify from hope_country_workspace.config import env +from hope_country_workspace.utils import get_or_create_defaults_group if TYPE_CHECKING: from argparse import ArgumentParser @@ -94,7 +93,9 @@ def get_options(self, options: dict[str, Any]) -> None: self.debug = options["debug"] self.admin_email = str(options["admin_email"] or env("ADMIN_EMAIL", "")) - self.admin_password = str(options["admin_password"] or env("ADMIN_PASSWORD", "")) + self.admin_password = str( + options["admin_password"] or env("ADMIN_PASSWORD", "") + ) def halt(self, e: Exception) -> None: # pragma: no cover self.stdout.write(str(e), style_func=self.style.ERROR) @@ -128,7 +129,9 @@ def handle(self, *args: Any, **options: Any) -> None: # noqa call_command("check", deploy=True, verbosity=self.verbosity - 1) if self.static: static_root = Path(env("STATIC_ROOT")) - echo(f"Run collectstatic to: '{static_root}' - '{static_root.absolute()}") + echo( + f"Run collectstatic to: '{static_root}' - '{static_root.absolute()}" + ) if not static_root.exists(): static_root.mkdir(parents=True) call_command("collectstatic", **extra) @@ -161,13 +164,19 @@ def handle(self, *args: Any, **options: Any) -> None: # noqa interactive=False, ) - echo("Create default group") Group.objects.get_or_create(name=settings.ANALYST_GROUP_NAME) echo("Sync Country Offices") - CountryOffice.objects.get_or_create(slug=slugify(settings.TENANT_HQ, ), name=settings.TENANT_HQ) + CountryOffice.objects.get_or_create( + slug=slugify( + settings.TENANT_HQ, + ), + name=settings.TENANT_HQ, + ) CountryOffice.objects.sync() echo("Upgrade completed", style_func=self.style.SUCCESS) + + get_or_create_defaults_group() except ValidationError as e: self.halt(Exception("\n- ".join(["Wrong argument(s):", *e.messages]))) except (CommandError, SystemCheckError) as e: diff --git a/src/hope_country_workspace/middleware/exception.py b/src/hope_country_workspace/middleware/exception.py index 80c4f31..222e1d9 100644 --- a/src/hope_country_workspace/middleware/exception.py +++ b/src/hope_country_workspace/middleware/exception.py @@ -1,17 +1,23 @@ -from typing import TYPE_CHECKING - import logging +from typing import TYPE_CHECKING from django.core.exceptions import PermissionDenied -from django.http import HttpRequest, HttpResponse, HttpResponseForbidden, HttpResponseRedirect +from django.http import ( + HttpRequest, + HttpResponse, + HttpResponseForbidden, + HttpResponseRedirect, +) from django.urls import reverse -from hope_country_workspace.tenant.exceptions import InvalidTenantError, SelectTenantException +from hope_country_workspace.tenant.exceptions import ( + InvalidTenantError, + SelectTenantException, +) if TYPE_CHECKING: - from typing import TYPE_CHECKING - from collections.abc import Callable + from typing import TYPE_CHECKING logger = logging.getLogger(__name__) @@ -21,7 +27,9 @@ class ExceptionMiddleware: def __init__(self, get_response: "Callable[[HttpRequest],HttpResponse]") -> None: self.get_response = get_response - def process_exception(self, request: "HttpRequest", exception: BaseException) -> HttpResponse: + def process_exception( + self, request: "HttpRequest", exception: BaseException + ) -> HttpResponse: if isinstance(exception, (PermissionDenied,)): return HttpResponseForbidden() if isinstance(exception, (SelectTenantException, InvalidTenantError)): diff --git a/src/hope_country_workspace/middleware/state.py b/src/hope_country_workspace/middleware/state.py index f61fe57..878bf9c 100644 --- a/src/hope_country_workspace/middleware/state.py +++ b/src/hope_country_workspace/middleware/state.py @@ -1,6 +1,5 @@ -from typing import TYPE_CHECKING - import logging +from typing import TYPE_CHECKING from django.http import HttpRequest, HttpResponse @@ -9,10 +8,10 @@ from hope_country_workspace.tenant.utils import RequestHandler if TYPE_CHECKING: - from typing import TYPE_CHECKING - from collections.abc import Callable + from hope_country_workspace.types.http import AuthHttpRequest + logger = logging.getLogger(__name__) diff --git a/src/hope_country_workspace/models/household.py b/src/hope_country_workspace/models/household.py index 7a63f21..83a59bc 100644 --- a/src/hope_country_workspace/models/household.py +++ b/src/hope_country_workspace/models/household.py @@ -24,6 +24,7 @@ class Household(TenantModel): country_office = models.ForeignKey(CountryOffice, on_delete=models.CASCADE) + name = models.CharField(_("Name"), max_length=255) flex_fields = models.JSONField(default=dict, blank=True) class Meta: diff --git a/src/hope_country_workspace/security/models.py b/src/hope_country_workspace/security/models.py index d5b0593..edd75f5 100644 --- a/src/hope_country_workspace/security/models.py +++ b/src/hope_country_workspace/security/models.py @@ -11,6 +11,9 @@ def sync(self): class CountryOffice(models.Model): HQ = "HQ" + + hope_id = models.CharField(max_length=100, blank=True, null=True) + code = models.CharField(max_length=100, blank=True, null=True) name = models.CharField(max_length=100, blank=True, null=True) slug = models.SlugField(max_length=100, blank=True, null=True) objects = CountryOfficeManager() diff --git a/src/hope_country_workspace/state.py b/src/hope_country_workspace/state.py index 39ee29d..d19742b 100644 --- a/src/hope_country_workspace/state.py +++ b/src/hope_country_workspace/state.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from collections.abc import Iterator from typing import Any, List + from hope_country_workspace.types.http import AnyResponse not_set = object() @@ -76,7 +77,16 @@ def add_cookies( httponly: bool = False, samesite: "Any" = None, ) -> None: - self.cookies[key] = [value, max_age, expires, path, domain, secure, httponly, samesite] + self.cookies[key] = [ + value, + max_age, + expires, + path, + domain, + secure, + httponly, + samesite, + ] def set_cookies(self, response: "AnyResponse") -> None: for name, args in self.cookies.items(): diff --git a/src/hope_country_workspace/tenant/sites.py b/src/hope_country_workspace/tenant/sites.py index 333b6c5..19bf064 100644 --- a/src/hope_country_workspace/tenant/sites.py +++ b/src/hope_country_workspace/tenant/sites.py @@ -1,8 +1,9 @@ from asyncio import iscoroutinefunction from collections.abc import Callable from functools import update_wrapper, wraps -from typing import Any +from typing import TYPE_CHECKING, Any +from django.db.models import Model from django.http import HttpRequest, HttpResponse, HttpResponseRedirect from django.shortcuts import redirect from django.template.response import TemplateResponse @@ -14,13 +15,10 @@ from smart_admin.site import SmartAdminSite from .forms import SelectTenantForm -from .utils import ( - get_selected_tenant, - is_tenant_valid, - must_tenant, - set_selected_tenant, -) +from .utils import get_selected_tenant, is_tenant_valid, set_selected_tenant +if TYPE_CHECKING: + from hope_country_workspace.types.http import AuthHttpRequest class TenantAutocompleteJsonView(SmartAutocompleteJsonView): @@ -34,15 +32,13 @@ class TenantAutocompleteJsonView(SmartAutocompleteJsonView): # def get_context_data(self, **kwargs: Any) -> dict[str, Any]: # return super().get_context_data(**kwargs) # - def has_perm(self, request: "AuthHttpRequest", obj: "AnyModel|None" = None) -> bool: + def has_perm(self, request: "HttpRequest", obj: "Model|None" = None) -> bool: return request.user.is_active # def get(self, request, *args, **kwargs): # return JsonResponse({"t": state.tenant.slug}) - - def force_tenant(view_func): """ Decorator that adds headers to a response so that it will never be cached. @@ -52,7 +48,7 @@ def force_tenant(view_func): async def _view_wrapper(request, *args, **kwargs): if not is_tenant_valid() and "+select" not in request.path: # TODO: Dry - return redirect(f"admin:select_tenant") + return redirect("admin:select_tenant") response = await view_func(request, *args, **kwargs) return response @@ -60,12 +56,13 @@ async def _view_wrapper(request, *args, **kwargs): def _view_wrapper(request, *args, **kwargs): if not is_tenant_valid() and "+select" not in request.path: # TODO: Dry - return redirect(f"admin:select_tenant") + return redirect("admin:select_tenant") response = view_func(request, *args, **kwargs) return response return wraps(view_func)(_view_wrapper) + class TenantAdminSite(SmartAdminSite): enable_nav_sidebar = False @@ -87,14 +84,15 @@ def each_context(self, request: "HttpRequest") -> "dict[str, Any]": initial={"tenant": selected_tenant}, request=request ) ret["active_tenant"] = selected_tenant - # ret["tenant"] = selected_tenant + # ret["tenant"] = selected_tenant # else: # ret["active_tenant"] = None return ret # type: ignore def is_smart_enabled(self, request: "AuthHttpRequest") -> bool: - # if must_tenant(): + # if must_tenant(): return False + # return super().is_smart_enabled(request) def autocomplete_view(self, request: "HttpRequest") -> HttpResponse: @@ -165,6 +163,8 @@ def select_tenant(self, request: "HttpRequest") -> "HttpResponse": set_selected_tenant(form.cleaned_data["tenant"]) return HttpResponseRedirect(reverse("admin:index")) - form = SelectTenantForm(request=request, initial={"next": reverse("admin:index")}) + form = SelectTenantForm( + request=request, initial={"next": reverse("admin:index")} + ) context["form"] = form return TemplateResponse(request, "tenant_admin/select_tenant.html", context) diff --git a/src/hope_country_workspace/tenant/utils.py b/src/hope_country_workspace/tenant/utils.py index f3d3af1..60d0021 100644 --- a/src/hope_country_workspace/tenant/utils.py +++ b/src/hope_country_workspace/tenant/utils.py @@ -11,9 +11,11 @@ from ..security.models import CountryOffice, User if TYPE_CHECKING: + class AuthHttpRequest(HttpRequest): user: "User" = None + logger = logging.getLogger(__name__) @@ -76,7 +78,7 @@ def process_request(self, request: "AuthHttpRequest") -> State: return state def process_response( - self, request: "AuthHttpRequest", response: "HttpResponse|None" + self, request: "AuthHttpRequest", response: "HttpResponse|None" ) -> None: if response: state.set_cookies(response) diff --git a/src/hope_country_workspace/types/__init__.py b/src/hope_country_workspace/types/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/hope_country_workspace/types/django.pyi b/src/hope_country_workspace/types/django.pyi new file mode 100644 index 0000000..4395cfc --- /dev/null +++ b/src/hope_country_workspace/types/django.pyi @@ -0,0 +1,9 @@ +from typing import TypeAlias, TypeVar + +from django.contrib.auth.base_user import AbstractBaseUser +from django.contrib.auth.models import AnonymousUser +from django.db.models import Model + +AnyModel = TypeVar("AnyModel", bound=Model, covariant=True) + +AnyUser: TypeAlias = AbstractBaseUser | AnonymousUser diff --git a/src/hope_country_workspace/types/hope.pyi b/src/hope_country_workspace/types/hope.pyi new file mode 100644 index 0000000..e69de29 diff --git a/src/hope_country_workspace/types/http.pyi b/src/hope_country_workspace/types/http.pyi new file mode 100644 index 0000000..cb49dc0 --- /dev/null +++ b/src/hope_country_workspace/types/http.pyi @@ -0,0 +1,26 @@ +from typing import TypeVar, Union + +from django.db.models import Model +from django.http import ( + HttpRequest, + HttpResponse, + HttpResponseBase, + HttpResponseRedirect, + StreamingHttpResponse, +) + +from admin_extra_buttons.utils import HttpResponseRedirectToReferrer + +from hope_country_workspace.security.models import User + +AnyRequest = TypeVar("AnyRequest", bound=HttpRequest, covariant=True) +AnyResponse = TypeVar("AnyResponse", bound=HttpResponseBase, covariant=True) +RedirectOrResponse = Union[ + HttpResponseRedirect, + HttpResponseRedirectToReferrer, + HttpResponse, + StreamingHttpResponse, +] + +class AuthHttpRequest(HttpRequest): + user: User diff --git a/src/hope_country_workspace/utils/__init__.py b/src/hope_country_workspace/utils/__init__.py index e69de29..442f6c4 100644 --- a/src/hope_country_workspace/utils/__init__.py +++ b/src/hope_country_workspace/utils/__init__.py @@ -0,0 +1,13 @@ +from django.conf import settings +from django.contrib.auth.models import Group, Permission + + +def get_or_create_defaults_group() -> "Group": + reporter, created = Group.objects.get_or_create(name=settings.ANALYST_GROUP_NAME) + # if created: + for perm in Permission.objects.order_by("codename").filter( + content_type__app_label="hope_country_workspace", codename__startswith="view_" + ): + reporter.permissions.add(perm) + + return reporter diff --git a/src/hope_country_workspace/utils/flags.py b/src/hope_country_workspace/utils/flags.py index c7e23f6..237cc75 100644 --- a/src/hope_country_workspace/utils/flags.py +++ b/src/hope_country_workspace/utils/flags.py @@ -1,6 +1,5 @@ -from typing import TYPE_CHECKING - import contextlib +from typing import TYPE_CHECKING from django.conf import settings from django.core.exceptions import ValidationError @@ -10,9 +9,11 @@ from flags.conditions import conditions if TYPE_CHECKING: - from typing import Any + from hope_country_workspace.types.http import AuthHttpRequest +if TYPE_CHECKING: from collections.abc import Iterator + from typing import Any from django.http import HttpRequest @@ -25,7 +26,18 @@ def enable_flag(name: str) -> "Iterator[Any]": def validate_bool(value: str) -> None: - if not value.lower() in ["true", "1", "yes", "t", "y", "false", "0", "no", "f", "n"]: + if not value.lower() in [ + "true", + "1", + "yes", + "t", + "y", + "false", + "0", + "no", + "f", + "n", + ]: raise ValidationError("Enter a valid bool") diff --git a/src/hope_country_workspace/web/templatetags/filters.py b/src/hope_country_workspace/web/templatetags/filters.py index 1f26517..cb9df79 100644 --- a/src/hope_country_workspace/web/templatetags/filters.py +++ b/src/hope_country_workspace/web/templatetags/filters.py @@ -1,5 +1,4 @@ from typing import Any - from urllib.parse import urlencode from django.template import Library @@ -11,7 +10,9 @@ @register.simple_tag(takes_context=True) -def build_filter_url(context: dict[str, Any], field: str | None = None, value: str | None = None) -> str: +def build_filter_url( + context: dict[str, Any], field: str | None = None, value: str | None = None +) -> str: params = context["request"].GET.copy() if field: if value: diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..1c4b5b0 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,89 @@ +import os +import sys +from pathlib import Path + +import django + +import pytest +import responses +from constance import config + + +here = Path(__file__).parent +sys.path.insert(0, str(here / "../src")) +sys.path.insert(0, str(here / "extras")) + +def pytest_configure(config): + + os.environ.update(DJANGO_SETTINGS_MODULE="hope_country_workspace.config.settings") + os.environ.setdefault("MEDIA_ROOT", "/tmp/static/") + os.environ.setdefault("STATIC_ROOT", "/tmp/media/") + os.environ.setdefault("TEST_EMAIL_SENDER", "sender@example.com") + os.environ.setdefault("TEST_EMAIL_RECIPIENT", "recipient@example.com") + + os.environ["MAILJET_API_KEY"] = "11" + os.environ["MAILJET_SECRET_KEY"] = "11" + os.environ["FILE_STORAGE_DEFAULT"] = ( + "django.core.files.storage.FileSystemStorage?location=/tmp/hde/storage/" + ) + os.environ["FILE_STORAGE_STATIC"] = ( + "django.core.files.storage.FileSystemStorage?location=/tmp/hde/static/" + ) + os.environ["FILE_STORAGE_MEDIA"] = ( + "django.core.files.storage.FileSystemStorage?location=/tmp/hde/storage/" + ) + os.environ["FILE_STORAGE_HOPE"] = ( + "django.core.files.storage.FileSystemStorage?location=/tmp/hde/hope/" + ) + os.environ["SOCIAL_AUTH_REDIRECT_IS_HTTPS"] = "0" + os.environ["CELERY_TASK_ALWAYS_EAGER"] = "0" + os.environ["SECURE_HSTS_PRELOAD"] = "0" + os.environ["SECRET_KEY"] = "kugiugiuygiuygiuygiuhgiuhgiuhgiugiu" + + os.environ["GMAIL_USER"] = "11" + os.environ["GMAIL_PASSWORD"] = "11" + from django.conf import settings + + settings.ALLOWED_HOSTS = ["127.0.0.1", "localhost"] + settings.MEDIA_ROOT = "/tmp/media" + settings.STATIC_ROOT = "/tmp/static" + os.makedirs(settings.MEDIA_ROOT, exist_ok=True) + os.makedirs(settings.STATIC_ROOT, exist_ok=True) + + from django.core.management import CommandError, call_command + + django.setup() + + try: + call_command("env", check=True) + except CommandError: + pytest.exit("FATAL: Environment variables missing") + + +@pytest.fixture(autouse=True) +def setup(db): + from testutils.factories import GroupFactory + + GroupFactory(name=config.NEW_USER_DEFAULT_GROUP) + + +@pytest.fixture() +def mocked_responses(): + with responses.RequestsMock(assert_all_requests_are_fired=False) as rsps: + yield rsps + + + +@pytest.fixture() +def afghanistan(db): + from testutils.factories import CountryOfficeFactory + + return CountryOfficeFactory(name="Afghanistan") + + + +@pytest.fixture +def reporters(db, afghanistan, user): + from hope_country_workspace.utils import get_or_create_defaults_group + + return get_or_create_defaults_group() diff --git a/tests/extras/testutils/__init__.py b/tests/extras/testutils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/extras/testutils/decorators.py b/tests/extras/testutils/decorators.py new file mode 100644 index 0000000..2755f1b --- /dev/null +++ b/tests/extras/testutils/decorators.py @@ -0,0 +1,15 @@ +import os + +import pytest + + +def requires_env(*envs): + missing = [] + + for env in envs: + if os.environ.get(env, None) is None: + missing.append(env) + + return pytest.mark.skipif( + len(missing) > 0, reason=f"Not suitable environment {missing} for current test" + ) diff --git a/tests/extras/testutils/factories/__init__.py b/tests/extras/testutils/factories/__init__.py new file mode 100644 index 0000000..5ed822b --- /dev/null +++ b/tests/extras/testutils/factories/__init__.py @@ -0,0 +1,49 @@ +import importlib +import pkgutil +from pathlib import Path + +from factory.django import DjangoModelFactory +from pytest_factoryboy import register + +from .base import ( + AutoRegisterModelFactory, + TAutoRegisterModelFactory, + factories_registry, +) +from .household import HouseholdFactory +from .django_celery_beat import PeriodicTaskFactory # noqa +from .social import SocialAuthUserFactory # noqa +from .user import ( # noqa + CountryOfficeFactory, + GroupFactory, + SuperUserFactory, + User, + UserFactory, +) +from .userrole import UserRole, UserRoleFactory # noqa + +for _, name, _ in pkgutil.iter_modules([str(Path(__file__).parent)]): + importlib.import_module(f".{name}", __package__) + + +django_model_factories = { + factory._meta.model: factory for factory in DjangoModelFactory.__subclasses__() +} + + +def get_factory_for_model( + _model, +) -> type[TAutoRegisterModelFactory] | type[DjangoModelFactory]: + class Meta: + model = _model + + bases = (AutoRegisterModelFactory,) + if _model in factories_registry: + return factories_registry[_model] # noqa + + if _model in django_model_factories: + return django_model_factories[_model] + + return register( + type(f"{_model._meta.model_name}AutoCreatedFactory", bases, {"Meta": Meta}) + ) # noqa diff --git a/tests/extras/testutils/factories/base.py b/tests/extras/testutils/factories/base.py new file mode 100644 index 0000000..601ca5d --- /dev/null +++ b/tests/extras/testutils/factories/base.py @@ -0,0 +1,23 @@ +import typing + +import factory +from factory.base import FactoryMetaClass + +TAutoRegisterModelFactory = typing.TypeVar( + "TAutoRegisterModelFactory", bound="AutoRegisterModelFactory" +) + +factories_registry: dict[str, TAutoRegisterModelFactory] = {} + + +class AutoRegisterFactoryMetaClass(FactoryMetaClass): + def __new__(mcs, class_name, bases, attrs): + new_class = super().__new__(mcs, class_name, bases, attrs) + factories_registry[new_class._meta.model] = new_class + return new_class + + +class AutoRegisterModelFactory( + factory.django.DjangoModelFactory, metaclass=AutoRegisterFactoryMetaClass +): + pass diff --git a/tests/extras/testutils/factories/contenttypes.py b/tests/extras/testutils/factories/contenttypes.py new file mode 100644 index 0000000..848bdaf --- /dev/null +++ b/tests/extras/testutils/factories/contenttypes.py @@ -0,0 +1,12 @@ +from django.contrib.contenttypes.models import ContentType + +from .base import AutoRegisterModelFactory + + +class ContentTypeFactory(AutoRegisterModelFactory): + app_label = "auth" + model = "user" + + class Meta: + model = ContentType + django_get_or_create = ("app_label", "model") diff --git a/tests/extras/testutils/factories/django_auth.py b/tests/extras/testutils/factories/django_auth.py new file mode 100644 index 0000000..0e3e8e3 --- /dev/null +++ b/tests/extras/testutils/factories/django_auth.py @@ -0,0 +1,32 @@ +from django.contrib.auth.models import Group, Permission + +import factory + +from .base import AutoRegisterModelFactory +from .contenttypes import ContentTypeFactory + + +class PermissionFactory(AutoRegisterModelFactory): + content_type = factory.SubFactory(ContentTypeFactory) + + class Meta: + model = Permission + + +class GroupFactory(AutoRegisterModelFactory): + name = factory.Sequence(lambda n: "group %s" % n) + + class Meta: + model = Group + django_get_or_create = ("name",) + + @factory.post_generation + def permissions(self, create, extracted, **kwargs): + if not create: + # Simple build, do nothing. + return + + if extracted: + # A list of groups were passed in, use them + for perm in extracted: + self.permissions.add(perm) diff --git a/tests/extras/testutils/factories/django_celery_beat.py b/tests/extras/testutils/factories/django_celery_beat.py new file mode 100644 index 0000000..c691ad4 --- /dev/null +++ b/tests/extras/testutils/factories/django_celery_beat.py @@ -0,0 +1,47 @@ +from django.utils import timezone + +import factory +from django_celery_beat.models import ( + SOLAR_SCHEDULES, + ClockedSchedule, + IntervalSchedule, + PeriodicTask, + SolarSchedule, +) +from factory.fuzzy import FuzzyChoice + +from .base import AutoRegisterModelFactory + + +class IntervalScheduleFactory(AutoRegisterModelFactory): + every = 1 + period = IntervalSchedule.HOURS + + class Meta: + model = IntervalSchedule + + +class SolarScheduleFactory(AutoRegisterModelFactory): + event = FuzzyChoice([x[0] for x in SOLAR_SCHEDULES]) + + latitude = 10.1 + longitude = 10.1 + + class Meta: + model = SolarSchedule + + +class ClockedScheduleFactory(AutoRegisterModelFactory): + clocked_time = timezone.now() + + class Meta: + model = ClockedSchedule + + +class PeriodicTaskFactory(AutoRegisterModelFactory): + name = factory.Sequence(lambda n: "PeriodicTask%03d" % n) + interval = factory.SubFactory(IntervalScheduleFactory) + task = "hope_dedup_engine.tasks.process" + + class Meta: + model = PeriodicTask diff --git a/tests/extras/testutils/factories/household.py b/tests/extras/testutils/factories/household.py new file mode 100644 index 0000000..81ecb72 --- /dev/null +++ b/tests/extras/testutils/factories/household.py @@ -0,0 +1,11 @@ +import factory + +from hope_country_workspace.models import Household + +from .base import AutoRegisterModelFactory + +class HouseholdFactory(AutoRegisterModelFactory): + name = factory.Sequence(lambda n: f"Household {n}") + class Meta: + model = Household + django_get_or_create = ("name",) diff --git a/tests/extras/testutils/factories/log.py b/tests/extras/testutils/factories/log.py new file mode 100644 index 0000000..15267bd --- /dev/null +++ b/tests/extras/testutils/factories/log.py @@ -0,0 +1,11 @@ +from django.contrib.admin.models import LogEntry + +from .base import AutoRegisterModelFactory + + +class LogEntryFactory(AutoRegisterModelFactory): + level = "INFO" + message = "Message for {{ event.name }} on channel {{channel.name}}" + + class Meta: + model = LogEntry diff --git a/tests/extras/testutils/factories/social.py b/tests/extras/testutils/factories/social.py new file mode 100644 index 0000000..f703c03 --- /dev/null +++ b/tests/extras/testutils/factories/social.py @@ -0,0 +1,12 @@ +from uuid import uuid4 + +import factory +from social_django.models import UserSocialAuth + +from .user import UserFactory + + +class SocialAuthUserFactory(UserFactory): + @factory.post_generation + def sso(obj, create, extracted, **kwargs): + UserSocialAuth.objects.get_or_create(user=obj, provider="test", uid=uuid4()) diff --git a/tests/extras/testutils/factories/user.py b/tests/extras/testutils/factories/user.py new file mode 100644 index 0000000..21239a4 --- /dev/null +++ b/tests/extras/testutils/factories/user.py @@ -0,0 +1,49 @@ +from django.contrib.auth.models import Group + +import factory.fuzzy + +from hope_country_workspace.security.models import User, CountryOffice + +from .base import AutoRegisterModelFactory + + +class CountryOfficeFactory(AutoRegisterModelFactory): + name = factory.Iterator(["Afghanistan", "Ukraine", "Niger", "South Sudan"]) + code = factory.Sequence(lambda x: str(x)) + + class Meta: + model = CountryOffice + django_get_or_create = ("name",) + + +class UserFactory(AutoRegisterModelFactory): + _password = "password" + username = factory.Sequence(lambda n: "m%03d@example.com" % n) + password = factory.django.Password(_password) + email = factory.Sequence(lambda n: "m%03d@example.com" % n) + + class Meta: + model = User + django_get_or_create = ("username",) + + @classmethod + def _create(cls, model_class, *args, **kwargs): + ret = super()._create(model_class, *args, **kwargs) + ret._password = cls._password + return ret + + +class SuperUserFactory(UserFactory): + username = factory.Sequence(lambda n: "superuser%03d@example.com" % n) + email = factory.Sequence(lambda n: "superuser%03d@example.com" % n) + is_superuser = True + is_staff = True + is_active = True + + +class GroupFactory(AutoRegisterModelFactory): + name = factory.Sequence(lambda n: "Group-%03d" % n) + + class Meta: + model = Group + django_get_or_create = ("name",) diff --git a/tests/extras/testutils/factories/userrole.py b/tests/extras/testutils/factories/userrole.py new file mode 100644 index 0000000..71e4977 --- /dev/null +++ b/tests/extras/testutils/factories/userrole.py @@ -0,0 +1,23 @@ +import factory + +from hope_country_workspace.security.models import CountryOffice, UserRole + +from .base import AutoRegisterModelFactory +from .django_auth import GroupFactory +from .user import UserFactory + + +class CountryOfficeFactory(AutoRegisterModelFactory): + class Meta: + model = CountryOffice + django_get_or_create = ("name",) + + +class UserRoleFactory(AutoRegisterModelFactory): + class Meta: + model = UserRole + django_get_or_create = ("user", "group", "country_office") + + user = factory.SubFactory(UserFactory) + group = factory.SubFactory(GroupFactory) + country_office = factory.SubFactory(CountryOfficeFactory) diff --git a/tests/extras/testutils/perms.py b/tests/extras/testutils/perms.py new file mode 100644 index 0000000..b2faea0 --- /dev/null +++ b/tests/extras/testutils/perms.py @@ -0,0 +1,169 @@ +from contextlib import ContextDecorator +from random import choice + +from unittest.mock import Mock + +from django.conf import settings +from django.contrib.auth.models import Group, Permission + +from faker import Faker + +from hope_country_workspace.security.models import UserRole +from hope_country_workspace.state import state + +from .factories import GroupFactory + +whitespace = " \t\n\r\v\f" +lowercase = "abcdefghijklmnopqrstuvwxyz" +uppercase = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" +letters = lowercase + uppercase +ascii_lowercase = lowercase +ascii_uppercase = uppercase +ascii_letters = ascii_lowercase + ascii_uppercase + +faker = Faker() + + +def text(length, choices=ascii_letters): + """returns a random (fixed length) string + + :param length: string length + :param choices: string containing all the chars can be used to build the string + + .. seealso:: + :py:func:`rtext` + """ + return "".join(choice(choices) for x in range(length)) + + +def get_group(name=None, permissions=None): + group = GroupFactory(name=(name or text(5))) + permission_names = permissions or [] + for permission_name in permission_names: + try: + app_label, codename = permission_name.split(".") + except ValueError: + raise ValueError(f"Invalid permission name `{permission_name}`") + try: + permission = Permission.objects.get(content_type__app_label=app_label, codename=codename) + except Permission.DoesNotExist: + raise Permission.DoesNotExist("Permission `{0}` does not exists", permission_name) + + group.permissions.add(permission) + return group + + +class set_current_user(ContextDecorator): # noqa + def __init__(self, user): + self.user = user + + def __enter__(self): + r = Mock() + r.user = self.user + self.state = state.set(request=r) + self.state.__enter__() + + def __exit__(self, e_typ, e_val, trcbak): + self.state.__exit__(e_typ, e_val, trcbak) + if e_typ: + raise e_typ(e_val).with_traceback(trcbak) + + +class user_grant_role(ContextDecorator): # noqa + caches = [ + "_group_perm_cache", + "_user_perm_cache", + "_dsspermissionchecker", + "_officepermissionchecker", + "_perm_cache", + "_dss_acl_cache", + ] + + def __init__(self, user, country_office, group=settings.ANALYST_GROUP_NAME): + self.user = user + if isinstance(group, str): + self.group = Group.objects.get(name=settings.ANALYST_GROUP_NAME) + else: + self.group = group + self.country_office = country_office + + def __enter__(self): + for cache in self.caches: + if hasattr(self.user, cache): + delattr(self.user, cache) + if self.country_office: + cache_name = "_power_query_%s_perm_cache" % self.country_office.pk + if hasattr(self.user, cache_name): + delattr(self.user, cache_name) + __, self.is_added = UserRole.objects.get_or_create( + country_office=self.country_office, user=self.user, group=self.group + ) + return self + + def __exit__(self, e_typ, e_val, trcbak): + if self.is_added: + self.user.groups.remove(self.group) + + if e_typ: + raise e_val.with_traceback(trcbak) + + def start(self): + """Activate a patch, returning any created mock.""" + result = self.__enter__() + return result + + def stop(self): + """Stop an active patch.""" + return self.__exit__(None, None, None) + + +class user_grant_permissions(ContextDecorator): # noqa + caches = [ + "_group_perm_cache", + "_user_perm_cache", + "_dsspermissionchecker", + "_officepermissionchecker", + "_perm_cache", + "_dss_acl_cache", + ] + + def __init__(self, user, permissions=None, country_office=None, group_name=None): + self.user = user + if not isinstance(permissions, (list, tuple)): + permissions = [permissions] + self.permissions = permissions + self.group_name = group_name + self.group = None + self.country_office = country_office + + def __enter__(self): + for cache in self.caches: + if hasattr(self.user, cache): + delattr(self.user, cache) + if self.country_office: + cache_name = "_power_query_%s_perm_cache" % self.country_office.pk + if hasattr(self.user, cache_name): + delattr(self.user, cache_name) + + self.group = get_group(name=self.group_name, permissions=self.permissions or []) + self.user.groups.add(self.group) + if self.country_office: + UserRole.objects.get_or_create(country_office=self.country_office, user=self.user, group=self.group) + return self + + def __exit__(self, e_typ, e_val, trcbak): + if self.group: + self.user.groups.remove(self.group) + self.group.delete() + + if e_typ: + raise e_val.with_traceback(trcbak) + + def start(self): + """Activate a patch, returning any created mock.""" + result = self.__enter__() + return result + + def stop(self): + """Stop an active patch.""" + return self.__exit__(None, None, None) diff --git a/tests/tenant/test_tenant_backend.py b/tests/tenant/test_tenant_backend.py new file mode 100644 index 0000000..37624f7 --- /dev/null +++ b/tests/tenant/test_tenant_backend.py @@ -0,0 +1,127 @@ +from typing import TYPE_CHECKING + +from collections import namedtuple + +import pytest + +from django.apps import apps +from django.contrib.auth.models import AnonymousUser + +from testutils.perms import user_grant_permissions + +from hope_country_workspace.tenant.backend import TenantBackend +from hope_country_workspace.state import state + +if TYPE_CHECKING: + from hope_country_workspace.security.models import CountryOffice + +_DATA = namedtuple("_DATA", "afg,ukr") + +from pytest_factoryboy import LazyFixture, register +from testutils.factories import UserFactory + +register(UserFactory) + + +@pytest.fixture() +def data() -> _DATA: + from testutils.factories import CountryOfficeFactory + + with state.set(must_tenant=False): + co1: "CountryOffice" = CountryOfficeFactory(name="Afghanistan") + co2: "CountryOffice" = CountryOfficeFactory(name="Ukraine") + return _DATA(co1, co2) + + +def pytest_generate_tests(metafunc): + if "hope_model" in metafunc.fixturenames: + models = list(apps.get_app_config("hope").get_models()) + metafunc.parametrize("hope_model", models) + + +@pytest.fixture() +def backend(): + return TenantBackend() + + +def test_aaa(backend, user): + assert not backend.has_perm(user, "aaaa") + + +def test_has_get_all_permissions_no_active_tenant(backend, data, user, admin_user): + assert backend.get_all_permissions(user) == set() + + +def test_get_all_permissions_anonymous(backend, data, user, admin_user): + with state.set(tenant=data.afg): + assert backend.get_all_permissions(AnonymousUser()) == set() + + +def test_get_all_permissions_no_enabled_tenant(backend, data, user, admin_user): + with state.set(tenant=data.afg): + assert backend.get_all_permissions(user) == set() + + +def test_get_all_permissions_no_current_tenant(backend, data, user, admin_user): + with state.set(tenant=data.afg): + with user_grant_permissions(user, "hope_country_workspace.view_household", country_office=data.ukr): + assert backend.get_all_permissions(user) == set() + + +def test_get_all_permissions_valid_tenant(backend, data, user, admin_user): + with state.set(tenant=data.afg): + with user_grant_permissions(user, "hope_country_workspace.view_household", country_office=data.afg): + assert backend.get_all_permissions(user) == {"hope_country_workspace.view_household"} + # test cache + assert backend.get_all_permissions(user) == {"hope_country_workspace.view_household"} + + +def test_get_all_permissions_superuser(backend, data, user, admin_user): + with state.set(tenant=data.afg): + assert backend.get_all_permissions(admin_user) + + +def test_get_available_modules(backend, data, user, admin_user): + with state.set(tenant=data.afg): + with user_grant_permissions(user, "hope_country_workspace.view_household", country_office=data.afg): + assert backend.get_available_modules(user) == {"hope_country_workspace"} + + +def test_has_module_perms_no_active_tenant(backend, data, user, admin_user): + with user_grant_permissions(user, "hope_country_workspace.view_household", country_office=data.afg): + assert not backend.has_module_perms(user, "hope_country_workspace") + + +def test_has_module_perms(backend, data, user, admin_user): + with state.set(tenant=data.afg): + with user_grant_permissions(user, "hope_country_workspace.view_household", country_office=data.afg): + assert backend.has_module_perms(user, "hope_country_workspace") + + +def test_has_module_perms_superuser(backend, data, admin_user): + with state.set(tenant=data.afg): + assert backend.has_module_perms(admin_user, "hope_country_workspace") + + +def test_get_allowed_tenants_user(backend, data, user, rf): + request = rf.get("/") + request.user = user + with state.set(tenant=data.afg, request=request): + with user_grant_permissions(user, "hope_country_workspace.view_household", country_office=data.afg): + assert backend.get_allowed_tenants().count() == 1 + assert backend.get_allowed_tenants().first() == data.afg + + +def test_get_allowed_tenants_superuser(backend, data, admin_user, rf): + request = rf.get("/") + request.user = admin_user + with state.set(tenant=data.afg, request=request): + assert backend.get_allowed_tenants().count() == 2 + assert backend.get_allowed_tenants().first() == data.afg + + +def test_get_allowed_tenants_anon(backend, data, admin_user, rf): + request = rf.get("/") + request.user = AnonymousUser() + with state.set(tenant=data.afg, request=request): + assert backend.get_allowed_tenants().count() == 0 diff --git a/tests/tenant/test_tenant_filtering.py b/tests/tenant/test_tenant_filtering.py new file mode 100644 index 0000000..77fbd20 --- /dev/null +++ b/tests/tenant/test_tenant_filtering.py @@ -0,0 +1,38 @@ +from typing import TYPE_CHECKING + +import pytest +from django.conf import settings + +from hope_country_workspace.state import state + +if TYPE_CHECKING: + from hope_country_workspace.security.models import CountryOffice + + +@pytest.fixture() +def data(): + from testutils.factories import CountryOfficeFactory, HouseholdFactory, UserFactory, UserRoleFactory + + with state.set(must_tenant=False): + co1: "CountryOffice" = CountryOfficeFactory(name="Afghanistan") + co2: "CountryOffice" = CountryOfficeFactory(name="Niger") + co3: "CountryOffice" = CountryOfficeFactory(name="Sudan") + + HouseholdFactory(country_office=co1) + HouseholdFactory(country_office=co1) + HouseholdFactory(country_office=co2) + HouseholdFactory(country_office=co2) + + user = UserFactory(username="user", is_staff=False, is_superuser=False, is_active=True) + UserRoleFactory(country_office=co1, group__name=settings.ANALYST_GROUP_NAME, user=user) + + return {"Afghanistan": co1, "Niger": co2, "Sudan": co3} + + +@pytest.mark.parametrize("co,expected", [("Afghanistan", 2), ("Niger", 2), ("Sudan", 0)]) +def test_filtering(data, co, expected): + from hope_country_workspace.models import Household + + assert Household.objects.count() == 4 + with state.set(tenant=data[co], must_tenant=True): + assert Household.objects.count() == expected diff --git a/tests/tenant/test_tenant_manager.py b/tests/tenant/test_tenant_manager.py new file mode 100644 index 0000000..9296072 --- /dev/null +++ b/tests/tenant/test_tenant_manager.py @@ -0,0 +1,84 @@ +from typing import TYPE_CHECKING + +import pytest +from unittest import mock + +from hope_country_workspace.tenant.db import TenantManager +from hope_country_workspace.tenant.exceptions import InvalidTenantError +from hope_country_workspace.state import state + +if TYPE_CHECKING: + from hope_country_workspace.security.models import CountryOffice + + +@pytest.fixture() +def manager(): + return TenantManager() + + +@pytest.fixture() +def household(afghanistan: "CountryOffice"): + from testutils.factories import HouseholdFactory + + return HouseholdFactory(country_office=afghanistan) + + +def test_get_tenant_filter_no_active_tenant(manager): + from hope_country_workspace.models import Household + + manager.model = Household + assert manager.get_tenant_filter() == {} + + +def test_get_tenant_filter_invalid_tenant(manager, afghanistan): + from hope_country_workspace.models import Household + + manager.model = Household + with pytest.raises(InvalidTenantError): + with state.set(must_tenant=True): + manager.get_tenant_filter() + + +def test_get_tenant_filter_valid_tenant(manager, afghanistan): + from hope_country_workspace.models import Household + + manager.model = Household + with state.set(must_tenant=True, tenant=afghanistan): + assert manager.get_tenant_filter() == {"country_office": afghanistan.hope_id} + + +def test_get_tenant_filter_invalid_model(manager, afghanistan): + from hope_country_workspace.models import Household + + manager.model = Household + with mock.patch("hope_country_workspace.models.Household.Tenant.tenant_filter_field", ""): + with pytest.raises(ValueError): + with state.set(must_tenant=True, tenant=afghanistan): + manager.get_tenant_filter() + + +def test_get_tenant_filter_all(manager, afghanistan): + from hope_country_workspace.models import Household + + manager.model = Household + with mock.patch("hope_country_workspace.models.Household.Tenant.tenant_filter_field", "__all__"): + with state.set(must_tenant=True, tenant=afghanistan): + assert manager.get_tenant_filter() == {} + + +# +# +# def test_get_queryset_active(manager, household: "Household"): +# from hope_country_workspace.models import Household +# +# manager.model = Household +# with state.set(must_tenant=True, tenant=household.business_area): +# assert manager.get_queryset() +# + +# +# def test_get_queryset_not_active(manager, household, country_office): +# from hope_country_workspace.models import Household +# manager.model = Household +# with state.set(must_tenant=True, tenant=country_office): +# assert not manager.get_queryset() diff --git a/tests/tenant/test_tenant_middleware.py b/tests/tenant/test_tenant_middleware.py new file mode 100644 index 0000000..3cd309b --- /dev/null +++ b/tests/tenant/test_tenant_middleware.py @@ -0,0 +1,21 @@ +from unittest.mock import MagicMock + +from django.http import HttpResponse + +from hope_country_workspace.middleware.state import StateClearMiddleware, StateSetMiddleware + + +def test_set(rf, user): + request = rf.get("/") + request.user = user + get_response = MagicMock(side_effect=HttpResponse("Ok")) + res = StateSetMiddleware(get_response)(request) + assert res == b"Ok" + + +def test_clear(rf, user): + request = rf.get("/") + request.user = user + get_response = MagicMock(side_effect=HttpResponse("Ok")) + res = StateClearMiddleware(get_response)(request) + assert res == b"Ok" diff --git a/tests/tenant/test_tenant_permissions.py b/tests/tenant/test_tenant_permissions.py new file mode 100644 index 0000000..906a97c --- /dev/null +++ b/tests/tenant/test_tenant_permissions.py @@ -0,0 +1,91 @@ +from typing import TYPE_CHECKING + +import pytest + +from hope_country_workspace.state import state + +if TYPE_CHECKING: + from django.contrib.auth.models import Group + + from hope_country_workspace.security.models import CountryOffice, UserRole + from hope_country_workspace.types.http import AuthHttpRequest + + +@pytest.fixture() +def anonymous(request): + from django.contrib.auth.models import AnonymousUser + + return AnonymousUser() + + +@pytest.fixture() +def tenant_user(request): + from testutils.factories import CountryOfficeFactory, UserRoleFactory + + from hope_country_workspace.utils import get_or_create_defaults_group + + if "afghanistan" in request.fixturenames: + co = request.getfixturevalue("afghanistan") + else: + co: "CountryOffice" = CountryOfficeFactory() + g: "Group" = get_or_create_defaults_group() + r: "UserRole" = UserRoleFactory(group=g, country_office=co) + return r.user + + +@pytest.fixture() +def req(request, rf) -> "AuthHttpRequest": + req: "AuthHttpRequest" = rf.get("/") + current = state.request + state.request = req + yield req + state.request = current + + +@pytest.mark.parametrize("u", ["tenant_user", "admin_user", "anonymous"]) +def test_tenant_backend_get_allowed_tenants(request, afghanistan, req, u, django_assert_max_num_queries): + from hope_country_workspace.tenant.backend import TenantBackend + + req.user = request.getfixturevalue(u) + b: TenantBackend = TenantBackend() + + # with django_assert_max_num_queries(1): + if req.user.is_authenticated: + assert list(b.get_allowed_tenants().values_list("pk", flat=True)) == [afghanistan.pk] + else: + assert not b.get_allowed_tenants().values_list("pk", flat=True) + + +@pytest.mark.parametrize("u", ["tenant_user", "admin_user", "anonymous"]) +def test_tenant_backend_get_all_permissions(request, afghanistan, req, u, django_assert_max_num_queries): + from hope_country_workspace.tenant.backend import TenantBackend + + req.user = request.getfixturevalue(u) + b: TenantBackend = TenantBackend() + if hasattr(req.user, "_tenant_%s_perm_cache" % afghanistan.pk): + delattr(req.user, "_tenant_%s_perm_cache" % afghanistan.pk) + + with state.set(tenant=afghanistan): + with django_assert_max_num_queries(2): + if req.user.is_authenticated: + assert b.get_all_permissions(req.user) + with django_assert_max_num_queries(0): + assert b.get_all_permissions(req.user) + else: + assert not b.get_all_permissions(req.user) + with django_assert_max_num_queries(0): + assert not b.get_all_permissions(req.user) + + +def test_tenant_backend_get_all_permissions_no_tenant( + request, afghanistan, req, tenant_user, django_assert_max_num_queries +): + from hope_country_workspace.tenant.backend import TenantBackend + + req.user = tenant_user + b: TenantBackend = TenantBackend() + if hasattr(req.user, "_tenant_%s_perm_cache" % afghanistan.pk): + delattr(req.user, "_tenant_%s_perm_cache" % afghanistan.pk) + + with django_assert_max_num_queries(1): + assert not b.get_all_permissions(req.user) From e1f1e47ed31f8d62233e4717847b795d87e7fdd5 Mon Sep 17 00:00:00 2001 From: saxix Date: Fri, 5 Jul 2024 17:18:39 +0200 Subject: [PATCH 03/16] updates --- .gitignore | 1 + ...tions_remove_household_address_and_more.py | 169 ++++++++++++++++++ .../migrations/0003_household_name.py | 22 +++ ...ryoffice_slug_userrole_expires_and_more.py | 34 ++++ .../migrations/0003_countryoffice_code.py | 18 ++ .../migrations/0004_countryoffice_hope_id.py | 18 ++ 6 files changed, 262 insertions(+) create mode 100644 src/hope_country_workspace/migrations/0002_alter_household_options_remove_household_address_and_more.py create mode 100644 src/hope_country_workspace/migrations/0003_household_name.py create mode 100644 src/hope_country_workspace/security/migrations/0002_countryoffice_slug_userrole_expires_and_more.py create mode 100644 src/hope_country_workspace/security/migrations/0003_countryoffice_code.py create mode 100644 src/hope_country_workspace/security/migrations/0004_countryoffice_hope_id.py diff --git a/.gitignore b/.gitignore index bbbc9d4..97f719d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ !.pdm-python !.github Makefile +coverage.xml diff --git a/src/hope_country_workspace/migrations/0002_alter_household_options_remove_household_address_and_more.py b/src/hope_country_workspace/migrations/0002_alter_household_options_remove_household_address_and_more.py new file mode 100644 index 0000000..eaf5083 --- /dev/null +++ b/src/hope_country_workspace/migrations/0002_alter_household_options_remove_household_address_and_more.py @@ -0,0 +1,169 @@ +# Generated by Django 5.0.6 on 2024-07-05 14:28 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("hope_country_workspace", "0001_initial"), + ] + + operations = [ + migrations.AlterModelOptions( + name="household", + options={"verbose_name": "Household"}, + ), + migrations.RemoveField( + model_name="household", + name="address", + ), + migrations.RemoveField( + model_name="household", + name="admin1", + ), + migrations.RemoveField( + model_name="household", + name="admin2", + ), + migrations.RemoveField( + model_name="household", + name="admin3", + ), + migrations.RemoveField( + model_name="household", + name="admin4", + ), + migrations.RemoveField( + model_name="household", + name="admin_area", + ), + migrations.RemoveField( + model_name="household", + name="children_count", + ), + migrations.RemoveField( + model_name="household", + name="children_disabled_count", + ), + migrations.RemoveField( + model_name="household", + name="country", + ), + migrations.RemoveField( + model_name="household", + name="female_age_group_0_5_count", + ), + migrations.RemoveField( + model_name="household", + name="female_age_group_0_5_disabled_count", + ), + migrations.RemoveField( + model_name="household", + name="female_age_group_12_17_count", + ), + migrations.RemoveField( + model_name="household", + name="female_age_group_12_17_disabled_count", + ), + migrations.RemoveField( + model_name="household", + name="female_age_group_18_59_count", + ), + migrations.RemoveField( + model_name="household", + name="female_age_group_18_59_disabled_count", + ), + migrations.RemoveField( + model_name="household", + name="female_age_group_60_count", + ), + migrations.RemoveField( + model_name="household", + name="female_age_group_60_disabled_count", + ), + migrations.RemoveField( + model_name="household", + name="female_age_group_6_11_count", + ), + migrations.RemoveField( + model_name="household", + name="female_age_group_6_11_disabled_count", + ), + migrations.RemoveField( + model_name="household", + name="female_children_count", + ), + migrations.RemoveField( + model_name="household", + name="female_children_disabled_count", + ), + migrations.RemoveField( + model_name="household", + name="male_age_group_0_5_count", + ), + migrations.RemoveField( + model_name="household", + name="male_age_group_0_5_disabled_count", + ), + migrations.RemoveField( + model_name="household", + name="male_age_group_12_17_count", + ), + migrations.RemoveField( + model_name="household", + name="male_age_group_12_17_disabled_count", + ), + migrations.RemoveField( + model_name="household", + name="male_age_group_18_59_count", + ), + migrations.RemoveField( + model_name="household", + name="male_age_group_18_59_disabled_count", + ), + migrations.RemoveField( + model_name="household", + name="male_age_group_60_count", + ), + migrations.RemoveField( + model_name="household", + name="male_age_group_60_disabled_count", + ), + migrations.RemoveField( + model_name="household", + name="male_age_group_6_11_count", + ), + migrations.RemoveField( + model_name="household", + name="male_age_group_6_11_disabled_count", + ), + migrations.RemoveField( + model_name="household", + name="male_children_count", + ), + migrations.RemoveField( + model_name="household", + name="male_children_disabled_count", + ), + migrations.RemoveField( + model_name="household", + name="pregnant_count", + ), + migrations.RemoveField( + model_name="household", + name="representatives", + ), + migrations.RemoveField( + model_name="household", + name="residence_status", + ), + migrations.RemoveField( + model_name="household", + name="size", + ), + migrations.RemoveField( + model_name="household", + name="zip_code", + ), + ] diff --git a/src/hope_country_workspace/migrations/0003_household_name.py b/src/hope_country_workspace/migrations/0003_household_name.py new file mode 100644 index 0000000..ab3f689 --- /dev/null +++ b/src/hope_country_workspace/migrations/0003_household_name.py @@ -0,0 +1,22 @@ +# Generated by Django 5.0.6 on 2024-07-05 15:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ( + "hope_country_workspace", + "0002_alter_household_options_remove_household_address_and_more", + ), + ] + + operations = [ + migrations.AddField( + model_name="household", + name="name", + field=models.CharField(default=1, max_length=255, verbose_name="Name"), + preserve_default=False, + ), + ] diff --git a/src/hope_country_workspace/security/migrations/0002_countryoffice_slug_userrole_expires_and_more.py b/src/hope_country_workspace/security/migrations/0002_countryoffice_slug_userrole_expires_and_more.py new file mode 100644 index 0000000..b454813 --- /dev/null +++ b/src/hope_country_workspace/security/migrations/0002_countryoffice_slug_userrole_expires_and_more.py @@ -0,0 +1,34 @@ +# Generated by Django 5.0.6 on 2024-07-05 14:28 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("security", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="countryoffice", + name="slug", + field=models.SlugField(blank=True, max_length=100, null=True), + ), + migrations.AddField( + model_name="userrole", + name="expires", + field=models.DateField(blank=True, null=True), + ), + migrations.AlterField( + model_name="userrole", + name="user", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="roles", + to=settings.AUTH_USER_MODEL, + ), + ), + ] diff --git a/src/hope_country_workspace/security/migrations/0003_countryoffice_code.py b/src/hope_country_workspace/security/migrations/0003_countryoffice_code.py new file mode 100644 index 0000000..81083c3 --- /dev/null +++ b/src/hope_country_workspace/security/migrations/0003_countryoffice_code.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.6 on 2024-07-05 14:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("security", "0002_countryoffice_slug_userrole_expires_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="countryoffice", + name="code", + field=models.CharField(blank=True, max_length=100, null=True), + ), + ] diff --git a/src/hope_country_workspace/security/migrations/0004_countryoffice_hope_id.py b/src/hope_country_workspace/security/migrations/0004_countryoffice_hope_id.py new file mode 100644 index 0000000..d3a54cd --- /dev/null +++ b/src/hope_country_workspace/security/migrations/0004_countryoffice_hope_id.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.6 on 2024-07-05 15:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("security", "0003_countryoffice_code"), + ] + + operations = [ + migrations.AddField( + model_name="countryoffice", + name="hope_id", + field=models.CharField(blank=True, max_length=100, null=True), + ), + ] From 735e8220ad53bce1a2b2e204b6c033c02ae4b99c Mon Sep 17 00:00:00 2001 From: saxix Date: Fri, 5 Jul 2024 17:21:32 +0200 Subject: [PATCH 04/16] updates --- .flake8 | 13 +++++++++++++ .gitignore | 5 +++-- 2 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..c3fe134 --- /dev/null +++ b/.flake8 @@ -0,0 +1,13 @@ +[flake8] +max-complexity = 12 +max-line-length = 120 +exclude = + .*/ + __pycache__ + docs + ~build + dist + *.md + +per-file-ignores = + src/**/migrations/*.py:E501 diff --git a/.gitignore b/.gitignore index 97f719d..f02d35f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ .* ~* +Makefile +coverage.xml !.dockerignore !.gitignore !.pdm-python !.github -Makefile -coverage.xml +!.flake8 From c1bc0a9f902e47fb80dbb1123cf0a2d1307bde0d Mon Sep 17 00:00:00 2001 From: saxix Date: Fri, 5 Jul 2024 17:23:12 +0200 Subject: [PATCH 05/16] updates --- .flake8 | 1 + 1 file changed, 1 insertion(+) diff --git a/.flake8 b/.flake8 index c3fe134..784c638 100644 --- a/.flake8 +++ b/.flake8 @@ -11,3 +11,4 @@ exclude = per-file-ignores = src/**/migrations/*.py:E501 + src/**/upgrade.py:E731 From 78ef8dfbd19918d13b9755c1f94b160e910ed2d4 Mon Sep 17 00:00:00 2001 From: saxix Date: Fri, 5 Jul 2024 17:31:35 +0200 Subject: [PATCH 06/16] lint --- src/hope_country_workspace/state.py | 1 + src/hope_country_workspace/tenant/config.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hope_country_workspace/state.py b/src/hope_country_workspace/state.py index d19742b..da9abb5 100644 --- a/src/hope_country_workspace/state.py +++ b/src/hope_country_workspace/state.py @@ -10,6 +10,7 @@ if TYPE_CHECKING: from collections.abc import Iterator from typing import Any, List + from hope_country_workspace.types.http import AnyResponse not_set = object() diff --git a/src/hope_country_workspace/tenant/config.py b/src/hope_country_workspace/tenant/config.py index d12327d..a3fa191 100644 --- a/src/hope_country_workspace/tenant/config.py +++ b/src/hope_country_workspace/tenant/config.py @@ -39,7 +39,7 @@ def __init__(self, prefix: str): setting_changed.connect(self._on_setting_changed) def _set_attr(self, prefixed_name: str, value: "Any") -> None: - name = prefixed_name[(len(self.prefix) + 1):] + name = prefixed_name[(len(self.prefix) + 1) :] setattr(self, name, value) @cached_property From 3d179d527911e0fc2f439a4be97f87ded9b7c9d7 Mon Sep 17 00:00:00 2001 From: saxix Date: Fri, 5 Jul 2024 17:40:40 +0200 Subject: [PATCH 07/16] updates --- src/hope_country_workspace/tenant/config.py | 2 +- src/hope_country_workspace/tenant/db.py | 2 +- tests/conftest.py | 10 +++-- tests/extras/testutils/factories/__init__.py | 2 +- tests/extras/testutils/factories/household.py | 2 + tests/extras/testutils/factories/user.py | 2 +- tests/extras/testutils/perms.py | 13 ++++-- tests/tenant/test_tenant_backend.py | 42 ++++++++++++------- tests/tenant/test_tenant_filtering.py | 22 +++++++--- tests/tenant/test_tenant_manager.py | 14 ++++--- tests/tenant/test_tenant_middleware.py | 5 ++- tests/tenant/test_tenant_permissions.py | 12 ++++-- 12 files changed, 89 insertions(+), 39 deletions(-) diff --git a/src/hope_country_workspace/tenant/config.py b/src/hope_country_workspace/tenant/config.py index a3fa191..d12327d 100644 --- a/src/hope_country_workspace/tenant/config.py +++ b/src/hope_country_workspace/tenant/config.py @@ -39,7 +39,7 @@ def __init__(self, prefix: str): setting_changed.connect(self._on_setting_changed) def _set_attr(self, prefixed_name: str, value: "Any") -> None: - name = prefixed_name[(len(self.prefix) + 1) :] + name = prefixed_name[(len(self.prefix) + 1):] setattr(self, name, value) @cached_property diff --git a/src/hope_country_workspace/tenant/db.py b/src/hope_country_workspace/tenant/db.py index 227886b..5274bde 100644 --- a/src/hope_country_workspace/tenant/db.py +++ b/src/hope_country_workspace/tenant/db.py @@ -29,7 +29,7 @@ def get_tenant_filter(self) -> "Dict[str, Any]": active_tenant = get_selected_tenant() if not active_tenant: raise InvalidTenantError("State does not have any active tenant") - return {tenant_filter_field: state.tenant.hope_id} + return {tenant_filter_field: state.tenant} def get_queryset(self): flt = self.get_tenant_filter() diff --git a/tests/conftest.py b/tests/conftest.py index 1c4b5b0..3199348 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,13 +8,12 @@ import responses from constance import config - here = Path(__file__).parent sys.path.insert(0, str(here / "../src")) sys.path.insert(0, str(here / "extras")) -def pytest_configure(config): +def pytest_configure(config): os.environ.update(DJANGO_SETTINGS_MODULE="hope_country_workspace.config.settings") os.environ.setdefault("MEDIA_ROOT", "/tmp/static/") os.environ.setdefault("STATIC_ROOT", "/tmp/media/") @@ -73,6 +72,12 @@ def mocked_responses(): yield rsps +@pytest.fixture() +def user(db): + from testutils.factories import UserFactory + + return UserFactory() + @pytest.fixture() def afghanistan(db): @@ -81,7 +86,6 @@ def afghanistan(db): return CountryOfficeFactory(name="Afghanistan") - @pytest.fixture def reporters(db, afghanistan, user): from hope_country_workspace.utils import get_or_create_defaults_group diff --git a/tests/extras/testutils/factories/__init__.py b/tests/extras/testutils/factories/__init__.py index 5ed822b..f0c37f5 100644 --- a/tests/extras/testutils/factories/__init__.py +++ b/tests/extras/testutils/factories/__init__.py @@ -10,8 +10,8 @@ TAutoRegisterModelFactory, factories_registry, ) -from .household import HouseholdFactory from .django_celery_beat import PeriodicTaskFactory # noqa +from .household import HouseholdFactory # noqa from .social import SocialAuthUserFactory # noqa from .user import ( # noqa CountryOfficeFactory, diff --git a/tests/extras/testutils/factories/household.py b/tests/extras/testutils/factories/household.py index 81ecb72..67f7d55 100644 --- a/tests/extras/testutils/factories/household.py +++ b/tests/extras/testutils/factories/household.py @@ -4,8 +4,10 @@ from .base import AutoRegisterModelFactory + class HouseholdFactory(AutoRegisterModelFactory): name = factory.Sequence(lambda n: f"Household {n}") + class Meta: model = Household django_get_or_create = ("name",) diff --git a/tests/extras/testutils/factories/user.py b/tests/extras/testutils/factories/user.py index 21239a4..71347ca 100644 --- a/tests/extras/testutils/factories/user.py +++ b/tests/extras/testutils/factories/user.py @@ -2,7 +2,7 @@ import factory.fuzzy -from hope_country_workspace.security.models import User, CountryOffice +from hope_country_workspace.security.models import CountryOffice, User from .base import AutoRegisterModelFactory diff --git a/tests/extras/testutils/perms.py b/tests/extras/testutils/perms.py index b2faea0..a97f484 100644 --- a/tests/extras/testutils/perms.py +++ b/tests/extras/testutils/perms.py @@ -1,6 +1,5 @@ from contextlib import ContextDecorator from random import choice - from unittest.mock import Mock from django.conf import settings @@ -45,9 +44,13 @@ def get_group(name=None, permissions=None): except ValueError: raise ValueError(f"Invalid permission name `{permission_name}`") try: - permission = Permission.objects.get(content_type__app_label=app_label, codename=codename) + permission = Permission.objects.get( + content_type__app_label=app_label, codename=codename + ) except Permission.DoesNotExist: - raise Permission.DoesNotExist("Permission `{0}` does not exists", permission_name) + raise Permission.DoesNotExist( + "Permission `{0}` does not exists", permission_name + ) group.permissions.add(permission) return group @@ -148,7 +151,9 @@ def __enter__(self): self.group = get_group(name=self.group_name, permissions=self.permissions or []) self.user.groups.add(self.group) if self.country_office: - UserRole.objects.get_or_create(country_office=self.country_office, user=self.user, group=self.group) + UserRole.objects.get_or_create( + country_office=self.country_office, user=self.user, group=self.group + ) return self def __exit__(self, e_typ, e_val, trcbak): diff --git a/tests/tenant/test_tenant_backend.py b/tests/tenant/test_tenant_backend.py index 37624f7..635b49f 100644 --- a/tests/tenant/test_tenant_backend.py +++ b/tests/tenant/test_tenant_backend.py @@ -1,23 +1,21 @@ -from typing import TYPE_CHECKING - from collections import namedtuple - -import pytest +from typing import TYPE_CHECKING from django.apps import apps from django.contrib.auth.models import AnonymousUser +import pytest from testutils.perms import user_grant_permissions -from hope_country_workspace.tenant.backend import TenantBackend from hope_country_workspace.state import state +from hope_country_workspace.tenant.backend import TenantBackend if TYPE_CHECKING: from hope_country_workspace.security.models import CountryOffice _DATA = namedtuple("_DATA", "afg,ukr") -from pytest_factoryboy import LazyFixture, register +from pytest_factoryboy import register from testutils.factories import UserFactory register(UserFactory) @@ -64,16 +62,24 @@ def test_get_all_permissions_no_enabled_tenant(backend, data, user, admin_user): def test_get_all_permissions_no_current_tenant(backend, data, user, admin_user): with state.set(tenant=data.afg): - with user_grant_permissions(user, "hope_country_workspace.view_household", country_office=data.ukr): + with user_grant_permissions( + user, "hope_country_workspace.view_household", country_office=data.ukr + ): assert backend.get_all_permissions(user) == set() def test_get_all_permissions_valid_tenant(backend, data, user, admin_user): with state.set(tenant=data.afg): - with user_grant_permissions(user, "hope_country_workspace.view_household", country_office=data.afg): - assert backend.get_all_permissions(user) == {"hope_country_workspace.view_household"} + with user_grant_permissions( + user, "hope_country_workspace.view_household", country_office=data.afg + ): + assert backend.get_all_permissions(user) == { + "hope_country_workspace.view_household" + } # test cache - assert backend.get_all_permissions(user) == {"hope_country_workspace.view_household"} + assert backend.get_all_permissions(user) == { + "hope_country_workspace.view_household" + } def test_get_all_permissions_superuser(backend, data, user, admin_user): @@ -83,18 +89,24 @@ def test_get_all_permissions_superuser(backend, data, user, admin_user): def test_get_available_modules(backend, data, user, admin_user): with state.set(tenant=data.afg): - with user_grant_permissions(user, "hope_country_workspace.view_household", country_office=data.afg): + with user_grant_permissions( + user, "hope_country_workspace.view_household", country_office=data.afg + ): assert backend.get_available_modules(user) == {"hope_country_workspace"} def test_has_module_perms_no_active_tenant(backend, data, user, admin_user): - with user_grant_permissions(user, "hope_country_workspace.view_household", country_office=data.afg): + with user_grant_permissions( + user, "hope_country_workspace.view_household", country_office=data.afg + ): assert not backend.has_module_perms(user, "hope_country_workspace") def test_has_module_perms(backend, data, user, admin_user): with state.set(tenant=data.afg): - with user_grant_permissions(user, "hope_country_workspace.view_household", country_office=data.afg): + with user_grant_permissions( + user, "hope_country_workspace.view_household", country_office=data.afg + ): assert backend.has_module_perms(user, "hope_country_workspace") @@ -107,7 +119,9 @@ def test_get_allowed_tenants_user(backend, data, user, rf): request = rf.get("/") request.user = user with state.set(tenant=data.afg, request=request): - with user_grant_permissions(user, "hope_country_workspace.view_household", country_office=data.afg): + with user_grant_permissions( + user, "hope_country_workspace.view_household", country_office=data.afg + ): assert backend.get_allowed_tenants().count() == 1 assert backend.get_allowed_tenants().first() == data.afg diff --git a/tests/tenant/test_tenant_filtering.py b/tests/tenant/test_tenant_filtering.py index 77fbd20..7437b62 100644 --- a/tests/tenant/test_tenant_filtering.py +++ b/tests/tenant/test_tenant_filtering.py @@ -1,8 +1,9 @@ from typing import TYPE_CHECKING -import pytest from django.conf import settings +import pytest + from hope_country_workspace.state import state if TYPE_CHECKING: @@ -11,7 +12,12 @@ @pytest.fixture() def data(): - from testutils.factories import CountryOfficeFactory, HouseholdFactory, UserFactory, UserRoleFactory + from testutils.factories import ( + CountryOfficeFactory, + HouseholdFactory, + UserFactory, + UserRoleFactory, + ) with state.set(must_tenant=False): co1: "CountryOffice" = CountryOfficeFactory(name="Afghanistan") @@ -23,13 +29,19 @@ def data(): HouseholdFactory(country_office=co2) HouseholdFactory(country_office=co2) - user = UserFactory(username="user", is_staff=False, is_superuser=False, is_active=True) - UserRoleFactory(country_office=co1, group__name=settings.ANALYST_GROUP_NAME, user=user) + user = UserFactory( + username="user", is_staff=False, is_superuser=False, is_active=True + ) + UserRoleFactory( + country_office=co1, group__name=settings.ANALYST_GROUP_NAME, user=user + ) return {"Afghanistan": co1, "Niger": co2, "Sudan": co3} -@pytest.mark.parametrize("co,expected", [("Afghanistan", 2), ("Niger", 2), ("Sudan", 0)]) +@pytest.mark.parametrize( + "co,expected", [("Afghanistan", 2), ("Niger", 2), ("Sudan", 0)] +) def test_filtering(data, co, expected): from hope_country_workspace.models import Household diff --git a/tests/tenant/test_tenant_manager.py b/tests/tenant/test_tenant_manager.py index 9296072..9839997 100644 --- a/tests/tenant/test_tenant_manager.py +++ b/tests/tenant/test_tenant_manager.py @@ -1,11 +1,11 @@ from typing import TYPE_CHECKING +from unittest import mock import pytest -from unittest import mock +from hope_country_workspace.state import state from hope_country_workspace.tenant.db import TenantManager from hope_country_workspace.tenant.exceptions import InvalidTenantError -from hope_country_workspace.state import state if TYPE_CHECKING: from hope_country_workspace.security.models import CountryOffice @@ -44,14 +44,16 @@ def test_get_tenant_filter_valid_tenant(manager, afghanistan): manager.model = Household with state.set(must_tenant=True, tenant=afghanistan): - assert manager.get_tenant_filter() == {"country_office": afghanistan.hope_id} + assert manager.get_tenant_filter() == {"country_office": afghanistan} def test_get_tenant_filter_invalid_model(manager, afghanistan): from hope_country_workspace.models import Household manager.model = Household - with mock.patch("hope_country_workspace.models.Household.Tenant.tenant_filter_field", ""): + with mock.patch( + "hope_country_workspace.models.Household.Tenant.tenant_filter_field", "" + ): with pytest.raises(ValueError): with state.set(must_tenant=True, tenant=afghanistan): manager.get_tenant_filter() @@ -61,7 +63,9 @@ def test_get_tenant_filter_all(manager, afghanistan): from hope_country_workspace.models import Household manager.model = Household - with mock.patch("hope_country_workspace.models.Household.Tenant.tenant_filter_field", "__all__"): + with mock.patch( + "hope_country_workspace.models.Household.Tenant.tenant_filter_field", "__all__" + ): with state.set(must_tenant=True, tenant=afghanistan): assert manager.get_tenant_filter() == {} diff --git a/tests/tenant/test_tenant_middleware.py b/tests/tenant/test_tenant_middleware.py index 3cd309b..867d097 100644 --- a/tests/tenant/test_tenant_middleware.py +++ b/tests/tenant/test_tenant_middleware.py @@ -2,7 +2,10 @@ from django.http import HttpResponse -from hope_country_workspace.middleware.state import StateClearMiddleware, StateSetMiddleware +from hope_country_workspace.middleware.state import ( + StateClearMiddleware, + StateSetMiddleware, +) def test_set(rf, user): diff --git a/tests/tenant/test_tenant_permissions.py b/tests/tenant/test_tenant_permissions.py index 906a97c..cd7a40f 100644 --- a/tests/tenant/test_tenant_permissions.py +++ b/tests/tenant/test_tenant_permissions.py @@ -43,7 +43,9 @@ def req(request, rf) -> "AuthHttpRequest": @pytest.mark.parametrize("u", ["tenant_user", "admin_user", "anonymous"]) -def test_tenant_backend_get_allowed_tenants(request, afghanistan, req, u, django_assert_max_num_queries): +def test_tenant_backend_get_allowed_tenants( + request, afghanistan, req, u, django_assert_max_num_queries +): from hope_country_workspace.tenant.backend import TenantBackend req.user = request.getfixturevalue(u) @@ -51,13 +53,17 @@ def test_tenant_backend_get_allowed_tenants(request, afghanistan, req, u, django # with django_assert_max_num_queries(1): if req.user.is_authenticated: - assert list(b.get_allowed_tenants().values_list("pk", flat=True)) == [afghanistan.pk] + assert list(b.get_allowed_tenants().values_list("pk", flat=True)) == [ + afghanistan.pk + ] else: assert not b.get_allowed_tenants().values_list("pk", flat=True) @pytest.mark.parametrize("u", ["tenant_user", "admin_user", "anonymous"]) -def test_tenant_backend_get_all_permissions(request, afghanistan, req, u, django_assert_max_num_queries): +def test_tenant_backend_get_all_permissions( + request, afghanistan, req, u, django_assert_max_num_queries +): from hope_country_workspace.tenant.backend import TenantBackend req.user = request.getfixturevalue(u) From 4c353f3c2913643b0959025a40c68f2e39cf3670 Mon Sep 17 00:00:00 2001 From: saxix Date: Fri, 5 Jul 2024 17:41:52 +0200 Subject: [PATCH 08/16] updates --- src/hope_country_workspace/tenant/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hope_country_workspace/tenant/config.py b/src/hope_country_workspace/tenant/config.py index d12327d..a3fa191 100644 --- a/src/hope_country_workspace/tenant/config.py +++ b/src/hope_country_workspace/tenant/config.py @@ -39,7 +39,7 @@ def __init__(self, prefix: str): setting_changed.connect(self._on_setting_changed) def _set_attr(self, prefixed_name: str, value: "Any") -> None: - name = prefixed_name[(len(self.prefix) + 1):] + name = prefixed_name[(len(self.prefix) + 1) :] setattr(self, name, value) @cached_property From d85ed395b71dab022bcdf8757776646302e0ecd2 Mon Sep 17 00:00:00 2001 From: saxix Date: Fri, 5 Jul 2024 18:25:53 +0200 Subject: [PATCH 09/16] updates --- .flake8 | 3 +- .github/codeql/codeql-config.yml | 2 +- .pdm-python | 2 +- docker/bin/wait-for-it.sh | 2 +- docker/conf/mime.types | 2 +- manage.py | 4 +- pdm.lock | 90 ++++++++++++++++++- pyproject.toml | 1 + src/hope_country_workspace/config/__init__.py | 13 ++- .../config/fragments/celery.py | 1 - .../config/fragments/csp.py | 3 +- src/hope_country_workspace/config/settings.py | 1 - src/hope_country_workspace/tenant/config.py | 2 +- src/hope_country_workspace/tenant/db.py | 3 +- src/hope_country_workspace/tenant/utils.py | 6 -- tests/tenant/test_tenant_backend.py | 4 +- 16 files changed, 116 insertions(+), 23 deletions(-) diff --git a/.flake8 b/.flake8 index 784c638..15c6267 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,6 @@ [flake8] max-complexity = 12 -max-line-length = 120 +max-line-length = 88 exclude = .*/ __pycache__ @@ -12,3 +12,4 @@ exclude = per-file-ignores = src/**/migrations/*.py:E501 src/**/upgrade.py:E731 + src/**/upgrade.py:E731 diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml index 979cf37..955fcce 100644 --- a/.github/codeql/codeql-config.yml +++ b/.github/codeql/codeql-config.yml @@ -1,4 +1,4 @@ name: 'App CodeQL Config' paths-ignore: - - '**/tests/**' \ No newline at end of file + - '**/tests/**' diff --git a/.pdm-python b/.pdm-python index a9ff23a..4d4f30c 100644 --- a/.pdm-python +++ b/.pdm-python @@ -1 +1 @@ -/Users/sax/Documents/data/PROGETTI/UNICEF/hope-country-workspace/.venv/bin/python \ No newline at end of file +/Users/sax/Documents/data/PROGETTI/UNICEF/hope-country-workspace/.venv/bin/python diff --git a/docker/bin/wait-for-it.sh b/docker/bin/wait-for-it.sh index 5d868ec..4d473c4 100644 --- a/docker/bin/wait-for-it.sh +++ b/docker/bin/wait-for-it.sh @@ -179,4 +179,4 @@ if [[ $WAITFORIT_CLI != "" ]]; then exec "${WAITFORIT_CLI[@]}" else exit $WAITFORIT_RESULT -fi \ No newline at end of file +fi diff --git a/docker/conf/mime.types b/docker/conf/mime.types index 8c6daa1..ec6469d 100644 --- a/docker/conf/mime.types +++ b/docker/conf/mime.types @@ -2184,4 +2184,4 @@ video/x-sgi-movie movie x-conference/x-cooltalk ice x-epoc/x-sisx-app sisx -x-world/x-vrml vrm vrml wrl \ No newline at end of file +x-world/x-vrml vrm vrml wrl diff --git a/manage.py b/manage.py index 4b0471f..84aeee7 100755 --- a/manage.py +++ b/manage.py @@ -6,7 +6,9 @@ sys.path.insert(0, SRC) if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "hope_country_workspace.config.settings") + os.environ.setdefault( + "DJANGO_SETTINGS_MODULE", "hope_country_workspace.config.settings" + ) from django.core.management import execute_from_command_line diff --git a/pdm.lock b/pdm.lock index a3f16e9..f4a60d6 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.1" -content_hash = "sha256:8744d30ad35acc7ffb7119daa600cc8b2557e66f380d27f20ecc3104ba587ae6" +content_hash = "sha256:bff7331d24703aa007cb86ab745ffa3e16edf0d199c28ea41d9f12278aa422e7" [[package]] name = "amqp" @@ -147,6 +147,17 @@ files = [ {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, ] +[[package]] +name = "cfgv" +version = "3.4.0" +requires_python = ">=3.8" +summary = "Validate configuration and produce human readable error messages." +groups = ["dev"] +files = [ + {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"}, + {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, +] + [[package]] name = "charset-normalizer" version = "3.3.2" @@ -378,6 +389,16 @@ files = [ {file = "defusedxml-0.8.0rc2.tar.gz", hash = "sha256:138c7d540a78775182206c7c97fe65b246a2f40b29471e1a2f1b0da76e7a3942"}, ] +[[package]] +name = "distlib" +version = "0.3.8" +summary = "Distribution utilities" +groups = ["dev"] +files = [ + {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, + {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, +] + [[package]] name = "django" version = "5.0.6" @@ -736,6 +757,17 @@ files = [ {file = "fancycompleter-0.9.1.tar.gz", hash = "sha256:09e0feb8ae242abdfd7ef2ba55069a46f011814a80fe5476be48f51b00247272"}, ] +[[package]] +name = "filelock" +version = "3.15.4" +requires_python = ">=3.8" +summary = "A platform independent file lock." +groups = ["dev"] +files = [ + {file = "filelock-3.15.4-py3-none-any.whl", hash = "sha256:6ca1fffae96225dab4c6eaf1c4f4f28cd2568d3ec2a44e15a08520504de468e7"}, + {file = "filelock-3.15.4.tar.gz", hash = "sha256:2207938cbc1844345cb01a5a95524dae30f0ce089eba5b00378295a17e3e90cb"}, +] + [[package]] name = "flake8" version = "7.1.0" @@ -766,6 +798,17 @@ files = [ {file = "freezegun-1.5.1.tar.gz", hash = "sha256:b29dedfcda6d5e8e083ce71b2b542753ad48cfec44037b3fc79702e2980a89e9"}, ] +[[package]] +name = "identify" +version = "2.5.36" +requires_python = ">=3.8" +summary = "File identification library for Python" +groups = ["dev"] +files = [ + {file = "identify-2.5.36-py2.py3-none-any.whl", hash = "sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa"}, + {file = "identify-2.5.36.tar.gz", hash = "sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d"}, +] + [[package]] name = "idna" version = "3.7" @@ -847,6 +890,17 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "nodeenv" +version = "1.9.1" +requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +summary = "Node.js virtual environment builder" +groups = ["dev"] +files = [ + {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"}, + {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, +] + [[package]] name = "oauthlib" version = "3.2.2" @@ -917,6 +971,24 @@ files = [ {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] +[[package]] +name = "pre-commit" +version = "3.7.1" +requires_python = ">=3.9" +summary = "A framework for managing and maintaining multi-language pre-commit hooks." +groups = ["dev"] +dependencies = [ + "cfgv>=2.0.0", + "identify>=1.0.0", + "nodeenv>=0.11.1", + "pyyaml>=5.1", + "virtualenv>=20.10.0", +] +files = [ + {file = "pre_commit-3.7.1-py2.py3-none-any.whl", hash = "sha256:fae36fd1d7ad7d6a5a1c0b0d5adb2ed1a3bda5a21bf6c3e5372073d7a11cd4c5"}, + {file = "pre_commit-3.7.1.tar.gz", hash = "sha256:8ca3ad567bc78a4972a3f1a477e94a79d4597e8140a6e0b651c5e33899c3654a"}, +] + [[package]] name = "prompt-toolkit" version = "3.0.47" @@ -1516,6 +1588,22 @@ files = [ {file = "vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0"}, ] +[[package]] +name = "virtualenv" +version = "20.26.3" +requires_python = ">=3.7" +summary = "Virtual Python Environment builder" +groups = ["dev"] +dependencies = [ + "distlib<1,>=0.3.7", + "filelock<4,>=3.12.2", + "platformdirs<5,>=3.9.1", +] +files = [ + {file = "virtualenv-20.26.3-py3-none-any.whl", hash = "sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589"}, + {file = "virtualenv-20.26.3.tar.gz", hash = "sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a"}, +] + [[package]] name = "waitress" version = "3.0.0" diff --git a/pyproject.toml b/pyproject.toml index 1f4ecab..08097c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ dev = [ "responses>=0.25.3", "pytest-factoryboy>=2.7.0", "django-webtest>=1.9.11", + "pre-commit>=3.7.1", ] [tool.black] diff --git a/src/hope_country_workspace/config/__init__.py b/src/hope_country_workspace/config/__init__.py index a99bde3..59d648d 100644 --- a/src/hope_country_workspace/config/__init__.py +++ b/src/hope_country_workspace/config/__init__.py @@ -17,6 +17,13 @@ def setting(anchor: str) -> str: return f"@see {DJANGO_HELP_BASE}#{anchor}" +def celery_doc(anchor: str) -> str: + return ( + f"@see https://docs.celeryq.dev/en/stable/" + f"userguide/configuration.html#{anchor}" + ) + + class Group(Enum): DJANGO = 1 @@ -46,18 +53,18 @@ class Group(Enum): "CELERY_TASK_ALWAYS_EAGER": ( bool, False, - "https://docs.celeryq.dev/en/stable/userguide/configuration.html#std-setting-task_always_eager", + f"{celery_doc}#std-setting-task_always_eager", True, ), "CELERY_TASK_EAGER_PROPAGATES": ( bool, True, - "https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-eager-propagates", + f"{celery_doc}#task-eager-propagates", ), "CELERY_VISIBILITY_TIMEOUT": ( int, 1800, - "https://docs.celeryq.dev/en/stable/userguide/configuration.html#broker-transport-options", + f"{celery_doc}#broker-transport-options", ), "CSRF_COOKIE_SECURE": (bool, True, setting("csrf-cookie-secure")), "DATABASE_URL": ( diff --git a/src/hope_country_workspace/config/fragments/celery.py b/src/hope_country_workspace/config/fragments/celery.py index ee15ce5..2d192f3 100644 --- a/src/hope_country_workspace/config/fragments/celery.py +++ b/src/hope_country_workspace/config/fragments/celery.py @@ -1,7 +1,6 @@ from ..settings import env # type: ignore[attr-defined] CELERY_ACCEPT_CONTENT = ["pickle", "json", "application/text", "application/json"] -# CELERY_BROKER_TRANSPORT_OPTIONS = {"visibility_timeout": int(CELERY_BROKER_VISIBILITY_VAR)} CELERY_BROKER_URL = env("CELERY_BROKER_URL") CELERY_BROKER_VISIBILITY_VAR = env("CELERY_VISIBILITY_TIMEOUT") diff --git a/src/hope_country_workspace/config/fragments/csp.py b/src/hope_country_workspace/config/fragments/csp.py index a0e02fb..773433f 100644 --- a/src/hope_country_workspace/config/fragments/csp.py +++ b/src/hope_country_workspace/config/fragments/csp.py @@ -1,4 +1,5 @@ -# CSP_DEFAULT_SRC = ["'self'", "'unsafe-inline'", "'same-origin'", "fonts.googleapis.com", 'fonts.gstatic.com', 'data:', +# CSP_DEFAULT_SRC = ["'self'", "'unsafe-inline'", "'same-origin'", +# "fonts.googleapis.com", 'fonts.gstatic.com', 'data:', # 'blob:', "cdn.redoc.ly"] CSP_DEFAULT_SRC = ["'self'", "'unsafe-inline'"] CSP_STYLE_SRC = [ diff --git a/src/hope_country_workspace/config/settings.py b/src/hope_country_workspace/config/settings.py index 089e6f1..822cbcf 100644 --- a/src/hope_country_workspace/config/settings.py +++ b/src/hope_country_workspace/config/settings.py @@ -190,7 +190,6 @@ DEFAULT_FROM_EMAIL = "hope@unicef.org" -# EMAIL_BACKEND = "djcelery_email.backends.CeleryEmailBackend" # TODO: when ready, add djcelery_email EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" EMAIL_HOST = env("EMAIL_HOST", default="") EMAIL_HOST_USER = env("EMAIL_HOST_USER", default="") diff --git a/src/hope_country_workspace/tenant/config.py b/src/hope_country_workspace/tenant/config.py index a3fa191..2d73be7 100644 --- a/src/hope_country_workspace/tenant/config.py +++ b/src/hope_country_workspace/tenant/config.py @@ -39,7 +39,7 @@ def __init__(self, prefix: str): setting_changed.connect(self._on_setting_changed) def _set_attr(self, prefixed_name: str, value: "Any") -> None: - name = prefixed_name[(len(self.prefix) + 1) :] + name = prefixed_name[(len(self.prefix) + 1) :] # noqa setattr(self, name, value) @cached_property diff --git a/src/hope_country_workspace/tenant/db.py b/src/hope_country_workspace/tenant/db.py index 5274bde..d62a6c5 100644 --- a/src/hope_country_workspace/tenant/db.py +++ b/src/hope_country_workspace/tenant/db.py @@ -18,7 +18,8 @@ def get_tenant_filter(self) -> "Dict[str, Any]": tenant_filter_field = self.model.Tenant.tenant_filter_field if not tenant_filter_field: raise ValueError( - f"Set 'tenant_filter_field' on {self} or override `get_queryset()` to enable queryset filtering" + f"Set 'tenant_filter_field' on {self} or override " + f"`get_queryset()` to enable queryset filtering" ) if tenant_filter_field == "__all__": return {} diff --git a/src/hope_country_workspace/tenant/utils.py b/src/hope_country_workspace/tenant/utils.py index 60d0021..aa49137 100644 --- a/src/hope_country_workspace/tenant/utils.py +++ b/src/hope_country_workspace/tenant/utils.py @@ -83,9 +83,3 @@ def process_response( if response: state.set_cookies(response) state.reset() - - # @contextlib.contextmanager - # def context(self, request: "AuthHttpRequest", response: "HttpResponse|None" = None) -> "Iterator[None]": - # self.process_request(request) - # yield - # self.process_response(request, response) diff --git a/tests/tenant/test_tenant_backend.py b/tests/tenant/test_tenant_backend.py index 635b49f..ecaf793 100644 --- a/tests/tenant/test_tenant_backend.py +++ b/tests/tenant/test_tenant_backend.py @@ -5,6 +5,8 @@ from django.contrib.auth.models import AnonymousUser import pytest +from pytest_factoryboy import register +from testutils.factories import UserFactory from testutils.perms import user_grant_permissions from hope_country_workspace.state import state @@ -15,8 +17,6 @@ _DATA = namedtuple("_DATA", "afg,ukr") -from pytest_factoryboy import register -from testutils.factories import UserFactory register(UserFactory) From 392e43d0fe7a60698deef755187443e09a72f09f Mon Sep 17 00:00:00 2001 From: saxix Date: Fri, 5 Jul 2024 18:30:51 +0200 Subject: [PATCH 10/16] updates --- .github/workflows/delete_image.yml | 49 ------------------------------ 1 file changed, 49 deletions(-) delete mode 100644 .github/workflows/delete_image.yml diff --git a/.github/workflows/delete_image.yml b/.github/workflows/delete_image.yml deleted file mode 100644 index 7d51ec6..0000000 --- a/.github/workflows/delete_image.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Branch Deleted -on: delete -jobs: - delete: - if: github.event.ref_type == 'branch' - runs-on: ubuntu-latest - steps: - - name: Install regctl - uses: regclient/actions/regctl-installer@main - - name: regctl login - uses: regclient/actions/regctl-login@main - with: - registry: docker.io - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - shell: bash - run: | - ref="${{github.event.ref}}" - tag=$(echo $ref | sed -e "s#refs/heads/##g" | sed -e s#/#-##g) - name="${{vars.DOCKER_IMAGE}}:test-${{github.event.ref}}" - echo "Delete $name" -# - name: Delete Test Docker Image -# shell: bash -# run: | -# name="${{vars.DOCKER_IMAGE}}:test-${{github.event.ref}}" -# registry="https://registry-1.docker.io" -# curl -v -sSL -X DELETE "http://${registry}/v2/${name}/manifests/$( -# curl -sSL -I \ -# -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ -# "http://${registry}/v2/${name}/manifests/$( -# curl -sSL "http://${registry}/v2/${name}/tags/list" | jq -r '.tags[0]' -# )" \ -# | awk '$1 == "Docker-Content-Digest:" { print $2 }' \ -# | tr -d $'\r' \ -# )" -# - name: Delete linked Docker Image -# shell: bash -# run: | -# name="${{vars.DOCKER_IMAGE}}:${{github.event.ref}}" -# registry="https://registry-1.docker.io" -# curl -v -sSL -X DELETE "http://${registry}/v2/${name}/manifests/$( -# curl -sSL -I \ -# -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ -# "http://${registry}/v2/${name}/manifests/$( -# curl -sSL "http://${registry}/v2/${name}/tags/list" | jq -r '.tags[0]' -# )" \ -# | awk '$1 == "Docker-Content-Digest:" { print $2 }' \ -# | tr -d $'\r' \ -# )" From 411d2a07aa5d16762ee4f1ed7d78ca37691af588 Mon Sep 17 00:00:00 2001 From: saxix Date: Fri, 5 Jul 2024 18:33:33 +0200 Subject: [PATCH 11/16] updates --- .github/actions/docker_build/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/docker_build/action.yml b/.github/actions/docker_build/action.yml index a37dc4a..3a19717 100644 --- a/.github/actions/docker_build/action.yml +++ b/.github/actions/docker_build/action.yml @@ -64,7 +64,9 @@ runs: key: ${{ runner.os }}-regclient - name: Install regctl if: steps.cache-regclient.outputs.cache-hit != 'true' - uses: regclient/actions/regctl-installer@main + uses: regclient/actions/regctl-installer@1.2 + with: + regctl-release: v0.4.7 # optional - name: Cache regclient id: cache-regclient-save uses: actions/cache/save@v4 From cef527269a21ee35c8f403c6114d3f25a35c761c Mon Sep 17 00:00:00 2001 From: saxix Date: Fri, 5 Jul 2024 18:35:14 +0200 Subject: [PATCH 12/16] updates --- .github/actions/docker_build/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/docker_build/action.yml b/.github/actions/docker_build/action.yml index 3a19717..cf7a721 100644 --- a/.github/actions/docker_build/action.yml +++ b/.github/actions/docker_build/action.yml @@ -64,7 +64,7 @@ runs: key: ${{ runner.os }}-regclient - name: Install regctl if: steps.cache-regclient.outputs.cache-hit != 'true' - uses: regclient/actions/regctl-installer@1.2 + uses: regclient/actions/regctl-installer@v1.2 with: regctl-release: v0.4.7 # optional - name: Cache regclient From fc91a0dac14e05c7f7ea950719856ae9468372d0 Mon Sep 17 00:00:00 2001 From: saxix Date: Fri, 5 Jul 2024 18:36:33 +0200 Subject: [PATCH 13/16] updates --- .github/actions/docker_build/action.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/actions/docker_build/action.yml b/.github/actions/docker_build/action.yml index cf7a721..a37dc4a 100644 --- a/.github/actions/docker_build/action.yml +++ b/.github/actions/docker_build/action.yml @@ -64,9 +64,7 @@ runs: key: ${{ runner.os }}-regclient - name: Install regctl if: steps.cache-regclient.outputs.cache-hit != 'true' - uses: regclient/actions/regctl-installer@v1.2 - with: - regctl-release: v0.4.7 # optional + uses: regclient/actions/regctl-installer@main - name: Cache regclient id: cache-regclient-save uses: actions/cache/save@v4 From e2a7501853a690b03ea4b9ae376292f809736440 Mon Sep 17 00:00:00 2001 From: saxix Date: Fri, 5 Jul 2024 18:43:48 +0200 Subject: [PATCH 14/16] updates --- docker/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 12de2cc..0c1ed06 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -234,12 +234,12 @@ LABEL distro="final" LABEL maintainer="hope@unicef.org" LABEL org.opencontainers.image.authors="hope@unicef.org" LABEL org.opencontainers.image.created="$BUILD_DATE" -LABEL org.opencontainers.image.description="App runtime image" -LABEL org.opencontainers.image.documentation="https://github.com/saxix/trash" +LABEL org.opencontainers.image.description="Hope Country Workspace" +LABEL org.opencontainers.image.documentation="https://github.com/unicef/hope-country-workspace/" LABEL org.opencontainers.image.licenses="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${SOURCE_COMMIT:-master}/LICENSE" LABEL org.opencontainers.image.revision=$SOURCE_COMMIT LABEL org.opencontainers.image.source="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/tree/${SOURCE_COMMIT:-master}/" -LABEL org.opencontainers.image.title="Hope Deduplication Engine" +LABEL org.opencontainers.image.title="Hope Country Workspace" LABEL org.opencontainers.image.version="$VERSION" #LABEL org.opencontainers.image.url="https://app.io/" #LABEL org.opencontainers.image.vendor="App ltd" From cbeacf07d5c6c15bb34b9beaec688c2d050b2cd3 Mon Sep 17 00:00:00 2001 From: saxix Date: Fri, 5 Jul 2024 18:48:51 +0200 Subject: [PATCH 15/16] updates --- .github/actions/docker_build/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/docker_build/action.yml b/.github/actions/docker_build/action.yml index a37dc4a..e44afee 100644 --- a/.github/actions/docker_build/action.yml +++ b/.github/actions/docker_build/action.yml @@ -90,6 +90,7 @@ runs: else echo "TAG_PREFIX=test-" >> $GITHUB_ENV fi + echo "IMAGE=${{ inputs.image }}" >> $GITHUB_ENV - id: last_commit uses: ./.github/actions/last_commit - name: Docker meta From ad5db1fb6eb64cd0ed73f5160984358b9ba06107 Mon Sep 17 00:00:00 2001 From: saxix Date: Fri, 5 Jul 2024 18:57:39 +0200 Subject: [PATCH 16/16] updates README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 499a48a..52ee908 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # hope-country-workspace +[![Test](https://github.com/unicef/hope-country-workspace/actions/workflows/test.yml/badge.svg)](https://github.com/unicef/hope-country-workspace/actions/workflows/test.yml) +[![Lint](https://github.com/unicef/hope-country-workspace/actions/workflows/lint.yml/badge.svg)](https://github.com/unicef/hope-country-workspace/actions/workflows/lint.yml) +[![codecov](https://codecov.io/github/unicef/hope-country-workspace/graph/badge.svg?token=FBUB7HML5S)](https://codecov.io/github/unicef/hope-country-workspace) + ## Spreadsheet import ```mermaid