[CR][ENG-5997] merge develop into b-and-i branch (CenterForOpenScienc…

…e#10691)

## Purpose

Bring `feature/b-and-i-24-14` up to date with `develop` (and the upgrade to python v3.12)

## Changes

* Some small conflict resolutions
* One change to make a `.format()` string an f-string

## QA Notes

This may merit a retesting, but I have reviewed the changes in the branch and they all seem pretty compatible with the py3.12 updates. 

## Documentation

No documentation changes needed.

## Side Effects

Devs will need to update their envs for the new py3.12 base.  That'll involve pulling images, rebuilding requirements, and maybe something else?

## Ticket

https://openscience.atlassian.net/browse/ENG-5997

.github/actions/gen-report/action.yml

@@ -0,0 +1,15 @@
+name: 'Generate Report'
+description: 'Generate test report from junit xml file'
+runs:
+  using: "composite"
+  steps:
+  - name: Generate Report
+    uses: dorny/test-reporter@v1
+    if: success() || failure()    # run this step even if previous step failed
+    with:
+      name: ${{github.job}} REPORT   # Name of the check run which will be created
+      path: report.xml      # Path to test results
+      reporter: java-junit  # Format of test results
+      max-annotations: 49
+      list-tests: failed
+      list-suites: failed

.github/actions/start-build/action.yml

@@ -3,6 +3,27 @@ description: 'Last composite action before tests are run'
 runs:
   using: "composite"
   steps:
+  - id: cache-objects
+    uses: actions/cache@v2
+    with:
+      path: ~/.cache
+      key: reqs_${{ hashFiles('poetry.lock') }}
+      restore-keys: reqs_
+  - uses: ./.github/actions/build-es
+    with:
+      ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
+  - uses: ./.github/actions/build-es6
+    with:
+      ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
+  - name: Set up Python 3.12
+    uses: actions/setup-python@v5
+    with:
+      python-version: '3.12'
+  - name: Install lxml
+    shell: bash
+    run: |
+      sudo apt update
+      sudo apt-get install libxml2-dev libxslt-dev python-dev
   - name: Copy Settings
     shell: bash
     run: |
@@ -13,19 +34,15 @@ runs:
   - name: PIP install
     shell: bash
     run: |
-      pip install --upgrade pip
-      pip install invoke==0.13.0
-      pip install flake8==2.4.0 --force-reinstall --upgrade
-      pip install wheel
-      invoke wheelhouse --dev --addons
+      # pinned pip to 21.0a and setuptools<58.0.0 as current dependencies do not build on newer versions of pip and setuptools
+      pip install poetry==1.8.0
+      poetry install --no-root --without release
   - name: Other installs
     shell: bash
     run: |
-      pip install psycopg2==2.7.3 --no-binary psycopg2
-      invoke travis_addon_settings
-      invoke requirements --dev --addons
+      # bumped psycopg to match requirements.txt, as otherwise build would fail
+      poetry run python3 -m invoke travis-addon-settings
       pip uninstall uritemplate.py --yes
-      pip install uritemplate.py==0.3.0
       # use yarn add --exact to match versions in yarn.lock w/o installing all deps
       yarn add --exact bower@^1.8.8
       yarn add --exact @centerforopenscience/list-of-licenses@^1.1.0

.github/workflows/test-build.yml

@@ -33,10 +33,11 @@ jobs:
   addons:
     runs-on: ubuntu-20.04
     needs: build-cache
+    permissions:
+      checks: write
     services:
       postgres:
         image: postgres
-
         env:
           POSTGRES_PASSWORD: postgres
         options: >-
@@ -49,37 +50,21 @@ jobs:
           - 5432:5432
     steps:
     - uses: actions/checkout@v2
-    - id: cache-objects
-      uses: actions/cache@v2
-      with:
-        path: ~/.cache
-        key: reqs_${{ hashFiles('**/requirements.txt') }}
-        restore-keys: reqs_
-    - uses: ./.github/actions/build-es
-      with:
-        ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
-    - uses: ./.github/actions/build-es6
-      with:
-        ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
-    - name: Set up Python 3.6
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.6
-    - name: Install lxml
-      run: |
-        sudo apt update
-        sudo apt-get install libxml2-dev libxslt-dev python-dev
     - uses: ./.github/actions/start-build
     - name: Run tests
-      run: invoke test_travis_addons -n 1
+      run: poetry run python3 -m invoke test-travis-addons -n 1 --junit
+    - name: Upload report
+      if: (github.event_name != 'pull_request') && (success() || failure())    # run this step even if previous step failed
+      uses: ./.github/actions/gen-report
 
   website:
     runs-on: ubuntu-20.04
     needs: build-cache
+    permissions:
+      checks: write
     services:
       postgres:
         image: postgres
-
         env:
           POSTGRES_PASSWORD: postgres
         options: >-
@@ -92,37 +77,21 @@ jobs:
           - 5432:5432
     steps:
     - uses: actions/checkout@v2
-    - id: cache-objects
-      uses: actions/cache@v2
-      with:
-        path: ~/.cache
-        key: reqs_${{ hashFiles('**/requirements.txt') }}
-        restore-keys: reqs_
-    - uses: ./.github/actions/build-es
-      with:
-        ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
-    - uses: ./.github/actions/build-es6
-      with:
-        ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
-    - name: Set up Python 3.6
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.6
-    - name: Install lxml
-      run: |
-        sudo apt update
-        sudo apt-get install libxml2-dev libxslt-dev python-dev
     - uses: ./.github/actions/start-build
     - name: Run tests
-      run: invoke test_travis_website -n 1
+      run: poetry run python3 -m invoke test-travis-website -n 1 --junit
+    - name: Upload report
+      if: (github.event_name != 'pull_request') && (success() || failure())    # run this step even if previous step failed
+      uses: ./.github/actions/gen-report
 
   api1_and_js:
     runs-on: ubuntu-20.04
     needs: build-cache
+    permissions:
+      checks: write
     services:
       postgres:
         image: postgres
-
         env:
           POSTGRES_PASSWORD: postgres
         options: >-
@@ -135,40 +104,23 @@ jobs:
           - 5432:5432
     steps:
       - uses: actions/checkout@v2
-      - id: cache-objects
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache
-          key: reqs_${{ hashFiles('**/requirements.txt') }}
-          restore-keys: reqs_
-      - uses: ./.github/actions/build-es
-        with:
-          ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
-      - uses: ./.github/actions/build-es6
-        with:
-          ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
-      - name: Set up Python 3.6
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.6
-      - name: Install lxml
-        run: |
-          sudo apt update
-          sudo apt-get install libxml2-dev libxslt-dev python-dev
       - uses: ./.github/actions/start-build
       - name: NVM & yarn install
-        run: |
-          invoke assets --dev
+        run: poetry run python3 -m invoke assets --dev
       - name: Run test
-        run: invoke test_travis_api1_and_js -n 1
+        run: poetry run python3 -m invoke test-travis-api1-and-js -n 1 --junit
+      - name: Upload report
+        if: (github.event_name != 'pull_request') && (success() || failure())    # run this step even if previous step failed
+        uses: ./.github/actions/gen-report
 
   api2:
     runs-on: ubuntu-20.04
     needs: build-cache
+    permissions:
+      checks: write
     services:
       postgres:
         image: postgres
-
         env:
           POSTGRES_PASSWORD: postgres
         options: >-
@@ -181,32 +133,17 @@ jobs:
           - 5432:5432
     steps:
     - uses: actions/checkout@v2
-    - id: cache-objects
-      uses: actions/cache@v2
-      with:
-        path: ~/.cache
-        key: reqs_${{ hashFiles('**/requirements.txt') }}
-        restore-keys: reqs_
-    - uses: ./.github/actions/build-es
-      with:
-        ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
-    - uses: ./.github/actions/build-es6
-      with:
-        ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
-    - name: Set up Python 3.6
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.6
-    - name: Install lxml
-      run: |
-        sudo apt update
-        sudo apt-get install libxml2-dev libxslt-dev python-dev
     - uses: ./.github/actions/start-build
     - name: Run tests
-      run: invoke test_travis_api2 -n 1
+      run: poetry run python3 -m invoke test-travis-api2 -n 1 --junit
+    - name: Upload report
+      if: (github.event_name != 'pull_request') && (success() || failure())    # run this step even if previous step failed
+      uses: ./.github/actions/gen-report
 
   api3_and_osf:
     runs-on: ubuntu-20.04
+    permissions:
+      checks: write
     needs: build-cache
     services:
       postgres:
@@ -224,27 +161,9 @@ jobs:
           - 5432:5432
     steps:
     - uses: actions/checkout@v2
-    - id: cache-objects
-      uses: actions/cache@v2
-      with:
-        path: ~/.cache
-        key: reqs_${{ hashFiles('**/requirements.txt') }}
-        restore-keys: reqs_
-    - uses: ./.github/actions/build-es
-      with:
-        ELASTICSEARCH_ARCHIVE: ${{ env.ELASTICSEARCH_ARCHIVE }}
-    - uses: ./.github/actions/build-es6
-      with:
-        ELASTICSEARCH6_ARCHIVE: ${{ env.ELASTICSEARCH6_ARCHIVE }}
-    - name: Set up Python 3.6
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.6
-    - name: Install lxml
-      run: |
-        sudo apt update
-        sudo apt-get install libxml2-dev libxslt-dev python-dev
     - uses: ./.github/actions/start-build
     - name: Run tests
-      run: invoke test_travis_api3_and_osf -n 1
-
+      run: poetry run python3 -m invoke test-travis-api3-and-osf -n 1 --junit
+    - name: Upload report
+      if: (github.event_name != 'pull_request') && (success() || failure())    # run this step even if previous step failed
+      uses: ./.github/actions/gen-report

.pre-commit-config.yaml

@@ -1,21 +1,24 @@
 default_language_version:
-  python: python3.6
+  python: python3.12
 repos:
 - repo: https://github.com/asottile/add-trailing-comma
-  rev: v0.7.0
+  rev: v3.1.0
   hooks:
   - id: add-trailing-comma
     # TODO: Remove this line. For now, we only format the api/ directory
     files: ^api/
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v1.4.0
+  rev: v4.6.0
   hooks:
   - id: double-quote-string-fixer
   - id: trailing-whitespace
     exclude: "^(website/static/vendor/|osf/metadata/definitions/.*\\.xsd)"
-  - id: flake8
-    additional_dependencies: ["flake8==3.6.0", "flake8-mutable==1.2.0"]
+-   repo: https://github.com/pycqa/flake8
+    rev: '7.0.0'  # pick a git hash / tag to point to
+    hooks:
+    -   id: flake8
+        additional_dependencies: ["flake8-mutable==1.2.0"]
 - repo: https://github.com/pre-commit/mirrors-jshint
-  rev: v2.9.6
+  rev: v2.13.6
   hooks:
   - id: jshint

.travis.yml

@@ -3,7 +3,7 @@
 language: python
 
 python:
-    - "3.6"
+    - "3.10"
 
 dist: trusty
 
@@ -132,7 +132,7 @@ install:
 
     - travis_retry pip install --upgrade pip
     - travis_retry pip install invoke==0.13.0
-    - travis_retry pip install flake8==2.4.0 --force-reinstall --upgrade
+    - travis_retry pip install flake8==7.0.0 --force-reinstall --upgrade
     - travis_retry invoke wheelhouse --dev --addons
 
     - |
@@ -145,7 +145,8 @@ install:
       fi
 
     - travis_retry invoke travis_addon_settings
-    - travis_retry pip install psycopg2==2.7.3 --no-binary psycopg2
+    # bumped psycopg to match requirements.txt, as otherwise build would fail
+    - travis_retry pip install psycopg2==2.9.9 --no-binary psycopg2
     - travis_retry invoke requirements --dev --addons
     # Hack to fix package conflict between uritemplate and uritemplate.py (dependency of github3.py)
     - pip uninstall uritemplate.py --yes

CHANGELOG

@@ -2,7 +2,16 @@
 
 We follow the CalVer (https://calver.org/) versioning scheme: YY.MINOR.MICRO.
 
-24.03.0 (2024-07-08)
+24.05.0 (2024-07-22)
+====================
+- Bump base python version from py3.6 to py3.12.
+- Switch to using poetry for dependency management
+- Bump most (not all) dependencies to their maximum version as of mid-March.
+- Significantly update Dockerfile
+- Upgrade Django to v4.2
+- Generate test summary reports in CI
+
+24.04.0 (2024-07-08)
 ====================
 - Preprints into Ember OSF Web, Phase 2 Backend Release