Merge pull request #1553 from ScilifelabDataCentre/dev

New release v2.8.0 - No more Python 3.7
ScilifelabDataCentre · Sep 24, 2024 · 93b485f · 93b485f
2 parents e83cf75 + 22173ec
commit 93b485f
Show file tree

Hide file tree

Showing 52 changed files with 3,773 additions and 111 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1,2 @@
+# Team Hermes are the owners of all files in this repository
+* @ScilifelabDataCentre/teamhermes
diff --git a/.github/workflows/docker-compose-tests.yml b/.github/workflows/docker-compose-tests.yml
@@ -22,7 +22,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Run tests against database container
-        run: docker-compose -f docker-compose.yml -f tests/docker-compose-test.yml up --build --exit-code-from backend
+        run: docker compose -f docker-compose.yml -f tests/docker-compose-test.yml up --build --exit-code-from backend
 
       - name: Setup upterm session
         uses: lhotari/action-upterm@v1

diff --git a/.github/workflows/publish_and_trivyscan.yml b/.github/workflows/publish_and_trivyscan.yml
@@ -15,7 +15,6 @@
 name: Publish to GHCR (+ Trivy scan)
 on:
   workflow_dispatch:
-    branches: [dev]
   pull_request:
   push:
     branches:
@@ -24,7 +23,56 @@ on:
   release:
     types: [published]
 jobs:
+  build_tech_overview:
+    name: Build technical overview
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+      - name: Build tech overview PDF
+        uses: docker://pandoc/latex:3.2
+        with:
+          entrypoint: /bin/sh
+          args: >-
+            -c "
+            tlmgr update --self &&
+            tlmgr install cm-super fontaxes lato pdflscape xkeyval &&
+            updmap-sys &&
+            pandoc
+            --output=dds_web/static/dds-technical-overview.pdf
+            doc/technical-overview.md
+            "
+      - name: Upload technical overview PDF
+        uses: actions/upload-artifact@v4
+        with:
+          name: technical-overview-pdf
+          path: dds_web/static/dds-technical-overview.pdf
+  build_troubleshooting:
+    name: Build troubleshooting guide
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+      - name: Build troubleshooting guide
+        uses: docker://pandoc/latex:3.2
+        with:
+          entrypoint: /bin/sh
+          args: >-
+            -c "
+            tlmgr update --self &&
+            tlmgr install cm-super fontaxes lato xkeyval &&
+            updmap-sys &&
+            pandoc
+            --output=dds_web/static/dds-troubleshooting.pdf
+            doc/troubleshooting.md
+            "
+      - name: Upload troubleshooting PDF
+        uses: actions/upload-artifact@v4
+        with:
+          name: troubleshooting-pdf
+          path: dds_web/static/dds-troubleshooting.pdf
   push_to_registry:
+    needs: [build_tech_overview, build_troubleshooting]
     if: github.repository == 'ScilifelabDataCentre/dds_web'
     name: Push image
     runs-on: ubuntu-latest
@@ -37,23 +85,33 @@ jobs:
       cancel-in-progress: true
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Log in to Github Container Repository
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Download technical overview PDF
+        uses: actions/download-artifact@v4
+        with:
+          name: technical-overview-pdf
+          path: dds_web/static/dds-technical-overview.pdf
+      - name: Download troubleshooting PDF
+        uses: actions/download-artifact@v4
+        with:
+          name: troubleshooting-pdf
+          path: dds_web/static/dds-troubleshooting.pdf
       - name: Docker metadata
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: |
             ghcr.io/scilifelabdatacentre/dds-backend
       - name: Ensure lowercase name
         run: echo IMAGE_REPOSITORY=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
       - name: Build for scan
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           file: Dockerfiles/backend.Dockerfile
           context: .
@@ -67,7 +125,7 @@ jobs:
           output: "trivy-results.sarif"
           severity: "CRITICAL,HIGH"
       - name: Upload Trivy scan results to Github Security tab
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: "trivy-results.sarif"
           category: trivy-build

diff --git a/.github/workflows/trivy-scan-branch.yml b/.github/workflows/trivy-scan-branch.yml
@@ -31,7 +31,7 @@ jobs:
           severity: "CRITICAL,HIGH"
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: "trivy-results.sarif"
           category: trivy
diff --git a/.github/workflows/trivy-scheduled-dev.yml b/.github/workflows/trivy-scheduled-dev.yml
@@ -6,10 +6,7 @@
 # ---------------------------------
 name: Trivy - ghcr image scan - dev
 on:
-  workflow_dispatch:
-    branches:
-      - dev
-  schedule:
+  schedule: # Since dev is the default branch of the repo don't specify
     - cron: "0 9,12,15 * * *"
 jobs:
   scan:
@@ -26,15 +23,15 @@ jobs:
         run: echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr "[:upper:]" "[:lower:]") >> $GITHUB_ENV
 
       - name: Run Trivy on latest dev image
-        uses: aquasecurity/trivy-action@0.7.1
+        uses: aquasecurity/trivy-action@0.24.0
         with:
           image-ref: "ghcr.io/${{ env.REPOSITORY_OWNER }}/dds-backend:dev"
           format: "sarif"
           output: "trivy-results-dev.sarif"
           severity: "CRITICAL,HIGH"
 
       - name: Upload Trivy scan results to dev branch GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: "trivy-results-dev.sarif"
           category: trivy-dev
diff --git a/.github/workflows/trivy-scheduled-master.yml b/.github/workflows/trivy-scheduled-master.yml
@@ -6,9 +6,6 @@
 # ---------------------------------
 name: Trivy - ghcr image scan - master
 on:
-  workflow_dispatch:
-    branches:
-      - master
   schedule:
     - cron: "0 7,15 * * *"
 jobs:
@@ -28,15 +25,15 @@ jobs:
         run: echo REPOSITORY_OWNER=$(echo ${{ github.repository_owner }} | tr "[:upper:]" "[:lower:]") >> $GITHUB_ENV
 
       - name: Run Trivy on latest release image
-        uses: aquasecurity/trivy-action@0.7.1
+        uses: aquasecurity/trivy-action@0.24.0
         with:
           image-ref: "ghcr.io/${{ env.REPOSITORY_OWNER }}/dds-backend:latest"
           format: "sarif"
           output: "trivy-results-master.sarif"
           severity: "CRITICAL,HIGH"
 
       - name: Upload Trivy scan results to master branch GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: "trivy-results-master.sarif"
           category: trivy-master
diff --git a/.prettierignore b/.prettierignore
@@ -1,3 +1,6 @@
 *.html
 dds_web/static/css
 dds_web/static/node_modules
+# Would like to enable this but prettier isn't correctly formatting grid tables for pandoc
+# We need grid table markdown due to having tables with multi-column/row spans
+doc/technical-overview.md
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,6 +1,22 @@
 Changelog
 ==========
 
+.. _2.8.0:
+
+2.8.0 - 2024-09-24
+~~~~~~~~~~~~~~~~~~~~~~~
+
+- New features:
+    - Technical Overview and Troubleshooting updated and made available as pdf.
+    - Added Flask command for updating units quota.
+- Dependencies:
+    - `certifi` from `2023.07.22` to `2024.7.4`
+    - `requests` from `2.31.0` to `2.32.0`
+    - `wrapt` from `1.13.3` to `1.14.0`
+- Dependencies (tests):
+    - `pyfakefs` from `4.5.5` to `5.3.0`
+- Update base image for the docker containers from `python:3.11-alpine` to `python:3.12-alpine`
+
 .. _2.7.1:
 
 2.7.1 - 2024-06-26

diff --git a/Dockerfiles/backend.Dockerfile b/Dockerfiles/backend.Dockerfile
@@ -3,7 +3,7 @@
 #############################
 
 # Set official image -- parent image
-FROM python:3.11-alpine as base
+FROM python:3.12-alpine as base
 
 ARG USERNAME=dds-user
 ARG USER_UID=1001

diff --git a/Dockerfiles/cli.Dockerfile b/Dockerfiles/cli.Dockerfile
@@ -1,5 +1,5 @@
 # Set official image
-FROM python:3.11-alpine as base
+FROM python:3.12-alpine as base
 
 # Update and upgrade
 RUN apk update && apk upgrade

diff --git a/Makefile b/Makefile
@@ -0,0 +1,22 @@
+# For Linux and MacOS users only and assumes brew is already installed
+TECH_MARKDOWN_FILE := doc/technical-overview.md
+TECH_OVERVIEW_PDF := dds_web/static/dds-technical-overview.pdf
+TROUBLE_MARKDOWN_FILE := doc/troubleshooting.md
+TROUBLE_PDF := dds_web/static/dds-troubleshooting.pdf
+
+install_pandoc:
+	brew install pandoc basictex librsvg
+
+update_tlmgr:
+	tlmgr update --self
+	tlmgr install cm-super fontaxes lato pdflscape xkeyval
+
+build_tech_overview_pdf: update_tlmgr $(TECH_MARKDOWN_FILE)
+	pandoc $(TECH_MARKDOWN_FILE) -o $(TECH_OVERVIEW_PDF)
+
+build_troubleshooting_pdf: update_tlmgr $(TROUBLE_MARKDOWN_FILE)
+	pandoc $(TROUBLE_MARKDOWN_FILE) -o $(TROUBLE_PDF)
+
+build_docs: build_tech_overview_pdf build_troubleshooting_pdf
+
+.PHONY: update_tlmgr build_tech_overview_pdf build_troubleshooting_pdf build_docs all
diff --git a/README.md b/README.md
@@ -31,13 +31,13 @@
    <a href="https://scilifelabdatacentre.github.io/dds_cli/">
       <img alt="Documentation" src="https://img.shields.io/badge/-Documentation-222222?logo=github-pages">
    </a>
-   <a href="https://github.com/ScilifelabDataCentre/dds_web/blob/master/doc/Technical-Overview.pdf">
+   <a href="https://github.com/ScilifelabDataCentre/dds_web/blob/master/doc/technical-overview.md">
       <img alt="Technical Overview" src="https://img.shields.io/badge/-Technical%20Overview-informational?logo=github">
    </a>
    <a href="https://github.com/ScilifelabDataCentre/dds_web/wiki/Architecture-Decision-Record,-ADR">
       <img alt="Architecture Decision Record" src="https://img.shields.io/badge/-ADR-000000?logo=github">
    </a>
-   <a href="https://github.com/ScilifelabDataCentre/dds_web/blob/master/doc/Troubleshooting.pdf">
+   <a href="https://github.com/ScilifelabDataCentre/dds_web/blob/master/doc/troubleshooting.md">
       <img alt="Troubleshooting" src="https://img.shields.io/badge/-Troubleshooting%20Guide-red?logo=github">
    </a>
    <a href="https://github.com/ScilifelabDataCentre/dds_cli">

diff --git a/SPRINTLOG.md b/SPRINTLOG.md
@@ -231,11 +231,11 @@ Please add a _short_ line describing the PR you make, if the PR implements a spe
 
 # 2023-03-17 - 2023-03-31
 
-_Nothing merged in CLI during this sprint_
+_Nothing merged during this sprint_
 
 # 2023-03-31 - 2023-04-14
 
-_Nothing merged in CLI during this sprint_
+_Nothing merged during this sprint_
 
 # 2023-04-14 - 2023-04-28
 
@@ -404,3 +404,29 @@ _Nothing merged in CLI during this sprint_
 - Update pymysql to address cve ([#1534](https://github.com/ScilifelabDataCentre/dds_web/pull/1534))
 - Update authlib to address cve ([#1535](https://github.com/ScilifelabDataCentre/dds_web/pull/1535))
 - Update node packages to address cve ([#1536](https://github.com/ScilifelabDataCentre/dds_web/pull/1536))
+
+# 2024-07-15 - 2024-07-26
+
+- Move raw Technical Overview doc to repo, add page numbers ([#1539](https://github.com/ScilifelabDataCentre/dds_web/pull/1539))
+- Small updates to Technical Overview contents ([#1540](https://github.com/ScilifelabDataCentre/dds_web/pull/1540))
+- Build Technical Overview PDF in GitHub Actions, rename to include DDS and remove option to view on GitHub ([#1541](https://github.com/ScilifelabDataCentre/dds_web/pull/1541/))
+- Fixed index out of range when listing files from root ([#1543](https://github.com/ScilifelabDataCentre/dds_web/pull/1543/))
+- Update Trivy GitHub Actions ([#1545](https://github.com/ScilifelabDataCentre/dds_web/pull/1545))
+
+# 2024-07-29 - 2024-08-09
+
+- Move raw troubleshooting doc to repo and make small updates ([#1546](https://github.com/ScilifelabDataCentre/dds_web/pull/1546))
+
+# 2024-08-12 - 2024-08-23
+
+_Nothing merged during this sprint_
+
+# 2024-08-26 - 2024-09-06
+
+- Update certifi to remove GLOBALISSUER certicates ([#1549](https://github.com/ScilifelabDataCentre/dds_web/pull/1549))
+- Add CODEOWNERS file in order to define Team Hermes as owners of all files in repository ([#708](https://github.com/ScilifelabDataCentre/dds_web/pull/708))
+
+# 2024-09-09 - 2024-09-20
+
+- Flask command to update unit quotas ([#1551](https://github.com/ScilifelabDataCentre/dds_web/pull/1551))
+- Bump python base image to 3.12 and related libraries in both web and client([#1548](https://github.com/ScilifelabDataCentre/dds_web/pull/1548))
diff --git a/dds_web/__init__.py b/dds_web/__init__.py
@@ -282,13 +282,15 @@ def load_user(user_id):
             send_usage,
             collect_stats,
             monitor_usage,
-            update_unit,
+            update_unit_sto4,
+            update_unit_quota,
         )
 
         # Add flask commands - general
         app.cli.add_command(fill_db_wrapper)
         app.cli.add_command(create_new_unit)
-        app.cli.add_command(update_unit)
+        app.cli.add_command(update_unit_sto4)
+        app.cli.add_command(update_unit_quota)
         app.cli.add_command(update_uploaded_file_with_log)
         app.cli.add_command(lost_files_s3_db)