tools: adding pip requirements check workflow

* The workflow merges the requirements from nrf, mcuboot and zephyr * Checks if the requirements-fixed and new generated file match * Packages that where only in the current requirements-fixed are added to requirements-ci.txt * Following packages will be added to requirements-fixed.txt with this change Deprecated Markdown PyGithub PyJWT PyNaCl azure-core azure-storage-blob capstone dill editdistance exceptiongroup gitlint-core graphviz grpcio importlib-resources isodate jsonschema-specifications lark libusb-package m2r2 mistune msgpack pathspec pkgutil_resolve_name platformdirs pypng referencing rpds-py sphinx-copybutton sphinx-markdown-tables sphinx-ncs-theme sphinx-notfound-page sphinx-togglebutton sphinxcontrib-jquery sphinxcontrib-plantuml Signed-off-by: Sebastian Wezel <[email protected]>
DatGizmo · Aug 10, 2023 · 9d5d2cc · 9d5d2cc
1 parent 8d16584
commit 9d5d2cc
Show file tree

Hide file tree

Showing 6 changed files with 300 additions and 90 deletions.
diff --git a/.github/workflows/pip-requirements.yml b/.github/workflows/pip-requirements.yml
@@ -0,0 +1,81 @@
+name: Validate pip requirements-fixed.txt
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    branches:
+      - main
+      - 'v*-branch'
+    paths:
+      - 'scripts/requirements*.txt'
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: ncs/nrf
+          fetch-depth: 0
+
+      - name: Get python version
+        id: pyv
+        run: |
+          sudo snap install yq
+          PYTHON_VERSION=$(yq '.python.version' ./ncs/nrf/scripts/tools-versions-linux.yml)
+          echo "python_version=$PYTHON_VERSION" >> $GITHUB_OUTPUT
+
+      - name: Setup python version
+        uses: actions/setup-python@v4
+        with:
+          python-version: '${{ steps.pyv.outputs.python_version }}'
+
+      - name: Setup environment
+        working-directory: ncs
+        run: |
+          pip3 install --user -U setuptools wheel pip virtualenv virtualenvwrapper
+          pip3 install -r nrf/scripts/requirements-base.txt
+          west init -l nrf
+          west update mcuboot zephyr
+
+      - name: Generate new requirements-fixed.txt
+        working-directory: ncs
+        run: nrf/scripts/fix-requirements.sh requirements-fixed.txt
+
+      - name: Store requirements-fixed
+        uses: actions/upload-artifact@v3
+        with:
+          name: requirements-fixed
+          path: ncs/requirements-fixed.txt
+
+      - name: Compare requirements-fixed
+        id: diff
+        run: |
+          NEW=ncs/requirements-fixed.txt
+          OLD=ncs/nrf/scripts/requirements-fixed.txt
+          echo -e "$NEW\t\t\t\t\t"$OLD
+          diff -y $NEW $OLD
+
+      - name: Post summary
+        if: ${{ !cancelled() }}
+        run: |
+          if [[ "failure" == "${{ steps.diff.outcome }}" ]]; then
+            echo -e 'New requirements-fixed.txt file differs from the current file.
+            Please add the newly created file to the PR.' >> $GITHUB_STEP_SUMMARY
+
+          fi
+          if [[ -e ncs/requirements-fixed.txt ]]; then
+            echo -e '
+            <details>
+              <summary>requirements-fixed.txt</summary>
+
+            ```' >> $GITHUB_STEP_SUMMARY
+            cat ncs/requirements-fixed.txt >> $GITHUB_STEP_SUMMARY
+            echo -e '```
+
+            </details>' >> $GITHUB_STEP_SUMMARY
+          else
+            echo -e 'Error during workflow, requirements-fixed file was not created' >> GITHUB_STEP_SUMMARY
+          fi
diff --git a/scripts/fix-requirements.sh b/scripts/fix-requirements.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# This script merges the py-reqs from mcuboot, zephyr, and nrf.
+
+OUT_FILE=$1
+[ -z "$OUT_FILE" ] && echo "Error output file not provided" && exit 1
+echo "Writing frozen requirements to: $OUT_FILE"
+echo "Log python version: $(python --version)"
+
+TOPDIR=$(west topdir)
+cd $TOPDIR
+
+echo "#####       Generated Python Requirements             #####" > $OUT_FILE
+echo "#####       Do not edit this file manually            #####" >> $OUT_FILE
+echo "###########################################################" >> $OUT_FILE
+echo "##### Merged inputs:                                  #####" >> $OUT_FILE
+echo "#####     bootloader/mcuboot/scripts/requirements.txt #####" >> $OUT_FILE
+echo "#####     zephyr/scripts/requirements.txt             #####" >> $OUT_FILE
+echo "#####     nrf/scripts/requirements.txt                #####" >> $OUT_FILE
+echo "" >> $OUT_FILE
+
+source ~/.local/bin/virtualenvwrapper.sh
+[[ $? != 0 ]] && echo "error sourcing virtualenvwrapper" && exit 1
+
+rmvirtualenv pip-fixed-venv
+mkvirtualenv pip-fixed-venv
+workon pip-fixed-venv
+pip3 install --isolated \
+    -r bootloader/mcuboot/scripts/requirements.txt \
+    -r zephyr/scripts/requirements.txt  \
+    -r nrf/scripts/requirements.txt
+pip3 freeze --all | LC_ALL=C sort --ignore-case >> $OUT_FILE
+#Set LC_ALL=C to have the same sorting behaviour on all systems
+
+deactivate
+rmvirtualenv pip-fixed-venv
diff --git a/scripts/fix.sh b/scripts/fix.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# This script merges the py-reqs from mcuboot, zephyr, and nrf.
+
+
+OUT_FILE=$1
+[ -z "$OUT_FILE" ] && echo "Error output file not provided" && exit 1
+
+echo "#####       Generated Python Requirements             #####" > $OUT_FILE
+echo "#####       Do not edit this file manually            #####" >> $OUT_FILE
+echo "###########################################################" >> $OUT_FILE
+echo "##### Merged inputs:                                  #####" >> $OUT_FILE
+echo "#####     bootloader/mcuboot/scripts/requirements.txt #####" >> $OUT_FILE
+echo "#####     zephyr/scripts/requirements.txt             #####" >> $OUT_FILE
+echo "#####     nrf/scripts/requirements.txt                #####" >> $OUT_FILE
+echo "" >> $OUT_FILE
+
+source ~/.local/bin/virtualenvwrapper.sh
+[[ $? != 0 ]] && echo "error sourcing virtualenvwrapper" && exit 1
+
+rmvirtualenv pip-fixed-venv
+mkvirtualenv pip-fixed-venv
+workon pip-fixed-venv
+pip3 install --isolated \
+    -r ~/tmp/west/bootloader/mcuboot/scripts/requirements.txt \
+    -r ~/tmp/west/zephyr/scripts/requirements.txt  \
+    -r ./requirements.txt
+pip3 freeze --all | LC_ALL=C sort --ignore-case >> $OUT_FILE
+#Set LC_ALL=C to have the same sorting behaviour on all systems
+
+deactivate
+rmvirtualenv pip-fixed-venv
diff --git a/scripts/requirements-ci.txt b/scripts/requirements-ci.txt
@@ -0,0 +1,15 @@
+aenum
+bitarray
+chardet
+gitlint==0.18.0
+jsonschema
+milksnake
+py
+pyparsing
+python-stdnum
+pytz
+pyusb
+qrcode
+typed-ast
+toml
+wget