feature(#315): Support python 3.10+

Revamp CI workflow, condense files and jobs down, using matrices where possible. Main idea: test with {earliest python+OS version, latest python+OS version}, and assume that if those work then everything in between will work too. - Fix subtle nasty bug with pipeline where a nested usage of the actions/checkout@v3 action caused inner the publish job always to run against the master branch. - Fix issue in upload-artifact excluding dotfiles by default (who introduces breaking changes without major version increment???). - Add caching for {homebrew, apt} packages and build files to try to get the pipeline execution time down.
jharwell · Sep 16, 2024 · 2d43f36 · 2d43f36
1 parent 0588c2b
commit 2d43f36
Show file tree

Hide file tree

Showing 40 changed files with 802 additions and 805 deletions.
diff --git a/.github/actions/publish/action.yml b/.github/actions/publish/action.yml
@@ -11,7 +11,6 @@ inputs:
 runs:
   using: 'composite'
   steps:
-    - uses: actions/checkout@v3
     - uses: ./.github/actions/sierra-setup
     - name: Build package
       shell: bash
@@ -37,7 +36,7 @@ runs:
 
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
-        skip_existing: true
+        skip-existing: true
         user: __token__
         password: ${{ inputs.testpypi-token }}
         repository_url: https://test.pypi.org/legacy/

diff --git a/.github/actions/sample-project-setup/action.yml b/.github/actions/sample-project-setup/action.yml
@@ -7,18 +7,26 @@ inputs:
   rosdistro:
     description: 'The version of ROS to test with'
     default: 'noetic'
-# outputs:
-#   time: # id of output
-#     description: 'The time we greeted you'
+
 runs:
   using: 'composite'
   steps:
+    ############################################################################
+    # Common setup
+    ############################################################################
+    - name: Setup python (all OSes)
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: 'pip'
+
     - name: Clone sample project
       shell: bash
-      working-directory: ../../
+      working-directory: ${{ github.workspace }} # Needed to match with scripts/argos-integration-tests.sh
+
       run: |
         if [ ! -d sierra-sample-project ]; then
-        git clone https://github.com/swarm-robotics/sierra-sample-project.git
+        git clone https://github.com/jharwell/sierra-sample-project.git
         cd sierra-sample-project
         else
         cd sierra-sample-project
@@ -30,46 +38,73 @@ runs:
       uses: SimenB/github-actions-cpu-cores@v1
       id: cpu-cores
 
+    - uses: kenchan0130/actions-system-info@master
+      id: system-info
+
+    - name: Cache build files
+      uses: hendrikmuhs/[email protected]
+      with:
+        key: ${{ runner.os }}-${{ steps.system-info.outputs.release }}-build-objects
+        verbose: 0
+
     ############################################################################
     # ARGoS setup
     ############################################################################
-    - name: Install ARGoS dependencies
-      shell: bash
-      working-directory: ../../
-      run: |
-        # 2023/01/16: || true needed because homebrew installation of whatever
-        # dependencies these packages requires manual intervention to fix, and I
-        # don't have a mac.
-        if [ "${{ runner.os }}" == "macOS" ]; then
-          brew install pkg-config cmake libpng freeimage lua qt \
-          docbook asciidoc graphviz doxygen|| true
-        elif [ "${{ runner.os }}" == "Linux" ]; then
-          sudo apt-get update
-          sudo apt-get install cmake libfreeimage-dev libfreeimageplus-dev \
-          qt5-default freeglut3-dev libxi-dev libxmu-dev liblua5.3-dev \
-          lua5.3 doxygen graphviz libgraphviz-dev asciidoc
-        fi
+    - uses: nick-fields/retry@v2
+      name: Install ARGoS deps (OSX)
+      if: runner.os == 'macOS'
+      with:
+        timeout_minutes: 10000
+        max_attempts: 3
+        shell: bash
+        command: |
+          # 2023/01/16: Caching doesn't work reliably, for reasons unknown :-(.
+          #
+          # The {xquartz, mactex, qt} deps are not required as long as the ARGoS
+          # integration tests for OSX don't run stage 4+ or try to do video
+          # capture. Installing them takes ~30 min in github actions, so cutting
+          # this particular corner saves a TON of time.
+          #
+          # Note that by omitting Xquartz you MUST also omit Qt, otherwise CI
+          # fails (I think ARGoS segfaults).
+          brew install pkg-config cmake libpng freeimage lua
+
+    - name: Install and cache ARGoS dependencies (Linux)
+      if: runner.os == 'Linux'
+      uses: awalsh128/cache-apt-pkgs-action@latest
+      with:
+        packages: libfreeimage-dev libfreeimageplus-dev qtbase5-dev freeglut3-dev libxi-dev libxmu-dev liblua5.3-dev lua5.3 libgraphviz-dev ccache
+        version: ${{ runner.os }}
 
     - name: Install ARGoS
       if: inputs.platform == 'argos'
       shell: bash
-      working-directory: ../../
       run: |
         git clone https://github.com/ilpincy/argos3.git
         cd argos3
+
         mkdir -p build && cd build
-        cmake -DARGOS_DOCUMENTATION=OFF ../src
+        cmake \
+          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+          -DARGOS_DOCUMENTATION=OFF \
+          ../src
         make -j ${{ steps.cpu-cores.outputs.count }}
         sudo make install
 
     - name: Build SIERRA project for ARGoS
       if: inputs.platform == 'argos'
-      working-directory: ../../
       shell: bash
+      working-directory: ${{ github.workspace }} # Needed to match with scripts/argos-integration-tests.sh
+
       run: |
         cd sierra-sample-project/argos
         mkdir -p build && cd build
-        cmake -DARGOS_INSTALL_DIR=/usr/local ..
+        cmake \
+          -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+          -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+          -DARGOS_INSTALL_DIR=/usr/local \
+          ..
         make -j ${{ steps.cpu-cores.outputs.count }}
 
     ############################################################################
@@ -81,39 +116,44 @@ runs:
       with:
         required-ros-distributions: ${{ inputs.rosdistro }}
 
-    - name: Install SIERRA ROSbridge dependencies
+    - name: Install SIERRA ROSbridge system dependencies (Linux)
+      if: inputs.platform == 'ros1gazebo'
+      uses: awalsh128/cache-apt-pkgs-action@latest
+      with:
+        packages: ros-${{ inputs.rosdistro }}-desktop-full ros-${{ inputs.rosdistro }}-turtlebot3-description ros-${{ inputs.rosdistro }}-turtlebot3-msgs ros-${{ inputs.rosdistro }}-turtlebot3-gazebo ros-${{ inputs.rosdistro }}-turtlebot3-bringup python${{ matrix.python-version }} python${{ matrix.python-version }}-dev python${{ matrix.python-version }}-venv python3-apt python3-cairo intltool python3-wheel python3-rospkg python3-empy
+        version: ${{ runner.os }}
+
+    - name: Install SIERRA ROSbridge python dependencies (Linux)
       if: inputs.platform == 'ros1gazebo'
-      working-directory: ../..
       shell: bash
       run: |
-        sudo apt-get install ros-${{ inputs.rosdistro }}-desktop-full
-        sudo apt-get install ros-${{ inputs.rosdistro }}-turtlebot3-description
-        sudo apt-get install ros-${{ inputs.rosdistro }}-turtlebot3-msgs
-        sudo apt-get install ros-${{ inputs.rosdistro }}-turtlebot3-gazebo
-        sudo apt-get install ros-${{ inputs.rosdistro }}-turtlebot3-bringup
-        sudo apt-get install python${{ matrix.python-version }} python${{ matrix.python-version }}-dev
-        sudo apt-get install python${{ matrix.python-version }}-venv python3-apt
-        sudo apt-get install python3-pip python3-cairo intltool python3-wheel python3-rospkg python3-empy
-
         # 2023/1/18: You still need install these even after installing stuff
         # with apt AND if the sysem python matches your current python. I don't
         # know why.
-        pip3 install empy
-        pip3 install rospkg wheel pyparsing pyqt5
-        pip3 install  pyqt5 pysip numpy rospkg
-        pip3 install  matplotlib pyyaml psutil
-        pip3 install  pysip defusedxml pyparsing pydev
-        pip3 install  pyopengl opencv-python
+        pip3 install empy \
+                     rospkg \
+                     wheel \
+                     pyparsing \
+                     pyqt5 \
+                     pysip \
+                     numpy \
+                     matplotlib \
+                     pyyaml \
+                     psutil \
+                     defusedxml \
+                     pyparsing \
+                     pydev \
+                     pyopengl \
+                     opencv-python \
+                     catkin_tools
 
     - name: Install SIERRA ROSBridge dependency (ubuntu)
       if: inputs.platform == 'ros1gazebo'
-      working-directory: ../..
       shell: bash
       run: |
         source /opt/ros/${{ inputs.rosdistro }}/setup.bash
 
-        pip3 install catkin_tools
-        git clone https://github.com/swarm-robotics/sierra_rosbridge.git
+        git clone https://github.com/jharwell/sierra_rosbridge.git
         cd sierra_rosbridge
         git checkout devel
         catkin init

diff --git a/.github/actions/sierra-setup/action.yml b/.github/actions/sierra-setup/action.yml
@@ -1,45 +1,52 @@
 name: 'Setup SIERRA'
 description: 'Setup SIERRA for CI and testing'
-# inputs:
-  # who-to-greet:  # id of input
-  #   description: 'Who to greet'
-  #   required: true
-  #   default: 'World'
-# outputs:
-#   time: # id of output
-#     description: 'The time we greeted you'
 runs:
   using: 'composite'
   steps:
 
     ############################################################################
-    # Ubuntu setup
+    # Common setup
     ############################################################################
-    - name: Setup python (ubuntu)
-      uses: actions/setup-python@v4
-      if: runner.os == 'Linux'
+    - name: Setup python (all OSes)
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        cache: 'pip'
 
-    - name: Install system deps (ubuntu)
-      if: runner.os == 'Linux'
+    - name: Install python packages (all OSes)
       shell: bash
       run: |
-        sudo apt-get update
-        sudo apt-get install parallel cm-super texlive-fonts-recommended
-        sudo apt-get install texlive-latex-extra dvipng pssh ffmpeg xvfb
-        python3 -m pip install --upgrade pip
         python3 -m pip install wheel
+        python3 -m pip install -r docs/requirements.txt
+
+    - uses: kenchan0130/actions-system-info@master
+      id: system-info
 
     ############################################################################
-    # OSX setup
+    # Ubuntu setup
     ############################################################################
+    - name: Install+cache system packages (ubuntu)
+      if: runner.os == 'Linux'
+      uses: awalsh128/cache-apt-pkgs-action@latest
+      with:
+        packages: parallel cm-super dvipng pssh ffmpeg xvfb
+        version: ${{ runner.os }}
 
-    - name: Setup python (OSX)
-      uses: actions/setup-python@v4
+    # 2024/09/24: Needed because caching + texlive don't play well
+    # together. There IS an "install texlive" action, but that requires manual
+    # specification of all packages needed, and it seem easier to just use the
+    # package manager to recursively pull in everything I would need.
+    - name: Install system packages (redundant, ubuntu)
+      if: runner.os == 'Linux'
+      run: |
+        sudo apt-get install -y texlive-fonts-recommended
+
+    ############################################################################
+    # OSX setup
+    ############################################################################
+    - name: Enable homebrew (OSX)
       if: runner.os == 'macOS'
-      with:
-        python-version: ${{ matrix.python-version }}
+      uses: Homebrew/actions/setup-homebrew@master
 
     - uses: nick-fields/retry@v2
       name: Install system deps (OSX)
@@ -49,22 +56,14 @@ runs:
         max_attempts: 3
         shell: bash
         command: |
-          # 2023/01/16: || true needed because homebrew installation of whatever
-          # dependencies these packages requires manual intervention to fix, and I
-          # don't have a mac.
-          brew update && brew install parallel pssh ffmpeg
-          brew update && brew install --cask mactex
-          brew update && brew install --cask xquartz
-          python -m pip install --upgrade pip
-          python -m pip install wheel
+          brew install parallel pssh
 
     ############################################################################
     # SIERRA install
     ############################################################################
     - name: Install SIERRA
       shell: bash
       run: |
-        python -m pip install -r docs/requirements.txt
         cd docs && make man && cd ..
         python3 -m pip install .
         python3 -m pip install .[devel]

diff --git a/.github/workflows/analysis-top.yml b/.github/workflows/analysis-top.yml
@@ -0,0 +1,37 @@
+name: Analysis Top-level
+
+# Run:
+#
+# - Every month. Scheduling helps keep drift/issues with updates
+#   to runners at bay.
+# - On pushes.
+on:
+  push:
+    paths:
+      - 'sierra/**'
+
+  schedule:
+    - cron: '0 0 1 * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze-ci-ubuntu-2004:
+    uses: ./.github/workflows/analyze-nox.yml
+    secrets: inherit
+    with:
+      os: ubuntu-20.04
+
+  analyze-ci-ubuntu-2204:
+    uses: ./.github/workflows/analyze-nox.yml
+    secrets: inherit
+    with:
+      os: ubuntu-22.04
+
+  analyze-ci-osx:
+    uses: ./.github/workflows/analyze-nox.yml
+    secrets: inherit
+    with:
+      os: macos-13