From 3e2d5d2b04ed70cbb296291e0ff8eb165adc8117 Mon Sep 17 00:00:00 2001 From: John Harwell Date: Wed, 21 Aug 2024 11:21:23 -0500 Subject: [PATCH] feature(#315): Support python 3.10 - Preliminary --- .../actions/sample-project-setup/action.yml | 79 +++++-- .github/actions/sierra-setup/action.yml | 13 +- .github/workflows/analysis.yml | 33 +++ .github/workflows/analyze-nox.yml | 43 ++++ .github/workflows/argos.yml | 177 ---------------- .github/workflows/coverage.yml | 60 +----- .github/workflows/exec-env-plugins.yml | 81 ------- .github/workflows/integration-all.yml | 43 ++-- .github/workflows/plugin.yml | 190 +++++++++++++++++ .github/workflows/ros1gazebo.yml | 68 ------ .github/workflows/ros1robot.yml | 66 ------ .github/workflows/sierra-core.yml | 200 ++++++++++++------ .github/workflows/static-analysis.yml | 38 ---- .github/workflows/unit-tests.yml | 41 +++- docs/conf.py | 6 +- docs/requirements.txt | 4 +- docs/src/getting_started.rst | 1 + docs/src/requirements.rst | 4 +- docs/src/trial.rst | 3 +- docs/src/usage/subprograms.rst | 4 + noxfile.py | 19 +- setup.py | 1 + sierra/core/config.py | 2 +- sierra/core/graphs/heatmap.py | 2 +- sierra/core/startup.py | 4 +- sierra/core/utils.py | 2 +- sierra/version.py | 2 +- 27 files changed, 561 insertions(+), 625 deletions(-) create mode 100644 .github/workflows/analysis.yml create mode 100644 .github/workflows/analyze-nox.yml delete mode 100644 .github/workflows/argos.yml delete mode 100644 .github/workflows/exec-env-plugins.yml create mode 100644 .github/workflows/plugin.yml delete mode 100644 .github/workflows/ros1gazebo.yml delete mode 100644 .github/workflows/ros1robot.yml delete mode 100644 .github/workflows/static-analysis.yml diff --git a/.github/actions/sample-project-setup/action.yml b/.github/actions/sample-project-setup/action.yml index 02dbaeef..49e07c22 100644 --- a/.github/actions/sample-project-setup/action.yml +++ b/.github/actions/sample-project-setup/action.yml @@ -18,7 +18,7 @@ runs: working-directory: ../../ run: | if [ ! -d sierra-sample-project ]; then - git clone https://github.com/swarm-robotics/sierra-sample-project.git + git clone https://github.com/jharwell/sierra-sample-project.git cd sierra-sample-project else cd sierra-sample-project @@ -41,13 +41,34 @@ runs: # dependencies these packages requires manual intervention to fix, and I # don't have a mac. if [ "${{ runner.os }}" == "macOS" ]; then - brew install pkg-config cmake libpng freeimage lua qt \ - docbook asciidoc graphviz doxygen|| true + brew install \ + pkg-config \ + cmake \ + libpng \ + freeimage \ + lua \ + qt \ + docbook \ + asciidoc \ + graphviz \ + doxygen \ + || true elif [ "${{ runner.os }}" == "Linux" ]; then sudo apt-get update - sudo apt-get install cmake libfreeimage-dev libfreeimageplus-dev \ - qt5-default freeglut3-dev libxi-dev libxmu-dev liblua5.3-dev \ - lua5.3 doxygen graphviz libgraphviz-dev asciidoc + sudo apt-get install -y \ + cmake \ + libfreeimage-dev \ + libfreeimageplus-dev \ + qtbase5-dev \ + freeglut3-dev \ + libxi-dev \ + libxmu-dev \ + liblua5.3-dev \ + lua5.3 \ + doxygen \ + graphviz \ + libgraphviz-dev \ + asciidoc fi - name: Install ARGoS @@ -86,24 +107,42 @@ runs: working-directory: ../.. shell: bash run: | - sudo apt-get install ros-${{ inputs.rosdistro }}-desktop-full - sudo apt-get install ros-${{ inputs.rosdistro }}-turtlebot3-description - sudo apt-get install ros-${{ inputs.rosdistro }}-turtlebot3-msgs - sudo apt-get install ros-${{ inputs.rosdistro }}-turtlebot3-gazebo - sudo apt-get install ros-${{ inputs.rosdistro }}-turtlebot3-bringup - sudo apt-get install python${{ matrix.python-version }} python${{ matrix.python-version }}-dev - sudo apt-get install python${{ matrix.python-version }}-venv python3-apt - sudo apt-get install python3-pip python3-cairo intltool python3-wheel python3-rospkg python3-empy + sudo apt-get install -y \ + ros-${{ inputs.rosdistro }}-desktop-full \ + ros-${{ inputs.rosdistro }}-turtlebot3-description \ + ros-${{ inputs.rosdistro }}-turtlebot3-msgs \ + ros-${{ inputs.rosdistro }}-turtlebot3-gazebo \ + ros-${{ inputs.rosdistro }}-turtlebot3-bringup \ + python${{ matrix.python-version }} \ + python${{ matrix.python-version }}-dev \ + python${{ matrix.python-version }}-venv \ + python3-apt \ + python3-pip \ + python3-cairo \ + intltool \ + python3-wheel \ + python3-rospkg \ + python3-empy + # 2023/1/18: You still need install these even after installing stuff # with apt AND if the sysem python matches your current python. I don't # know why. - pip3 install empy - pip3 install rospkg wheel pyparsing pyqt5 - pip3 install pyqt5 pysip numpy rospkg - pip3 install matplotlib pyyaml psutil - pip3 install pysip defusedxml pyparsing pydev - pip3 install pyopengl opencv-python + pip3 install empy \ + rospkg \ + wheel \ + pyparsing \ + pyqt5 \ + pysip \ + numpy \ + matplotlib \ + pyyaml \ + psutil \ + defusedxml \ + pyparsing \ + pydev \ + pyopengl \ + opencv-python - name: Install SIERRA ROSBridge dependency (ubuntu) if: inputs.platform == 'ros1gazebo' diff --git a/.github/actions/sierra-setup/action.yml b/.github/actions/sierra-setup/action.yml index d5c7abca..71f01049 100644 --- a/.github/actions/sierra-setup/action.yml +++ b/.github/actions/sierra-setup/action.yml @@ -16,7 +16,7 @@ runs: # Ubuntu setup ############################################################################ - name: Setup python (ubuntu) - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 if: runner.os == 'Linux' with: python-version: ${{ matrix.python-version }} @@ -26,8 +26,9 @@ runs: shell: bash run: | sudo apt-get update - sudo apt-get install parallel cm-super texlive-fonts-recommended - sudo apt-get install texlive-latex-extra dvipng pssh ffmpeg xvfb + sudo apt-get install parallel cm-super texlive-fonts-recommended -y + sudo apt-get install texlive-latex-extra dvipng pssh ffmpeg xvfb -y + python3 -m pip install --upgrade pip python3 -m pip install wheel @@ -36,7 +37,7 @@ runs: ############################################################################ - name: Setup python (OSX) - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 if: runner.os == 'macOS' with: python-version: ${{ matrix.python-version }} @@ -55,8 +56,8 @@ runs: brew update && brew install parallel pssh ffmpeg brew update && brew install --cask mactex brew update && brew install --cask xquartz - python -m pip install --upgrade pip - python -m pip install wheel + python3 -m pip install --upgrade pip + python3 -m pip install wheel ############################################################################ # SIERRA install diff --git a/.github/workflows/analysis.yml b/.github/workflows/analysis.yml new file mode 100644 index 00000000..de70d7c1 --- /dev/null +++ b/.github/workflows/analysis.yml @@ -0,0 +1,33 @@ +name: Static Analysis Top-level + +# Run: +# +# - Every month. Scheduling helps keep drift/issues with updates +# to runners at bay. +# - On pushes. +on: + push: + paths: + - 'sierra/**' + + schedule: + - cron: '0 0 1 * *' + +jobs: + analyze-ci-ubuntu-2004: + uses: ./.github/workflows/analyze-nox.yml + secrets: inherit + with: + os: ubuntu-20.04 + + analyze-ci-ubuntu-2204: + uses: ./.github/workflows/analyze-nox.yml + secrets: inherit + with: + os: ubuntu-22.04 + + analyze-ci-osx: + uses: ./.github/workflows/analyze-nox.yml + secrets: inherit + with: + os: macos-12 diff --git a/.github/workflows/analyze-nox.yml b/.github/workflows/analyze-nox.yml new file mode 100644 index 00000000..110ea0c5 --- /dev/null +++ b/.github/workflows/analyze-nox.yml @@ -0,0 +1,43 @@ +name: OS Static Analysis + +on: + workflow_call: + inputs: + os: + required: True + type: string + +jobs: + nox: + runs-on: ${{ inputs.os }} + strategy: + matrix: + os: + - ${{ inputs.os }} + + # 2024/08/26: Test the earliest and latest supported version, to speed + # up CI. + python-version: + - 3.8 + - '3.10' + + session: + - lint + - docs + - analyze_pytype + - analyze_mypy + + exclude: + - python-version: '3.10' + os: ubuntu-20.04 + + - python-version: 3.8 + os: ubuntu-22.04 + + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/sierra-setup + + - name: Run nox + run: | + nox --pythons ${{ matrix.python-version }} --sessions ${{ matrix.session }} diff --git a/.github/workflows/argos.yml b/.github/workflows/argos.yml deleted file mode 100644 index 0366ab59..00000000 --- a/.github/workflows/argos.yml +++ /dev/null @@ -1,177 +0,0 @@ -# Only stuff that actually tests the platform should be here: stage 1 and 2. -name: ARGoS CI Tests - -on: - workflow_call: - inputs: - os: - required: True - type: string - -jobs: - bc-univar-sanity: - runs-on: ${{ inputs.os }} - strategy: - matrix: - python-version: - - 3.8 - - 3.9 - - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/sierra-setup - - uses: ./.github/actions/sample-project-setup - with: - platform: argos - - - uses: nick-fields/retry@v2 - name: Integration tests - with: - timeout_minutes: 3600 - max_attempts: 3 - shell: bash - command: ./scripts/argos-integration-tests.sh -f bc_univar_sanity_test - - - uses: actions/upload-artifact@v3 - with: - name: ci-platform-argos-bc-univar-sanity-${{ runner.os }}-${{ matrix.python-version }}-coverage - path: .coverage.* - - physics-engines: - runs-on: ${{ inputs.os }} - strategy: - matrix: - python-version: - - 3.8 - - 3.9 - - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/sierra-setup - - uses: ./.github/actions/sample-project-setup - with: - platform: argos - - - uses: nick-fields/retry@v2 - name: Integration tests - with: - timeout_minutes: 3600 - max_attempts: 3 - shell: bash - command: ./scripts/argos-integration-tests.sh -f physics_engines_test - - - uses: actions/upload-artifact@v3 - with: - name: ci-platform-argos-physics-engines-${{ runner.os }}-${{ matrix.python-version }}-coverage - path: .coverage.* - - stage1-univar: - runs-on: ${{ inputs.os }} - strategy: - matrix: - python-version: - - 3.8 - - 3.9 - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/sierra-setup - - uses: ./.github/actions/sample-project-setup - with: - platform: argos - - - uses: nick-fields/retry@v2 - name: Integration tests - with: - timeout_minutes: 3600 - max_attempts: 3 - shell: bash - command: ./scripts/argos-integration-tests.sh -f stage1_univar_test - - - uses: actions/upload-artifact@v3 - with: - name: ci-platform-argos-stage1-${{ runner.os }}-${{ matrix.python-version }}-coverage - path: .coverage.* - - stage2-univar: - runs-on: ${{ inputs.os }} - strategy: - matrix: - python-version: - - 3.8 - - 3.9 - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/sierra-setup - - uses: ./.github/actions/sample-project-setup - with: - platform: argos - - - uses: nick-fields/retry@v2 - name: Integration tests - with: - timeout_minutes: 3600 - max_attempts: 3 - shell: bash - command: ./scripts/argos-integration-tests.sh -f stage2_univar_test -e hpc.local - - - uses: actions/upload-artifact@v3 - with: - name: ci-platform-argos-stage2-${{ runner.os }}-${{ matrix.python-version }}-coverage - path: .coverage.* - - - visual-capture: - runs-on: ${{ inputs.os }} - strategy: - matrix: - python-version: - - 3.8 - - 3.9 - - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/sierra-setup - - uses: ./.github/actions/sample-project-setup - with: - platform: argos - - - uses: nick-fields/retry@v2 - name: Integration tests - with: - timeout_minutes: 3600 - max_attempts: 3 - shell: bash - command: ./scripts/argos-integration-tests.sh -f vc_test - - - uses: actions/upload-artifact@v3 - with: - name: ci-platform-argos-vc-${{ runner.os }}-${{ matrix.python-version }}-coverage - path: .coverage.* - - cmdline: - runs-on: ${{ inputs.os }} - strategy: - matrix: - python-version: - - 3.8 - - 3.9 - - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/sierra-setup - - uses: ./.github/actions/sample-project-setup - with: - platform: argos - - - uses: nick-fields/retry@v2 - name: Integration tests - with: - timeout_minutes: 3600 - max_attempts: 3 - shell: bash - command: ./scripts/argos-integration-tests.sh -f cmdline_test - - - uses: actions/upload-artifact@v3 - with: - name: ci-platform-argos-cmdline-${{ runner.os }}-${{ matrix.python-version }}-coverage - path: .coverage.* diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 1cc0fe35..3b6f01ed 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -1,58 +1,26 @@ -name: Coverage CI +name: Coverage Top-level # Run: # # - Every month. Scheduling helps keep drift/issues with updates # to runners at bay. # - On pushes. +# +# This is a dependent workflow that collects the results of other workflows to +# avoid duplicating things and speed up CI. on: - push: - paths: - - 'sierra/**' - schedule: - - cron: '0 0 1 * *' + worflow_run: + workflows: + - "Unit Tests Top-level" + - "Integration Top-level" + types: + - completed concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: - argos-ci: - uses: ./.github/workflows/argos.yml - secrets: inherit - with: - os: ubuntu-20.04 - - ros1gazebo-ci: - uses: ./.github/workflows/ros1gazebo.yml - secrets: inherit - with: - os: ubuntu-20.04 - - sierra-core-ci: - uses: ./.github/workflows/sierra-core.yml - secrets: inherit - with: - os: ubuntu-20.04 - - ros1robot-ci: - uses: ./.github/workflows/ros1robot.yml - secrets: inherit - with: - os: ubuntu-20.04 - - exec-env-ci: - uses: ./.github/workflows/exec-env-plugins.yml - secrets: inherit - with: - os: ubuntu-20.04 - - unit-tests: - uses: ./.github/workflows/unit-tests.yml - secrets: inherit - with: - os: ubuntu-20.04 - coverage: runs-on: ubuntu-latest strategy: @@ -60,14 +28,6 @@ jobs: python-version: - 3.8 - needs: - - unit-tests - - argos-ci - - ros1gazebo-ci - - ros1robot-ci - - exec-env-ci - - sierra-core-ci - steps: - uses: actions/checkout@v3 - uses: ./.github/actions/sierra-setup diff --git a/.github/workflows/exec-env-plugins.yml b/.github/workflows/exec-env-plugins.yml deleted file mode 100644 index f0367572..00000000 --- a/.github/workflows/exec-env-plugins.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: --exec-env plugin CI - -on: - workflow_call: - inputs: - os: - required: True - type: string - - # push: - # branches: - # # - master - # # - devel - # paths: - # - 'sierra/**' -jobs: - adhoc: - runs-on: ${{ inputs.os }} - strategy: - matrix: - # You CANNOT use python 3.9 with ubuntu 20.04 with ROS, because 3.8 is - # the system python and things just don't work otherwise... - python-version: - - 3.8 - - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/sierra-setup - - uses: ./.github/actions/sample-project-setup - with: - platform: argos - - - uses: ./.github/actions/sample-project-setup - with: - platform: ros1gazebo - rosdistro: noetic - - - name: Adhoc - run: | - ./scripts/argos-integration-tests.sh -f stage2_univar_test -e hpc.adhoc - ./scripts/ros1gazebo-integration-tests.sh -f stage2_univar_test -e hpc.adhoc - - - uses: actions/upload-artifact@v3 - with: - name: ci-exec-env-adhoc-${{ runner.os }}-${{ matrix.python-version }}-coverage - path: .coverage.* - - slurm: - runs-on: ${{ inputs.os }} - strategy: - matrix: - # You CANNOT use python 3.9 with ubuntu 20.04 with ROS, because 3.8 is - # the system python and things just don't work otherwise... - python-version: - - 3.8 - - steps: - - uses: actions/checkout@v3 - - - uses: ./.github/actions/sierra-setup - - - uses: ./.github/actions/slurm-setup - - - uses: ./.github/actions/sample-project-setup - with: - platform: argos - - - uses: ./.github/actions/sample-project-setup - with: - platform: ros1gazebo - rosdistro: noetic - - - name: SLURM - run: | - ./scripts/argos-integration-tests.sh -f stage2_univar_test -e hpc.slurm - ./scripts/ros1gazebo-integration-tests.sh -f stage2_univar_test -e hpc.slurm - - - uses: actions/upload-artifact@v3 - with: - name: ci-exec-env-slurm-${{ runner.os }}-${{ matrix.python-version }}-coverage - path: .coverage.* diff --git a/.github/workflows/integration-all.yml b/.github/workflows/integration-all.yml index 1eff0f23..415e15ae 100644 --- a/.github/workflows/integration-all.yml +++ b/.github/workflows/integration-all.yml @@ -1,4 +1,4 @@ -name: Integration Tests +name: Integration Top-level # Run: # @@ -14,23 +14,17 @@ on: - cron: '0 0 1 * *' jobs: - argos-ci-ubuntu: - uses: ./.github/workflows/argos.yml + sierra-core-ci-ubuntu-2004: + uses: ./.github/workflows/sierra-core.yml secrets: inherit with: os: ubuntu-20.04 - argos-ci-osx: - uses: ./.github/workflows/argos.yml - secrets: inherit - with: - os: macos-12 - - sierra-core-ci-ubuntu: + sierra-core-ci-ubuntu-2204: uses: ./.github/workflows/sierra-core.yml secrets: inherit with: - os: ubuntu-20.04 + os: ubuntu-22.04 sierra-core-ci-osx: uses: ./.github/workflows/sierra-core.yml @@ -38,23 +32,23 @@ jobs: with: os: macos-12 - ros1gazebo-ci-ubuntu: - uses: ./.github/workflows/ros1gazebo.yml + plugin-ci-ubuntu-2004: + uses: ./.github/workflows/plugin.yml secrets: inherit with: os: ubuntu-20.04 - ros1robot-ci-ubuntu: - uses: ./.github/workflows/ros1robot.yml + plugin-ci-ubuntu-2204: + uses: ./.github/workflows/plugin.yml secrets: inherit with: - os: ubuntu-20.04 + os: ubuntu-22.04 - exec-env-ci-ubuntu: - uses: ./.github/workflows/exec-env-plugins.yml + plugin-ci-osx: + uses: ./.github/workflows/plugin.yml secrets: inherit with: - os: ubuntu-20.04 + os: macos-12 publish: runs-on: ubuntu-latest @@ -63,13 +57,12 @@ jobs: python-version: - 3.8 needs: - - argos-ci-ubuntu - - argos-ci-osx - - sierra-core-ci-ubuntu + - plugin-ci-ubuntu-2004 + - plugin-ci-ubuntu-2204 + - plugin-ci-osx + - sierra-core-ci-ubuntu-2004 + - sierra-core-ci-ubuntu-2204 - sierra-core-ci-osx - - ros1gazebo-ci-ubuntu - - ros1robot-ci-ubuntu - - exec-env-ci-ubuntu steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/plugin.yml b/.github/workflows/plugin.yml new file mode 100644 index 00000000..fb3350ff --- /dev/null +++ b/.github/workflows/plugin.yml @@ -0,0 +1,190 @@ +# Only stuff that actually tests the platform should be here: stage 1 and 2. +name: ARGoS Plugin CI + +on: + workflow_call: + inputs: + os: + required: True + type: string + +jobs: + plugin-argos: + runs-on: ${{ inputs.os }} + strategy: + matrix: + os: + - ${{ inputs.os }} + + # 2024/08/26: Test the earliest and latest supported version, to speed + # up CI. + python-version: + - 3.8 + - '3.10' + + function: + - bc_univar_sanity_test + - physics_engines_test + - stage1_univar_test + - stage2_univar_test + - vc_test + - cmdline_test + + exclude: + - python-version: '3.10' + os: ubuntu-20.04 + + - python-version: 3.8 + os: ubuntu-22.04 + + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/sierra-setup + - uses: ./.github/actions/sample-project-setup + with: + platform: argos + + - uses: nick-fields/retry@v2 + name: Integration tests + with: + timeout_minutes: 3600 + max_attempts: 3 + shell: bash + command: ./scripts/argos-integration-tests.sh -f ${{ matrix.function }} + + - uses: actions/upload-artifact@v3 + with: + name: ci-platform-argos-${{ matrix.function }}-${{ runner.os }}-${{ matrix.python-version }}-coverage + path: .coverage.* + + plugin-ros1robot: + runs-on: ${{ inputs.os }} + strategy: + matrix: + python-version: + - 3.8 + - 3.9 + + function: + - bc_univar_sanity_test + - stage1_test + + os: + - ${{ inputs.os }} + + exclude: + os: + - ubuntu-22.04 + - macos-12 + + steps: + - uses: actions/checkout@v3 + + - uses: ./.github/actions/sierra-setup + + - uses: ./.github/actions/sample-project-setup + with: + platform: ros1robot + rosdistro: noetic + + - name: Integration tests + shell: bash + run: | + ./scripts/ros1robot-integration-tests.sh -f ${{ matrix.function }} + + - uses: actions/upload-artifact@v3 + with: + name: ci-platform-ros1robot-${{ matrix.function }}-${{ runner.os }}-${{ matrix.python-version }}-coverage + path: .coverage.* + + plugin-ros1gazebo: + runs-on: ${{ inputs.os }} + strategy: + matrix: + # You CANNOT use python 3.9 with ubuntu 20.04 with ROS, because 3.8 is + # the system python and things just don't work otherwise... + python-version: + - 3.8 + + function: + - bc_univar_sanity_test + - stage2_univar_test + + os: + - ${{ inputs.os }} + + exclude: + os: + - ubuntu-22.04 + - macos-12 + + steps: + - uses: actions/checkout@v3 + + - uses: ./.github/actions/sierra-setup + + - uses: ./.github/actions/sample-project-setup + with: + platform: ros1gazebo + rosdistro: noetic + + - name: Integration tests + shell: bash + run: | + ./scripts/ros1gazebo-integration-tests.sh -f ${{ matrix.function }} + + - uses: actions/upload-artifact@v3 + with: + name: ci-platform-ros1gazebo-${{ matrix.function }}-${{ runner.os }}-${{ matrix.python-version }}-coverage + path: .coverage.* + + plugin-exec-env: + runs-on: ${{ inputs.os }} + strategy: + matrix: + # You CANNOT use python 3.9 with ubuntu 20.04 with ROS, because 3.8 is + # the system python and things just don't work otherwise... + python-version: + - 3.8 + + platform: + - argos + - ros1gazebo + + exec-env: + - hpc.adhoc + - hpc.slurm + + os: + - ${{ inputs.os }} + + exclude: + os: + - ubuntu-22.04 + - macos-12 + + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/sierra-setup + - uses: ./.github/actions/sample-project-setup + if: matrix.platform == 'argos' + with: + platform: argos + + - uses: ./.github/actions/sample-project-setup + if: matrix.platform == 'ros1gazebo' + with: + platform: ros1gazebo + rosdistro: noetic + + - uses: ./.github/actions/slurm-setup + if: matrix.exec-env == 'hpc.slurm' + + - name: Integration Tests + run: | + ./scripts/${{ matrix.platform }}-integration-tests.sh -f stage2_univar_test -e ${{ matrix.exec-env }} + + - uses: actions/upload-artifact@v3 + with: + name: ci-${{ matrix.platform }}-${{ matrix.exec-env }}-${{ runner.os }}-${{ matrix.python-version }}-coverage + path: .coverage.* diff --git a/.github/workflows/ros1gazebo.yml b/.github/workflows/ros1gazebo.yml deleted file mode 100644 index 57b27d3a..00000000 --- a/.github/workflows/ros1gazebo.yml +++ /dev/null @@ -1,68 +0,0 @@ -# Only stuff that actually tests the platform should be here: stage 1 and 2. -name: ROS1+Gazebo CI Tests - -on: - workflow_call: - inputs: - os: - required: True - type: string - -jobs: - bc-univar-sanity: - runs-on: ${{ inputs.os }} - strategy: - matrix: - # You CANNOT use python 3.9 with ubuntu 20.04 with ROS, because 3.8 is - # the system python and things just don't work otherwise... - python-version: - - 3.8 - - steps: - - uses: actions/checkout@v3 - - - uses: ./.github/actions/sierra-setup - - - uses: ./.github/actions/sample-project-setup - with: - platform: ros1gazebo - rosdistro: noetic - - - name: Integration tests - shell: bash - run: | - ./scripts/ros1gazebo-integration-tests.sh -f bc_univar_sanity_test - - - uses: actions/upload-artifact@v3 - with: - name: ci-platform-ros1gazebo-bc-univar-sanity-${{ runner.os }}-${{ matrix.python-version }}-coverage - path: .coverage.* - - stage2-univar: - runs-on: ${{ inputs.os }} - strategy: - matrix: - # You CANNOT use python 3.9 with ubuntu 20.04 with ROS, because 3.8 is - # the system python and things just don't work otherwise... - python-version: - - 3.8 - - steps: - - uses: actions/checkout@v3 - - - uses: ./.github/actions/sierra-setup - - - uses: ./.github/actions/sample-project-setup - with: - platform: ros1gazebo - rosdistro: noetic - - - name: Integration tests - shell: bash - run: | - ./scripts/ros1gazebo-integration-tests.sh -f stage2_univar_test -e hpc.local - - - uses: actions/upload-artifact@v3 - with: - name: ci-platform-ros1gazebo-stage2-${{ runner.os }}-${{ matrix.python-version }}-coverage - path: .coverage.* diff --git a/.github/workflows/ros1robot.yml b/.github/workflows/ros1robot.yml deleted file mode 100644 index 3cde7a14..00000000 --- a/.github/workflows/ros1robot.yml +++ /dev/null @@ -1,66 +0,0 @@ -# Only stuff that actually tests the platform should be here: stage 1 and 2. -name: ROS1+Robot CI Tests - -on: - workflow_call: - inputs: - os: - required: True - type: string - -jobs: - bc-univar-sanity: - runs-on: ${{ inputs.os }} - strategy: - matrix: - python-version: - - 3.8 - - 3.9 - - steps: - - uses: actions/checkout@v3 - - - uses: ./.github/actions/sierra-setup - - - uses: ./.github/actions/sample-project-setup - with: - platform: ros1robot - rosdistro: noetic - - - name: Integration tests - shell: bash - run: | - ./scripts/ros1robot-integration-tests.sh -f bc_univar_sanity_test - - - uses: actions/upload-artifact@v3 - with: - name: ci-platform-ros1robot-bc-sanity-${{ runner.os }}-${{ matrix.python-version }}-coverage - path: .coverage.* - - stage1: - runs-on: ${{ inputs.os }} - strategy: - matrix: - python-version: - - 3.8 - - 3.9 - - steps: - - uses: actions/checkout@v3 - - - uses: ./.github/actions/sierra-setup - - - uses: ./.github/actions/sample-project-setup - with: - platform: ros1robot - rosdistro: noetic - - - name: Integration tests - shell: bash - run: | - ./scripts/ros1robot-integration-tests.sh -f stage1_test -e hpc.local - - - uses: actions/upload-artifact@v3 - with: - name: ci-platform-ros1robot-stage1-${{ runner.os }}-${{ matrix.python-version }}-coverage - path: .coverage.* diff --git a/.github/workflows/sierra-core.yml b/.github/workflows/sierra-core.yml index 041cdebc..38ae3125 100644 --- a/.github/workflows/sierra-core.yml +++ b/.github/workflows/sierra-core.yml @@ -8,41 +8,29 @@ on: type: string jobs: - cmdline-opts: + argos: runs-on: ${{ inputs.os }} strategy: matrix: + os: + - ${{ inputs.os }} + + # 2024/08/26: Test the earliest and latest supported version, to speed + # up CI. python-version: - 3.8 - - 3.9 - - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/sierra-setup - - uses: ./.github/actions/sample-project-setup - with: - platform: argos + - '3.10' - - uses: nick-fields/retry@v2 - name: Cmdline options - with: - timeout_minutes: 3600 - max_attempts: 3 - shell: bash - command: ./scripts/core-integration-tests.sh -f cmdline_opts_test + function: + - cmdline_opts_test + - env_vars_test - - uses: actions/upload-artifact@v3 - with: - name: ci-core-cmdline-opts-${{ runner.os }}-${{ matrix.python-version }}-coverage - path: .coverage.* + exclude: + - python-version: '3.10' + os: ubuntu-20.04 - env-vars: - runs-on: ${{ inputs.os }} - strategy: - matrix: - python-version: - - 3.8 - - 3.9 + - python-version: 3.8 + os: ubuntu-22.04 steps: - uses: actions/checkout@v3 @@ -52,111 +40,138 @@ jobs: platform: argos - uses: nick-fields/retry@v2 - name: Environment variables with: timeout_minutes: 3600 max_attempts: 3 shell: bash - command: ./scripts/core-integration-tests.sh -f env_vars_test + command: ./scripts/core-integration-tests.sh -f ${{ matrix.function }} - uses: actions/upload-artifact@v3 with: - name: ci-core-env-vars-${{ runner.os }}-${{ matrix.python-version }}-coverage + name: ci-core-${{ matrix.function }}-${{ runner.os }}-${{ matrix.python-version }}-coverage path: .coverage.* + + bc-bivar-sanity: runs-on: ${{ inputs.os }} strategy: matrix: + os: + - ${{ inputs.os }} + + platform: + - argos + - ros1gazebo + python-version: - 3.8 - 3.9 + exclude: + os: macos12 + steps: - uses: actions/checkout@v3 - uses: ./.github/actions/sierra-setup - uses: ./.github/actions/sample-project-setup + if: matrix.platform == 'argos' with: platform: argos - uses: ./.github/actions/sample-project-setup - if: runner.os == 'Linux' + if: matrix.platform == 'ros1gazebo' with: platform: ros1gazebo rosdistro: noetic - uses: nick-fields/retry@v2 - name: Bivariate batch criteria sanity (ARGoS) + name: Bivariate batch criteria sanity (${{ matrix.platform }}) with: timeout_minutes: 3600 max_attempts: 3 shell: bash - command: ./scripts/argos-integration-tests.sh -f bc_bivar_sanity_test + command: ./scripts/${{ matrix.platform }}-integration-tests.sh -f bc_bivar_sanity_test - - uses: nick-fields/retry@v2 - if: runner.os == 'Linux' - name: Bivariate batch criteria sanity (ROS1+Gazebo) - with: - timeout_minutes: 3600 - max_attempts: 3 - shell: bash - command: ./scripts/ros1gazebo-integration-tests.sh -f bc_bivar_sanity_test - uses: actions/upload-artifact@v3 with: - name: ci-core-bc-bivar-sanity-${{ runner.os }}-${{ matrix.python-version }}-coverage + name: ci-core-${{ matrix.platform }}-bc-bivar-sanity-${{ runner.os }}-${{ matrix.python-version }}-coverage path: .coverage.* stage1-bivar: runs-on: ${{ inputs.os }} strategy: matrix: + os: + - ${{ inputs.os }} + + # 2024/08/26: Test the earliest and latest supported version, to speed + # up CI. python-version: - 3.8 - - 3.9 + - '3.10' + + platform: + - argos + - ros1gazebo + + exclude: + - python-version: '3.10' + os: ubuntu-20.04 + + - python-version: 3.8 + os: ubuntu-22.04 + + - os: macos12 steps: - uses: actions/checkout@v3 - uses: ./.github/actions/sierra-setup - uses: ./.github/actions/sample-project-setup + if: matrix.platform == 'argos' with: platform: argos - uses: ./.github/actions/sample-project-setup - if: runner.os == 'Linux' + if: matrix.platform == 'ros1gazebo' with: platform: ros1gazebo rosdistro: noetic - uses: nick-fields/retry@v2 - name: Bivariate batch criteria stage 1 (ARGoS) - with: - timeout_minutes: 3600 - max_attempts: 3 - shell: bash - command: ./scripts/argos-integration-tests.sh -f stage1_bivar_test - - - uses: nick-fields/retry@v2 - if: runner.os == 'Linux' - name: Bivariate batch criteria stage 1 (ROS1+Gazebo) + name: Bivariate batch criteria stage 1 (${{ matrix.platform }} with: timeout_minutes: 3600 max_attempts: 3 shell: bash - command: ./scripts/ros1gazebo-integration-tests.sh -f stage1_bivar_test + command: ./scripts/${{ matrix.platform }}-integration-tests.sh -f stage1_bivar_test - uses: actions/upload-artifact@v3 with: - name: ci-core-stage1-bivar-${{ runner.os }}-${{ matrix.python-version }}-coverage + name: ci-core-${{ matrix.platform }}-stage1-bivar-${{ runner.os }}-${{ matrix.python-version }}-coverage path: .coverage.* stage3-univar: runs-on: ${{ inputs.os }} strategy: matrix: + os: + - ${{ inputs.os }} + + # 2024/08/26: Test the earliest and latest supported version, to speed + # up CI. python-version: - 3.8 - - 3.9 + - '3.10' + + exclude: + - python-version: '3.10' + os: ubuntu-20.04 + + - python-version: 3.8 + os: ubuntu-22.04 + steps: - uses: actions/checkout@v3 - uses: ./.github/actions/sierra-setup @@ -174,16 +189,29 @@ jobs: - uses: actions/upload-artifact@v3 with: - name: ci-core-stage3-univar-${{ runner.os }}-${{ matrix.python-version }}-coverage + name: ci-core-argos-stage3-univar-${{ runner.os }}-${{ matrix.python-version }}-coverage path: .coverage.* stage3-bivar: runs-on: ${{ inputs.os }} strategy: matrix: + os: + - ${{ inputs.os }} + + # 2024/08/26: Test the earliest and latest supported version, to speed + # up CI. python-version: - 3.8 - - 3.9 + - '3.10' + + exclude: + - python-version: '3.10' + os: ubuntu-20.04 + + - python-version: 3.8 + os: ubuntu-22.04 + steps: - uses: actions/checkout@v3 - uses: ./.github/actions/sierra-setup @@ -201,16 +229,29 @@ jobs: - uses: actions/upload-artifact@v3 with: - name: ci-core-stage3-univar-${{ runner.os }}-${{ matrix.python-version }}-coverage + name: ci-core-argos-stage3-univar-${{ runner.os }}-${{ matrix.python-version }}-coverage path: .coverage.* stage4-univar: runs-on: ${{ inputs.os }} strategy: matrix: + os: + - ${{ inputs.os }} + + # 2024/08/26: Test the earliest and latest supported version, to speed + # up CI. python-version: - 3.8 - - 3.9 + - '3.10' + + exclude: + - python-version: '3.10' + os: ubuntu-20.04 + + - python-version: 3.8 + os: ubuntu-22.04 + steps: - uses: actions/checkout@v3 - uses: ./.github/actions/sierra-setup @@ -228,16 +269,29 @@ jobs: - uses: actions/upload-artifact@v3 with: - name: ci-core-stage4-univar-${{ runner.os }}-${{ matrix.python-version }}-coverage + name: ci-core-argos-stage4-univar-${{ runner.os }}-${{ matrix.python-version }}-coverage path: .coverage.* stage5-univar: runs-on: ${{ inputs.os }} strategy: matrix: + os: + - ${{ inputs.os }} + + # 2024/08/26: Test the earliest and latest supported version, to speed + # up CI. python-version: - 3.8 - - 3.9 + - '3.10' + + exclude: + - python-version: '3.10' + os: ubuntu-20.04 + + - python-version: 3.8 + os: ubuntu-22.04 + steps: - uses: actions/checkout@v3 - uses: ./.github/actions/sierra-setup @@ -255,16 +309,30 @@ jobs: - uses: actions/upload-artifact@v3 with: - name: ci-core-stage5-univar-${{ runner.os }}-${{ matrix.python-version }}-coverage + name: ci-core-argos-stage5-univar-${{ runner.os }}-${{ matrix.python-version }}-coverage path: .coverage.* stage5-bivar: runs-on: ${{ inputs.os }} strategy: matrix: + + os: + - ${{ inputs.os }} + + # 2024/08/26: Test the earliest and latest supported version, to speed + # up CI. python-version: - 3.8 - - 3.9 + - '3.10' + + exclude: + - python-version: '3.10' + os: ubuntu-20.04 + + - python-version: 3.8 + os: ubuntu-22.04 + steps: - uses: actions/checkout@v3 - uses: ./.github/actions/sierra-setup @@ -282,5 +350,5 @@ jobs: - uses: actions/upload-artifact@v3 with: - name: ci-core-stage5-bivar-${{ runner.os }}-${{ matrix.python-version }}-coverage + name: ci-core-argos-stage5-bivar-${{ runner.os }}-${{ matrix.python-version }}-coverage path: .coverage.* diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml deleted file mode 100644 index 884ef641..00000000 --- a/.github/workflows/static-analysis.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Static Analysis - -# Run: -# -# - Every month. Scheduling helps keep drift/issues with updates -# to runners at bay. -# - On pushes. -on: - push: - paths: - - 'sierra/**' - schedule: - - cron: '0 0 1 * *' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - analyze: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: - - ubuntu-20.04 # ubuntu-latest, as of 8/22 - - macos-12 - - python-version: - - 3.8 - - 3.9 - - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/sierra-setup - - - name: Run nox - run: | - nox --pythons ${{ matrix.python-version }} --sessions lint analysis docs diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index f9537b80..54b6a0bd 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,20 +1,43 @@ -name: Unit Tests +name: Unit Tests Top-level +# Run: +# +# - Every month. Scheduling helps keep drift/issues with updates +# to runners at bay. +# - On pushes. on: - workflow_call: - inputs: - os: - required: True - type: string + push: + paths: + - 'sierra/**' + schedule: + - cron: '0 0 1 * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true jobs: - unit-tests: - runs-on: ${{ inputs.os }} + ci-unit-tests: + runs-on: ${{ matrix.os }} strategy: matrix: + os: + - ubuntu-20.04 + - ubuntu-22.04 + - macos-12 + + # 2024/08/26: Test the earliest and latest supported version, to speed + # up CI. python-version: - 3.8 - - 3.9 + - '3.10' + + exclude: + - python-version: '3.10' + os: ubuntu-20.04 + + - python-version: 3.8 + os: ubuntu-22.04 steps: - uses: actions/checkout@v3 diff --git a/docs/conf.py b/docs/conf.py index 8662810f..bd90beec 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -160,8 +160,8 @@ "https://www.cs.unm.edu/~wjust/CS523/S2018/Readings/Hecker_Beyond_Pheromones_Swarm_Intelligence.pdf"), "Rosenfeld2006": ("Rosenfeld2006", "http://users.umiacs.umd.edu/~sarit/data/articles/rosenfeldetalbook06.pdf"), - "SIERRA_GITHUB": ("https://github.com:jharwell/sierra.git", - "https://github.com:jharwell/sierra.git"), + "SIERRA_GITHUB": ("https://github.com/jharwell/sierra.git", + "https://github.com/jharwell/sierra.git"), "SIERRA_SAMPLE_PROJECT": ("https://github.com:jharwell/sierra-sample-project.git", "https://github.com:jharwell/sierra-sample-project.git"), "SIERRA_DOCS": ("https://sierra.readthedocs.io/en/master", @@ -310,7 +310,7 @@ 'sphinx': ('https://www.sphinx-doc.org/en/stable/', None), 'pandas': ('https://pandas.pydata.org/pandas-docs/dev', None), 'implements': ('https://implements.readthedocs.io/en/latest/', None), - 'libra': ('https://libra2.readthedocs.io/en/master', None) + 'libra': ('https://jharwell.github.io/libra', None) } # This is the expected signature of the handler for this event, cf doc diff --git a/docs/requirements.txt b/docs/requirements.txt index 64c9b616..72106916 100755 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,7 +1,7 @@ -e . -sphinx==4.4.0 -docutils==0.16 +sphinx==6.0.0 +docutils==0.19 sphinx-rtd-theme sphinx-argparse sphinx-tabs diff --git a/docs/src/getting_started.rst b/docs/src/getting_started.rst index 9c434bf9..25d5361c 100755 --- a/docs/src/getting_started.rst +++ b/docs/src/getting_started.rst @@ -48,6 +48,7 @@ Basic Setup - ``pssh`` - ``ffmpeg`` - ``xvfb`` + - ``psmisc`` .. group-tab:: OSX diff --git a/docs/src/requirements.rst b/docs/src/requirements.rst index 602328bb..68f44cd2 100644 --- a/docs/src/requirements.rst +++ b/docs/src/requirements.rst @@ -34,8 +34,8 @@ One of the following: Python Requirements =================== -Python 3.8+. Tested with 3.8-3.9. It may work for newer versions, probably won't -for older. +Python 3.8+. Tested with 3.8-3.10. It may work for newer versions, probably +won't for older. .. _ln-sierra-req-exp: diff --git a/docs/src/trial.rst b/docs/src/trial.rst index 8d81019a..f925bfe1 100755 --- a/docs/src/trial.rst +++ b/docs/src/trial.rst @@ -30,6 +30,7 @@ you clone things somewhere else. - ``pssh`` - ``ffmpeg`` - ``xvfb`` + - ``psmisc`` .. group-tab:: OSX @@ -92,7 +93,7 @@ you clone things somewhere else. Where ```` is replaced by your ROS distro. - #. Install the `SIERRA ROSBridge `_:: + #. Install the `SIERRA ROSBridge `_:: pip3 install catkin_tools git clone https://github.com/jharwell/sierra_rosbridge.git diff --git a/docs/src/usage/subprograms.rst b/docs/src/usage/subprograms.rst index fc70700a..ffdedb87 100755 --- a/docs/src/usage/subprograms.rst +++ b/docs/src/usage/subprograms.rst @@ -20,3 +20,7 @@ depending on what you are doing. - :program:`parallel-rsync` - Used during stage 1 when generating experiments experiments (:term:`ROS1+Robot` platform). + +- :program:`killall` - Used during stage 2 when running :term: + :term:`ARGoS`, :term:`ROS1+Gazebo` experiments to cleanup after each + experimental run. diff --git a/noxfile.py b/noxfile.py index 03ce9735..12f5fbcb 100644 --- a/noxfile.py +++ b/noxfile.py @@ -23,7 +23,7 @@ # Project packages -@nox.session(python=['3.8', '3.9']) +@nox.session(python=['3.8', '3.9', '3.10']) def lint(session): session.install('.') # same as 'pip3 install .' session.install('.[devel]') # same as 'pip3 install .[devel]' @@ -52,8 +52,8 @@ def lint(session): # venv argument needed so the apt module can be found in the nox venv on linux -@nox.session(python=['3.8', '3.9']) -def analysis(session): +@nox.session(python=['3.8', '3.9', '3.10']) +def analyze_pytype(session): session.install('.') # same as 'pip3 install .' session.install('.[devel]') # same as 'pip3 install .[devel]' @@ -65,6 +65,15 @@ def analysis(session): 'sierra', external=True) + +# venv argument needed so the apt module can be found in the nox venv on linux +@nox.session(python=['3.8', '3.9', '3.10']) +def analyze_mypy(session): + session.install('.') # same as 'pip3 install .' + session.install('.[devel]') # same as 'pip3 install .[devel]' + + cores = psutil.cpu_count() + session.run('mypyrun', '--select', # No syntax errors @@ -126,7 +135,7 @@ def analysis(session): 'sierra') -@nox.session(python=['3.8', '3.9']) +@nox.session(python=['3.8', '3.9', '3.10']) def docs(session): session.install('.') # same as 'pip3 install .' session.install('.[devel]') # same as 'pip3 install .[devel]' @@ -147,7 +156,7 @@ def docs(session): 'sierra') -@nox.session(python=['3.8', '3.9']) +@nox.session(python=['3.8', '3.9', '3.10']) def unit_tests(session): session.install('.') # same as 'pip3 install .' session.install('.[devel]') # same as 'pip3 install .[devel]' diff --git a/setup.py b/setup.py index fc151c29..30d92571 100644 --- a/setup.py +++ b/setup.py @@ -56,6 +56,7 @@ # "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "License :: OSI Approved :: MIT License", "Environment :: Console", "Operating System :: MacOS :: MacOS X", diff --git a/sierra/core/config.py b/sierra/core/config.py index 9f760c0a..7b6d1956 100755 --- a/sierra/core/config.py +++ b/sierra/core/config.py @@ -45,7 +45,7 @@ def mpl_init(): import matplotlib.pyplot as plt # Set MPL style - plt.style.use('seaborn-colorblind') + plt.style.use('seaborn-v0_8-colorblind') # Actually initialize matplotlib diff --git a/sierra/core/graphs/heatmap.py b/sierra/core/graphs/heatmap.py index 5559a558..0fbf5c09 100755 --- a/sierra/core/graphs/heatmap.py +++ b/sierra/core/graphs/heatmap.py @@ -211,7 +211,7 @@ def generate(self) -> None: # Scaffold graph. We can use either dataframe for setting the graph # size; we assume they have the same dimensions. # - fig, axes = plt.subplots(ncols=2) + fig, axes = plt.subplots(nrows=1, ncols=2) Heatmap.set_graph_size(dfs[0], fig) y = np.arange(len(dfs[0].columns)) diff --git a/sierra/core/startup.py b/sierra/core/startup.py index 977d84c9..ecb7fb58 100644 --- a/sierra/core/startup.py +++ b/sierra/core/startup.py @@ -74,8 +74,8 @@ def _linux_pkg_checks() -> None: dist = distro.id() os_info = distro.os_release_info() - if any(candidate in os_info['id_like'] for candidate in ['debian', - 'ubuntu']): + if any(candidate in os_info['id'] for candidate in ['debian', + 'ubuntu']): _apt_pkg_checks(dist) else: logging.warning(("Unknown Linux distro '%s' detected: skipping package " diff --git a/sierra/core/utils.py b/sierra/core/utils.py index c2708bb0..156a200f 100755 --- a/sierra/core/utils.py +++ b/sierra/core/utils.py @@ -365,7 +365,7 @@ def df_fill(df: pd.DataFrame, policy: str) -> pd.DataFrame: @retry(OSError, tries=10, delay=0.100, backoff=1.1) # type:ignore -def pickle_dump(obj: object, f: tp.IO) -> None: +def pickle_dump(obj: object, f) -> None: pickle.dump(obj, f) diff --git a/sierra/version.py b/sierra/version.py index 80604bc8..d4be2b36 100644 --- a/sierra/version.py +++ b/sierra/version.py @@ -9,4 +9,4 @@ # Project packages -__version__ = "1.3.5" +__version__ = "1.3.6"