task(#309): Add ROS1+Robot integration tests

- Bivar batch criteria now supports one criteria adding XML tags and one changing them. - Fix several bugs in bivariate batch criteria which arose when I tried to add the new batch criteria in the sample project - ros1+gazebo plugin now respects --exec-devnull - Make all documentation examples refer to the SIERRA sample project--better cohesion. - Expand inter-experiment graphs to include heatmaps - Expand rendering capabilities to include bivariate heatmaps - Rework stats extensions so that I can just iterate from config stored in ONE PLACE, rather than having to enumerate explicit keys, which is error-prone. - Fix warning from DataFrame.append() deprecation (I think). - Fix bug where all exps in a batch got the same pickle file containing contain definitions for ALL exps, rather than just the current one. - Fix bugs in stage 5 graph comparison, univariate and bivariate - Document/make limitation about linegraphs from bivariate batch criteria not supporting statistics clear
jharwell · Aug 20, 2022 · 6fb94fa · 6fb94fa
1 parent bbad273
commit 6fb94fa
Show file tree

Hide file tree

Showing 85 changed files with 2,938 additions and 1,159 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -19,3 +19,5 @@ omit =
      */site-packages/*
 
      /usr/*
+
+     */docs/*
diff --git a/.github/actions/publish/action.yml b/.github/actions/publish/action.yml
@@ -1,30 +1,33 @@
 name: Publishing
 description: Publish SIERRA to PyPI and Github
-
-on:
-  push:
-    branches:
-      - devel
-      - master
+inputs:
+  github-token:
+    required: true
+  pypi-token:
+    required: true
+  testpypi-token:
+    required: true
 
 runs:
   using: 'composite'
   steps:
     - uses: actions/checkout@v3
     - uses: ./.github/actions/sierra-setup
     - name: Build package
+      shell: bash
       run: |
         python3 -m build
 
     - name: Set tag
       if: github.ref == 'refs/heads/devel'
+      shell: bash
       run: echo "RELEASE_TAG=v$(python3 setup.py --version).beta" >> $GITHUB_ENV
 
     - name: Github beta release
       if: github.ref == 'refs/heads/devel'
       uses: "marvinpinto/action-automatic-releases@latest"
       with:
-        repo_token: "${{ secrets.GITHUB_TOKEN }}"
+        repo_token: "${{ inputs.github-token }}"
         automatic_release_tag: ${{ env.RELEASE_TAG }}
         prerelease: true
         title: "Development release ${{ env.RELEASE_TAG }}"
@@ -36,18 +39,19 @@ runs:
       with:
         skip_existing: true
         user: __token__
-        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+        password: ${{ inputs.testpypi-token }}
         repository_url: https://test.pypi.org/legacy/
 
     - name: Set tag
       if: github.ref == 'refs/heads/master'
+      shell: bash
       run: echo "RELEASE_TAG=v$(python3 setup.py --version)" >> $GITHUB_ENV
 
     - name: Github release
       if: github.ref == 'refs/heads/master'
       uses: "marvinpinto/action-automatic-releases@latest"
       with:
-        repo_token: "${{ secrets.GITHUB_TOKEN }}"
+        repo_token: "${{ inputs.github-token }}"
         automatic_release_tag: ${{ env.RELEASE_TAG }}
         prerelease: false
         title: "Release ${{ env.RELEASE_TAG }}"
@@ -58,4 +62,4 @@ runs:
       with:
         skip_existing: true
         user: __token__
-        password: ${{ secrets.PYPI_API_TOKEN }}
+        password: ${{ inputs.pypi-token }}
diff --git a/.github/actions/sample-project-setup/action.yml b/.github/actions/sample-project-setup/action.yml
@@ -105,3 +105,9 @@ runs:
         catkin config --extend /opt/ros/${{ inputs.rosdistro }}
         catkin config --install -DCMAKE_INSTALL_PREFIX=$HOME/.local
         catkin build
+
+    ############################################################################
+    # ROS1+Robot setup
+    ############################################################################
+
+    # None needed for the moment--stage 1 XML generation only.
diff --git a/.github/workflows/argos.yml b/.github/workflows/argos.yml
@@ -1,3 +1,4 @@
+# Only stuff that actually tests the platform should be here: stage 1 and 2.
 name: ARGoS CI Tests
 
 on:
@@ -7,15 +8,8 @@ on:
         required: True
         type: string
 
-  # push:
-  #   branches:
-  #     # - master
-  #     # - devel
-  #   paths:
-  #     - 'sierra/**'
-
 jobs:
-  batch-criteria:
+  bc-univar-sanity:
     runs-on: ${{ inputs.os }}
     strategy:
       matrix:
@@ -33,11 +27,11 @@ jobs:
       - name: Integration tests
         shell: bash
         run: |
-          ./scripts/argos-integration-tests.sh -f batch_criteria_test
+          ./scripts/argos-integration-tests.sh -f bc_univar_sanity_test
 
       - uses: actions/upload-artifact@v3
         with:
-          name: ci-platform-argos-batch-criteria-${{ runner.os }}-${{ matrix.python-version }}-coverage
+          name: ci-platform-argos-bc-univar-sanity-${{ runner.os }}-${{ matrix.python-version }}-coverage
           path: .coverage.*
 
   physics-engines:
@@ -65,7 +59,7 @@ jobs:
           name: ci-platform-argos-physics-engines-${{ runner.os }}-${{ matrix.python-version }}-coverage
           path: .coverage.*
 
-  stage1:
+  stage1-univar:
     runs-on: ${{ inputs.os }}
     strategy:
       matrix:
@@ -82,14 +76,14 @@ jobs:
       - name: Integration tests
         shell: bash
         run: |
-          ./scripts/argos-integration-tests.sh -f stage1_outputs_test
+          ./scripts/argos-integration-tests.sh -f stage1_univar_test
 
       - uses: actions/upload-artifact@v3
         with:
           name: ci-platform-argos-stage1-${{ runner.os }}-${{ matrix.python-version }}-coverage
           path: .coverage.*
 
-  stage2:
+  stage2-univar:
     runs-on: ${{ inputs.os }}
     strategy:
       matrix:
@@ -106,84 +100,13 @@ jobs:
       - name: Integration tests
         shell: bash
         run: |
-          ./scripts/argos-integration-tests.sh -f stage2_outputs_test -e hpc.local
+          ./scripts/argos-integration-tests.sh -f stage2_univar_test -e hpc.local
 
       - uses: actions/upload-artifact@v3
         with:
           name: ci-platform-argos-stage2-${{ runner.os }}-${{ matrix.python-version }}-coverage
           path: .coverage.*
 
-  stage3:
-    runs-on: ${{ inputs.os }}
-    strategy:
-      matrix:
-        python-version:
-          - 3.8
-          - 3.9
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/sierra-setup
-      - uses: ./.github/actions/sample-project-setup
-        with:
-          platform: argos
-
-      - name: Integration tests
-        shell: bash
-        run: |
-          ./scripts/argos-integration-tests.sh -f stage3_outputs_test
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ci-platform-argos-stage3-${{ runner.os }}-${{ matrix.python-version }}-coverage
-          path: .coverage.*
-
-  stage4:
-    runs-on: ${{ inputs.os }}
-    strategy:
-      matrix:
-        python-version:
-          - 3.8
-          - 3.9
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/sierra-setup
-      - uses: ./.github/actions/sample-project-setup
-        with:
-          platform: argos
-
-      - name: Integration tests
-        shell: bash
-        run: |
-          ./scripts/argos-integration-tests.sh -f stage4_outputs_test
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ci-platform-argos-stage4-${{ runner.os }}-${{ matrix.python-version }}-coverage
-          path: .coverage.*
-
-  stage5:
-    runs-on: ${{ inputs.os }}
-    strategy:
-      matrix:
-        python-version:
-          - 3.8
-          - 3.9
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/sierra-setup
-      - uses: ./.github/actions/sample-project-setup
-        with:
-          platform: argos
-
-      - name: Integration tests
-        shell: bash
-        run: |
-          ./scripts/argos-integration-tests.sh -f stage5_outputs_test
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ci-platform-argos-stage5-${{ runner.os }}-${{ matrix.python-version }}-coverage
-          path: .coverage.*
 
   visual-capture:
     runs-on: ${{ inputs.os }}

diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -2,9 +2,6 @@ name: Coverage CI
 
 on:
   push:
-    branches:
-      # - master
-      # - devel
     paths:
       - 'sierra/**'
 
@@ -21,6 +18,18 @@ jobs:
     with:
       os: ubuntu-20.04
 
+  sierra-core-ci:
+    uses: ./.github/workflows/sierra-core.yml
+    secrets: inherit
+    with:
+      os: ubuntu-20.04
+
+  ros1robot-ci:
+    uses: ./.github/workflows/ros1robot.yml
+    secrets: inherit
+    with:
+      os: ubuntu-20.04
+
   exec-env-ci:
     uses: ./.github/workflows/exec-env-plugins.yml
     secrets: inherit
@@ -33,6 +42,29 @@ jobs:
     with:
       os: ubuntu-20.04
 
+  # docs:
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       python-version:
+  #         - 3.9
+  #   steps:
+  #     - uses: actions/checkout@v3
+  #     - uses: ./.github/actions/sierra-setup
+
+  #     - name: Build docs
+  #       shell: bash
+  #       run: |
+  #          cd docs && coverage run $(which sphinx-build) -M html "." "_build"
+  #          cd docs && coverage combine
+  #          cd docs && coverage report
+  #          cd docs && coverage xml
+
+  #     - uses: actions/upload-artifact@v3
+  #       with:
+  #         name: ci-sphinx-docs-${{ runner.os }}-${{ matrix.python-version }}-coverage
+  #         path: docs/.coverage.*
+
   coverage:
     runs-on: ubuntu-latest
     strategy:
@@ -41,16 +73,17 @@ jobs:
           - 3.9
 
     needs:
+      # - docs
+      - unit-tests
       - argos-ci
       - ros1gazebo-ci
+      - ros1robot-ci
       - exec-env-ci
-      - unit-tests
+      - sierra-core-ci
 
     steps:
       - uses: actions/checkout@v3
-
       - uses: ./.github/actions/sierra-setup
-
       - uses: actions/download-artifact@v3
 
       - name: Generate coverage report

diff --git a/.github/workflows/exec-env-plugins.yml b/.github/workflows/exec-env-plugins.yml
@@ -38,8 +38,8 @@ jobs:
 
       - name: Adhoc
         run: |
-          ./scripts/argos-integration-tests.sh -f stage2_outputs_test -e hpc.adhoc
-          ./scripts/ros1gazebo-integration-tests.sh -f stage2_test -e hpc.adhoc
+          ./scripts/argos-integration-tests.sh -f stage2_univar_test -e hpc.adhoc
+          ./scripts/ros1gazebo-integration-tests.sh -f stage2_univar_test -e hpc.adhoc
 
       - uses: actions/upload-artifact@v3
         with:
@@ -72,8 +72,8 @@ jobs:
 
       - name: SLURM
         run: |
-          ./scripts/argos-integration-tests.sh -f stage2_outputs_test -e hpc.slurm
-          ./scripts/ros1gazebo-integration-tests.sh -f stage2_test -e hpc.slurm
+          ./scripts/argos-integration-tests.sh -f stage2_univar_test -e hpc.slurm
+          ./scripts/ros1gazebo-integration-tests.sh -f stage2_univar_test -e hpc.slurm
 
       - uses: actions/upload-artifact@v3
         with: