Analyses Snapshot Testing (#14277)

.eslintignore

@@ -31,3 +31,5 @@ hardware-testing/**
 
 # app-testing don't format the json protocols
 app-testing/files
+# app testing don't format the snapshots
+app-testing/tests/__snapshots__

.github/workflows/analyses-snapshot-test.yaml

@@ -0,0 +1,71 @@
+name: Analyses Snapshot Test
+
+on:
+  workflow_dispatch:
+    inputs:
+      TARGET:
+        description: 'Target branch or tag'
+        required: true
+        default: 'edge'
+      TEST_SOURCE:
+        description: 'Target for the test code'
+        required: true
+        default: 'edge'
+  schedule:
+    - cron:  '0 7-8 * * *' # Random time between 2-3 AM EST (7-8 AM UTC)
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+    env:
+      TARGET: ${{ github.event.inputs.TARGET || 'edge' }}
+      TEST_SOURCE: ${{ github.event.inputs.TEST_SOURCE || 'edge' }}
+
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ env.TEST_SOURCE }}
+
+    - name: Docker Build
+      working-directory: app-testing
+      run: make build-opentrons-analysis
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+        cache: 'pipenv'
+        cache-dependency-path: app-testing/Pipfile.lock
+
+    - name: Setup Python Dependencies
+      working-directory: app-testing
+      run: make setup
+
+    - name: Run Test
+      id: run_test
+      working-directory: app-testing
+      run: make snapshot-test
+
+    - name: Upload Report
+      if: '!cancelled()'
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-report
+        path: app-testing/results/
+
+    - name: Handle Test Failure
+      if: failure()
+      working-directory: app-testing
+      run: make snapshot-test-update
+
+    - name: Create Snapshot update Request
+      if: failure()
+      uses: peter-evans/create-pull-request@v5
+      with:
+          commit-message: 'fix(app-testing): snapshot failure capture'
+          title: 'Evaluate Analyses Snapshot Update ${{ env.TARGET }}'
+          body: 'This PR is an automated snapshot update request. Please review the changes and merge if they are acceptable or find you bug and fix it.'
+          branch: 'app-testing/${{ env.TARGET }}-from-${{ env.TEST_SOURCE}}'
+          base: ${{ env.TEST_SOURCE}}
+

.github/workflows/app-testing-lint.yaml

@@ -19,20 +19,18 @@ jobs:
   lint:
     name: 'app-testing lint'
     timeout-minutes: 5
-    runs-on: 'ubuntu-22.04'
+    runs-on: 'ubuntu-latest'
     steps:
       - name: Checkout opentrons repo
-        uses: 'actions/checkout@v3'
+        uses: 'actions/checkout@v4'
 
       - name: Setup Python
-        uses: 'actions/setup-python@v4'
+        uses: 'actions/setup-python@v5'
         with:
-          python-version: '3.11'
+          python-version: '3.12'
           cache: 'pipenv'
           cache-dependency-path: app-testing/Pipfile.lock
-      - name: Install Pipenv
-        run: pip install -U pipenv
-      - name: Pipenv Install
+      - name: Setup
         id: install
         working-directory: ./app-testing
         run: make setup

app-testing/.gitignore

@@ -1,2 +1,3 @@
 .env
 results
+analysis_results/*.json

app-testing/Makefile

@@ -1,22 +1,22 @@
 .PHONY: black
 black:
-	pipenv run python -m black .
+	python -m pipenv run python -m black .
 
 .PHONY: black-check
 black-check:
-	pipenv run python -m black . --check
+	python -m pipenv run python -m black . --check
 
 .PHONY: ruff
 ruff:
-	pipenv run python -m ruff . --fix
+	python -m pipenv run python -m ruff . --fix --unsafe-fixes
 
 .PHONY: ruff-check
 ruff-check:
-	pipenv run python -m ruff .
+	python -m pipenv run python -m ruff .
 
 .PHONY: mypy
 mypy:
-	pipenv run python -m mypy conftest.py automation tests
+	python -m pipenv run python -m mypy conftest.py automation tests citools
 
 .PHONY: lint
 lint:
@@ -32,24 +32,44 @@ format:
 
 .PHONY: test-ci
 test-ci:
-	pipenv run python -m pytest -m "emulated_alpha"
+	python -m pipenv run python -m pytest -m "emulated_alpha"
 
 .PHONY: test-protocol-analysis
 test-protocol-analysis:
 	pipenv run python -m pytest -v tests/protocol_analyze_test.py
 
 .PHONY: setup
-setup:
-	pipenv install
+setup: install-pipenv
+	python -m pipenv install
 
 .PHONY: teardown
 teardown:
-	pipenv --rm
+	python -m pipenv --rm
 
 .PHONY: format-readme
 format-readme:
 	yarn prettier --ignore-path .eslintignore --write app-testing/**/*.md
 
 .PHONY: print-protocols
 print-protocols:
-	pipenv run python print_protocols.py
+	python -m pipenv run python print_protocols.py
+
+.PHONY: install-pipenv
+install-pipenv:
+	python -m pip install -U pipenv
+
+.PHONY: snapshot-test
+snapshot-test:
+	python -m pipenv run pytest -k analyses_snapshot_test -vv
+
+.PHONY: snapshot-test-update
+snapshot-test-update:
+	python -m pipenv run pytest -k analyses_snapshot_test --snapshot-update
+
+TARGET ?= edge
+
+.PHONY: build-opentrons-analysis
+build-opentrons-analysis:
+	@echo "Building docker image for $(TARGET)"
+	@echo "If you want to build a different version, run 'make build-docker TARGET=<version>'"
+	docker build --build-arg OPENTRONS_VERSION=$(TARGET) -t opentrons-analysis:$(TARGET) citools/.

app-testing/Pipfile

@@ -4,21 +4,24 @@ url = "https://pypi.org/simple"
 verify_ssl = true
 
 [packages]
-pytest = "==7.2.0"
-black = "==22.12.0"
-selenium = "==4.7.2"
-importlib-metadata = "==5.0.0"
-requests = "==2.28.1"
-python-dotenv = "==0.21.0"
-pytest-xdist = "==3.0.2"
-mypy = "==0.991"
-types-requests = "==2.28.11.5"
-rich = "==13.0.0"
+pytest = "==7.4.3"
+black = "==23.11.0"
+selenium = "==4.15.2"
+importlib-metadata = "==6.8.0"
+requests = "==2.31.0"
+python-dotenv = "==1.0.0"
+pytest-xdist = "==3.5.0"
+mypy = "==1.7.1"
+types-requests = "==2.31.0.10"
+rich = "==13.7.0"
 atomicwrites = "==1.4.1"
 pyreadline3 = "==3.4.1"
-pydantic = "==1.10.4"
-pygithub = "==1.57"
-ruff = "==0.0.236"
+pydantic = "==2.5.2"
+pygithub = "==2.1.1"
+ruff = "==0.1.6"
+docker = "==6.1.3"
+syrupy = "==4.6.0"
+pytest-html = "==4.1.1"
 
 [requires]
-python_version = "3.11"
+python_version = "3.12"