test: 7.2.0 Analysis Snapshot Protocols

app-testing/README.md

@@ -82,5 +82,59 @@ This allows us to run one or many.
 
 ### Analyses Snapshot Test
 
-- run the Workflow Dispatch job
-  - `gh workflow run 'Analyses Snapshot Test' --ref 7.1-analyses-battery -f TARGET=v7.1.0-alpha.10 -f TEST_SOURCE=7.1-analyses-battery`
+The analyses snapshot test runs protocol analysis using `TARGET` branch or tag then compares them against snapshots found on `TEST_SOURCE` branch or tag.
+
+#### Protocol Files Location
+
+The set of protocols to analyze is defined inside of `app-testing/.env` file, under the `APP_ANALYSIS_TEST_PROTOCOLS` variable. These protocols must exist inside of `app-testing/files/protocols` folder.
+
+- Protocol Designer protocols go in the `json` folder
+- Python protocols go in the `python` folder.
+
+#### Analysis Snapshots Location
+
+Analysis snapshots are located inside of `app-testing/tests/__snapshots__/analyses_snapshot_test` folder.
+
+#### Running Analysis Snapshot Tests Locally
+
+To run analysis snapshot tests locally, you must first build the Docker image by running the following command:
+
+```bash
+TARGET="<target-branch-or-tag>" make build-opentrons-analysis
+```
+
+Then to run the analysis snapshot test, you can run the following command:
+
+```bash
+TARGET="<target-branch-or-tag>" make snapshot-test
+```
+
+This will run the analyses snapshot test using the `TARGET` branch or tag, and compare the results against your local analysis snapshots located inside `app-testing/tests/__snapshots__/analyses_snapshot_test`.
+
+#### Updating Analysis Snapshots
+
+If you want to update the analysis snapshots, you can run the following command:
+
+```bash
+TARGET="<target-branch-or-tag>" make snapshot-test-update
+```
+
+This will take the results of the analysis snapshot test using the `TARGET` branch or tag, and update the local analysis snapshots located inside `app-testing/tests/__snapshots__/analyses_snapshot_test`.
+
+#### Running Analysis Snapshot Tests on CI
+
+To run analysis snapshot tests on CI, you need to run the `Analyses Snapshot Test` workflow dispatch job. This job requires two inputs, `TARGET` and `TEST_SOURCE`.
+
+Given the scenario that you want to see if the latest version of `chore_release-v7.2.0` release branch has any differences compared to the current analysis snapshots.
+
+`TARGET` - is chore_release-v7.2.0. "I want to run analysis against `chore_release-v7.2.0`"
+
+`TEST_SOURCE` - This one varies a bit on what it can be. The question to ask is, "Where are the snapshots that you want to compare against?"
+
+- If you want to compare against the current analysis snapshots for this release, then TEST_SOURCE is chore_release-v7.2.0.
+- If you want to compare against the previous release branch, then TEST_SOURCE is chore_release-v7.1.0.
+- If you want to compare your in-progress release branch against the previous release branch, then TEST_SOURCE is `<your-branch-name>`.
+
+run the Workflow Dispatch job
+
+- `gh workflow run 'Analyses Snapshot Test' --ref chore_release-v7.2.0 -f TARGET=chore_release-v7.2.0 -f TEST_SOURCE=chore_release-v7.1.0`

app-testing/automation/data/protocol_files.py

@@ -2,61 +2,87 @@
 from typing import Literal
 
 names = Literal[
-    "OT2_None_None_2_12_Python310SyntaxRobotAnalysisOnlyError",
-    "OT2_None_None_2_13_PythonSyntaxError",
-    "OT2_P1000SLeft_None_6_1_SimpleTransfer",
-    "OT2_P10S_P300M_TC1_TM_MM_2_11_Swift",
-    "OT2_P20S_None_2_7_Walkthrough",
-    "OT2_P20S_P300M_HS_6_1_HS_WithCollision_Error",
-    "OT2_P20S_P300M_NoMods_6_1_TransferReTransferLiquid",
-    "OT2_P20SRight_None_6_1_SimpleTransferError",
-    "OT2_P300M_P20S_2_16_aspirateDispenseMix0Volume",
-    "OT2_P300M_P20S_HS_6_1_Smoke620release",
-    "OT2_P300M_P20S_MM_HS_TD_TC_6_1_AllMods_Error",
-    "OT2_P300M_P20S_MM_TM_TC1_5_2_6_PD40",
-    "OT2_P300M_P20S_MM_TM_TC1_5_2_6_PD40Error",
-    "OT2_P300M_P20S_NoMod_6_1_MixTransferManyLiquids",
-    "OT2_P300M_P20S_None_2_12_FailOnRun",
-    "OT2_P300M_P20S_TC_HS_TM_2_13_SmokeTestV3",
-    "OT2_P300M_P20S_TC_HS_TM_2_14_SmokeTestV3",
-    "OT2_P300M_P20S_TC_HS_TM_2_15_SmokeTestV3",
-    "OT2_P300M_P20S_TC_HS_TM_2_16_SmokeTestV3",
-    "OT2_P300M_P20S_TC_MM_TM_2_13_Smoke620Release",
-    "OT2_P300M_P300S_HS_6_1_HS_NormalUseWithTransfer",
-    "OT2_P300MLeft_MM_TM_2_4_Zymo",
-    "OT2_P300S_Thermocycler_Moam_Error",
-    "OT2_P300S_Twinning_Error",
-    "OT2_P300SG1_None_5_2_6_Gen1PipetteSimple",
-    "OT2_P300SLeft_MM_TM_TM_5_2_6_MOAMTemps",
-    "OT2_P300SLeft_MM1_MM_2_2_EngageMagHeightFromBase",
     "OT2_P300SLeft_MM1_MM_TM_2_3_Mix",
+    "OT2_P300SLeft_MM1_MM_2_2_EngageMagHeightFromBase",
+    "OT2_P300SLeft_MM_TM_TM_5_2_6_MOAMTemps",
+    "OT2_P300SG1_None_5_2_6_Gen1PipetteSimple",
+    "OT2_P300S_Twinning_Error",
+    "OT2_P300S_Thermocycler_Moam_Error",
+    "OT2_P300S_None_2_16_verifyNoFloatingPointErrorInPipetting",
+    "OT2_P300MLeft_MM_TM_2_4_Zymo",
+    "OT2_P300M_P300S_HS_6_1_HS_NormalUseWithTransfer",
+    "OT2_P300M_P20S_TC_MM_TM_2_13_Smoke620Release",
+    "OT2_P300M_P20S_TC_HS_TM_2_17_SmokeTestV3",
+    "OT2_P300M_P20S_TC_HS_TM_2_17_dispense_changes",
+    "OT2_P300M_P20S_TC_HS_TM_2_16_SmokeTestV3",
+    "OT2_P300M_P20S_TC_HS_TM_2_16_dispense_changes",
     "OT2_P300M_P20S_TC_HS_TM_2_16_aspirateDispenseMix0Volume",
-    "Flex_None_None_2_16_AnalysisError_AccessToFixedTrashProp",
-    "Flex_None_None_2_16_AnalysisError_TrashBinInCol2",
-    "Flex_None_None_2_16_AnalysisError_TrashBinInStagingAreaCol3",
-    "Flex_None_None_2_16_AnalysisError_TrashBinInStagingAreaCol4",
-    "Flex_None_None_MM_2_16_AnalysisError_MagneticModuleInFlexProtocol",
-    "Flex_None_None_TM_2_16_AnalysisError_ModuleInStagingAreaCol3",
+    "OT2_P300M_P20S_TC_HS_TM_2_15_SmokeTestV3",
+    "OT2_P300M_P20S_TC_HS_TM_2_15_dispense_changes",
+    "OT2_P300M_P20S_TC_HS_TM_2_14_SmokeTestV3",
+    "OT2_P300M_P20S_TC_HS_TM_2_13_SmokeTestV3",
+    "OT2_P300M_P20S_None_2_12_FailOnRun",
+    "OT2_P300M_P20S_NoMod_6_1_MixTransferManyLiquids",
+    "OT2_P300M_P20S_MM_TM_TC1_5_2_6_PD40Error",
+    "OT2_P300M_P20S_MM_TM_TC1_5_2_6_PD40",
+    "OT2_P300M_P20S_MM_HS_TD_TC_6_1_AllMods_Error",
+    "OT2_P300M_P20S_HS_6_1_Smoke620release",
+    "OT2_P300M_P20S_2_16_aspirateDispenseMix0Volume",
+    "OT2_P20SRight_None_6_1_SimpleTransferError",
+    "OT2_P20S_P300M_NoMods_6_1_TransferReTransferLiquid",
+    "OT2_P20S_P300M_HS_6_1_HS_WithCollision_Error",
+    "OT2_P20S_None_2_7_Walkthrough",
+    "OT2_P10S_P300M_TC1_TM_MM_2_11_Swift",
+    "OT2_P1000SLeft_None_6_1_SimpleTransfer",
+    "OT2_None_None_TC_2_17_VerifyThermocyclerLoadedSlots",
+    "OT2_None_None_TC_2_16_VerifyThermocyclerLoadedSlots",
+    "OT2_None_None_TC_2_15_VerifyThermocyclerLoadedSlots",
+    "OT2_None_None_TC_2_14_VerifyThermocyclerLoadedSlots",
+    "OT2_None_None_HS_2_16_AnalysisError_HeaterShakerConflictWithTrashBin2",
+    "OT2_None_None_HS_2_16_AnalysisError_HeaterShakerConflictWithTrashBin1",
+    "OT2_None_None_2_16_verifyDoesNotDeadlock",
+    "OT2_None_None_2_13_PythonSyntaxError",
+    "OT2_None_None_2_12_Python310SyntaxRobotAnalysisOnlyError",
+    "Flex_P50MLeft_P1000MRight_None_2_15_ABRKAPALibraryQuantLongv2",
+    "Flex_P300Gen2_None_2_16_AnalysisError_OT2PipetteInFlexProtocol",
+    "Flex_P1000SRight_None_2_15_ABR_Simple_Normalize_Long_Right",
+    "Flex_P1000MLeft_P50MRight_HS_TM_MM_TC_2_15_ABR4_Illumina_DNA_Prep_24x",
+    "Flex_P1000MLeft_P50MRight_HS_MM_TC_TM_2_15_ABR3_Illumina_DNA_Enrichment",
+    "Flex_P1000MLeft_P50MRight_HS_MM_TC_TM_2_15_ABR3_Illumina_DNA_Enrichment_v4",
+    "Flex_P1000_96_TM_2_16_AnalysisError_ModuleAndWasteChuteConflict",
+    "Flex_P1000_96_TC_2_16_PartialTipPickupSingle",
+    "Flex_P1000_96_TC_2_16_PartialTipPickupColumn",
+    "Flex_P1000_96_TC_2_16_AnalysisError_PartialTipPickupTryToReturnTip",
+    "Flex_P1000_96_TC_2_16_AnalysisError_PartialTipPickupThermocyclerLidConflict",
+    "Flex_P1000_96_None_TC_2_16_AnalysisError_pipetteCollisionWithThermocyclerLidClips",
+    "Flex_P1000_96_None_TC_2_16_AnalysisError_pipetteCollisionWithThermocyclerLid",
+    "Flex_P1000_96_None_2_16_AnalysisError_TrashBinInStagingAreaCol3",
+    "Flex_P1000_96_None_2_15_ABR5_6_IDT_xGen_EZ_96x_Head_PART_I_III_ABR",
+    "Flex_P1000_96_HS_TM_TC_MM_2_15_ABR5_6_Illumina_DNA_Prep_96x_Head_PART_III",
+    "Flex_P1000_96_HS_TM_MM_2_15_MagMaxRNACells96Ch",
+    "Flex_P1000_96_HS_TM_MM_2_15_ABR5_6_HDQ_Bacteria_ParkTips_96_channel",
+    "Flex_P1000_96_Gripper_TC_TM_HS_prepareForMountMovement",
+    "Flex_P1000_96_Gripper_TC_TM_HS_AnalysisError_GripperCollisionWithTips",
+    "Flex_P1000_96_GRIPPER_HS_TM_TC_MB_2_16_Smoke",
+    "Flex_P1000_96_GRIPPER_HS_TM_TC_MB_2_16_DeckConfiguration1",
+    "Flex_P1000_96_GRIPPER_HS_TM_TC_MB_2_16_DeckConfiguration1_NoModulesNoFixtures",
+    "Flex_P1000_96_GRIPPER_HS_TM_TC_MB_2_16_DeckConfiguration1_NoModules",
+    "Flex_P1000_96_GRIPPER_HS_TM_TC_MB_2_16_DeckConfiguration1_NoFixtures",
+    "Flex_P1000_96_GRIPPER_2_16_AnalysisError_DropLabwareIntoTrashBin",
+    "Flex_P1000_96_2_16_AnalysisError_DropTipsWithNoTrash",
+    "Flex_P100_96_HS_TM_2_15_Quick_Zymo_RNA_Bacteria",
     "Flex_None_None_TM_2_16_AnalysisError_ModuleInStagingAreaCol4",
+    "Flex_None_None_TM_2_16_AnalysisError_ModuleInStagingAreaCol3",
     "Flex_None_None_TM_2_16_AnalysisError_ModuleInCol2",
-    "Flex_P100_96_HS_TM_2_15_Quick_Zymo_RNA_Bacteria",
-    "Flex_P1000_96_2_16_AnalysisError_DropTipsWithNoTrash",
-    "Flex_P1000_96_GRIPPER_2_16_AnalysisError_DropLabwareIntoTrashBin",
-    "Flex_P1000_96_GRIPPER_HS_TM_TC_MB_2_16_DeckConfiguration1_NoFixtures",
-    "Flex_P1000_96_GRIPPER_HS_TM_TC_MB_2_16_DeckConfiguration1_NoModules",
-    "Flex_P1000_96_GRIPPER_HS_TM_TC_MB_2_16_DeckConfiguration1_NoModulesNoFixtures",
-    "Flex_P1000_96_GRIPPER_HS_TM_TC_MB_2_16_DeckConfiguration1",
-    "Flex_P1000_96_GRIPPER_HS_TM_TC_MB_2_16_Smoke",
-    "Flex_P1000_96_HS_TM_MM_2_15_ABR5_6_HDQ_Bacteria_ParkTips_96_channel",
-    "Flex_P1000_96_HS_TM_MM_2_15_MagMaxRNACells96Ch",
-    "Flex_P1000_96_HS_TM_TC_MM_2_15_ABR5_6_Illumina_DNA_Prep_96x_Head_PART_III",
-    "Flex_P1000_96_None_2_15_ABR5_6_IDT_xGen_EZ_96x_Head_PART_I_III_ABR",
-    "Flex_P1000_96_None_2_16_AnalysisError_TrashBinInStagingAreaCol3",
-    "Flex_P1000_96_TM_2_16_AnalysisError_ModuleAndWasteChuteConflict",
-    "Flex_P1000MLeft_P50MRight_HS_MM_TC_TM_2_15_ABR3_Illumina_DNA_Enrichment_v4",
-    "Flex_P1000MLeft_P50MRight_HS_MM_TC_TM_2_15_ABR3_Illumina_DNA_Enrichment",
-    "Flex_P1000MLeft_P50MRight_HS_TM_MM_TC_2_15_ABR4_Illumina_DNA_Prep_24x",
-    "Flex_P1000SRight_None_2_15_ABR_Simple_Normalize_Long_Right",
-    "Flex_P300Gen2_None_2_16_AnalysisError_OT2PipetteInFlexProtocol",
-    "Flex_P50MLeft_P1000MRight_None_2_15_ABRKAPALibraryQuantLongv2",
+    "Flex_None_None_TC_2_17_verifyThermocyclerLoadedSlots",
+    "Flex_None_None_TC_2_16_verifyThermocyclerLoadedSlots",
+    "Flex_None_None_TC_2_16_AnalysisError_TrashBinAndThermocyclerConflict",
+    "Flex_None_None_TC_2_15_verifyThermocyclerLoadedSlots",
+    "Flex_None_None_TC_2_14_verifyThermocyclerLoadedSlots",
+    "Flex_None_None_MM_2_16_AnalysisError_MagneticModuleInFlexProtocol",
+    "Flex_None_None_2_16_AnalysisError_TrashBinInStagingAreaCol4",
+    "Flex_None_None_2_16_AnalysisError_TrashBinInStagingAreaCol3",
+    "Flex_None_None_2_16_AnalysisError_TrashBinInCol2",
+    "Flex_None_None_2_16_AnalysisError_AccessToFixedTrashProp",
+    "Flex_P1000_96_Gripper_2_16_TriggerPrepareForMountMovement",
 ]