wip

api/src/opentrons/protocol_engine/actions/actions.py

@@ -7,6 +7,7 @@
 from datetime import datetime
 from enum import Enum
 from typing import Optional, Union
+from opentrons.protocol_engine.error_recovery_policy import ErrorRecoveryType
 
 from opentrons.protocols.models import LabwareDefinition
 from opentrons.hardware_control.types import DoorState
@@ -122,18 +123,13 @@ class UpdateCommandAction:
 
 @dataclass(frozen=True)
 class FailCommandAction:
-    """Mark a given command as failed.
+    """Mark a given command as failed."""
 
-    The given command and all currently queued commands will be marked
-    as failed due to the given error.
-    """
-
-    # TODO(mc, 2021-11-12): we'll likely need to add the command params
-    # to this payload for state reaction purposes
     command_id: str
     error_id: str
     failed_at: datetime
     error: EnumeratedError
+    type: ErrorRecoveryType
 
 
 @dataclass(frozen=True)

api/src/opentrons/protocol_engine/error_recovery_policy.py

@@ -0,0 +1,47 @@
+import enum
+from typing import Protocol
+
+from opentrons_shared_data.errors import EnumeratedError, ErrorCodes
+
+from opentrons.config import feature_flags as ff
+
+
+class ErrorRecoveryType(enum.Enum):
+    """Ways to handle a command failure."""
+
+    FAIL_RUN = enum.auto()
+    """Permanently fail the entire run."""
+
+    WAIT_FOR_RECOVERY = enum.auto()
+    """Stop and wait for the error to be recovered from manually."""
+
+    CONTINUE = enum.auto()
+    """Continue with the run, as if the command never failed."""
+
+
+class ErrorRecoveryPolicy(Protocol):
+    @staticmethod
+    def __call__(exception: Exception) -> ErrorRecoveryType:
+        ...
+
+
+def error_recovery_by_ff(exception: Exception) -> ErrorRecoveryType:
+    """Use API feature flags to decide how to handle an error.
+
+    This is just for development. This should be replaced by a proper config
+    system exposed through robot-server's HTTP API.
+    """
+    if _is_recoverable(exception) and ff.enable_error_recovery_experiments():
+        return ErrorRecoveryType.WAIT_FOR_RECOVERY
+    else:
+        return ErrorRecoveryType.FAIL_RUN
+
+
+def _is_recoverable(exception: Exception) -> bool:
+    if isinstance(exception, EnumeratedError):
+        return exception.code in {
+            ErrorCodes.TIP_PICKUP_FAILED,
+            ErrorCodes.PIPETTE_OVERPRESSURE,
+        }
+    else:
+        return False

api/src/opentrons/protocol_engine/execution/command_executor.py

@@ -11,6 +11,8 @@
     PythonException,
 )
 
+from opentrons.protocol_engine.error_recovery_policy import ErrorRecoveryPolicy
+
 from ..state import StateStore
 from ..resources import ModelUtils
 from ..commands import (
@@ -57,6 +59,7 @@ def __init__(
         run_control: RunControlHandler,
         rail_lights: RailLightsHandler,
         status_bar: StatusBarHandler,
+        error_recovery_policy: ErrorRecoveryPolicy,
         model_utils: Optional[ModelUtils] = None,
     ) -> None:
         """Initialize the CommandExecutor with access to its dependencies."""
@@ -71,8 +74,9 @@ def __init__(
         self._tip_handler = tip_handler
         self._run_control = run_control
         self._rail_lights = rail_lights
-        self._model_utils = model_utils or ModelUtils()
         self._status_bar = status_bar
+        self._error_recovery_policy = error_recovery_policy
+        self._model_utils = model_utils or ModelUtils()
 
     async def execute(self, command_id: str) -> None:
         """Run a given command's execution procedure.
@@ -135,6 +139,7 @@ async def execute(self, command_id: str) -> None:
                     command_id=command_id,
                     error_id=self._model_utils.generate_id(),
                     failed_at=self._model_utils.get_timestamp(),
+                    type=self._error_recovery_policy(error),
                 )
             )
         else:

api/src/opentrons/protocol_engine/execution/create_queue_worker.py

@@ -1,5 +1,6 @@
 """QueueWorker and dependency factory."""
 from opentrons.hardware_control import HardwareControlAPI
+from opentrons.protocol_engine.error_recovery_policy import ErrorRecoveryPolicy
 from opentrons.protocol_engine.execution.rail_lights import RailLightsHandler
 
 from ..state import StateStore
@@ -20,13 +21,15 @@ def create_queue_worker(
     hardware_api: HardwareControlAPI,
     state_store: StateStore,
     action_dispatcher: ActionDispatcher,
+    error_recovery_policy: ErrorRecoveryPolicy,
 ) -> QueueWorker:
     """Create a ready-to-use QueueWorker instance.
 
     Arguments:
         hardware_api: Hardware control API to pass down to dependencies.
         state_store: StateStore to pass down to dependencies.
         action_dispatcher: ActionDispatcher to pass down to dependencies.
+        error_recovery_policy: ErrorRecoveryPolicy to pass down to dependencies.
     """
     gantry_mover = create_gantry_mover(
         hardware_api=hardware_api,
@@ -85,6 +88,7 @@ def create_queue_worker(
         run_control=run_control_handler,
         rail_lights=rail_lights_handler,
         status_bar=status_bar_handler,
+        error_recovery_policy=error_recovery_policy,
     )
 
     return QueueWorker(

api/src/opentrons/protocol_engine/protocol_engine.py

@@ -2,6 +2,10 @@
 from contextlib import AsyncExitStack
 from logging import getLogger
 from typing import Dict, Optional, Union
+from opentrons.protocol_engine.error_recovery_policy import (
+    ErrorRecoveryPolicy,
+    error_recovery_by_ff,
+)
 
 from opentrons.protocols.models import LabwareDefinition
 from opentrons.hardware_control import HardwareControlAPI
@@ -89,6 +93,7 @@ def __init__(
         hardware_stopper: Optional[HardwareStopper] = None,
         door_watcher: Optional[DoorWatcher] = None,
         module_data_provider: Optional[ModuleDataProvider] = None,
+        error_recovery_policy: ErrorRecoveryPolicy = error_recovery_by_ff,
     ) -> None:
         """Initialize a ProtocolEngine instance.
 
@@ -112,6 +117,7 @@ def __init__(
             hardware_api=hardware_api,
             state_store=self._state_store,
             action_dispatcher=self._action_dispatcher,
+            error_recovery_policy=error_recovery_policy,
         )
         self._hardware_stopper = hardware_stopper or HardwareStopper(
             hardware_api=hardware_api,