Put the run into recovery mode when there's a recoverable error. Allo…

…w exiting with `resume-from-recovery` and `stop`.

api/src/opentrons/protocol_engine/state/commands.py

@@ -6,6 +6,7 @@
 from dataclasses import dataclass
 from datetime import datetime
 from typing import Dict, List, Optional, Union
+from typing_extensions import assert_never
 
 from opentrons_shared_data.errors import EnumeratedError, ErrorCodes, PythonException
 
@@ -306,13 +307,25 @@ def handle_action(self, action: Action) -> None:  # noqa: C901
                         command_id=id, failed_at=action.failed_at, error_occurrence=None
                     )
                 self._state.queued_setup_command_ids.clear()
+            elif (
+                prev_entry.command.intent == CommandIntent.PROTOCOL
+                or prev_entry.command.intent is None
+            ):
+                if action.type == ErrorRecoveryType.WAIT_FOR_RECOVERY:
+                    self._state.queue_status = QueueStatus.AWAITING_RECOVERY
+                elif action.type == ErrorRecoveryType.FAIL_RUN:
+                    other_command_ids_to_fail = self._state.queued_command_ids
+                    for id in other_command_ids_to_fail:
+                        self._update_to_failed(
+                            command_id=id,
+                            failed_at=action.failed_at,
+                            error_occurrence=None,
+                        )
+                    self._state.queued_command_ids.clear()
+                else:
+                    assert_never(action.type)
             else:
-                other_command_ids_to_fail = self._state.queued_command_ids
-                for id in other_command_ids_to_fail:
-                    self._update_to_failed(
-                        command_id=id, failed_at=action.failed_at, error_occurrence=None
-                    )
-                self._state.queued_command_ids.clear()
+                assert_never(prev_entry.command.intent)
 
             if self._state.running_command_id == action.command_id:
                 self._state.running_command_id = None
@@ -331,6 +344,9 @@ def handle_action(self, action: Action) -> None:  # noqa: C901
         elif isinstance(action, PauseAction):
             self._state.queue_status = QueueStatus.PAUSED
 
+        elif isinstance(action, ResumeFromRecoveryAction):
+            self._state.queue_status = QueueStatus.RUNNING
+
         elif isinstance(action, StopAction):
             if not self._state.run_result:
                 self._state.queue_status = QueueStatus.PAUSED
@@ -377,6 +393,8 @@ def handle_action(self, action: Action) -> None:  # noqa: C901
             if self._config.block_on_door_open:
                 if action.door_state == DoorState.OPEN:
                     self._state.is_door_blocking = True
+                    # todo(mm, 2024-03-19): It's unclear how the door should interact
+                    # with error recovery (QueueStatus.AWAITING_RECOVERY).
                     if self._state.queue_status != QueueStatus.SETUP:
                         self._state.queue_status = QueueStatus.PAUSED
                 elif action.door_state == DoorState.CLOSED:
@@ -802,6 +820,11 @@ def get_status(self) -> EngineStatus:
             else:
                 return EngineStatus.PAUSED
 
+        elif self._state.queue_status == QueueStatus.AWAITING_RECOVERY:
+            return EngineStatus.AWAITING_RECOVERY
+
+        # todo(mm, 2024-03-19): Does this intentionally return idle if QueueStatus is
+        # SETUP and we're currently a setup command?
         return EngineStatus.IDLE
 
     def get_latest_command_hash(self) -> Optional[str]: