feat(robot_server): send and save csv rtp files for runs (#15857)

re AUTH-591

<!--
Thanks for taking the time to open a Pull Request (PR)! Please make sure
you've read the "Opening Pull Requests" section of our Contributing
Guide:


https://github.com/Opentrons/opentrons/blob/edge/CONTRIBUTING.md#opening-pull-requests

GitHub provides robust markdown to format your PR. Links, diagrams,
pictures, and videos along with text formatting make it possible to
create a rich and informative PR. For more information on GitHub
markdown, see:


https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax

To ensure your code is reviewed quickly and thoroughly, please fill out
the sections below to the best of your ability!
-->

# Overview

<!--
Describe your PR at a high level. State acceptance criteria and how this
PR fits into other work. Link issues, PRs, and other relevant resources.
-->

Send and save the CSV RTPs used in protocol runs to the new table
created in the database, and hook it up to the GET endpoint that fetches
all the data files used in analysis and run.

## Test Plan and Hands on Testing

<!--
Describe your testing of the PR. Emphasize testing not reflected in the
code. Attach protocols, logs, screenshots and any other assets that
support your testing.
-->
- run test files to ensure that csv rtps get insert to the new
run_csv_rtp_table and able to get them from the table

## Changelog

<!--
List changes introduced by this PR considering future developers and the
end user. Give careful thought and clear documentation to breaking
changes.
-->

- Added `run_csv_rtp_table` to schema_6.py file
- Added runTimeParameterFiles field to the run creation request model 
- Passed rtp files input onto the Run Orchestrator’s load process
- Added `insert_csv_rtp()` and `get_all_csv_rtp()` function
- Updated `get_reference_data_files()` to add file referenced in runs
- Updated tests

---------

Co-authored-by: shiyaochen <[email protected]>
Co-authored-by: Sanniti <[email protected]>

robot-server/robot_server/protocols/protocol_store.py

@@ -24,6 +24,7 @@
     analysis_primitive_type_rtp_table,
     analysis_csv_rtp_table,
     data_files_table,
+    run_csv_rtp_table,
     ProtocolKindSQLEnum,
 )
 from robot_server.protocols.protocol_models import ProtocolKind
@@ -310,28 +311,46 @@ def get_usage_info(self) -> List[ProtocolUsageInfo]:
     # TODO (spp, 2024-07-22): get files referenced in runs as well
     async def get_referenced_data_files(self, protocol_id: str) -> List[DataFile]:
         """Get a list of data files referenced in specified protocol's analyses and runs."""
-        # Get analyses of protocol_id
+        # Get analyses and runs of protocol_id
         select_referencing_analysis_ids = sqlalchemy.select(analysis_table.c.id).where(
             analysis_table.c.protocol_id == protocol_id
         )
+        select_referencing_run_ids = sqlalchemy.select(run_table.c.id).where(
+            run_table.c.protocol_id == protocol_id
+        )
         # Get all entries in csv table that match the analyses
-        csv_file_ids = sqlalchemy.select(analysis_csv_rtp_table.c.file_id).where(
+        analysis_csv_file_ids = sqlalchemy.select(
+            analysis_csv_rtp_table.c.file_id
+        ).where(
             analysis_csv_rtp_table.c.analysis_id.in_(select_referencing_analysis_ids)
         )
+        run_csv_file_ids = sqlalchemy.select(run_csv_rtp_table.c.file_id).where(
+            run_csv_rtp_table.c.run_id.in_(select_referencing_run_ids)
+        )
         # Get list of data file IDs from the entries
-        select_data_file_rows_statement = data_files_table.select().where(
-            data_files_table.c.id.in_(csv_file_ids)
+        select_analysis_data_file_rows_statement = data_files_table.select().where(
+            data_files_table.c.id.in_(analysis_csv_file_ids)
+        )
+        select_run_data_file_rows_statement = data_files_table.select().where(
+            data_files_table.c.id.in_(run_csv_file_ids)
         )
         with self._sql_engine.begin() as transaction:
-            data_files_rows = transaction.execute(select_data_file_rows_statement).all()
+            analysis_data_files_rows = transaction.execute(
+                select_analysis_data_file_rows_statement
+            ).all()
+            run_data_files_rows = transaction.execute(
+                select_run_data_file_rows_statement
+            ).all()
+
+        combine_data_file_rows = set(analysis_data_files_rows + run_data_files_rows)
 
         return [
             DataFile(
                 id=sql_row.id,
                 name=sql_row.name,
                 createdAt=sql_row.created_at,
             )
-            for sql_row in data_files_rows
+            for sql_row in combine_data_file_rows
         ]
 
     def get_referencing_run_ids(self, protocol_id: str) -> List[str]:

robot-server/robot_server/runs/router/base_router.py

@@ -176,6 +176,9 @@ async def create_run(
     rtp_values = (
         request_body.data.runTimeParameterValues if request_body is not None else None
     )
+    rtp_files = (
+        request_body.data.runTimeParameterFiles if request_body is not None else None
+    )
     protocol_resource = None
 
     deck_configuration = await deck_configuration_store.get_deck_configuration()
@@ -206,6 +209,7 @@ async def create_run(
             labware_offsets=offsets,
             deck_configuration=deck_configuration,
             run_time_param_values=rtp_values,
+            run_time_param_files=rtp_files,
             protocol=protocol_resource,
             notify_publishers=notify_publishers,
         )

robot-server/robot_server/runs/run_data_manager.py

@@ -12,7 +12,10 @@
     CommandPointer,
     Command,
 )
-from opentrons.protocol_engine.types import PrimitiveRunTimeParamValuesType
+from opentrons.protocol_engine.types import (
+    CSVRunTimeParamFilesType,
+    PrimitiveRunTimeParamValuesType,
+)
 
 from robot_server.protocols.protocol_store import ProtocolResource
 from robot_server.service.task_runner import TaskRunner
@@ -156,6 +159,7 @@ async def create(
         labware_offsets: List[LabwareOffsetCreate],
         deck_configuration: DeckConfigurationType,
         run_time_param_values: Optional[PrimitiveRunTimeParamValuesType],
+        run_time_param_files: Optional[CSVRunTimeParamFilesType],
         notify_publishers: Callable[[], None],
         protocol: Optional[ProtocolResource],
     ) -> Union[Run, BadRun]:
@@ -168,6 +172,7 @@ async def create(
             deck_configuration: A mapping of fixtures to cutout fixtures the deck will be loaded with.
             notify_publishers: Utilized by the engine to notify publishers of state changes.
             run_time_param_values: Any runtime parameter values to set.
+            run_time_param_files: Any runtime parameter values to set.
             protocol: The protocol to load the runner with, if any.
 
         Returns:
@@ -192,6 +197,7 @@ async def create(
             deck_configuration=deck_configuration,
             protocol=protocol,
             run_time_param_values=run_time_param_values,
+            run_time_param_files=run_time_param_files,
             notify_publishers=notify_publishers,
         )
         run_resource = self._run_store.insert(
@@ -210,7 +216,7 @@ async def create(
             run_resource=run_resource,
             state_summary=state_summary,
             current=True,
-            run_time_parameters=[],
+            run_time_parameters=self._run_orchestrator_store.get_run_time_parameters(),
         )
 
     def get(self, run_id: str) -> Union[Run, BadRun]:

robot-server/robot_server/runs/run_models.py

@@ -19,6 +19,7 @@
     CommandNote,
 )
 from opentrons.protocol_engine.types import (
+    CSVRunTimeParamFilesType,
     RunTimeParameter,
     PrimitiveRunTimeParamValuesType,
 )
@@ -252,6 +253,10 @@ class RunCreate(BaseModel):
         None,
         description="Key-value pairs of run-time parameters defined in a protocol.",
     )
+    runTimeParameterFiles: Optional[CSVRunTimeParamFilesType] = Field(
+        None,
+        description="Key-fileId pairs of CSV run-time parameters defined in a protocol.",
+    )
 
 
 class RunUpdate(BaseModel):

robot-server/robot_server/runs/run_orchestrator_store.py

@@ -4,7 +4,11 @@
 from typing import List, Optional, Callable
 
 from opentrons.protocol_engine.errors.exceptions import EStopActivatedError
-from opentrons.protocol_engine.types import PostRunHardwareState, RunTimeParameter
+from opentrons.protocol_engine.types import (
+    CSVRunTimeParamFilesType,
+    PostRunHardwareState,
+    RunTimeParameter,
+)
 
 from opentrons_shared_data.labware.labware_definition import LabwareDefinition
 from opentrons_shared_data.robot.types import RobotType
@@ -188,6 +192,7 @@ async def create(
         notify_publishers: Callable[[], None],
         protocol: Optional[ProtocolResource],
         run_time_param_values: Optional[PrimitiveRunTimeParamValuesType] = None,
+        run_time_param_files: Optional[CSVRunTimeParamFilesType] = None,
     ) -> StateSummary:
         """Create and store a ProtocolRunner and ProtocolEngine for a given Run.
 
@@ -198,6 +203,7 @@ async def create(
             notify_publishers: Utilized by the engine to notify publishers of state changes.
             protocol: The protocol to load the runner with, if any.
             run_time_param_values: Any runtime parameter values to set.
+            run_time_param_files: Any runtime parameter files to set.
 
         Returns:
             The initial equipment and status summary of the engine.
@@ -243,8 +249,7 @@ async def create(
             await self.run_orchestrator.load(
                 protocol.source,
                 run_time_param_values=run_time_param_values,
-                # TODO (spp, 2024-07-16): update this once runs accept csv params
-                run_time_param_files={},
+                run_time_param_files=run_time_param_files,
                 parse_mode=ParseMode.ALLOW_LEGACY_METADATA_AND_REQUIREMENTS,
             )
         else:

robot-server/robot_server/runs/run_store.py

@@ -25,6 +25,7 @@
     run_table,
     run_command_table,
     action_table,
+    run_csv_rtp_table,
 )
 from robot_server.persistence.pydantic import (
     json_to_pydantic,
@@ -85,6 +86,15 @@ class BadStateSummary:
     dataError: EnumeratedError
 
 
+@dataclass
+class CSVParameterRunResource:
+    """A CSV runtime parameter from a completed run, storable in a SQL database."""
+
+    run_id: str
+    parameter_variable_name: str
+    file_id: Optional[str]
+
+
 class CommandNotFoundError(ValueError):
     """Error raised when a given command ID is not found in the store."""
 
@@ -198,6 +208,39 @@ def insert_action(self, run_id: str, action: RunAction) -> None:
 
         self._clear_caches()
 
+    def get_all_csv_rtp(self) -> List[CSVParameterRunResource]:
+        """Get all of the csv rtp from the run_csv_rtp_table."""
+        select_all_csv_rtp = sqlalchemy.select(run_csv_rtp_table).order_by(
+            sqlite_rowid.asc()
+        )
+
+        with self._sql_engine.begin() as transaction:
+            csv_rtps = transaction.execute(select_all_csv_rtp).all()
+
+        return [_covert_row_to_csv_rtp(row) for row in csv_rtps]
+
+    def insert_csv_rtp(
+        self, run_id: str, run_time_parameters: List[RunTimeParameter]
+    ) -> None:
+        """Save csv rtp to the run_csv_rtp_table."""
+        insert_csv_rtp = sqlalchemy.insert(run_csv_rtp_table)
+
+        with self._sql_engine.begin() as transaction:
+            if not self._run_exists(run_id, transaction):
+                raise RunNotFoundError(run_id=run_id)
+            for run_time_param in run_time_parameters:
+                if run_time_param.type == "csv_file":
+                    transaction.execute(
+                        insert_csv_rtp,
+                        {
+                            "run_id": run_id,
+                            "parameter_variable_name": run_time_param.variableName,
+                            "file_id": run_time_param.file.id
+                            if run_time_param.file
+                            else None,
+                        },
+                    )
+
     def insert(
         self,
         run_id: str,
@@ -531,6 +574,22 @@ def _clear_caches(self) -> None:
 _run_columns = [run_table.c.id, run_table.c.protocol_id, run_table.c.created_at]
 
 
+def _covert_row_to_csv_rtp(
+    row: sqlalchemy.engine.Row,
+) -> CSVParameterRunResource:
+    run_id = row.run_id
+    parameter_variable_name = row.parameter_variable_name
+    file_id = row.file_id
+
+    assert isinstance(run_id, str)
+    assert isinstance(parameter_variable_name, str)
+    assert isinstance(file_id, str) or file_id is None
+
+    return CSVParameterRunResource(
+        run_id=run_id, parameter_variable_name=parameter_variable_name, file_id=file_id
+    )
+
+
 def _convert_row_to_run(
     row: sqlalchemy.engine.Row,
     action_rows: List[sqlalchemy.engine.Row],