Merge branch 'main' into feat(mm)/import-custom-csv

lib/workload/components/event-workflowdraftrunstatechange-to-workflowrunstatechange-ready/lambdas/fill_placeholders_in_event_payload_data_py/fill_placeholders_in_event_payload_data.py

@@ -243,35 +243,67 @@ def handler(event, context):
 # PRIMARY ANALYSIS TEST
 #########################
 
-if __name__ == '__main__':
-    import json
-
-    print(
-        json.dumps(
-            handler(
-                {
-                    "portal_run_id": "20240530abcd1234",
-                    "workflow_name": "bsshFastqCopy",
-                    "workflow_version": "4.2.4",
-                    "event_data_inputs": {
-                        "instrumentRunId": "240229_7001234_1234_AHJLJLDS",
-                    },
-                    "engine_parameters": {
-                        "outputUri": "icav2://development/primary_data/__instrument_run_id__/__portal_run_id__/",
-                        "logsUri": "",
-                        "cacheUri": ""
-                    }
-                },
-                None
-            ),
-            indent=2
-        )
-    )
-    # {
-    #   "engine_parameters_updated": {
-    #     "outputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/primary_data/240229_7001234_1234_AHJLJLDS/20240530abcd1234/"
-    #   }
-    # }
+## DEV
+# if __name__ == '__main__':
+#     import json
+#
+#     print(
+#         json.dumps(
+#             handler(
+#                 {
+#                     "portal_run_id": "20240530abcd1234",
+#                     "workflow_name": "bsshFastqCopy",
+#                     "workflow_version": "4.2.4",
+#                     "event_data_inputs": {
+#                         "instrumentRunId": "240229_7001234_1234_AHJLJLDS",
+#                     },
+#                     "engine_parameters": {
+#                         "outputUri": "icav2://development/primary_data/__instrument_run_id__/__portal_run_id__/",
+#                         "logsUri": "",
+#                         "cacheUri": ""
+#                     }
+#                 },
+#                 None
+#             ),
+#             indent=2
+#         )
+#     )
+#     # {
+#     #     "engine_parameters_updated": {
+#     #         "outputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/primary_data/240229_7001234_1234_AHJLJLDS/20240530abcd1234/"
+#     #     }
+#     # }
+
+## PROD
+# if __name__ == '__main__':
+#     import json
+#     print(
+#         json.dumps(
+#             handler(
+#                 {
+#                     "portal_run_id": "202409134661e8d7",
+#                     "workflow_version": "2024.05.24",
+#                     "workflow_name": "bsshFastqCopy",
+#                     "engine_parameters": {
+#                         "outputUri": "icav2://eba5c946-1677-441d-bbce-6a11baadecbb/primary/__instrument_run_id__/__portal_run_id__/"
+#                     },
+#                     "event_data_inputs": {
+#                         "bsshAnalysisId": "8569c8b4-83a7-46f9-bae1-c22025e86861",
+#                         "bsshProjectId": "9ec02c1f-53ba-47a5-854d-e6b53101adb7",
+#                         "instrumentRunId": "240424_A01052_0193_BH7JMMDRX5"
+#                     }
+#                 },
+#                 None
+#             ),
+#             indent=2
+#         )
+#     )
+#
+#     {
+#         "engine_parameters_updated": {
+#             "outputUri": "s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/primary/240424_A01052_0193_BH7JMMDRX5/202409134661e8d7/"
+#         }
+#     }
 
 # #########################
 # # SECONDARY ANALYSIS TEST
@@ -290,8 +322,8 @@ def handler(event, context):
 #                         "fastqListRows": ["foo", "bar"],
 #                     },
 #                     "engine_parameters": {
-#                         "outputUri": "icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/analysis_data/__workflow_name__/__workflow_version__/__portal_run_id__/",
-#                         "logsUri": "icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/analysis_logs/__workflow_name__/__workflow_version__/__portal_run_id__/",
+#                         "outputUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/analysis/__workflow_name__/__workflow_version__/__portal_run_id__/",
+#                         "logsUri": "icav2://ea19a3f5-ec7c-4940-a474-c31cd91dbad4/analysis/__workflow_name__/__workflow_version__/__portal_run_id__/",
 #                     }
 #                 },
 #                 None
@@ -302,7 +334,7 @@ def handler(event, context):
 #
 #     # {
 #     #   "engine_parameters_updated": {
-#     #     "outputUri": "icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/analysis_data/cttsov2/2-1-1/20240530abcd1234/",
-#     #     "logsUri": "icav2://7595e8f2-32d3-4c76-a324-c6a85dae87b5/analysis_logs/cttsov2/2-1-1/20240530abcd1234/"
+#     #     "outputUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/2-1-1/20240530abcd1234/",
+#     #     "logsUri": "s3://pipeline-dev-cache-503977275616-ap-southeast-2/byob-icav2/development/analysis/cttsov2/2-1-1/20240530abcd1234/"
 #     #   }
 #     # }

lib/workload/stateless/stacks/bclconvert-manager/deps/requirements.txt

@@ -4,3 +4,4 @@ wrapica==2.27.1.post20240830140737
 xmltodict==0.13.0
 mypy_boto3_ssm>=1.16.0
 mypy_boto3_secretsmanager>=1.16.0
+more-itertools>=10.3.0

lib/workload/stateless/stacks/bclconvert-manager/translator_service/helper/icav2_analysis/analysis_helper.py

@@ -197,7 +197,8 @@ def collect_analysis_objects(project_id: str, analysis_id: str) -> Dict:
     logger.info("Reading in the samplesheet")
     samplesheet_dict = read_v2_samplesheet(
         project_id=project_id,
-        data_id=samplesheet_file_id
+        samplesheet_data_id=samplesheet_file_id,
+        runinfo_data_id=run_info_file_id
     )
 
     return {

lib/workload/stateless/stacks/bclconvert-manager/translator_service/helper/runinfo/__init__.py

@@ -1,5 +1,6 @@
-from .runinfo_helper import get_run_id_from_run_info
+from .runinfo_helper import get_run_id_from_run_info, get_num_lanes_from_run_info
 
 __all__ = [
+    'get_num_lanes_from_run_info',
     'get_run_id_from_run_info'
 ]

lib/workload/stateless/stacks/bclconvert-manager/translator_service/helper/runinfo/runinfo_helper.py

@@ -100,3 +100,12 @@ def get_run_id_from_run_info(project_id: str, data_id: str) -> str:
     """
     return read_runinfo_xml(project_id, data_id)['RunInfo']['Run']['@Id']
 
+
+def get_num_lanes_from_run_info(project_id: str, data_id: str) -> int:
+    """
+    Get the number of lanes in a run info object
+    :param project_id:
+    :param data_id:
+    :return:
+    """
+    return int(read_runinfo_xml(project_id, data_id)['RunInfo']['Run']['FlowcellLayout']['@LaneCount'])

lib/workload/stateless/stacks/bclconvert-manager/translator_service/helper/samplesheet/samplesheet_helper.py

@@ -7,6 +7,7 @@
 from io import StringIO
 from typing import Dict
 
+from more_itertools import flatten
 # UMCCR Libraries
 from v2_samplesheet_maker.functions.v2_samplesheet_reader import v2_samplesheet_reader
 from wrapica.project_data import read_icav2_file_contents_to_string
@@ -19,12 +20,17 @@
 logger.setLevel(logging.INFO)
 
 
-def read_v2_samplesheet(project_id: str, data_id: str) -> Dict:
+def read_v2_samplesheet(
+        project_id: str,
+        samplesheet_data_id: str,
+        runinfo_data_id: str
+) -> Dict:
     """
     Given a v2 samplesheet path, read in the file as a v2 samplesheet (we first convert to json)
 
     :param project_id:
-    :param data_id:
+    :param samplesheet_data_id:
+    :param runinfo_data_id
 
     :return: A dictionary
 
@@ -90,12 +96,43 @@ def read_v2_samplesheet(project_id: str, data_id: str) -> Dict:
         ]
     }
     """
+    from ..runinfo import get_num_lanes_from_run_info
 
-    return v2_samplesheet_reader(
+    v2_samplesheet_dict = v2_samplesheet_reader(
         StringIO(
             read_icav2_file_contents_to_string(
                 project_id=project_id,
-                data_id=data_id
+                data_id=samplesheet_data_id
             )
         )
     )
+
+    # Get bclconvert data from the v2 samplesheet dict
+    # And confirm that the lane column is present
+    if 'lane' in v2_samplesheet_dict['bclconvert_data'][0].keys():
+        # Return the samplesheet as is
+        return v2_samplesheet_dict
+
+    # Otherwise we read the runinfo file
+    num_lanes = get_num_lanes_from_run_info(
+        project_id=project_id,
+        data_id=runinfo_data_id
+    )
+
+    # And now append the lane attribute to every
+    v2_samplesheet_dict['bclconvert_data'] = flatten(
+        map(
+            lambda bclconvert_data_row_iter: list(
+                map(
+                    lambda lane_iter: {
+                        **bclconvert_data_row_iter,
+                        **{"lane": lane_iter + 1}
+                    },
+                    range(num_lanes)
+                )
+            ),
+            v2_samplesheet_dict['bclconvert_data']
+        ),
+    )
+
+    return v2_samplesheet_dict

lib/workload/stateless/stacks/bclconvert-manager/translator_service/icav2_event_translator.py

@@ -499,3 +499,5 @@ def parse_event_code(event_code):
 #     #   "statusCode": 200,
 #     #   "body": "\"Internal event sent to the event bus and both msg stored in the DynamoDB table.\""
 #     # }
+
+

lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/constructs/cttsov2-icav2-manager/index.ts

@@ -43,6 +43,7 @@ interface Cttsov2Icav2PipelineManagerConstructProps {
   setOutputJsonLambdaObj: PythonFunction;
   getVcfsLambdaObj: PythonFunction;
   compressVcfLambdaObj: DockerImageFunction;
+  checkSuccessSampleLambdaObj: PythonFunction;
   // ICAv2 Copy Batch State Machine Object
   icav2CopyFilesStateMachineObj: sfn.IStateMachine;
 }
@@ -119,6 +120,7 @@ export class Cttsov2Icav2PipelineManagerConstruct extends Construct {
       props.setOutputJsonLambdaObj,
       props.getVcfsLambdaObj,
       props.compressVcfLambdaObj,
+      props.checkSuccessSampleLambdaObj,
     ].forEach((lambda_obj) => {
       props.icav2AccessTokenSecretObj.grantRead(<iam.Role>lambda_obj.currentVersion.role);
     });
@@ -139,6 +141,8 @@ export class Cttsov2Icav2PipelineManagerConstruct extends Construct {
           props.getVcfsLambdaObj.currentVersion.functionArn,
         __compress_vcf_file_lambda_function_arn__:
           props.compressVcfLambdaObj.currentVersion.functionArn,
+        __check_successful_analysis_lambda_function_arn__:
+          props.checkSuccessSampleLambdaObj.currentVersion.functionArn,
       },
     });
 
@@ -151,6 +155,7 @@ export class Cttsov2Icav2PipelineManagerConstruct extends Construct {
       props.setOutputJsonLambdaObj,
       props.getVcfsLambdaObj,
       props.compressVcfLambdaObj,
+      props.checkSuccessSampleLambdaObj,
     ].forEach((lambda_obj) => {
       lambda_obj.currentVersion.grantInvoke(<iam.IRole>configure_outputs_sfn.role);
     });

lib/workload/stateless/stacks/cttso-v2-pipeline-manager/deploy/stack.ts

@@ -197,6 +197,24 @@ export class Cttsov2Icav2PipelineManagerStack extends cdk.Stack {
       },
     });
 
+    // Check success lambda
+    const check_success_lambda_function = new PythonFunction(
+      this,
+      'check_success_lambda_function',
+      {
+        entry: path.join(__dirname, '../lambdas/check_success_py'),
+        runtime: lambda.Runtime.PYTHON_3_12,
+        architecture: lambda.Architecture.ARM_64,
+        index: 'check_success.py',
+        handler: 'handler',
+        memorySize: 1024,
+        timeout: Duration.seconds(60),
+        environment: {
+          ICAV2_ACCESS_TOKEN_SECRET_ID: icav2_access_token_secret_obj.secretName,
+        },
+      }
+    );
+
     // Create the state machine to launch the nextflow workflow on ICAv2
     const cttso_v2_launch_state_machine = new Cttsov2Icav2PipelineManagerConstruct(this, id, {
       /* Stack Objects */
@@ -211,6 +229,7 @@ export class Cttsov2Icav2PipelineManagerStack extends cdk.Stack {
       setOutputJsonLambdaObj: set_output_json_lambda_function,
       getVcfsLambdaObj: get_vcfs_lambda_function,
       compressVcfLambdaObj: compress_vcf_lambda_function,
+      checkSuccessSampleLambdaObj: check_success_lambda_function,
       /* Step function templates */
       generateInputJsonSfnTemplatePath: path.join(
         __dirname,