Skip to content

Commit

Permalink
Added ora compression and ora decompression services
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexiswl
alexiswl committed Nov 1, 2024
1 parent 1f88e76 commit 89b6b41
Show file tree
Hide file tree
Showing 42 changed files with 2,401 additions and 72 deletions.
1 change: 1 addition & 0 deletions .gitattributes
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/.yarn/releases/** binary
/.yarn/plugins/** binary
*.json.gz filter=lfs diff=lfs merge=lfs -text
*.tar.gz filter=lfs diff=lfs merge=lfs -text
240 changes: 240 additions & 0 deletions cdk.context.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,86 @@
}
]
},
"vpc-provider:account=843407916570:filter.tag:Name=main-vpc:region=ap-southeast-2:returnAsymmetricSubnets=true": {
"vpcId": "vpc-00eafc63c0dfca266",
"vpcCidrBlock": "10.2.0.0/16",
"ownerAccountId": "843407916570",
"availabilityZones": [],
"subnetGroups": [
{
"name": "database",
"type": "Isolated",
"subnets": [
{
"subnetId": "subnet-08edbca7f0eda37f6",
"cidr": "10.2.40.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-04f3e62b4f5c4f071"
},
{
"subnetId": "subnet-00e0de28430f6e3cc",
"cidr": "10.2.42.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-04f3e62b4f5c4f071"
},
{
"subnetId": "subnet-01009a3a383f4bff1",
"cidr": "10.2.44.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-04f3e62b4f5c4f071"
}
]
},
{
"name": "public",
"type": "Public",
"subnets": [
{
"subnetId": "subnet-0fab038b0341872f1",
"cidr": "10.2.0.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-0645235a1d1a53601"
},
{
"subnetId": "subnet-0e84dd3a07fb770f5",
"cidr": "10.2.2.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-0645235a1d1a53601"
},
{
"subnetId": "subnet-093aee876a555f218",
"cidr": "10.2.4.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-0645235a1d1a53601"
}
]
},
{
"name": "private",
"type": "Private",
"subnets": [
{
"subnetId": "subnet-050e6fb0f6028178b",
"cidr": "10.2.20.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-06aa800ce565746f4"
},
{
"subnetId": "subnet-07b944de33d98047e",
"cidr": "10.2.22.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-06aa800ce565746f4"
},
{
"subnetId": "subnet-0207da0a8acfcb732",
"cidr": "10.2.24.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-06aa800ce565746f4"
}
]
}
]
},
"vpc-provider:account=455634345446:filter.tag:Name=main-vpc:filter.tag:Stack=networking:region=ap-southeast-2:returnAsymmetricSubnets=true": {
"vpcId": "vpc-0e226f3aa0f9c5781",
"vpcCidrBlock": "10.2.0.0/16",
Expand Down Expand Up @@ -159,6 +239,86 @@
}
]
},
"vpc-provider:account=455634345446:filter.tag:Name=main-vpc:region=ap-southeast-2:returnAsymmetricSubnets=true": {
"vpcId": "vpc-0e226f3aa0f9c5781",
"vpcCidrBlock": "10.2.0.0/16",
"ownerAccountId": "455634345446",
"availabilityZones": [],
"subnetGroups": [
{
"name": "database",
"type": "Isolated",
"subnets": [
{
"subnetId": "subnet-016b98cc640954651",
"cidr": "10.2.40.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-014f5bf2c2d4af5e7"
},
{
"subnetId": "subnet-0c2c217219ed56bfc",
"cidr": "10.2.42.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-014f5bf2c2d4af5e7"
},
{
"subnetId": "subnet-03e6fba027a1e0427",
"cidr": "10.2.44.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-014f5bf2c2d4af5e7"
}
]
},
{
"name": "private",
"type": "Private",
"subnets": [
{
"subnetId": "subnet-01308be8bb704e5ef",
"cidr": "10.2.20.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-0ad5a1fddb41ed1cf"
},
{
"subnetId": "subnet-0ab125fef23f8feed",
"cidr": "10.2.22.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-0ad5a1fddb41ed1cf"
},
{
"subnetId": "subnet-02290481f7651e39b",
"cidr": "10.2.24.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-0ad5a1fddb41ed1cf"
}
]
},
{
"name": "public",
"type": "Public",
"subnets": [
{
"subnetId": "subnet-079ba5a17c7102446",
"cidr": "10.2.0.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-08cf65d720bf1e5f3"
},
{
"subnetId": "subnet-00df4e6910885d2f6",
"cidr": "10.2.2.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-08cf65d720bf1e5f3"
},
{
"subnetId": "subnet-027a37df34939ab32",
"cidr": "10.2.4.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-08cf65d720bf1e5f3"
}
]
}
]
},
"vpc-provider:account=472057503814:filter.tag:Name=main-vpc:filter.tag:Stack=networking:region=ap-southeast-2:returnAsymmetricSubnets=true": {
"vpcId": "vpc-0dc99f521ceaa3f2d",
"vpcCidrBlock": "10.2.0.0/16",
Expand Down Expand Up @@ -239,6 +399,86 @@
}
]
},
"vpc-provider:account=472057503814:filter.tag:Name=main-vpc:region=ap-southeast-2:returnAsymmetricSubnets=true": {
"vpcId": "vpc-0dc99f521ceaa3f2d",
"vpcCidrBlock": "10.2.0.0/16",
"ownerAccountId": "472057503814",
"availabilityZones": [],
"subnetGroups": [
{
"name": "public",
"type": "Public",
"subnets": [
{
"subnetId": "subnet-0a7fb9d501192f5ee",
"cidr": "10.2.0.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-0c10b927ab0264377"
},
{
"subnetId": "subnet-0d61be977ee60a5b5",
"cidr": "10.2.2.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-0c10b927ab0264377"
},
{
"subnetId": "subnet-0d8226a9b4af34507",
"cidr": "10.2.4.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-0c10b927ab0264377"
}
]
},
{
"name": "database",
"type": "Isolated",
"subnets": [
{
"subnetId": "subnet-03ac51c3ab4223a1a",
"cidr": "10.2.40.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-01266550d60b56ddb"
},
{
"subnetId": "subnet-06d14dcb934f22c5e",
"cidr": "10.2.42.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-01266550d60b56ddb"
},
{
"subnetId": "subnet-0f5a5386ddf295579",
"cidr": "10.2.44.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-01266550d60b56ddb"
}
]
},
{
"name": "private",
"type": "Private",
"subnets": [
{
"subnetId": "subnet-01be4c1109eca3446",
"cidr": "10.2.20.0/23",
"availabilityZone": "ap-southeast-2a",
"routeTableId": "rtb-067d123217c80f6bd"
},
{
"subnetId": "subnet-070a9acba78168239",
"cidr": "10.2.22.0/23",
"availabilityZone": "ap-southeast-2b",
"routeTableId": "rtb-067d123217c80f6bd"
},
{
"subnetId": "subnet-01ae2b4ad1eb584d7",
"cidr": "10.2.24.0/23",
"availabilityZone": "ap-southeast-2c",
"routeTableId": "rtb-067d123217c80f6bd"
}
]
}
]
},
"security-group:account=843407916570:region=ap-southeast-2:securityGroupName=OrcaBusSharedComputeSecurityGroup:vpcId=vpc-00eafc63c0dfca266": {
"securityGroupId": "sg-03abb47eba799e044",
"allowAllOutbound": false
Expand Down
9 changes: 9 additions & 0 deletions config/config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@ import {
getOncoanalyserPipelineManagerStackProps,
getOncoanalyserPipelineTableStackProps,
} from './stacks/oncoanalyser';
import {
getOraCompressionIcav2PipelineManagerStackProps,
getOraCompressionIcav2PipelineTableStackProps,
} from './stacks/oraCompressionPipelineManager';
import { getOraDecompressionManagerStackProps } from './stacks/oraDecompressionPipelineManager';

interface EnvironmentConfig {
name: string;
Expand Down Expand Up @@ -92,6 +97,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
wtsIcav2PipelineTableStackProps: getWtsIcav2PipelineTableStackProps(),
umccriseIcav2PipelineTableStackProps: getUmccriseIcav2PipelineTableStackProps(),
rnasumIcav2PipelineTableStackProps: getRnasumIcav2PipelineTableStackProps(),
oraCompressionIcav2PipelineTableStackProps: getOraCompressionIcav2PipelineTableStackProps(),
BclConvertTableStackProps: getBclConvertManagerTableStackProps(stage),
stackyStatefulTablesStackProps: getStatefulGlueStackProps(),
pierianDxPipelineTableStackProps: getPierianDxPipelineTableStackProps(),
Expand All @@ -115,6 +121,9 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
pieriandxPipelineManagerStackProps: getPierianDxPipelineManagerStackProps(stage),
oncoanalyserPipelineManagerStackProps: getOncoanalyserPipelineManagerStackProps(stage),
sashPipelineManagerStackProps: getSashPipelineManagerStackProps(stage),
oraCompressionIcav2PipelineManagerStackProps:
getOraCompressionIcav2PipelineManagerStackProps(stage),
oraDecompressionManagerStackProps: getOraDecompressionManagerStackProps(stage),
eventSchemaStackProps: getEventSchemaStackProps(),
dataSchemaStackProps: getDataSchemaStackProps(),
bclConvertManagerStackProps: getBclConvertManagerStackProps(stage),
Expand Down
50 changes: 50 additions & 0 deletions config/constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -813,3 +813,53 @@ export const stackyAnalysisLogsUriSsmParameterName = '/orcabus/stacky/analysis_l
// stg: s3://pipeline-stg-cache-503977275616-ap-southeast-2/byob-icav2/staging/cache/__workflow_name__/__portal_run_id__/
// prod: s3://pipeline-prod-cache-503977275616-ap-southeast-2/byob-icav2/production/cache/__workflow_name__/__portal_run_id__/
export const stackyAnalysisCacheUriSsmParameterName = '/orcabus/stacky/analysis_cache_uri';

/*
Resources generated by the ORA Compression pipeline
*/

export const oraCompressionSSMRoot = '/orcabus/ora_compression/';

export const oraCompressionIcav2PipelineManagerDynamodbTableName =
'oraCompressionICAv2AnalysesDynamoDBTable';

// Stateful
export const oraCompressionDynamoDbTableSSMName = path.join(
oraCompressionSSMRoot,
'dynamodb_table_name'
);
export const oraCompressionDynamoDbTableSSMArn = path.join(
oraCompressionSSMRoot,
'dynamodb_table_arn'
);

// Stateless
export const oraCompressionIcav2PipelineWorkflowType = 'ora-compression';
export const oraCompressionIcav2PipelineWorkflowTypeVersion = '4.2.4--v2';
export const oraCompressionIcav2ServiceVersion = '2024.07.01';

export const oraCompressionIcav2ReadyEventSource = 'orcabus.workflowmanager';
export const oraCompressionIcav2EventSource = 'orcabus.oracompression';
export const oraCompressionIcav2EventDetailType = 'WorkflowRunStateChange';
export const oraCompressionStateMachinePrefix = 'oraCompressionSfn';

/*
Resources used by the ora compression pipeline
*/

// Release can be found here: https://github.com/umccr/cwl-ica/releases/tag/dragen-instrument-run-fastq-to-ora-pipeline%2F4.2.4__20241030041958
// Pipeline ID is: ba8f618a-842f-4a2f-9b2f-a074c0472218 // FIXME not in stg/prod
export const oraCompressionIcav2PipelineIdSSMParameterPath =
'/icav2/umccr-prod/ora_compression_pipeline_id';

// Default Reference Uri for compressing ORA files // FIXME not in stg/prod
export const oraCompressionDefaultReferenceUriSSmParameterPath =
'/icav2/umccr-prod/ora_compression_default_reference_version_uri';

/*
Resources generated by the ora decompression manager
*/
export const oraDecompressionIcav2ReadyEventSource = 'orcabus.workflowmanager';
export const oraDecompressionIcav2EventSource = 'orcabus.oradecompression';
export const oraDecompressionIcav2EventDetailType = 'FastqListRowDecompressed';
export const oraDecompressionStateMachinePrefix = 'oraDecompressionSfn';
Loading

0 comments on commit 89b6b41

Please sign in to comment.