diff --git a/.github/workflows/run_release_tests.yml b/.github/workflows/run_release_tests.yml index a7a1b9bd11..feebd4bb3d 100644 --- a/.github/workflows/run_release_tests.yml +++ b/.github/workflows/run_release_tests.yml @@ -50,5 +50,4 @@ jobs: RESULTS_LOCATION: ${{ secrets.RESULTS_LOCATION }} DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }} S3_DATA_PATH: ${{ secrets.S3_DATA_PATH }} - S3_JARS_BUCKET: ${{ secrets.S3_JARS_BUCKET }} SPILL_BUCKET: ${{ secrets.SPILL_BUCKET }} diff --git a/validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/dynamo-stack.ts b/validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/dynamo-stack.ts index 0cd3bacb84..5c36662343 100644 --- a/validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/dynamo-stack.ts +++ b/validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/dynamo-stack.ts @@ -2,6 +2,7 @@ import * as cdk from 'aws-cdk-lib'; import * as glue from '@aws-cdk/aws-glue-alpha'; import * as ddb from 'aws-cdk-lib/aws-dynamodb'; import * as iam from 'aws-cdk-lib/aws-iam'; +import { Repository } from 'aws-cdk-lib/aws-ecr'; import { CfnInclude } from 'aws-cdk-lib/cloudformation-include'; import { Construct } from 'constructs'; import tpcdsJson from '../../resources/tpcds_specs.json' diff --git a/validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/opensearch-stack.ts b/validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/opensearch-stack.ts index 5b2378e815..1ca38debec 100644 --- a/validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/opensearch-stack.ts +++ b/validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/opensearch-stack.ts @@ -5,6 +5,7 @@ import * as ec2 from 'aws-cdk-lib/aws-ec2'; import * as glue from '@aws-cdk/aws-glue-alpha'; import * as s3 from 'aws-cdk-lib/aws-s3'; import * as iam from 'aws-cdk-lib/aws-iam'; +import { Repository } from 'aws-cdk-lib/aws-ecr'; import { CfnInclude } from 'aws-cdk-lib/cloudformation-include'; const path = require('path'); import {FederationStackProps} from './stack-props' diff --git a/validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/rds-generic-stack.ts b/validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/rds-generic-stack.ts index c6bde711fc..e0a97a2c13 100644 --- a/validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/rds-generic-stack.ts +++ b/validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/rds-generic-stack.ts @@ -3,6 +3,7 @@ import * as rds from 'aws-cdk-lib/aws-rds'; import * as ec2 from 'aws-cdk-lib/aws-ec2'; import * as glue from '@aws-cdk/aws-glue-alpha'; import * as iam from 'aws-cdk-lib/aws-iam' +import { Repository } from 'aws-cdk-lib/aws-ecr'; import { CfnInclude } from 'aws-cdk-lib/cloudformation-include'; import { Construct } from 'constructs'; const path = require('path') diff --git a/validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/redshift-stack.ts b/validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/redshift-stack.ts index 8f70345b02..f54b849e95 100644 --- a/validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/redshift-stack.ts +++ b/validation_testing/cdk_federation_infra_provisioning/app/lib/stacks/redshift-stack.ts @@ -7,6 +7,7 @@ import * as iam from 'aws-cdk-lib/aws-iam'; import * as s3 from 'aws-cdk-lib/aws-s3'; import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager'; import * as ssm from 'aws-cdk-lib/aws-ssm'; +import { Repository } from 'aws-cdk-lib/aws-ecr'; import { CfnInclude } from 'aws-cdk-lib/cloudformation-include'; import { Construct } from 'constructs'; const path = require('path'); diff --git a/validation_testing/deploy_infra.sh b/validation_testing/deploy_infra.sh index a02961918b..00bcbc6271 100644 --- a/validation_testing/deploy_infra.sh +++ b/validation_testing/deploy_infra.sh @@ -1,10 +1,24 @@ - CONNECTOR_NAME=$1 VALIDATION_TESTING_ROOT=$REPOSITORY_ROOT/validation_testing -# upload connector jar to s3 and update yaml to s3 uri, redirect to /dev/null to not log the s3 path -aws s3 cp $REPOSITORY_ROOT/athena-$CONNECTOR_NAME/target/athena-$CONNECTOR_NAME-2022.47.1.jar "$S3_JARS_BUCKET/" > /dev/null -sed -i "s#CodeUri: \"./target/athena-$CONNECTOR_NAME-2022.47.1.jar\"#CodeUri: \"$S3_JARS_BUCKET/athena-$CONNECTOR_NAME-2022.47.1.jar\"#" $REPOSITORY_ROOT/athena-$CONNECTOR_NAME/athena-$CONNECTOR_NAME.yaml +# ecr repository must be created and an image pushed before the cdk stack is deployed for the lambda to create successfully +# get the AWS account ID from the current roll (for use in ECR repo name) +ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" + +# create the ECR repository +aws ecr create-repository --repository-name athena-federation-repository-$CONNECTOR_NAME --region us-east-1 + +# push the ECR image for the connector to the ECR repository +aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $ACCOUNT_ID.dkr.ecr.us-east-1.amazonaws.com +docker build -t athena-federation-repository-$CONNECTOR_NAME $REPOSITORY_ROOT/athena-$CONNECTOR_NAME +docker tag athena-federation-repository-$CONNECTOR_NAME\:latest $ACCOUNT_ID.dkr.ecr.us-east-1.amazonaws.com/athena-federation-repository-$CONNECTOR_NAME\:latest +docker push $ACCOUNT_ID.dkr.ecr.us-east-1.amazonaws.com/athena-federation-repository-$CONNECTOR_NAME\:latest + +# update the template to use the correct ImageUri +sed -i "s|292517598671|$ACCOUNT_ID|g" "$REPOSITORY_ROOT/athena-$CONNECTOR_NAME/athena-$CONNECTOR_NAME.yaml" +sed -i "s#\(/athena-federation-repository-$CONNECTOR_NAME:\)[0-9]\{4\}\.[0-9]\{1,2\}\.[0-9]\{1\}#\1latest#" $REPOSITORY_ROOT/athena-$CONNECTOR_NAME/athena-$CONNECTOR_NAME.yaml + +echo "FINISHED PUSHING CONNECTOR IMAGE TO ECR REPOSITORY" # go to cdk dir, build/synth/deploy cd $(dirname $(find . -name ATHENA_INFRA_SPINUP_ROOT))/app; @@ -20,6 +34,4 @@ npm run build; npm run cdk synth; npm run cdk deploy ${CONNECTOR_NAME}CdkStack > /dev/null; -sed -i "s#CodeUri: \"$S3_JARS_BUCKET/athena-$CONNECTOR_NAME-2022.47.1.jar\"#CodeUri: \"./target/athena-$CONNECTOR_NAME-2022.47.1.jar\"#" $REPOSITORY_ROOT/athena-$CONNECTOR_NAME/athena-$CONNECTOR_NAME.yaml - echo "FINISHED DEPLOYING INFRA FOR ${CONNECTOR_NAME}." diff --git a/validation_testing/main.py b/validation_testing/main.py index bf03f13670..9bb8f10274 100644 --- a/validation_testing/main.py +++ b/validation_testing/main.py @@ -19,7 +19,6 @@ def assert_required_env_vars_set(): 'REPOSITORY_ROOT', 'DATABASE_PASSWORD', 'S3_DATA_PATH', - 'S3_JARS_BUCKET', 'SPILL_BUCKET' ] if not all([os.environ.get(env_var) for env_var in required_env_vars]): diff --git a/validation_testing/run_release_tests.sh b/validation_testing/run_release_tests.sh index 5b0a7c815b..47a272903a 100644 --- a/validation_testing/run_release_tests.sh +++ b/validation_testing/run_release_tests.sh @@ -7,9 +7,24 @@ CONNECTOR_NAME=$1 VALIDATION_TESTING_ROOT=$REPOSITORY_ROOT/validation_testing -# upload connector jar to s3 and update yaml to s3 uri, redirect to /dev/null to not log the s3 path -aws s3 cp $REPOSITORY_ROOT/athena-$CONNECTOR_NAME/target/athena-$CONNECTOR_NAME-2022.47.1.jar $S3_JARS_BUCKET > /dev/null -sed -i "s#CodeUri: \"./target/athena-$CONNECTOR_NAME-2022.47.1.jar\"#CodeUri: \"$S3_JARS_BUCKET/athena-$CONNECTOR_NAME-2022.47.1.jar\"#" $REPOSITORY_ROOT/athena-$CONNECTOR_NAME/athena-$CONNECTOR_NAME.yaml +# ecr repository must be created and an image pushed before the cdk stack is deployed for the lambda to create successfully +# get the AWS account ID from the current roll (for use in ECR repo name) +ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" + +# create the ECR repository +aws ecr create-repository --repository-name athena-federation-repository-$CONNECTOR_NAME --region us-east-1 + +# push the ECR image for the connector to the ECR repository +aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $ACCOUNT_ID.dkr.ecr.us-east-1.amazonaws.com +docker build -t athena-federation-repository-$CONNECTOR_NAME $REPOSITORY_ROOT/athena-$CONNECTOR_NAME +docker tag athena-federation-repository-$CONNECTOR_NAME\:latest $ACCOUNT_ID.dkr.ecr.us-east-1.amazonaws.com/athena-federation-repository-$CONNECTOR_NAME\:latest +docker push $ACCOUNT_ID.dkr.ecr.us-east-1.amazonaws.com/athena-federation-repository-$CONNECTOR_NAME\:latest + +# update the template to use the correct ImageUri +sed -i "s|292517598671|$ACCOUNT_ID|g" "$REPOSITORY_ROOT/athena-$CONNECTOR_NAME/athena-$CONNECTOR_NAME.yaml" +sed -i "s#\(/athena-federation-repository-$CONNECTOR_NAME:\)[0-9]\{4\}\.[0-9]\{1,2\}\.[0-9]\{1\}#\1latest#" $REPOSITORY_ROOT/athena-$CONNECTOR_NAME/athena-$CONNECTOR_NAME.yaml + +echo "FINISHED PUSHING CONNECTOR IMAGE TO ECR REPOSITORY" # go to cdk dir, build/synth/deploy cd $(dirname $(find . -name ATHENA_INFRA_SPINUP_ROOT))/app; @@ -25,8 +40,6 @@ npm run build; npm run cdk synth; npm run cdk deploy ${CONNECTOR_NAME}CdkStack > /dev/null; -sed -i "s#CodeUri: \"$S3_JARS_BUCKET/athena-$CONNECTOR_NAME-2022.47.1.jar\"#CodeUri: \"./target/athena-$CONNECTOR_NAME-2022.47.1.jar\"#" $REPOSITORY_ROOT/athena-$CONNECTOR_NAME/athena-$CONNECTOR_NAME.yaml - echo "FINISHED DEPLOYING INFRA FOR ${CONNECTOR_NAME}." # cd back to validation root @@ -61,6 +74,9 @@ cd $(dirname $(find . -name ATHENA_INFRA_SPINUP_ROOT))/app; # cannot use --force because npm is stripping the flags, so pipe yes through yes | npm run cdk destroy ${CONNECTOR_NAME}CdkStack; +# delete the ecr repository +aws ecr delete-repository --repository-name athena-federation-repository-$CONNECTOR_NAME --force + echo "FINISHED CLEANING UP RESOURCES FOR ${CONNECTOR_NAME}."