Refactor cicd pipeline

AWS_FOR_FLUENT_BIT_VERSION

@@ -1 +1 @@
-2.32.0
+2.32.0.20231031

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+### 2.32.0.20231031 Linux re-build
+
+*This release has the same Fluent Bit contents as 2.32.0, and is simply a linux-only re-build for recent patches in dependencies installed in the image. There are no windows images for this release.* 
+* Amazon Linux Base: [2.0.20231024.1](https://docs.aws.amazon.com/AL2/latest/relnotes/relnotes-20231024.html)
+
 ### 2.32.0
 This release includes:
 * Fluent Bit [1.9.10](https://github.com/fluent/fluent-bit/tree/v1.9.10)

buildspec_load_test.yml

@@ -8,6 +8,8 @@ env:
     LOG_STORAGE_STACK_NAME: 'load-test-fluent-bit-log-storage'
     EKS_CLUSTER_NAME: 'load-test-fluent-bit-eks-cluster'
     PREFIX: 'load-test-fluent-bit-'
+    # Use the following field to shorten the load tests
+    # LOAD_TEST_RUN_TIME_IN_SECONDS: 10 # Default 600
 
 phases:
   install:
@@ -61,17 +63,21 @@ phases:
       - export AWS_ACCESS_KEY_ID=$(echo "${CREDS}" | jq -r '.Credentials.AccessKeyId')
       - export AWS_SECRET_ACCESS_KEY=$(echo "${CREDS}" | jq -r '.Credentials.SecretAccessKey')
       - export AWS_SESSION_TOKEN=$(echo "${CREDS}" | jq -r '.Credentials.SessionToken')
-      # Create and set up related testing resources
-      - python ./load_tests/load_test.py create_testing_resources
       - source ./load_tests/setup_test_environment.sh
       - export AWS_ACCESS_KEY_ID=""
       - export AWS_SECRET_ACCESS_KEY=""
       - export AWS_SESSION_TOKEN=""
       # Run load tests on corresponding platform
-      - python ./load_tests/load_test.py ${PLATFORM}
-    finally:
-      # Clear up testing resources
-      - python ./load_tests/load_test.py delete_testing_resources
+      - |
+        for i in {1..2}; do
+          python ./load_tests/load_test.py ${PLATFORM} && break \
+          || (
+            if [ $i == 2 ]; then
+              echo "Load test retries failed. Exiting."
+              exit 1
+            fi
+            echo "Retrying load tests. Attempt" $(($i+1)))
+        done
 artifacts:
   files:
     - '**/*'

buildspec_load_test_create_infra_ecs.yml

@@ -0,0 +1,53 @@
+version: 0.2
+env:
+  shell: bash
+  variables: 
+    THROUGHPUT_LIST: '["20m", "25m", "30m"]'
+    CW_THROUGHPUT_LIST: '["1m", "2m", "3m"]'
+    TESTING_RESOURCES_STACK_NAME: 'load-test-fluent-bit-testing-resources'
+    LOG_STORAGE_STACK_NAME: 'load-test-fluent-bit-log-storage'
+    EKS_CLUSTER_NAME: 'load-test-fluent-bit-eks-cluster'
+    PREFIX: 'load-test-fluent-bit-'
+
+phases:
+  install:
+    runtime-versions:
+      golang: 1.16
+      python: 3.x
+      nodejs: 14
+  pre_build:
+    commands:
+      - echo Running the ecs load test infrastructure creation
+      # upgrade node version
+      - npm install -g n
+      - n stable
+      # install cdk
+      - npm config set prefix /usr/local
+      - npm install -g [email protected]
+      # install aws
+      - pip3 uninstall awscli -y
+      - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+      - unzip awscliv2.zip
+      - ./aws/install --bin-dir /usr/local/bin --install-dir /usr/local/bin --update
+      - which aws
+      - aws --version
+      # pre-config
+      - aws configure set default.region us-west-2
+  build:
+    commands:
+      # Activate Python virtual environment and install the AWS CDK core dependencies
+      - python -m venv venv
+      - source venv/bin/activate
+      - pip install -r ./load_tests/requirements.txt
+      - CREDS=$(aws sts assume-role --role-arn $LOAD_TEST_CFN_ROLE_ARN --role-session-name load-test-cfn --duration-seconds 3600)
+      - export CREDS
+      - export AWS_ACCESS_KEY_ID=$(echo "${CREDS}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${CREDS}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${CREDS}" | jq -r '.Credentials.SessionToken')
+      # Clean up testing resources
+      - python ./load_tests/load_test.py delete_testing_resources
+      # Create and set up related testing resources
+      - python ./load_tests/load_test.py create_testing_resources
+artifacts:
+  files:
+    - '**/*'

buildspec_load_test_delete_infra_ecs.yml

@@ -0,0 +1,51 @@
+version: 0.2
+env:
+  shell: bash
+  variables: 
+    THROUGHPUT_LIST: '["20m", "25m", "30m"]'
+    CW_THROUGHPUT_LIST: '["1m", "2m", "3m"]'
+    TESTING_RESOURCES_STACK_NAME: 'load-test-fluent-bit-testing-resources'
+    LOG_STORAGE_STACK_NAME: 'load-test-fluent-bit-log-storage'
+    EKS_CLUSTER_NAME: 'load-test-fluent-bit-eks-cluster'
+    PREFIX: 'load-test-fluent-bit-'
+
+phases:
+  install:
+    runtime-versions:
+      golang: 1.16
+      python: 3.x
+      nodejs: 14
+  pre_build:
+    commands:
+      - echo Running the load tests
+      # upgrade node version
+      - npm install -g n
+      - n stable
+      # install cdk
+      - npm config set prefix /usr/local
+      - npm install -g [email protected]
+      # install aws
+      - pip3 uninstall awscli -y
+      - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+      - unzip awscliv2.zip
+      - ./aws/install --bin-dir /usr/local/bin --install-dir /usr/local/bin --update
+      - which aws
+      - aws --version
+      # pre-config
+      - aws configure set default.region us-west-2
+  build:
+    commands:
+      # Activate Python virtual environment and install the AWS CDK core dependencies
+      - python -m venv venv
+      - source venv/bin/activate
+      - pip install -r ./load_tests/requirements.txt
+      - CREDS=$(aws sts assume-role --role-arn $LOAD_TEST_CFN_ROLE_ARN --role-session-name load-test-cfn --duration-seconds 3600)
+      - export CREDS
+      - export AWS_ACCESS_KEY_ID=$(echo "${CREDS}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${CREDS}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${CREDS}" | jq -r '.Credentials.SessionToken')
+      # Clean up testing resources
+      - python ./load_tests/load_test.py delete_testing_resources
+artifacts:
+  files:
+    - '**/*'

buildspec_publish_dockerhub.yml

@@ -38,7 +38,7 @@ phases:
       - 'DRY_RUN="false" ./scripts/publish.sh cicd-publish dockerhub'
 
       # Pull the image from dockerhub and verify
-      - './scripts/publish.sh cicd-verify dockerhub'
+      - './scripts/publish.sh cicd-verify-publish dockerhub'
 artifacts:
   files:
     - '**/*'

buildspec_publish_ecr.yml

@@ -36,7 +36,7 @@ phases:
         fi
 
       # Push the image to ECR
-      - './scripts/publish.sh cicd-publish ${REGION_TO_PUSH}'
+      - './scripts/publish.sh cicd-publish private-ecr'
 
       # Nullify the temporary credentials for the assumed role to publish
       - |
@@ -55,7 +55,7 @@ phases:
       - export AWS_SESSION_TOKEN=`echo $CREDS | jq -r .Credentials.SessionToken`
 
       # Verify from the verification account
-      - './scripts/publish.sh cicd-verify ${REGION_TO_PUSH}'
+      - './scripts/publish.sh cicd-verify-publish private-ecr'
 artifacts:
   files:
     - '**/*'

buildspec_publish_public_ecr.yml

@@ -61,7 +61,7 @@ phases:
         fi
 
       # Pull the image from Public ECR and verify
-      - './scripts/publish.sh cicd-verify public-ecr'
+      - './scripts/publish.sh cicd-verify-publish public-ecr'
 artifacts:
   files:
     - '**/*'

buildspec_publish_ssm.yml

@@ -11,7 +11,7 @@ phases:
     commands:
       # Enforce STS regional endpoints
       - export AWS_STS_REGIONAL_ENDPOINTS=regional
-      - './scripts/publish.sh cicd-publish-ssm ${AWS_REGION}'
+      - './scripts/publish.sh cicd-publish-ssm'
 
       # Assume role to verify, get the credentials, and set them as environment variables.
       # Verification should be done using the credentials from a different account. It ensures that

buildspec_sync.yml

@@ -14,34 +14,40 @@ phases:
       - aws --version
   build:
     commands:
+      # Note: The "sync" task is used both on the primary distribution account
+      #       and replica accounts. In the primary account, it is used for stable
+      #       tag updates exclusively and their respective ssm param updates.
+      #
+      #       In the replica accounts, it is used for replicating images to regional
+      #       Private ECRs from the primary Public ECR, and replicating ssm params.
+      - export AWS_ACCOUNT=$(aws sts get-caller-identity --query Account --output text)
+
       # Enforce STS regional endpoints
       - export AWS_STS_REGIONAL_ENDPOINTS=regional
-
-      - './scripts/publish.sh cicd-publish ${AWS_REGION} stable' 
 
-      # Publish stable tag to Dockerhub when AWS_REGION is us-west-2
-      - './scripts/publish.sh cicd-publish public-dockerhub-stable ${AWS_REGION}'
-
-      # Assume role to publish, get the credentials, and set them as environment variables
+      # Replica accounts: Private ECR sync
+      - './scripts/publish.sh cicd-replicate ${AWS_REGION}'
+
+      # Primary account: Private ECR stable update
+      - './scripts/publish.sh cicd-publish private-ecr-stable'
+      - './scripts/publish.sh cicd-publish dockerhub-stable'
+        # Assume role to publish, get the credentials, and set them as environment variables
       - |
         if [ "${PUBLISH_ROLE_ARN_PUBLIC_ECR}" != "" ]; then
           CREDS=`aws sts assume-role --role-arn ${PUBLISH_ROLE_ARN_PUBLIC_ECR} --role-session-name publicECR`
           export AWS_ACCESS_KEY_ID=`echo $CREDS | jq -r .Credentials.AccessKeyId`
           export AWS_SECRET_ACCESS_KEY=`echo $CREDS | jq -r .Credentials.SecretAccessKey`
           export AWS_SESSION_TOKEN=`echo $CREDS | jq -r .Credentials.SessionToken`
         fi
-
-      # Publish stable tag to Public ECR when AWS_REGION is us-west-2
-      - './scripts/publish.sh cicd-publish public-ecr-stable ${AWS_REGION}'
-
-      # Nullify the temporary credentials for the assumed role to publish
+      - './scripts/publish.sh cicd-publish public-ecr-stable'
+        # Nullify the temporary credentials for the assumed role to publish
       - |
         if [ "${PUBLISH_ROLE_ARN_PUBLIC_ECR}" != "" ]; then
           export AWS_ACCESS_KEY_ID=
           export AWS_SECRET_ACCESS_KEY=
           export AWS_SESSION_TOKEN=
         fi
-      
+
       # Assume role to verify, get the credentials, and set them as environment variables.
       # Verification should be done using the credentials from a different account. It ensures that
       # the images we published are public and accessible from any account.
@@ -50,17 +56,26 @@ phases:
       - export AWS_SECRET_ACCESS_KEY=`echo $CREDS | jq -r .Credentials.SecretAccessKey`
       - export AWS_SESSION_TOKEN=`echo $CREDS | jq -r .Credentials.SessionToken`
 
-      # Verify from the verification account
-      - './scripts/publish.sh cicd-verify ${AWS_REGION} stable'
-      - './scripts/publish.sh cicd-verify-ssm ${AWS_REGION} stable'
+      # Replica accounts: Verify from the verification account
+      - './scripts/publish.sh cicd-verify-replicate ${AWS_REGION}'
+
+      # Primary account: Verify stable update
+      - './scripts/publish.sh cicd-verify-publish private-ecr-stable'
 
-       # Nullify the temporary credentials for the assumed role to verify
+      # Note: stable updates on primary account and sync task includes SSM updates
+
+      # Replica & primary accounts: Verify ssm update
+      - './scripts/publish.sh cicd-verify-ssm ${AWS_REGION} true'
+
+      # Nullify the temporary credentials for the assumed role to verify
       - export AWS_ACCESS_KEY_ID=
       - export AWS_SECRET_ACCESS_KEY=
       - export AWS_SESSION_TOKEN=
 
-      # Verify the publishing on Public ECR and Dockerhub when AWS_REGION is us-west-2
-      - './scripts/publish.sh cicd-verify stable ${AWS_REGION}'
+      # Run verification in the pipeline account
+      - './scripts/publish.sh cicd-verify-publish dockerhub-stable'
+      - './scripts/publish.sh cicd-verify-publish public-ecr-stable'
+
 artifacts:
   files:
     - '**/*'

load_tests/.gitignore

@@ -1,5 +1,7 @@
 create_testing_resources/cdk.out
 create_testing_resources/kinesis_s3_firehose/cdk.out
+create_testing_resources/kinesis_s3_firehose/load-test-fluent-bit-
 task_definitions/*_*m.json
 __pycache__
-.venv
+.venv
+.env