From aa2d61af15c040503fc7f97cfca806aa09bedfff Mon Sep 17 00:00:00 2001 From: Saurabh Shrihar Date: Wed, 18 Sep 2024 10:28:25 +0400 Subject: [PATCH] Deployment fix for dev deployment through hosted/ PRs --- .github/assets/dev-taskdef.json | 6 +- .github/workflows/build_and_deploy.yml | 122 +++++++++++++++++++++++ .github/workflows/pr-deployment.yml | 16 +++ Dockerfile.review | 8 ++ build_branches.py | 132 +++++++++++++++++++++++++ nginx.conf | 25 +++++ 6 files changed, 306 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/build_and_deploy.yml create mode 100644 .github/workflows/pr-deployment.yml create mode 100755 Dockerfile.review create mode 100755 build_branches.py create mode 100644 nginx.conf diff --git a/.github/assets/dev-taskdef.json b/.github/assets/dev-taskdef.json index a06d210f3..122c0caf7 100644 --- a/.github/assets/dev-taskdef.json +++ b/.github/assets/dev-taskdef.json @@ -53,9 +53,9 @@ "networkMode": "awsvpc", "memory": "1024", "cpu": "512", - "executionRoleArn": "arn:aws:iam::605436358845:role/docs-dev-TaskRole", + "executionRoleArn": "arn:aws:iam::058264511034:role/docs-dev-TaskRole", "family": "docs-dev-taskdefinition", - "taskRoleArn": "arn:aws:iam::605436358845:role/docs-dev-TaskRole", + "taskRoleArn": "arn:aws:iam::058264511034:role/docs-dev-TaskRole", "runtimePlatform": { "operatingSystemFamily": "LINUX" }, @@ -81,7 +81,7 @@ }, { "key": "IAC", - "value": "terraform-workspace-aws-dev-applications-eu-west-1-apps-docs-dev-polygon-technology" + "value": "terraform-workspace-aws-dev-apps-eu-west-1-apps-docs-dev-polygon-technology" }, { "key": "Team", diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml new file mode 100644 index 000000000..1f9c4f01b --- /dev/null +++ b/.github/workflows/build_and_deploy.yml @@ -0,0 +1,122 @@ +on: + workflow_call: + inputs: + environment: + required: false + type: string + default: "dev" + core_app: + required: false + type: string + description: "Core app name" + default: "docs" + account_number: + required: false + type: string + description: "AWS Account number for deployment" + default: "058264511034" + region: + required: false + type: string + description: "AWS region for deployment" + default: "eu-west-1" + task_definition: + required: false + type: string + description: "Task Definition path for deployment" + default: ".github/assets/dev-taskdef.json" + cluster_name: + required: false + type: string + description: "Cluster name for deployment" + default: "frontend-dev-ecs-cluster" + +jobs: + build_site_data: + name: ${{ inputs.environment }} deployment + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + permissions: + id-token: write + contents: write + env: + AWS_REGION: ${{ inputs.region }} + ECR_REPOSITORY: ${{ inputs.core_app }}-${{ inputs.environment }}-ecr + ECS_SERVICE: ${{ inputs.core_app }}-${{ inputs.environment }}-ecs-service + ECS_CLUSTER: frontend-${{ inputs.environment }}-ecs-cluster + ECS_TASK_DEFINITION: ${{ inputs.task_definition }} + APP_NAME: ${{ inputs.core_app }}-${{ inputs.environment }} + steps: + - name: Checkout Code Repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.AWS_REGION }} + role-to-assume: arn:aws:iam::${{ inputs.account_number }}:role/${{ env.APP_NAME }}-GithubActionsRole + role-session-name: GithubActionsSession + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install pipenv + run: pip install pipenv + + - name: Install GitHub CLI + run: | + (type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) \ + && sudo mkdir -p -m 755 /etc/apt/keyrings \ + && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && sudo apt update \ + && sudo apt install gh -y + + - name: Authenticate GitHub CLI + run: gh auth login --with-token <<< "${{ secrets.GITHUB_TOKEN }}" + + - name: Build Site + run: | + python build_branches.py + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build, tag, and push image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + IMAGE_TAG: ${{ github.sha }}-${{ github.run_number }} + ECR_REPOSITORY: ${{ env.APP_NAME }}-ecr + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -f Dockerfile.review . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" >> $GITHUB_OUTPUT + + - name: Fill in the new image ID in the Amazon ECS task definition + id: task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: ${{ env.ECS_TASK_DEFINITION }} + container-name: ${{ env.APP_NAME }} + image: ${{ steps.build-image.outputs.image }} + + - name: Deploy Amazon ECS task definition + uses: aws-actions/amazon-ecs-deploy-task-definition@v1 + with: + task-definition: ${{ steps.task-def.outputs.task-definition }} + service: ${{ env.ECS_SERVICE }} + cluster: ${{ env.ECS_CLUSTER }} + wait-for-service-stability: true + + - name: Cloudflare Cache Purge + uses: nathanvaughn/actions-cloudflare-purge@master + with: + cf_zone: ${{ secrets.CLOUDFLARE_ZONE }} + cf_auth: ${{ secrets.CLOUDFLARE_AUTH_KEY }} + hosts: ${{ env.APP_NAME }}.polygon.technology diff --git a/.github/workflows/pr-deployment.yml b/.github/workflows/pr-deployment.yml new file mode 100644 index 000000000..86a961253 --- /dev/null +++ b/.github/workflows/pr-deployment.yml @@ -0,0 +1,16 @@ +name: hosted branch pr deployment +on: + pull_request: + types: [opened, edited, reopened] + branches: + - hosted/* + push: + branches: + - dev + workflow_dispatch: + +jobs: + deploy: + uses: ./.github/workflows/build_and_deploy.yml + secrets: inherit + \ No newline at end of file diff --git a/Dockerfile.review b/Dockerfile.review new file mode 100755 index 000000000..6743b610f --- /dev/null +++ b/Dockerfile.review @@ -0,0 +1,8 @@ +FROM nginx:alpine + +COPY nginx.conf /etc/nginx/nginx.conf +COPY app /app + +WORKDIR /app +EXPOSE 80 +CMD ["nginx", "-g", "daemon off;"] diff --git a/build_branches.py b/build_branches.py new file mode 100755 index 000000000..33302ca78 --- /dev/null +++ b/build_branches.py @@ -0,0 +1,132 @@ +import ast +import os +import shutil +import subprocess + + +def install_mkdocs_with_pipenv(): + """ + Builds a particular branch site. + Having a varying set of requirements can be handled by having each branch + build their dependencies and then running mkdocs build. + """ + folder = os.getcwd() + subprocess.run(["pipenv", "install", "--site-packages"], cwd=folder) + subprocess.run(["pipenv", "install", "-r", "requirements.txt"], cwd=folder) + subprocess.run(["pipenv", "run", "mkdocs", "build"], cwd=folder) + +def copy_folder(source_dir, target_dir): + """ + Copies contents from source directory to target directory + :param source_dir: Source directory from which contents are to be copied + :param target_dir: Target Directory where the contents are copied to. + """ + os.makedirs(target_dir, exist_ok=True) + + for item in os.listdir(source_dir): + source_path = os.path.join(source_dir, item) + target_path = os.path.join(target_dir, item) + + if os.path.isdir(source_path): + shutil.copytree(source_path, target_path, dirs_exist_ok=True) + else: + if os.path.exists(target_path): + os.remove(target_path) + shutil.copy2(source_path, target_path) + +def delete_folders(folder_paths): + """ + Cleans existing folders for app and branches before executing the builds + :param folder_paths: List of folders to be deleted under the current working directory + """ + for folder_path in folder_paths: + try: + shutil.rmtree(folder_path) + print(f"Folder {folder_path} deletion successful.") + except OSError as e: + print(f"Error deleting folder: {e}") + +def clone_data_to_branch_folder(branch_name, remote_url, parent_dir, pr_number=None): + """ + Clones data to branch folder in branch/ or branch/dev folder + :param branch_name: Branch to clone and build + :param remote_url: Remote url for the git repository + :param parent_dir: Parent directory to get context of where data is stored + :param pr_number: PR number for the branch to host data into the folder + """ + common_dir = "branch" + target_path = os.path.join(common_dir, pr_number) + os.makedirs(target_path, exist_ok=True) + os.chdir(target_path) + subprocess.run(["git", "init"]) + subprocess.run(["git", "remote", "add", "origin", remote_url]) + print(f"Checking out branch {branch_name}") + subprocess.run(["git", "fetch", "--depth", "1", "origin", branch_name]) + subprocess.run([ + "git", "checkout", "-b", branch_name, "--track", + f"origin/{branch_name}" + ]) + install_mkdocs_with_pipenv() + source_dir = os.path.join(os.getcwd(), "site") + copy_folder(source_dir, os.path.join(parent_dir, "app", pr_number)) + os.chdir(parent_dir) + + +def process_branch_folders(): + """ + Clones the branch specific code to hosted/ folder. + It then executes the build command and copy the built site to apps folder + under the same branch name + :return: PR numbers in str list where the site data is copied to + """ + delete_folders(["branch", "app"]) + + command = ["gh", "pr", "list", "--json", "number,headRefName"] + command_run_result = subprocess.run(command, capture_output=True, text=True).stdout.strip() + branches_data = ast.literal_eval(command_run_result) + remote_url = subprocess.run(["git", "remote", "get-url", "origin"], + capture_output=True, + text=True).stdout.strip() + parent_dir = os.getcwd() + clone_data_to_branch_folder("dev", remote_url, parent_dir, "dev") + pr_numbers = [] + for branch_data in branches_data: + if not branch_data["headRefName"].startswith("hosted/"): + continue + pr_number = str(branch_data["number"]) + clone_data_to_branch_folder(branch_data["headRefName"], remote_url, parent_dir, pr_number) + pr_numbers.append(pr_number) + + return pr_numbers + +def update_nginx_config(pr_numbers): + """ + Updates nginx.conf file with branches built information to host multiple versions + of software at the same time. + :param pr_numbers: pr numbers a str list of open pr numbers to be hosted + """ + config_file = os.path.join(os.getcwd(), "nginx.conf") + nginx_location_blocks = "" + + for pr_number in pr_numbers: + location_block = f"""location /{pr_number} {{ + alias /app/{pr_number}; + try_files $uri $uri/ /index.html; + error_page 404 /404.html; + }} + """ + nginx_location_blocks += location_block + + with open(config_file, "r+") as f: + content = f.read() + content = content.replace("#REPLACE_APPS", nginx_location_blocks) + f.seek(0) + f.write(content) + f.truncate() + + print("NGINX configuration updated successfully!") + +if __name__ == "__main__": + current_dir = os.getcwd() + pr_numbers = process_branch_folders() + update_nginx_config(pr_numbers) diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 000000000..0e7576393 --- /dev/null +++ b/nginx.conf @@ -0,0 +1,25 @@ +worker_processes auto; + +events { + worker_connections 1024; +} + +http { + include mime.types; + default_type application/octet-stream; + + sendfile on; + keepalive_timeout 65; + + server { + listen 80; + + #REPLACE_APPS + location / { + alias /app/dev/; + index index.html; + try_files $uri $uri/ /index.html; + error_page 404 /404.html; + } + } +}