From ac70861fbe6ddf3f279bb6444da72c5ff8cd251b Mon Sep 17 00:00:00 2001 From: Mishal Shah <100714644+mishah334@users.noreply.github.com> Date: Mon, 21 Nov 2022 09:05:10 +0530 Subject: [PATCH] Restructuring Repo (#70) * Restcturing repo * Fixing job * Typo fix * Removing not require job * Removing not require job * Adding dependency. * Fixing tf init command * Updating terraform module version. * Updating deployment_id * Typo fix * Using CIRCLE_WORKFLOW_ID replacing CIRCLE_SHA1 * Provisioning Gcloud credentials from token env. * Provisioning Gcloud credentials from token env. * Typo fix. * Updating script. * Now Source bash env * Typo fix. * Removing un-used scripts. * Adding storage role. * Using local to derive output value. * Adding help for make target. --- .circleci/config.yml | 168 +++++++----------- .terraform-version | 2 +- Makefile | 116 ++++++++++++ backend.tf.example | 7 - examples/from_scratch/.gitignore | 2 - examples/versions.tf | 40 ----- pipeline/lint.sh | 26 --- pipeline/run_terraform.sh | 43 ----- pipeline/update_cloud_module.sh | 32 ---- providers.tf.example | 19 -- terraform.tfvars.sample | 7 - bastion.tf => terraform/bastion.tf | 0 buckets.tf => terraform/buckets.tf | 0 db.tf => terraform/db.tf | 0 domain.tf => terraform/domain.tf | 0 firewalls.tf => terraform/firewalls.tf | 0 iam.tf => terraform/iam.tf | 3 +- locals.tf => terraform/locals.tf | 6 + main.tf => terraform/main.tf | 0 network.tf => terraform/network.tf | 0 node_pools.tf => terraform/node_pools.tf | 0 outputs.tf => terraform/outputs.tf | 13 +- .../service-accounts.tf | 0 spot.tf => terraform/spot.tf | 0 variables.tf => terraform/variables.tf | 0 versions.tf => terraform/versions.tf | 0 test/.gitignore | 1 - examples/from_scratch/main.tf => test/gcp.tf | 12 +- test/main.tf | 60 +++++++ test/variables.tf | 21 +++ 30 files changed, 277 insertions(+), 301 deletions(-) create mode 100644 Makefile delete mode 100644 backend.tf.example delete mode 100644 examples/from_scratch/.gitignore delete mode 100644 examples/versions.tf delete mode 100755 pipeline/lint.sh delete mode 100755 pipeline/run_terraform.sh delete mode 100755 pipeline/update_cloud_module.sh delete mode 100644 providers.tf.example delete mode 100644 terraform.tfvars.sample rename bastion.tf => terraform/bastion.tf (100%) rename buckets.tf => terraform/buckets.tf (100%) rename db.tf => terraform/db.tf (100%) rename domain.tf => terraform/domain.tf (100%) rename firewalls.tf => terraform/firewalls.tf (100%) rename iam.tf => terraform/iam.tf (97%) rename locals.tf => terraform/locals.tf (91%) rename main.tf => terraform/main.tf (100%) rename network.tf => terraform/network.tf (100%) rename node_pools.tf => terraform/node_pools.tf (100%) rename outputs.tf => terraform/outputs.tf (90%) rename service-accounts.tf => terraform/service-accounts.tf (100%) rename spot.tf => terraform/spot.tf (100%) rename variables.tf => terraform/variables.tf (100%) rename versions.tf => terraform/versions.tf (100%) delete mode 100644 test/.gitignore rename examples/from_scratch/main.tf => test/gcp.tf (77%) create mode 100644 test/main.tf create mode 100644 test/variables.tf diff --git a/.circleci/config.yml b/.circleci/config.yml index 6f22d1f..4ab7e0e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,135 +17,95 @@ executors: TFENV_AUTO_INSTALL: true workflows: - test: + pre-commit: jobs: - run_pre_commit terraform_pipeline: jobs: - - terraform_lint: + - terraform-apply: + region: us-east4 + project_id: astronomer-cloud-dev-236021 + tf_state_bucket: cloud2-dev-terraform + zonal: true context: - - slack - - terraform_apply: - requires: - - terraform_lint - context: - - slack - gcp-dev - - terraform_destroy: - requires: - - terraform_apply + - terraform-destroy: + region: us-east4 + project_id: astronomer-cloud-dev-236021 + tf_state_bucket: cloud2-dev-terraform + zonal: true context: - - slack - gcp-dev - #- git_tag: - # context: - # - github-repo - # - slack - # requires: - # - terraform_destroy - # filters: - # branches: - # only: - # - master + requires: + - terraform-apply jobs: run_pre_commit: executor: pre-commit-executor + resource_class: small steps: - checkout - pre-commit-commands - terraform_lint: - executor: pre-commit-executor - steps: - - checkout - - pre-commit-commands - - slack/notify: - event: fail - template: basic_fail_1 - - terraform_apply: + terraform-apply: executor: terraform-executor + resource_class: medium + parameters: + region: + type: string + project_id: + type: string + tf_state_bucket: + type: string + zonal: + type: boolean steps: - checkout - - run: EXAMPLE=from_scratch pipeline/run_terraform.sh + - terraform-install + - run: + name: Provision GCloud Credentials + command: | + echo "$GCP_TOKEN" > /tmp/account.json + echo 'export GOOGLE_APPLICATION_CREDENTIALS=/tmp/account.json' >>"$BASH_ENV" + source "$BASH_ENV" - run: - command: DESTROY=1 EXAMPLE=from_scratch pipeline/run_terraform.sh + name: Run Terraform Apply + command: | + DEPLOYMENT_ID=ci$(echo "${CIRCLE_PROJECT_REPONAME}${CIRCLE_WORKFLOW_ID}" | md5sum | awk '{print substr($1,0,5)}') + make init apply-auto-approve PROJECT_ID=<< parameters.project_id >> REGION=<< parameters.region >> DEPLOYMENT_ID=$DEPLOYMENT_ID ZONAL=<< parameters.zonal >> TF_STATE_BUCKET=<< parameters.tf_state_bucket >> + - run: + name: Run Terraform Destroy + command: | + DEPLOYMENT_ID=ci$(echo "${CIRCLE_PROJECT_REPONAME}${CIRCLE_WORKFLOW_ID}" | md5sum | awk '{print substr($1,0,5)}') + make init destroy-auto-approve PROJECT_ID=<< parameters.project_id >> REGION=<< parameters.region >> DEPLOYMENT_ID=$DEPLOYMENT_ID ZONAL=<< parameters.zonal >> TF_STATE_BUCKET=<< parameters.tf_state_bucket >> when: on_fail - - slack/notify: - event: fail - template: basic_fail_1 - - terraform_destroy: - executor: terraform-executor - steps: - - checkout - - run: DESTROY=1 EXAMPLE=from_scratch pipeline/run_terraform.sh - - slack/notify: - event: fail - template: basic_fail_1 - git_tag: + terraform-destroy: executor: terraform-executor + resource_class: medium + parameters: + region: + type: string + project_id: + type: string + tf_state_bucket: + type: string + zonal: + type: boolean steps: - checkout - - run: git remote set-url origin "https://astro-astronomer:${GITHUB_TOKEN}@github.com/astronomer/${CIRCLE_PROJECT_REPONAME}.git" - - run: git tag 1.2.<< pipeline.number >> - - run: git push origin << pipeline.git.branch >> --tags - - slack/notify: - event: fail - template: basic_fail_1 - - slack/notify: - event: pass - custom: | - { - "text": "", - "blocks": [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": "Release Successful! :tada:", - "emoji": true - } - }, - { - "type": "section", - "fields": [ - { - "type": "mrkdwn", - "text": "*Project*:\n$CIRCLE_PROJECT_REPONAME" - }, - { - "type": "mrkdwn", - "text": "*When*:\n$(date +'%m/%d/%Y %T')" - }, - { - "type": "mrkdwn", - "text": "*Tag*:\n1.2.<< pipeline.number >>" - } - ], - "accessory": { - "type": "image", - "image_url": "https://assets.brandfolder.com/otz5mn-bw4j2w-6jzqo8/original/circle-logo-badge-black.png", - "alt_text": "CircleCI logo" - } - }, - { - "type": "actions", - "elements": [ - { - "type": "button", - "text": { - "type": "plain_text", - "text": "View Job" - }, - "url": "${CIRCLE_BUILD_URL}" - } - ] - } - ] - } + - terraform-install + - run: + name: Provision GCloud Credentials + command: | + echo "$GCP_TOKEN" > /tmp/account.json + echo 'export GOOGLE_APPLICATION_CREDENTIALS=/tmp/account.json' >>"$BASH_ENV" + source "$BASH_ENV" + - run: + name: Run Terraform Destroy + command: | + DEPLOYMENT_ID=ci$(echo "${CIRCLE_PROJECT_REPONAME}${CIRCLE_WORKFLOW_ID}" | md5sum | awk '{print substr($1,0,5)}') + make init destroy-auto-approve PROJECT_ID=<< parameters.project_id >> REGION=<< parameters.region >> DEPLOYMENT_ID=$DEPLOYMENT_ID ZONAL=<< parameters.zonal >> TF_STATE_BUCKET=<< parameters.tf_state_bucket >> commands: pre-commit-commands: diff --git a/.terraform-version b/.terraform-version index c813fe1..d0149fe 100644 --- a/.terraform-version +++ b/.terraform-version @@ -1 +1 @@ -1.2.5 +1.3.4 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b55328f --- /dev/null +++ b/Makefile @@ -0,0 +1,116 @@ +# Defaults +REV:=$(shell git rev-parse --short HEAD) +DATE:=$(shell date +%Y.%m.%d-%H.%M.%S) +BRANCH:=$(shell git rev-parse --abbrev-ref HEAD) +COMMIT:=$(MODULE)_$(INFRA_VALUES)_$(DATE)_$(REV) + +CURRENT_DIR_PATH:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) + +# Inputs +REGION:=NO_REGION +PROJECT_ID:=PROJECT_ID +DEPLOYMENT_ID:=NO_DEPLOYMENT_ID +ZONAL:=NO_ZONAL + +TF_STATE_BUCKET:=NO_TF_STATE_BUCKET + +# Terraform Setup +TF_MODULE_PATH:=test +TF_PLAN_FILE:=test_$(DEPLOYMENT_ID)_$(REV) + + +.PHONY: help +help: ## Print Makefile help. + @grep -Eh '^[a-z.A-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-29s\033[0m %s\n", $$1, $$2}' + +.PHONY: pre-commit +pre-commit: ## It will run pre-commit target. It will run target `install`, `install-hooks` and then `run`. + pre-commit install + pre-commit install-hooks + pre-commit run --all-files --show-diff-on-failure + +.PHONY: install +install: ## It will install terraform version defined into `.terraform-version` file. + tfenv install + +.PHONY: init +init: clean ## It will run Terraform init. Required params are `DEPLOYMENT_ID` and `TF_STATE_BUCKET`. + terraform -chdir=$(TF_MODULE_PATH) init \ + -backend=true \ + -backend-config='bucket=$(TF_STATE_BUCKET)' \ + -backend-config='prefix=$(DEPLOYMENT_ID)/terraform.tfstate' + +.PHONY: upgrade +upgrade: clean ## It will run Terraform init with `-upgrade` flag. Required params are `DEPLOYMENT_ID` and `TF_STATE_BUCKET`. + terraform -chdir=$(TF_MODULE_PATH) init \ + -backend=true \ + -bucket='bucket=cloud2-dev-terraform' \ + -prefix='prefix=$(DEPLOYMENT_ID)/terraform.tfstate' + -upgrade + +.PHONY: validate +validate: ## It will run Terraform validate. Required params are `DEPLOYMENT_ID` and `TF_STATE_BUCKET`. + terraform -chdir=$(TF_MODULE_PATH) validate + +.PHONY: plan +plan: validate ## It will run Terraform plan. Required params are `PROJECT_ID`, `REGION`, `DEPLOYMENT_ID` and `ZONAL`. + terraform -chdir=$(TF_MODULE_PATH) plan \ + -var 'project_id=$(PROJECT_ID)' \ + -var 'region=$(REGION)' \ + -var 'deployment_id=$(DEPLOYMENT_ID)' \ + -var 'zonal=$(ZONAL)' \ + -out=$(TF_PLAN_FILE).tfplan + +.PHONY: plan-destroy +plan-destroy: validate ## It will run Terraform plan with `-destroy` flag to generate cleanup plan. Required params are `PROJECT_ID`, `REGION`, `DEPLOYMENT_ID` and `ZONAL`. + terraform -chdir=$(TF_MODULE_PATH) plan \ + -destroy \ + -var 'project_id=$(PROJECT_ID)' \ + -var 'region=$(REGION)' \ + -var 'deployment_id=$(DEPLOYMENT_ID)' \ + -var 'zonal=$(ZONAL)' \ + -out=$(TF_PLAN_FILE).tfplan + +.PHONY: apply +apply: validate ## It will run Terraform apply. Required params are `PROJECT_ID`, `REGION`, `DEPLOYMENT_ID` and `ZONAL`. + terraform -chdir=$(TF_MODULE_PATH) apply \ + -var 'project_id=$(PROJECT_ID)' \ + -var 'region=$(REGION)' \ + -var 'deployment_id=$(DEPLOYMENT_ID)' \ + -var 'zonal=$(ZONAL)' + +.PHONY: apply-auto-approve +apply-auto-approve: validate ## It will run Terraform apply with `-auto-approve` flag. Required params are `PROJECT_ID`, `REGION`, `DEPLOYMENT_ID` and `ZONAL`. + terraform -chdir=$(TF_MODULE_PATH) apply \ + -auto-approve \ + -var 'project_id=$(PROJECT_ID)' \ + -var 'region=$(REGION)' \ + -var 'deployment_id=$(DEPLOYMENT_ID)' \ + -var 'zonal=$(ZONAL)' + +.PHONY: apply-plan +apply-plan: ## It will run Terraform apply to plan. + terraform -chdir=$(TF_MODULE_PATH) apply $(TF_PLAN_FILE).tfplan + +.PHONY: destroy +destroy: validate ## It will run Terraform destroy. Required params are `PROJECT_ID`, `REGION`, `DEPLOYMENT_ID` and `ZONAL`. + terraform -chdir=$(TF_MODULE_PATH) destroy \ + -var 'project_id=$(PROJECT_ID)' \ + -var 'region=$(REGION)' \ + -var 'deployment_id=$(DEPLOYMENT_ID)' \ + -var 'zonal=$(ZONAL)' + +.PHONY: destroy-auto-approve +destroy-auto-approve: validate ## It will run Terraform destroy with `-auto-approve` flag. Required params are `PROJECT_ID`, `REGION`, `DEPLOYMENT_ID` and `ZONAL`. + terraform -chdir=$(TF_MODULE_PATH) destroy \ + -auto-approve \ + -var 'project_id=$(PROJECT_ID)' \ + -var 'region=$(REGION)' \ + -var 'deployment_id=$(DEPLOYMENT_ID)' \ + -var 'zonal=$(ZONAL)' + +.PHONY: clean +clean: ## Remove terraform init file and dir + find . -name "*.terraform" -type d -exec rm -rf {} + || true + find . -name "*.terraform.lock.hcl" -type f -delete || true + find . -name "*.tfplan" -type f -delete || true diff --git a/backend.tf.example b/backend.tf.example deleted file mode 100644 index 32af669..0000000 --- a/backend.tf.example +++ /dev/null @@ -1,7 +0,0 @@ -terraform { - required_version = ">= 0.13" - backend "gcs" { - bucket = "cloud2-dev-terraform" - prefix = "REPLACE/terraform.tfstate" - } -} diff --git a/examples/from_scratch/.gitignore b/examples/from_scratch/.gitignore deleted file mode 100644 index c7dc0c4..0000000 --- a/examples/from_scratch/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -backend.tf -kubeconfig* diff --git a/examples/versions.tf b/examples/versions.tf deleted file mode 100644 index adcbf0f..0000000 --- a/examples/versions.tf +++ /dev/null @@ -1,40 +0,0 @@ - -terraform { - required_version = ">= 0.13" - required_providers { - acme = { - source = "vancluever/acme" - version = "~> 1.4" - } - google = { - source = "hashicorp/google" - } - google-beta = { - source = "hashicorp/google-beta" - } - http = { - source = "hashicorp/http" - version = "~> 1.1" - } - local = { - source = "hashicorp/local" - version = "~> 1.3" - } - null = { - source = "hashicorp/null" - version = "~> 2.1" - } - random = { - source = "hashicorp/random" - version = "~> 2.2" - } - spotinst = { - source = "spotinst/spotinst" - version = "~> 1.17" - } - tls = { - source = "hashicorp/tls" - version = "~> 2.1" - } - } -} diff --git a/pipeline/lint.sh b/pipeline/lint.sh deleted file mode 100755 index d235e76..0000000 --- a/pipeline/lint.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -set -xe - -TERRAFORM="${TERRAFORM:-terraform-0.13.7}" - -"${TERRAFORM}" -v - -cp providers.tf.example providers.tf - -"${TERRAFORM}" init -"${TERRAFORM}" fmt -write=true -"${TERRAFORM}" fmt -check=true -"${TERRAFORM}" validate -var "deployment_id=validate" -var "dns_managed_zone=validate-fake.com" -var "email=fake@mailinator.com" - -find examples -maxdepth 1 -mindepth 1 -type d | while read -r example ; do - cp providers.tf "${example}" - cp versions.tf "${example}" - ( - cd "${example}" - pwd - echo "${example}" - "${TERRAFORM}" init - "${TERRAFORM}" fmt -check=true - "${TERRAFORM}" validate -var "deployment_id=citest" - ) -done diff --git a/pipeline/run_terraform.sh b/pipeline/run_terraform.sh deleted file mode 100755 index fc54998..0000000 --- a/pipeline/run_terraform.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -TERRAFORM="${TERRAFORM:-terraform}" - -"${TERRAFORM}" -v - -echo "$GCP_TOKEN" >/tmp/account.json - -set -xe - -if [ ! -f /tmp/account.json ]; then - echo "google credential json does not exists" -fi - -export GOOGLE_APPLICATION_CREDENTIALS='/tmp/account.json' - -# unique deployment ID to avoid collisions in CI -# needs to be 32 characters or less and start with letter -DEPLOYMENT_ID=ci$(echo "${CIRCLE_PROJECT_REPONAME}${CIRCLE_SHA1}" | md5sum | awk '{print substr($1,0,5)}') -ZONAL='true' - -if [[ "$REGIONAL" -eq 1 ]]; then - DEPLOYMENT_ID=regional$DEPLOYMENT_ID - ZONAL=false -fi - -echo "$DEPLOYMENT_ID" - -cp providers.tf.example "examples/$EXAMPLE/providers.tf" -cp backend.tf.example "examples/$EXAMPLE/backend.tf" -cp versions.tf "examples/$EXAMPLE/versions.tf" -cd "examples/$EXAMPLE" -sed -i "s/REPLACE/$DEPLOYMENT_ID/g" backend.tf - -"${TERRAFORM}" init - -if [[ "$DESTROY" -eq 1 ]]; then - "${TERRAFORM}" destroy --auto-approve -var "deployment_id=$DEPLOYMENT_ID" -var "zonal=$ZONAL" -lock=false -refresh=false -else - # this helps to fail fast in the pipeline, but it's not necessary - "${TERRAFORM}" apply --auto-approve -var "deployment_id=$DEPLOYMENT_ID" -var "zonal=$ZONAL" -lock=false --target=module.astronomer_gcp.google_service_networking_connection.private_vpc_connection - "${TERRAFORM}" apply --auto-approve -var "deployment_id=$DEPLOYMENT_ID" -var "zonal=$ZONAL" -lock=false -fi diff --git a/pipeline/update_cloud_module.sh b/pipeline/update_cloud_module.sh deleted file mode 100755 index 9f258b8..0000000 --- a/pipeline/update_cloud_module.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - - -apk add sed -# Install Terraform so we can do 'terraform fmt' -TERRAFORM_VERSION=0.12.4 -dir=$(pwd) -mkdir /opt/terraform_install && cd /opt/terraform_install && \ - wget https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ - unzip terraform_${TERRAFORM_VERSION}_linux_amd64.zip && \ - mv terraform /usr/local/bin/ -cd "$dir" - -cd /tmp -eval "$(ssh-agent)" -echo "$DEPLOY_KEY" | ssh-add - -mkdir -p "$HOME/.ssh" - -set -xe - -ssh-keyscan -t rsa github.com >> "$HOME"/.ssh/known_hosts -git clone git@github.com:astronomer/terraform-google-astronomer-cloud.git -cd terraform-google-astronomer-cloud -ls -sed -i "0,/version\\s*=\\s*\"[0-9]*\\.[0-9]*\\.[0-9]*\"/ s//version = \"$DRONE_TAG\"/" main.tf -terraform fmt -git add main.tf -git status -git config --global user.email "steven@astronomer.io" -git config --global user.name "Drone CI" -git commit -m "Drone CI: Update GCP infra module to $DRONE_TAG" -git push origin master diff --git a/providers.tf.example b/providers.tf.example deleted file mode 100644 index 1ce5b1a..0000000 --- a/providers.tf.example +++ /dev/null @@ -1,19 +0,0 @@ -provider "google" { - region = "us-east4" - zone = "us-east4-a" - project = "astronomer-cloud-dev-236021" -} - -provider "google-beta" { - region = "us-east4" - zone = "us-east4-a" - project = "astronomer-cloud-dev-236021" -} - -provider "acme" { - server_url = "https://acme-staging-v02.api.letsencrypt.org/directory" -} - -provider "spotinst" { - token = var.spotinist_token -} diff --git a/terraform.tfvars.sample b/terraform.tfvars.sample deleted file mode 100644 index b8cc97e..0000000 --- a/terraform.tfvars.sample +++ /dev/null @@ -1,7 +0,0 @@ -project = "astronomer-cloud-dev-236021" -bastion_admin_emails = ["steven@astronomer.io"] -bastion_user_emails = ["steven@astronomer.io"] -gke_secondary_ip_ranges_pods = "10.32.0.0/14" -gke_secondary_ip_ranges_services = "10.98.0.0/20" -deployment_id = "steven" -dns_managed_zone = "steven-zone" diff --git a/bastion.tf b/terraform/bastion.tf similarity index 100% rename from bastion.tf rename to terraform/bastion.tf diff --git a/buckets.tf b/terraform/buckets.tf similarity index 100% rename from buckets.tf rename to terraform/buckets.tf diff --git a/db.tf b/terraform/db.tf similarity index 100% rename from db.tf rename to terraform/db.tf diff --git a/domain.tf b/terraform/domain.tf similarity index 100% rename from domain.tf rename to terraform/domain.tf diff --git a/firewalls.tf b/terraform/firewalls.tf similarity index 100% rename from firewalls.tf rename to terraform/firewalls.tf diff --git a/iam.tf b/terraform/iam.tf similarity index 97% rename from iam.tf rename to terraform/iam.tf index e1820ef..1679799 100644 --- a/iam.tf +++ b/terraform/iam.tf @@ -47,7 +47,8 @@ resource "google_project_iam_custom_role" "velero_server" { "compute.snapshots.create", "compute.snapshots.useReadOnly", "compute.snapshots.delete", - "compute.zones.get" + "compute.zones.get", + "storage.objects.list" ] } diff --git a/locals.tf b/terraform/locals.tf similarity index 91% rename from locals.tf rename to terraform/locals.tf index 4b1e1c6..5ed526b 100644 --- a/locals.tf +++ b/terraform/locals.tf @@ -54,4 +54,10 @@ locals { # node_version = var.node_version == "" ? data.google_container_engine_versions.gke.latest_node_version : var.node_version gke_nodepool_network_tags = data.google_compute_instance.sample_instance.tags + + tls_key = var.lets_encrypt ? tls_private_key.cert_private_key.0.private_key_pem : "" + tls_cert = var.dns_managed_zone == "" ? "" : <