Restructuring Repo (#70)

* Restcturing repo * Fixing job * Typo fix * Removing not require job * Removing not require job * Adding dependency. * Fixing tf init command * Updating terraform module version. * Updating deployment_id * Typo fix * Using CIRCLE_WORKFLOW_ID replacing CIRCLE_SHA1 * Provisioning Gcloud credentials from token env. * Provisioning Gcloud credentials from token env. * Typo fix. * Updating script. * Now Source bash env * Typo fix. * Removing un-used scripts. * Adding storage role. * Using local to derive output value. * Adding help for make target.
astronomer · Nov 21, 2022 · ac70861 · ac70861
1 parent cc5178a
commit ac70861
Show file tree

Hide file tree

Showing 30 changed files with 277 additions and 301 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -17,135 +17,95 @@ executors:
           TFENV_AUTO_INSTALL: true
 
 workflows:
-  test:
+  pre-commit:
     jobs:
       - run_pre_commit
   terraform_pipeline:
     jobs:
-      - terraform_lint:
+      - terraform-apply:
+          region: us-east4
+          project_id: astronomer-cloud-dev-236021
+          tf_state_bucket: cloud2-dev-terraform
+          zonal: true
           context:
-            - slack
-      - terraform_apply:
-          requires:
-            - terraform_lint
-          context:
-            - slack
             - gcp-dev
-      - terraform_destroy:
-          requires:
-            - terraform_apply
+      - terraform-destroy:
+          region: us-east4
+          project_id: astronomer-cloud-dev-236021
+          tf_state_bucket: cloud2-dev-terraform
+          zonal: true
           context:
-            - slack
             - gcp-dev
-      #- git_tag:
-      #    context:
-      #      - github-repo
-      #      - slack
-      #    requires:
-      #      - terraform_destroy
-      #    filters:
-      #      branches:
-      #        only:
-      #          - master
+          requires:
+            - terraform-apply
 
 jobs:
   run_pre_commit:
     executor: pre-commit-executor
+    resource_class: small
     steps:
       - checkout
       - pre-commit-commands
 
-  terraform_lint:
-    executor: pre-commit-executor
-    steps:
-      - checkout
-      - pre-commit-commands
-      - slack/notify:
-          event: fail
-          template: basic_fail_1
-
-  terraform_apply:
+  terraform-apply:
     executor: terraform-executor
+    resource_class: medium
+    parameters:
+      region:
+        type: string
+      project_id:
+        type: string
+      tf_state_bucket:
+        type: string
+      zonal:
+        type: boolean
     steps:
       - checkout
-      - run: EXAMPLE=from_scratch pipeline/run_terraform.sh
+      - terraform-install
+      - run:
+          name: Provision GCloud Credentials
+          command: |
+            echo "$GCP_TOKEN" > /tmp/account.json
+            echo 'export GOOGLE_APPLICATION_CREDENTIALS=/tmp/account.json' >>"$BASH_ENV"
+            source "$BASH_ENV"
       - run:
-          command: DESTROY=1 EXAMPLE=from_scratch pipeline/run_terraform.sh
+          name: Run Terraform Apply
+          command: |
+            DEPLOYMENT_ID=ci$(echo "${CIRCLE_PROJECT_REPONAME}${CIRCLE_WORKFLOW_ID}" | md5sum | awk '{print substr($1,0,5)}')
+            make init apply-auto-approve PROJECT_ID=<< parameters.project_id >> REGION=<< parameters.region >> DEPLOYMENT_ID=$DEPLOYMENT_ID ZONAL=<< parameters.zonal >> TF_STATE_BUCKET=<< parameters.tf_state_bucket >>
+      - run:
+          name: Run Terraform Destroy
+          command: |
+            DEPLOYMENT_ID=ci$(echo "${CIRCLE_PROJECT_REPONAME}${CIRCLE_WORKFLOW_ID}" | md5sum | awk '{print substr($1,0,5)}')
+            make init destroy-auto-approve PROJECT_ID=<< parameters.project_id >> REGION=<< parameters.region >> DEPLOYMENT_ID=$DEPLOYMENT_ID ZONAL=<< parameters.zonal >> TF_STATE_BUCKET=<< parameters.tf_state_bucket >>
           when: on_fail
-      - slack/notify:
-          event: fail
-          template: basic_fail_1
-
-  terraform_destroy:
-    executor: terraform-executor
-    steps:
-      - checkout
-      - run: DESTROY=1 EXAMPLE=from_scratch pipeline/run_terraform.sh
-      - slack/notify:
-          event: fail
-          template: basic_fail_1
 
-  git_tag:
+  terraform-destroy:
     executor: terraform-executor
+    resource_class: medium
+    parameters:
+      region:
+        type: string
+      project_id:
+        type: string
+      tf_state_bucket:
+        type: string
+      zonal:
+        type: boolean
     steps:
       - checkout
-      - run: git remote set-url origin "https://astro-astronomer:${GITHUB_TOKEN}@github.com/astronomer/${CIRCLE_PROJECT_REPONAME}.git"
-      - run: git tag 1.2.<< pipeline.number >>
-      - run: git push origin << pipeline.git.branch >> --tags
-      - slack/notify:
-          event: fail
-          template: basic_fail_1
-      - slack/notify:
-          event: pass
-          custom: |
-            {
-              "text": "",
-              "blocks": [
-                {
-                  "type": "header",
-                  "text": {
-                    "type": "plain_text",
-                    "text": "Release Successful! :tada:",
-                    "emoji": true
-                  }
-                },
-                {
-                  "type": "section",
-                  "fields": [
-                    {
-                      "type": "mrkdwn",
-                      "text": "*Project*:\n$CIRCLE_PROJECT_REPONAME"
-                    },
-                    {
-                      "type": "mrkdwn",
-                      "text": "*When*:\n$(date +'%m/%d/%Y %T')"
-                    },
-                    {
-                      "type": "mrkdwn",
-                      "text": "*Tag*:\n1.2.<< pipeline.number >>"
-                    }
-                  ],
-                  "accessory": {
-                    "type": "image",
-                    "image_url": "https://assets.brandfolder.com/otz5mn-bw4j2w-6jzqo8/original/circle-logo-badge-black.png",
-                    "alt_text": "CircleCI logo"
-                  }
-                },
-                {
-                  "type": "actions",
-                  "elements": [
-                    {
-                      "type": "button",
-                      "text": {
-                        "type": "plain_text",
-                        "text": "View Job"
-                      },
-                      "url": "${CIRCLE_BUILD_URL}"
-                    }
-                  ]
-                }
-              ]
-            }
+      - terraform-install
+      - run:
+          name: Provision GCloud Credentials
+          command: |
+            echo "$GCP_TOKEN" > /tmp/account.json
+            echo 'export GOOGLE_APPLICATION_CREDENTIALS=/tmp/account.json' >>"$BASH_ENV"
+            source "$BASH_ENV"
+      - run:
+          name: Run Terraform Destroy
+          command: |
+            DEPLOYMENT_ID=ci$(echo "${CIRCLE_PROJECT_REPONAME}${CIRCLE_WORKFLOW_ID}" | md5sum | awk '{print substr($1,0,5)}')
+            make init destroy-auto-approve PROJECT_ID=<< parameters.project_id >> REGION=<< parameters.region >> DEPLOYMENT_ID=$DEPLOYMENT_ID ZONAL=<< parameters.zonal >> TF_STATE_BUCKET=<< parameters.tf_state_bucket >>
 
 commands:
   pre-commit-commands:

diff --git a/.terraform-version b/.terraform-version
@@ -1 +1 @@
-1.2.5
+1.3.4
diff --git a/Makefile b/Makefile
@@ -0,0 +1,116 @@
+# Defaults
+REV:=$(shell git rev-parse --short HEAD)
+DATE:=$(shell date +%Y.%m.%d-%H.%M.%S)
+BRANCH:=$(shell git rev-parse --abbrev-ref HEAD)
+COMMIT:=$(MODULE)_$(INFRA_VALUES)_$(DATE)_$(REV)
+
+CURRENT_DIR_PATH:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
+
+# Inputs
+REGION:=NO_REGION
+PROJECT_ID:=PROJECT_ID
+DEPLOYMENT_ID:=NO_DEPLOYMENT_ID
+ZONAL:=NO_ZONAL
+
+TF_STATE_BUCKET:=NO_TF_STATE_BUCKET
+
+# Terraform Setup
+TF_MODULE_PATH:=test
+TF_PLAN_FILE:=test_$(DEPLOYMENT_ID)_$(REV)
+
+
+.PHONY: help
+help: ## Print Makefile help.
+	@grep -Eh '^[a-z.A-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-29s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: pre-commit
+pre-commit: ## It will run pre-commit target. It will run target `install`, `install-hooks` and then `run`.
+	pre-commit install
+	pre-commit install-hooks
+	pre-commit run --all-files --show-diff-on-failure
+
+.PHONY: install
+install: ## It will install terraform version defined into `.terraform-version` file.
+	tfenv install
+
+.PHONY: init
+init: clean ## It will run Terraform init. Required params are `DEPLOYMENT_ID` and `TF_STATE_BUCKET`.
+	terraform -chdir=$(TF_MODULE_PATH) init \
+		-backend=true \
+		-backend-config='bucket=$(TF_STATE_BUCKET)' \
+		-backend-config='prefix=$(DEPLOYMENT_ID)/terraform.tfstate'
+
+.PHONY: upgrade
+upgrade: clean ## It will run Terraform init with `-upgrade` flag. Required params are `DEPLOYMENT_ID` and `TF_STATE_BUCKET`.
+	terraform -chdir=$(TF_MODULE_PATH) init \
+		-backend=true \
+		-bucket='bucket=cloud2-dev-terraform' \
+		-prefix='prefix=$(DEPLOYMENT_ID)/terraform.tfstate'
+		-upgrade
+
+.PHONY: validate
+validate: ## It will run Terraform validate. Required params are `DEPLOYMENT_ID` and `TF_STATE_BUCKET`.
+	terraform -chdir=$(TF_MODULE_PATH) validate
+
+.PHONY: plan
+plan: validate ## It will run Terraform plan. Required params are `PROJECT_ID`, `REGION`, `DEPLOYMENT_ID` and `ZONAL`.
+	terraform -chdir=$(TF_MODULE_PATH) plan \
+		-var 'project_id=$(PROJECT_ID)' \
+		-var 'region=$(REGION)' \
+		-var 'deployment_id=$(DEPLOYMENT_ID)' \
+		-var 'zonal=$(ZONAL)' \
+		-out=$(TF_PLAN_FILE).tfplan
+
+.PHONY: plan-destroy
+plan-destroy: validate ## It will run Terraform plan with `-destroy` flag to generate cleanup plan. Required params are `PROJECT_ID`, `REGION`, `DEPLOYMENT_ID` and `ZONAL`.
+	terraform -chdir=$(TF_MODULE_PATH) plan \
+		-destroy \
+		-var 'project_id=$(PROJECT_ID)' \
+		-var 'region=$(REGION)' \
+		-var 'deployment_id=$(DEPLOYMENT_ID)' \
+		-var 'zonal=$(ZONAL)' \
+		-out=$(TF_PLAN_FILE).tfplan
+
+.PHONY: apply
+apply: validate ## It will run Terraform apply. Required params are `PROJECT_ID`, `REGION`, `DEPLOYMENT_ID` and `ZONAL`.
+	terraform -chdir=$(TF_MODULE_PATH) apply \
+		-var 'project_id=$(PROJECT_ID)' \
+		-var 'region=$(REGION)' \
+		-var 'deployment_id=$(DEPLOYMENT_ID)' \
+		-var 'zonal=$(ZONAL)'
+
+.PHONY: apply-auto-approve
+apply-auto-approve: validate ## It will run Terraform apply with `-auto-approve` flag. Required params are `PROJECT_ID`, `REGION`, `DEPLOYMENT_ID` and `ZONAL`.
+	terraform -chdir=$(TF_MODULE_PATH) apply \
+		-auto-approve \
+		-var 'project_id=$(PROJECT_ID)' \
+		-var 'region=$(REGION)' \
+		-var 'deployment_id=$(DEPLOYMENT_ID)' \
+		-var 'zonal=$(ZONAL)'
+
+.PHONY: apply-plan
+apply-plan: ## It will run Terraform apply to plan.
+	terraform -chdir=$(TF_MODULE_PATH) apply $(TF_PLAN_FILE).tfplan
+
+.PHONY: destroy
+destroy: validate ## It will run Terraform destroy. Required params are `PROJECT_ID`, `REGION`, `DEPLOYMENT_ID` and `ZONAL`.
+	terraform -chdir=$(TF_MODULE_PATH) destroy \
+		-var 'project_id=$(PROJECT_ID)' \
+		-var 'region=$(REGION)' \
+		-var 'deployment_id=$(DEPLOYMENT_ID)' \
+		-var 'zonal=$(ZONAL)'
+
+.PHONY: destroy-auto-approve
+destroy-auto-approve: validate ## It will run Terraform destroy with `-auto-approve` flag. Required params are `PROJECT_ID`, `REGION`, `DEPLOYMENT_ID` and `ZONAL`.
+	terraform -chdir=$(TF_MODULE_PATH) destroy \
+		-auto-approve \
+		-var 'project_id=$(PROJECT_ID)' \
+		-var 'region=$(REGION)' \
+		-var 'deployment_id=$(DEPLOYMENT_ID)' \
+		-var 'zonal=$(ZONAL)'
+
+.PHONY: clean
+clean: ## Remove  terraform init file and dir
+	find . -name "*.terraform" -type d -exec rm -rf {} + || true
+	find . -name "*.terraform.lock.hcl" -type f -delete || true
+	find . -name "*.tfplan" -type f -delete || true
diff --git a/backend.tf.example b/backend.tf.example
diff --git a/examples/from_scratch/.gitignore b/examples/from_scratch/.gitignore
diff --git a/examples/versions.tf b/examples/versions.tf
diff --git a/pipeline/lint.sh b/pipeline/lint.sh