From 7211d87b8aadd359a575fb137cb693f37ec55404 Mon Sep 17 00:00:00 2001 From: eeaton Date: Wed, 5 Jun 2024 17:26:43 +0100 Subject: [PATCH] chore(monitoring): remove unused monitoring project (#1200) --- 0-bootstrap/README.md | 4 +- 0-bootstrap/terraform.example.tfvars | 9 ++-- 0-bootstrap/variables.tf | 14 ++--- 1-org/README.md | 4 +- 2-environments/README.md | 4 +- 2-environments/envs/development/README.md | 1 - 2-environments/envs/development/outputs.tf | 5 -- 2-environments/envs/nonproduction/README.md | 1 - 2-environments/envs/nonproduction/outputs.tf | 5 -- 2-environments/envs/production/README.md | 1 - 2-environments/envs/production/outputs.tf | 5 -- 2-environments/modules/env_baseline/README.md | 1 - 2-environments/modules/env_baseline/iam.tf | 25 --------- .../modules/env_baseline/monitoring.tf | 53 ------------------- .../modules/env_baseline/outputs.tf | 5 -- 2-environments/terraform.example.tfvars | 2 - 3-networks-dual-svpc/README.md | 2 +- 3-networks-hub-and-spoke/README.md | 2 +- 4-projects/README.md | 2 +- 5-app-infra/README.md | 2 +- README.md | 14 +---- .../foundation-deployer/global.tfvars.example | 1 - helpers/foundation-deployer/stages/data.go | 9 ++-- test/integration/envs/envs_test.go | 12 ----- test/setup/outputs.tf | 17 ++++-- 25 files changed, 34 insertions(+), 166 deletions(-) delete mode 100644 2-environments/modules/env_baseline/iam.tf delete mode 100644 2-environments/modules/env_baseline/monitoring.tf diff --git a/0-bootstrap/README.md b/0-bootstrap/README.md index 0acbf554b..268dc7f7b 100644 --- a/0-bootstrap/README.md +++ b/0-bootstrap/README.md @@ -15,7 +15,7 @@ stages. 1-org -Sets up top-level shared folders, monitoring and networking projects, and +Sets up top-level shared folders, networking projects, and organization-level logging, and sets baseline security settings through organizational policy. @@ -311,7 +311,7 @@ Each step has instructions for this change. | default\_region\_gcs | Case-Sensitive default region to create gcs resources where applicable. | `string` | `"US"` | no | | default\_region\_kms | Secondary default region to create kms resources where applicable. | `string` | `"us"` | no | | folder\_prefix | Name prefix to use for folders created. Should be the same in all steps. | `string` | `"fldr"` | no | -| groups | Contain the details of the Groups to be created. | 
object({
    create_required_groups = optional(bool, false)
    create_optional_groups = optional(bool, false)
    billing_project        = optional(string, null)
    required_groups = object({
      group_org_admins           = string
      group_billing_admins       = string
      billing_data_users         = string
      audit_data_users           = string
      monitoring_workspace_users = string
    })
    optional_groups = optional(object({
      gcp_security_reviewer    = optional(string, "")
      gcp_network_viewer       = optional(string, "")
      gcp_scc_admin            = optional(string, "")
      gcp_global_secrets_admin = optional(string, "")
      gcp_kms_admin            = optional(string, "")
    }), {})
  })
| n/a | yes | +| groups | Contain the details of the Groups to be created. | 
object({
    create_required_groups = optional(bool, false)
    create_optional_groups = optional(bool, false)
    billing_project        = optional(string, null)
    required_groups = object({
      group_org_admins     = string
      group_billing_admins = string
      billing_data_users   = string
      audit_data_users     = string
    })
    optional_groups = optional(object({
      gcp_security_reviewer    = optional(string, "")
      gcp_network_viewer       = optional(string, "")
      gcp_scc_admin            = optional(string, "")
      gcp_global_secrets_admin = optional(string, "")
      gcp_kms_admin            = optional(string, "")
    }), {})
  })
| n/a | yes | | initial\_group\_config | Define the group configuration when it is initialized. Valid values are: WITH\_INITIAL\_OWNER, EMPTY and INITIAL\_GROUP\_CONFIG\_UNSPECIFIED. | `string` | `"WITH_INITIAL_OWNER"` | no | | org\_id | GCP Organization ID | `string` | n/a | yes | | org\_policy\_admin\_role | Additional Org Policy Admin role for admin group. You can use this for testing purposes. | `bool` | `false` | no | diff --git a/0-bootstrap/terraform.example.tfvars b/0-bootstrap/terraform.example.tfvars index 88e93f7d7..9b85c95fa 100644 --- a/0-bootstrap/terraform.example.tfvars +++ b/0-bootstrap/terraform.example.tfvars @@ -25,11 +25,10 @@ groups = { # create_optional_groups = false # Change to true to create the optional_groups # billing_project = "REPLACE_ME" # Fill to create required or optional groups required_groups = { - group_org_admins = "REPLACE_ME" # example "gcp-organization-admins@example.com" - group_billing_admins = "REPLACE_ME" # example "gcp-billing-admins@example.com" - billing_data_users = "REPLACE_ME" # example "gcp-billing-data@example.com" - audit_data_users = "REPLACE_ME" # example "gcp-audit-data@example.com" - monitoring_workspace_users = "REPLACE_ME" # example "gcp-monitoring-workspace@example.com" + group_org_admins = "REPLACE_ME" # example "gcp-organization-admins@example.com" + group_billing_admins = "REPLACE_ME" # example "gcp-billing-admins@example.com" + billing_data_users = "REPLACE_ME" # example "gcp-billing-data@example.com" + audit_data_users = "REPLACE_ME" # example "gcp-audit-data@example.com" } # optional_groups = { # gcp_security_reviewer = "" #"gcp_security_reviewer_local_test@example.com" diff --git a/0-bootstrap/variables.tf b/0-bootstrap/variables.tf index ffd01bc83..39b993abb 100644 --- a/0-bootstrap/variables.tf +++ b/0-bootstrap/variables.tf @@ -100,11 +100,10 @@ variable "groups" { create_optional_groups = optional(bool, false) billing_project = optional(string, null) required_groups = object({ - group_org_admins = string - group_billing_admins = string - billing_data_users = string - audit_data_users = string - monitoring_workspace_users = string + group_org_admins = string + group_billing_admins = string + billing_data_users = string + audit_data_users = string }) optional_groups = optional(object({ gcp_security_reviewer = optional(string, "") @@ -139,11 +138,6 @@ variable "groups" { condition = var.groups.required_groups.audit_data_users != "" error_message = "The group audit_data_users is invalid, it must be a valid email" } - - validation { - condition = var.groups.required_groups.monitoring_workspace_users != "" - error_message = "The group monitoring_workspace_users is invalid, it must be a valid email" - } } variable "initial_group_config" { diff --git a/1-org/README.md b/1-org/README.md index 7c82c8606..1be58e521 100644 --- a/1-org/README.md +++ b/1-org/README.md @@ -15,7 +15,7 @@ stages. 1-org (this file) -Sets up top-level shared folders, monitoring and networking projects, and +Sets up top-level shared folders, networking projects, and organization-level logging, and sets baseline security settings through organizational policy. @@ -55,7 +55,7 @@ For an overview of the architecture and the parts, see the ## Purpose -The purpose of this step is to set up top-level shared folders, monitoring and networking projects, organization-level logging, and baseline security settings through organizational policies. +The purpose of this step is to set up top-level shared folders, networking projects, organization-level logging, and baseline security settings through organizational policies. ## Prerequisites diff --git a/2-environments/README.md b/2-environments/README.md index 05eb3e7ba..d0624593a 100644 --- a/2-environments/README.md +++ b/2-environments/README.md @@ -15,7 +15,7 @@ stages. 1-org -Sets up top level shared folders, monitoring and networking projects, and +Sets up top level shared folders, networking projects, and organization-level logging, and sets baseline security settings through organizational policy. @@ -61,8 +61,6 @@ The purpose of this step is to setup development, nonproduction, and production 1. 0-bootstrap executed successfully. 1. 1-org executed successfully. -1. Cloud Identity / Google Workspace group for monitoring admins. -1. Membership in the monitoring admins group for user running Terraform. ### Troubleshooting diff --git a/2-environments/envs/development/README.md b/2-environments/envs/development/README.md index 9a7b15a52..15e492c25 100644 --- a/2-environments/envs/development/README.md +++ b/2-environments/envs/development/README.md @@ -13,6 +13,5 @@ | env\_folder | Environment folder created under parent. | | env\_kms\_project\_id | Project for environment Cloud Key Management Service (KMS). | | env\_secrets\_project\_id | Project for environment related secrets. | -| monitoring\_project\_id | Project for monitoring infra. | diff --git a/2-environments/envs/development/outputs.tf b/2-environments/envs/development/outputs.tf index b38dee3db..5ffda4515 100644 --- a/2-environments/envs/development/outputs.tf +++ b/2-environments/envs/development/outputs.tf @@ -19,11 +19,6 @@ output "env_folder" { value = module.env.env_folder } -output "monitoring_project_id" { - description = "Project for monitoring infra." - value = module.env.monitoring_project_id -} - output "env_secrets_project_id" { description = "Project for environment related secrets." value = module.env.env_secrets_project_id diff --git a/2-environments/envs/nonproduction/README.md b/2-environments/envs/nonproduction/README.md index 9a7b15a52..15e492c25 100644 --- a/2-environments/envs/nonproduction/README.md +++ b/2-environments/envs/nonproduction/README.md @@ -13,6 +13,5 @@ | env\_folder | Environment folder created under parent. | | env\_kms\_project\_id | Project for environment Cloud Key Management Service (KMS). | | env\_secrets\_project\_id | Project for environment related secrets. | -| monitoring\_project\_id | Project for monitoring infra. | diff --git a/2-environments/envs/nonproduction/outputs.tf b/2-environments/envs/nonproduction/outputs.tf index b38dee3db..5ffda4515 100644 --- a/2-environments/envs/nonproduction/outputs.tf +++ b/2-environments/envs/nonproduction/outputs.tf @@ -19,11 +19,6 @@ output "env_folder" { value = module.env.env_folder } -output "monitoring_project_id" { - description = "Project for monitoring infra." - value = module.env.monitoring_project_id -} - output "env_secrets_project_id" { description = "Project for environment related secrets." value = module.env.env_secrets_project_id diff --git a/2-environments/envs/production/README.md b/2-environments/envs/production/README.md index 94b71c18d..99e479ae3 100644 --- a/2-environments/envs/production/README.md +++ b/2-environments/envs/production/README.md @@ -15,7 +15,6 @@ | env\_folder | Environment folder created under parent. | | env\_kms\_project\_id | Project for environment Cloud Key Management Service (KMS). | | env\_secrets\_project\_id | Project for environment related secrets. | -| monitoring\_project\_id | Project for monitoring infra. | diff --git a/2-environments/envs/production/outputs.tf b/2-environments/envs/production/outputs.tf index e3216e75d..822cdb4fa 100644 --- a/2-environments/envs/production/outputs.tf +++ b/2-environments/envs/production/outputs.tf @@ -19,11 +19,6 @@ output "env_folder" { value = module.env.env_folder } -output "monitoring_project_id" { - description = "Project for monitoring infra." - value = module.env.monitoring_project_id -} - output "env_secrets_project_id" { description = "Project for environment related secrets." value = module.env.env_secrets_project_id diff --git a/2-environments/modules/env_baseline/README.md b/2-environments/modules/env_baseline/README.md index 87d087f81..889e8a199 100644 --- a/2-environments/modules/env_baseline/README.md +++ b/2-environments/modules/env_baseline/README.md @@ -19,6 +19,5 @@ | env\_folder | Environment folder created under parent. | | env\_kms\_project\_id | Project for environment Cloud Key Management Service (KMS). | | env\_secrets\_project\_id | Project for environment secrets. | -| monitoring\_project\_id | Project for monitoring infra. | diff --git a/2-environments/modules/env_baseline/iam.tf b/2-environments/modules/env_baseline/iam.tf deleted file mode 100644 index ffe9acbc3..000000000 --- a/2-environments/modules/env_baseline/iam.tf +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************** - Monitoring - IAM -*****************************************/ - -resource "google_project_iam_member" "monitoring_viewer" { - project = module.monitoring_project.project_id - role = "roles/monitoring.viewer" - member = "group:${local.required_groups["monitoring_workspace_users"]}" -} diff --git a/2-environments/modules/env_baseline/monitoring.tf b/2-environments/modules/env_baseline/monitoring.tf deleted file mode 100644 index 01bc519d5..000000000 --- a/2-environments/modules/env_baseline/monitoring.tf +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/****************************************** - Projects for monitoring workspaces -*****************************************/ - -module "monitoring_project" { - source = "terraform-google-modules/project-factory/google" - version = "~> 15.0" - - random_project_id = true - random_project_id_length = 4 - name = "${local.project_prefix}-${var.environment_code}-monitoring" - org_id = local.org_id - billing_account = local.billing_account - folder_id = google_folder.env.id - disable_services_on_destroy = false - depends_on = [time_sleep.wait_60_seconds] - activate_apis = [ - "logging.googleapis.com", - "monitoring.googleapis.com", - "billingbudgets.googleapis.com" - ] - - labels = { - environment = var.env - application_name = "env-monitoring" - billing_code = "1234" - primary_contact = "example1" - secondary_contact = "example2" - business_code = "shared" - env_code = var.environment_code - vpc = "none" - } - budget_alert_pubsub_topic = var.project_budget.monitoring_alert_pubsub_topic - budget_alert_spent_percents = var.project_budget.monitoring_alert_spent_percents - budget_amount = var.project_budget.monitoring_budget_amount - budget_alert_spend_basis = var.project_budget.monitoring_budget_alert_spend_basis -} diff --git a/2-environments/modules/env_baseline/outputs.tf b/2-environments/modules/env_baseline/outputs.tf index 9dff9cd22..15cd2524f 100644 --- a/2-environments/modules/env_baseline/outputs.tf +++ b/2-environments/modules/env_baseline/outputs.tf @@ -19,11 +19,6 @@ output "env_folder" { value = google_folder.env.name } -output "monitoring_project_id" { - description = "Project for monitoring infra." - value = module.monitoring_project.project_id -} - output "env_secrets_project_id" { description = "Project for environment secrets." value = module.env_secrets.project_id diff --git a/2-environments/terraform.example.tfvars b/2-environments/terraform.example.tfvars index 1272eef0e..49cf2d8aa 100644 --- a/2-environments/terraform.example.tfvars +++ b/2-environments/terraform.example.tfvars @@ -14,6 +14,4 @@ * limitations under the License. */ -monitoring_workspace_users = "gcp-monitoring-admins@example.com" - remote_state_bucket = "REMOTE_STATE_BUCKET" diff --git a/3-networks-dual-svpc/README.md b/3-networks-dual-svpc/README.md index 604c72822..690e1157d 100644 --- a/3-networks-dual-svpc/README.md +++ b/3-networks-dual-svpc/README.md @@ -15,7 +15,7 @@ stages. 1-org -Sets up top level shared folders, monitoring and networking projects, and +Sets up top level shared folders, networking projects, and organization-level logging, and sets baseline security settings through organizational policy. diff --git a/3-networks-hub-and-spoke/README.md b/3-networks-hub-and-spoke/README.md index 743c275d7..aac22581e 100644 --- a/3-networks-hub-and-spoke/README.md +++ b/3-networks-hub-and-spoke/README.md @@ -15,7 +15,7 @@ stages. 1-org -Sets up top level shared folders, monitoring and networking projects, and +Sets up top level shared folders, networking projects, and organization-level logging, and sets baseline security settings through organizational policy. diff --git a/4-projects/README.md b/4-projects/README.md index b4a13e064..6ff36da28 100644 --- a/4-projects/README.md +++ b/4-projects/README.md @@ -15,7 +15,7 @@ stages. 1-org -Sets up top level shared folders, monitoring and networking projects, and +Sets up top level shared folders, networking projects, and organization-level logging, and sets baseline security settings through organizational policy. diff --git a/5-app-infra/README.md b/5-app-infra/README.md index c9174632d..e1b220850 100644 --- a/5-app-infra/README.md +++ b/5-app-infra/README.md @@ -15,7 +15,7 @@ stages. 1-org -Sets up top-level shared folders, monitoring and networking projects, +Sets up top-level shared folders, networking projects, organization-level logging, and baseline security settings through organizational policies. diff --git a/README.md b/README.md index d11a1d39e..84409c7c9 100644 --- a/README.md +++ b/README.md @@ -126,31 +126,22 @@ This stage only creates the projects and enables the correct APIs, the following ### [2. environments](./2-environments/) -The purpose of this stage is to set up the environments folders used for projects that contain monitoring and secrets projects. +The purpose of this stage is to set up the environments folders that contain shared projects for each environemnt. This will create the following folder and project structure: ``` example-organization └── fldr-development - ├── prj-d-monitoring ├── prj-p-kms └── prj-d-secrets └── fldr-nonproduction - ├── prj-n-monitoring ├── prj-n-kms └── prj-n-secrets └── fldr-production - ├── prj-p-monitoring ├── prj-p-kms └── prj-p-secrets ``` -#### Monitoring - -Under the environment folder, a project is created per environment (`development`, `nonproduction`, and `production`), which is intended to be used as a [Cloud Monitoring workspace](https://cloud.google.com/monitoring/workspaces) for all projects in that environment. -Please note that creating the [workspace and linking projects](https://cloud.google.com/monitoring/workspaces/create) can currently only be completed through the Cloud Console. -If you have strong IAM requirements for these monitoring workspaces, it is worth considering creating these at a more granular level, such as per business unit or per application. - #### KMS Under the environment folder, a project is created per environment (`development`, `nonproduction`, and `production`), which is intended to be used by [Cloud Key Management](https://cloud.google.com/security-key-management) for KMS resources shared by the environment. @@ -275,7 +266,6 @@ example-organization ├── prj-p-shared-base └── prj-p-shared-restricted └── fldr-development - ├── prj-d-monitoring ├── prj-d-kms └── prj-d-secrets └── fldr-development-bu1 @@ -291,7 +281,6 @@ example-organization ├── prj-d-bu2-sample-restrict └── prj-d-bu2-sample-peering └── fldr-nonproduction - ├── prj-n-monitoring ├── prj-n-kms └── prj-n-secrets └── fldr-nonproduction-bu1 @@ -307,7 +296,6 @@ example-organization ├── prj-n-bu2-sample-restrict └── prj-n-bu2-sample-peering └── fldr-production - ├── prj-p-monitoring ├── prj-p-kms └── prj-p-secrets └── fldr-production-bu1 diff --git a/helpers/foundation-deployer/global.tfvars.example b/helpers/foundation-deployer/global.tfvars.example index be6a63911..4ab6193e3 100644 --- a/helpers/foundation-deployer/global.tfvars.example +++ b/helpers/foundation-deployer/global.tfvars.example @@ -69,7 +69,6 @@ groups = { group_billing_admins = "REPLACE_ME" # "gcp-billing-admins@example.com" billing_data_users = "REPLACE_ME" #"billing_data_users_local_test@example.com" audit_data_users = "REPLACE_ME" #"audit_data_users_local_test@example.com" - monitoring_workspace_users = "REPLACE_ME" #"monitoring_workspace_users_local_test@example.com" } optional_groups = { gcp_security_reviewer = "" #"gcp_security_reviewer_local_test@example.com" diff --git a/helpers/foundation-deployer/stages/data.go b/helpers/foundation-deployer/stages/data.go index ab889921f..3a8adb1eb 100644 --- a/helpers/foundation-deployer/stages/data.go +++ b/helpers/foundation-deployer/stages/data.go @@ -96,11 +96,10 @@ type ServerAddress struct { } type RequiredGroups struct { - GroupOrgAdmins string `cty:"group_org_admins"` - GroupBillingAdmins string `cty:"group_billing_admins"` - BillingDataUsers string `cty:"billing_data_users"` - AuditDataUsers string `cty:"audit_data_users"` - MonitoringWorkspaceUsers string `cty:"monitoring_workspace_users"` + GroupOrgAdmins string `cty:"group_org_admins"` + GroupBillingAdmins string `cty:"group_billing_admins"` + BillingDataUsers string `cty:"billing_data_users"` + AuditDataUsers string `cty:"audit_data_users"` } type OptionalGroups struct { diff --git a/test/integration/envs/envs_test.go b/test/integration/envs/envs_test.go index 8a4caee98..ed06d1786 100644 --- a/test/integration/envs/envs_test.go +++ b/test/integration/envs/envs_test.go @@ -22,7 +22,6 @@ import ( "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" - "github.com/gruntwork-io/terratest/modules/terraform" "github.com/stretchr/testify/assert" "github.com/terraform-google-modules/terraform-example-foundation/test/integration/testutils" @@ -86,23 +85,12 @@ func TestEnvs(t *testing.T) { } assert.Subset([]string{envName}, fldrTagValue, fmt.Sprintf("tag value should be %s for %s env folder", envName, envName)) - monitoringWorkspaceUsers := terraform.OutputMap(t, bootstrap.GetTFOptions(), "required_groups")["monitoring_workspace_users"] for _, projectEnvOutput := range []struct { projectOutput string role string group string apis []string }{ - { - projectOutput: "monitoring_project_id", - role: "roles/monitoring.viewer", - group: monitoringWorkspaceUsers, - apis: []string{ - "logging.googleapis.com", - "monitoring.googleapis.com", - "billingbudgets.googleapis.com", - }, - }, { projectOutput: "env_kms_project_id", apis: []string{ diff --git a/test/setup/outputs.tf b/test/setup/outputs.tf index c21342b3b..bff0125e8 100644 --- a/test/setup/outputs.tf +++ b/test/setup/outputs.tf @@ -47,11 +47,18 @@ output "group_email" { output "groups" { value = { required_groups = { - group_org_admins = var.group_email - group_billing_admins = var.group_email - billing_data_users = var.group_email - audit_data_users = var.group_email - monitoring_workspace_users = var.group_email + group_org_admins = var.group_email + group_billing_admins = var.group_email + billing_data_users = var.group_email + audit_data_users = var.group_email + }, + optional_groups = { + gcp_security_reviewer = var.group_email + gcp_network_viewer = var.group_email + gcp_scc_admin = var.group_email + gcp_global_secrets_admin = var.group_email + gcp_kms_admin = var.group_email + } } }