From de49e875c27bbb8b12e0aaf53a5cc74ed53258ac Mon Sep 17 00:00:00 2001 From: jbeemster Date: Tue, 1 Aug 2023 13:32:57 +1000 Subject: [PATCH] Update terraform example for GCP pipeline (closes #67) --- terraform/gcp/iglu_server/default/README.md | 14 +- terraform/gcp/iglu_server/default/main.tf | 18 +- .../gcp/iglu_server/default/terraform.tfvars | 2 +- terraform/gcp/iglu_server/default/versions.tf | 6 +- terraform/gcp/iglu_server/secure/README.md | 14 +- terraform/gcp/iglu_server/secure/main.tf | 6 +- .../gcp/iglu_server/secure/terraform.tfvars | 2 +- terraform/gcp/iglu_server/secure/versions.tf | 6 +- terraform/gcp/pipeline/default/README.md | 32 +-- .../default/bigquery.terraform.tfvars | 56 ----- terraform/gcp/pipeline/default/main.tf | 201 +++--------------- terraform/gcp/pipeline/default/outputs.tf | 2 +- .../gcp/pipeline/default/target_bigquery.tf | 70 ++++++ .../gcp/pipeline/default/target_postgres.tf | 95 +++++++++ ...gres.terraform.tfvars => terraform.tfvars} | 33 +-- terraform/gcp/pipeline/default/versions.tf | 6 +- terraform/gcp/pipeline/secure/README.md | 32 +-- .../pipeline/secure/bigquery.terraform.tfvars | 55 ----- terraform/gcp/pipeline/secure/main.tf | 171 +-------------- terraform/gcp/pipeline/secure/outputs.tf | 2 +- .../gcp/pipeline/secure/target_bigquery.tf | 68 ++++++ .../gcp/pipeline/secure/target_postgres.tf | 91 ++++++++ ...gres.terraform.tfvars => terraform.tfvars} | 33 +-- terraform/gcp/pipeline/secure/versions.tf | 6 +- 24 files changed, 478 insertions(+), 543 deletions(-) delete mode 100644 terraform/gcp/pipeline/default/bigquery.terraform.tfvars create mode 100644 terraform/gcp/pipeline/default/target_bigquery.tf create mode 100644 terraform/gcp/pipeline/default/target_postgres.tf rename terraform/gcp/pipeline/default/{postgres.terraform.tfvars => terraform.tfvars} (83%) delete mode 100644 terraform/gcp/pipeline/secure/bigquery.terraform.tfvars create mode 100644 terraform/gcp/pipeline/secure/target_bigquery.tf create mode 100644 terraform/gcp/pipeline/secure/target_postgres.tf rename terraform/gcp/pipeline/secure/{postgres.terraform.tfvars => terraform.tfvars} (83%) diff --git a/terraform/gcp/iglu_server/default/README.md b/terraform/gcp/iglu_server/default/README.md index f176608..b21f7de 100644 --- a/terraform/gcp/iglu_server/default/README.md +++ b/terraform/gcp/iglu_server/default/README.md @@ -2,9 +2,9 @@ | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | ~> 1 | -| [google](#requirement\_google) | ~> 3.90.1 | -| [random](#requirement\_random) | ~> 3.1.0 | +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 3.90 | +| [random](#requirement\_random) | >= 3 | ## Providers @@ -14,9 +14,9 @@ No providers. | Name | Source | Version | |------|--------|---------| -| [iglu\_db](#module\_iglu\_db) | snowplow-devops/cloud-sql/google | 0.1.1 | -| [iglu\_lb](#module\_iglu\_lb) | snowplow-devops/lb/google | 0.1.0 | -| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ce/google | 0.3.2 | +| [iglu\_db](#module\_iglu\_db) | snowplow-devops/cloud-sql/google | 0.3.0 | +| [iglu\_lb](#module\_iglu\_lb) | snowplow-devops/lb/google | 0.3.0 | +| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ce/google | 0.4.0 | ## Resources @@ -46,4 +46,4 @@ No resources. | Name | Description | |------|-------------| -| [iglu\_server\_ip\_address](#output\_iglu\_server\_ip\_address) | The IP address for the Iglu Server | \ No newline at end of file +| [iglu\_server\_ip\_address](#output\_iglu\_server\_ip\_address) | The IP address for the Iglu Server | diff --git a/terraform/gcp/iglu_server/default/main.tf b/terraform/gcp/iglu_server/default/main.tf index 5e18395..301ded9 100644 --- a/terraform/gcp/iglu_server/default/main.tf +++ b/terraform/gcp/iglu_server/default/main.tf @@ -4,8 +4,10 @@ provider "google" { } module "iglu_db" { - source = "snowplow-devops/cloud-sql/google" - version = "0.1.1" + # source = "snowplow-devops/cloud-sql/google" + # version = "0.3.0" + + source = "git::https://github.com/snowplow-devops/terraform-google-cloud-sql.git?ref=release/0.3.0" name = "${var.prefix}-iglu-db" @@ -18,8 +20,10 @@ module "iglu_db" { } module "iglu_server" { - source = "snowplow-devops/iglu-server-ce/google" - version = "0.3.2" + # source = "snowplow-devops/iglu-server-ce/google" + # version = "0.4.0" + + source = "git::https://github.com/snowplow-devops/terraform-google-iglu-server-ce.git?ref=release/0.4.0" name = "${var.prefix}-iglu-server" @@ -46,8 +50,10 @@ module "iglu_server" { } module "iglu_lb" { - source = "snowplow-devops/lb/google" - version = "0.1.0" + # source = "snowplow-devops/lb/google" + # version = "0.3.0" + + source = "git::https://github.com/snowplow-devops/terraform-google-lb.git?ref=release/0.3.0" name = "${var.prefix}-iglu-lb" diff --git a/terraform/gcp/iglu_server/default/terraform.tfvars b/terraform/gcp/iglu_server/default/terraform.tfvars index f5f86a9..69ec6f3 100644 --- a/terraform/gcp/iglu_server/default/terraform.tfvars +++ b/terraform/gcp/iglu_server/default/terraform.tfvars @@ -31,7 +31,7 @@ ssh_key_pairs = [ iglu_db_name = "iglu" iglu_db_username = "iglu" # Change and keep this secret! -iglu_db_password = "Hell0W0rld!" +iglu_db_password = "" # Used for API actions on the Iglu Server # Change this to a new UUID and keep it secret! diff --git a/terraform/gcp/iglu_server/default/versions.tf b/terraform/gcp/iglu_server/default/versions.tf index 67f237f..81cb3d7 100644 --- a/terraform/gcp/iglu_server/default/versions.tf +++ b/terraform/gcp/iglu_server/default/versions.tf @@ -1,14 +1,14 @@ terraform { - required_version = "~> 1" + required_version = ">= 1.0.0" required_providers { google = { source = "hashicorp/google" - version = "~> 3.90.1" + version = ">= 3.90" } random = { source = "hashicorp/random" - version = "~> 3.1.0" + version = ">= 3" } } } diff --git a/terraform/gcp/iglu_server/secure/README.md b/terraform/gcp/iglu_server/secure/README.md index f176608..b21f7de 100644 --- a/terraform/gcp/iglu_server/secure/README.md +++ b/terraform/gcp/iglu_server/secure/README.md @@ -2,9 +2,9 @@ | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | ~> 1 | -| [google](#requirement\_google) | ~> 3.90.1 | -| [random](#requirement\_random) | ~> 3.1.0 | +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 3.90 | +| [random](#requirement\_random) | >= 3 | ## Providers @@ -14,9 +14,9 @@ No providers. | Name | Source | Version | |------|--------|---------| -| [iglu\_db](#module\_iglu\_db) | snowplow-devops/cloud-sql/google | 0.1.1 | -| [iglu\_lb](#module\_iglu\_lb) | snowplow-devops/lb/google | 0.1.0 | -| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ce/google | 0.3.2 | +| [iglu\_db](#module\_iglu\_db) | snowplow-devops/cloud-sql/google | 0.3.0 | +| [iglu\_lb](#module\_iglu\_lb) | snowplow-devops/lb/google | 0.3.0 | +| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ce/google | 0.4.0 | ## Resources @@ -46,4 +46,4 @@ No resources. | Name | Description | |------|-------------| -| [iglu\_server\_ip\_address](#output\_iglu\_server\_ip\_address) | The IP address for the Iglu Server | \ No newline at end of file +| [iglu\_server\_ip\_address](#output\_iglu\_server\_ip\_address) | The IP address for the Iglu Server | diff --git a/terraform/gcp/iglu_server/secure/main.tf b/terraform/gcp/iglu_server/secure/main.tf index a634c9a..3255a14 100644 --- a/terraform/gcp/iglu_server/secure/main.tf +++ b/terraform/gcp/iglu_server/secure/main.tf @@ -5,7 +5,7 @@ provider "google" { module "iglu_db" { source = "snowplow-devops/cloud-sql/google" - version = "0.1.1" + version = "0.3.0" name = "${var.prefix}-iglu-db" @@ -19,7 +19,7 @@ module "iglu_db" { module "iglu_server" { source = "snowplow-devops/iglu-server-ce/google" - version = "0.3.2" + version = "0.4.0" name = "${var.prefix}-iglu-server" @@ -49,7 +49,7 @@ module "iglu_server" { module "iglu_lb" { source = "snowplow-devops/lb/google" - version = "0.1.0" + version = "0.3.0" name = "${var.prefix}-iglu-lb" diff --git a/terraform/gcp/iglu_server/secure/terraform.tfvars b/terraform/gcp/iglu_server/secure/terraform.tfvars index 9ac847f..04f31d6 100644 --- a/terraform/gcp/iglu_server/secure/terraform.tfvars +++ b/terraform/gcp/iglu_server/secure/terraform.tfvars @@ -30,7 +30,7 @@ ssh_key_pairs = [ iglu_db_name = "iglu" iglu_db_username = "iglu" # Change and keep this secret! -iglu_db_password = "Hell0W0rld!" +iglu_db_password = "" # Used for API actions on the Iglu Server # Change this to a new UUID and keep it secret! diff --git a/terraform/gcp/iglu_server/secure/versions.tf b/terraform/gcp/iglu_server/secure/versions.tf index 67f237f..81cb3d7 100644 --- a/terraform/gcp/iglu_server/secure/versions.tf +++ b/terraform/gcp/iglu_server/secure/versions.tf @@ -1,14 +1,14 @@ terraform { - required_version = "~> 1" + required_version = ">= 1.0.0" required_providers { google = { source = "hashicorp/google" - version = "~> 3.90.1" + version = ">= 3.90" } random = { source = "hashicorp/random" - version = "~> 3.1.0" + version = ">= 3" } } } diff --git a/terraform/gcp/pipeline/default/README.md b/terraform/gcp/pipeline/default/README.md index b466426..4316680 100644 --- a/terraform/gcp/pipeline/default/README.md +++ b/terraform/gcp/pipeline/default/README.md @@ -2,31 +2,31 @@ | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | ~> 1 | -| [google](#requirement\_google) | ~> 3.90.1 | -| [random](#requirement\_random) | ~> 3.1.0 | +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 3.90 | +| [random](#requirement\_random) | >= 3 | ## Providers | Name | Version | |------|---------| -| [google](#provider\_google) | ~> 3.90.1 | +| [google](#provider\_google) | >= 3.90 | ## Modules | Name | Source | Version | |------|--------|---------| -| [bad\_1\_topic](#module\_bad\_1\_topic) | snowplow-devops/pubsub-topic/google | 0.1.0 | -| [bad\_rows\_topic](#module\_bad\_rows\_topic) | snowplow-devops/pubsub-topic/google | 0.1.0 | -| [bigquery\_loader](#module\_bigquery\_loader) | snowplow-devops/bigquery-loader-pubsub-ce/google | 0.1.0 | -| [collector\_lb](#module\_collector\_lb) | snowplow-devops/lb/google | 0.1.0 | -| [collector\_pubsub](#module\_collector\_pubsub) | snowplow-devops/collector-pubsub-ce/google | 0.2.2 | -| [enrich\_pubsub](#module\_enrich\_pubsub) | snowplow-devops/enrich-pubsub-ce/google | 0.1.2 | -| [enriched\_topic](#module\_enriched\_topic) | snowplow-devops/pubsub-topic/google | 0.1.0 | -| [postgres\_db](#module\_postgres\_db) | snowplow-devops/cloud-sql/google | 0.1.1 | -| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.2.1 | -| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.2.1 | -| [raw\_topic](#module\_raw\_topic) | snowplow-devops/pubsub-topic/google | 0.1.0 | +| [bad\_1\_topic](#module\_bad\_1\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 | +| [bigquery\_loader](#module\_bigquery\_loader) | snowplow-devops/bigquery-loader-pubsub-ce/google | 0.2.0 | +| [bq\_bad\_rows\_topic](#module\_bq\_bad\_rows\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 | +| [collector\_lb](#module\_collector\_lb) | snowplow-devops/lb/google | 0.3.0 | +| [collector\_pubsub](#module\_collector\_pubsub) | snowplow-devops/collector-pubsub-ce/google | 0.4.0 | +| [enrich\_pubsub](#module\_enrich\_pubsub) | snowplow-devops/enrich-pubsub-ce/google | 0.2.0 | +| [enriched\_topic](#module\_enriched\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 | +| [postgres\_db](#module\_postgres\_db) | snowplow-devops/cloud-sql/google | 0.3.0 | +| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.3.0 | +| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.3.0 | +| [raw\_topic](#module\_raw\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 | ## Resources @@ -71,4 +71,4 @@ | [bq\_loader\_dead\_letter\_bucket\_name](#output\_bq\_loader\_dead\_letter\_bucket\_name) | The name of the GCS bucket for dead letter events emitted from the BigQuery loader | | [collector\_ip\_address](#output\_collector\_ip\_address) | The IP address for the Pipeline Collector | | [postgres\_db\_ip\_address](#output\_postgres\_db\_ip\_address) | The IP address of the database where your data is being streamed | -| [postgres\_db\_port](#output\_postgres\_db\_port) | The port of the database where your data is being streamed | \ No newline at end of file +| [postgres\_db\_port](#output\_postgres\_db\_port) | The port of the database where your data is being streamed | diff --git a/terraform/gcp/pipeline/default/bigquery.terraform.tfvars b/terraform/gcp/pipeline/default/bigquery.terraform.tfvars deleted file mode 100644 index 979eec7..0000000 --- a/terraform/gcp/pipeline/default/bigquery.terraform.tfvars +++ /dev/null @@ -1,56 +0,0 @@ -# Will be prefixed to all resource names -# Use this to easily identify the resources created and provide entropy for subsequent environments -prefix = "sp" - -# The project to deploy the infrastructure into -project_id = "PROJECT_ID_TO_DEPLOY_INTO" - -# Where to deploy the infrastructure -region = "REGION_TO_DEPLOY_INTO" - -# --- Default Network -# Update to the network you would like to deploy into -# -# Note: If you opt to use your own network then you will need to define a subnetwork to deploy into as well -network = "default" -subnetwork = "" - -# --- SSH -# Update this to your IP Address -ssh_ip_allowlist = ["999.999.999.999/32"] -# Generate a new SSH key locally with `ssh-keygen` -# ssh-keygen -t rsa -b 4096 -ssh_key_pairs = [ - { - user_name = "snowplow" - public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQA0jSi9//bRsHW4M6czodTs6smCXsxZ0gijzth0aBmycE= snowplow@Snowplows-MacBook-Pro.local" - } -] - -# --- Iglu Server Configuration -# Iglu Server DNS output from the Iglu Server stack -iglu_server_dns_name = "http://CHANGE-TO-MY-IGLU-IP" -# Used for API actions on the Iglu Server -# Change this to the same UUID from when you created the Iglu Server -iglu_super_api_key = "00000000-0000-0000-0000-000000000000" - -# --- Snowplow BigQuery Loader -bigquery_db_enabled = true -# To use an existing bucket set this to false -bigquery_loader_dead_letter_bucket_deploy = true -# Must be globally unique so will need to be updated before applying -bigquery_loader_dead_letter_bucket_name = "sp-bq-loader-dead-letter" - -# See for more information: https://registry.terraform.io/modules/snowplow-devops/collector-pubsub-ce/google/latest#telemetry -# Telemetry principles: https://docs.snowplowanalytics.com/docs/open-source-quick-start/what-is-the-quick-start-for-open-source/telemetry-principles/ -user_provided_id = "" -telemetry_enabled = true - -# --- SSL Configuration (optional) -ssl_information = { - certificate_id = "" - enabled = false -} - -# --- Extra Labels to append to created resources (optional) -labels = {} diff --git a/terraform/gcp/pipeline/default/main.tf b/terraform/gcp/pipeline/default/main.tf index 9035efc..676e146 100644 --- a/terraform/gcp/pipeline/default/main.tf +++ b/terraform/gcp/pipeline/default/main.tf @@ -17,8 +17,10 @@ provider "google" { # 1. Deploy PubSub Topics module "raw_topic" { - source = "snowplow-devops/pubsub-topic/google" - version = "0.1.0" + # source = "snowplow-devops/pubsub-topic/google" + # version = "0.3.0" + + source = "git::https://github.com/snowplow-devops/terraform-google-pubsub-topic.git?ref=release/0.3.0" name = "${var.prefix}-raw-topic" @@ -26,8 +28,10 @@ module "raw_topic" { } module "bad_1_topic" { - source = "snowplow-devops/pubsub-topic/google" - version = "0.1.0" + # source = "snowplow-devops/pubsub-topic/google" + # version = "0.3.0" + + source = "git::https://github.com/snowplow-devops/terraform-google-pubsub-topic.git?ref=release/0.3.0" name = "${var.prefix}-bad-1-topic" @@ -35,8 +39,10 @@ module "bad_1_topic" { } module "enriched_topic" { - source = "snowplow-devops/pubsub-topic/google" - version = "0.1.0" + # source = "snowplow-devops/pubsub-topic/google" + # version = "0.3.0" + + source = "git::https://github.com/snowplow-devops/terraform-google-pubsub-topic.git?ref=release/0.3.0" name = "${var.prefix}-enriched-topic" @@ -45,11 +51,15 @@ module "enriched_topic" { # 2. Deploy Collector stack module "collector_pubsub" { - source = "snowplow-devops/collector-pubsub-ce/google" - version = "0.2.2" + # source = "snowplow-devops/collector-pubsub-ce/google" + # version = "0.4.0" + + source = "git::https://github.com/snowplow-devops/terraform-google-collector-pubsub-ce.git?ref=release/0.4.0" name = "${var.prefix}-collector-server" + project_id = var.project_id + network = var.network subnetwork = var.subnetwork region = var.region @@ -68,8 +78,10 @@ module "collector_pubsub" { } module "collector_lb" { - source = "snowplow-devops/lb/google" - version = "0.1.0" + # source = "snowplow-devops/lb/google" + # version = "0.3.0" + + source = "git::https://github.com/snowplow-devops/terraform-google-lb.git?ref=release/0.3.0" name = "${var.prefix}-collector-lb" @@ -83,180 +95,25 @@ module "collector_lb" { # 3. Deploy Enrichment module "enrich_pubsub" { - source = "snowplow-devops/enrich-pubsub-ce/google" - version = "0.1.2" - - name = "${var.prefix}-enrich-server" - - network = var.network - subnetwork = var.subnetwork - region = var.region - - ssh_ip_allowlist = var.ssh_ip_allowlist - ssh_key_pairs = var.ssh_key_pairs - - raw_topic_name = module.raw_topic.name - good_topic_id = module.enriched_topic.id - bad_topic_id = module.bad_1_topic.id - - # Linking in the custom Iglu Server here - custom_iglu_resolvers = local.custom_iglu_resolvers - - telemetry_enabled = var.telemetry_enabled - user_provided_id = var.user_provided_id - - labels = var.labels -} - -# 4. Deploy Postgres Loader -module "postgres_db" { - source = "snowplow-devops/cloud-sql/google" - version = "0.1.1" - - count = var.postgres_db_enabled ? 1 : 0 + # source = "snowplow-devops/enrich-pubsub-ce/google" + # version = "0.2.0" - name = "${var.prefix}-postgres-db" + source = "git::https://github.com/snowplow-devops/terraform-google-enrich-pubsub-ce.git?ref=release/0.2.0" - region = var.region - db_name = var.postgres_db_name - db_username = var.postgres_db_username - db_password = var.postgres_db_password + name = "${var.prefix}-enrich" - authorized_networks = var.postgres_db_authorized_networks - - tier = var.postgres_db_tier - - labels = var.labels -} - -module "postgres_loader_enriched" { - source = "snowplow-devops/postgres-loader-pubsub-ce/google" - version = "0.2.1" - - count = var.postgres_db_enabled ? 1 : 0 - - name = "${var.prefix}-pg-loader-enriched-server" - - network = var.network - subnetwork = var.subnetwork - region = var.region project_id = var.project_id - ssh_ip_allowlist = var.ssh_ip_allowlist - ssh_key_pairs = var.ssh_key_pairs - - in_topic_name = module.enriched_topic.name - purpose = "ENRICHED_EVENTS" - schema_name = "atomic" - - db_instance_name = join("", module.postgres_db.*.connection_name) - db_port = join("", module.postgres_db.*.port) - db_name = var.postgres_db_name - db_username = var.postgres_db_username - db_password = var.postgres_db_password - - # Linking in the custom Iglu Server here - custom_iglu_resolvers = local.custom_iglu_resolvers - - telemetry_enabled = var.telemetry_enabled - user_provided_id = var.user_provided_id - - labels = var.labels -} - -module "postgres_loader_bad" { - source = "snowplow-devops/postgres-loader-pubsub-ce/google" - version = "0.2.1" - - count = var.postgres_db_enabled ? 1 : 0 - - name = "${var.prefix}-pg-loader-bad-server" - network = var.network subnetwork = var.subnetwork region = var.region - project_id = var.project_id ssh_ip_allowlist = var.ssh_ip_allowlist ssh_key_pairs = var.ssh_key_pairs - in_topic_name = module.bad_1_topic.name - purpose = "JSON" - schema_name = "atomic_bad" - - db_instance_name = join("", module.postgres_db.*.connection_name) - db_port = join("", module.postgres_db.*.port) - db_name = var.postgres_db_name - db_username = var.postgres_db_username - db_password = var.postgres_db_password - - # Linking in the custom Iglu Server here - custom_iglu_resolvers = local.custom_iglu_resolvers - - telemetry_enabled = var.telemetry_enabled - user_provided_id = var.user_provided_id - - labels = var.labels -} - -# 5. Deploy BigQuery Loader -module "bad_rows_topic" { - source = "snowplow-devops/pubsub-topic/google" - version = "0.1.0" - - count = var.bigquery_db_enabled ? 1 : 0 - - name = "${var.prefix}-bq-bad-rows-topic" - - labels = var.labels -} - -resource "google_bigquery_dataset" "bigquery_db" { - count = var.bigquery_db_enabled ? 1 : 0 - - dataset_id = replace("${var.prefix}_pipeline_db", "-", "_") - location = var.region - - labels = var.labels -} - -resource "google_storage_bucket" "bq_loader_dead_letter_bucket" { - count = var.bigquery_db_enabled && var.bigquery_loader_dead_letter_bucket_deploy ? 1 : 0 - - name = var.bigquery_loader_dead_letter_bucket_name - location = var.region - force_destroy = true - - labels = var.labels -} - -locals { - bq_loader_dead_letter_bucket_name = coalesce( - join("", google_storage_bucket.bq_loader_dead_letter_bucket.*.name), - var.bigquery_loader_dead_letter_bucket_name, - ) -} - -module "bigquery_loader" { - source = "snowplow-devops/bigquery-loader-pubsub-ce/google" - version = "0.1.0" - - count = var.bigquery_db_enabled ? 1 : 0 - - name = "${var.prefix}-bq-loader-server" - - network = var.network - subnetwork = var.subnetwork - region = var.region - project_id = var.project_id - - ssh_ip_allowlist = var.ssh_ip_allowlist - ssh_key_pairs = var.ssh_key_pairs - - input_topic_name = module.enriched_topic.name - bad_rows_topic_name = join("", module.bad_rows_topic.*.name) - gcs_dead_letter_bucket_name = local.bq_loader_dead_letter_bucket_name - bigquery_dataset_id = join("", google_bigquery_dataset.bigquery_db.*.dataset_id) + raw_topic_name = module.raw_topic.name + good_topic_id = module.enriched_topic.id + bad_topic_id = module.bad_1_topic.id # Linking in the custom Iglu Server here custom_iglu_resolvers = local.custom_iglu_resolvers diff --git a/terraform/gcp/pipeline/default/outputs.tf b/terraform/gcp/pipeline/default/outputs.tf index 034f7a9..7c53128 100644 --- a/terraform/gcp/pipeline/default/outputs.tf +++ b/terraform/gcp/pipeline/default/outputs.tf @@ -25,5 +25,5 @@ output "bq_loader_dead_letter_bucket_name" { output "bq_loader_bad_rows_topic_name" { description = "The name of the topic for bad rows emitted from the BigQuery loader" - value = join("", module.bad_rows_topic.*.name) + value = join("", module.bq_bad_rows_topic.*.name) } diff --git a/terraform/gcp/pipeline/default/target_bigquery.tf b/terraform/gcp/pipeline/default/target_bigquery.tf new file mode 100644 index 0000000..4af1ec3 --- /dev/null +++ b/terraform/gcp/pipeline/default/target_bigquery.tf @@ -0,0 +1,70 @@ +module "bq_bad_rows_topic" { + # source = "snowplow-devops/pubsub-topic/google" + # version = "0.3.0" + + source = "git::https://github.com/snowplow-devops/terraform-google-pubsub-topic.git?ref=release/0.3.0" + + count = var.bigquery_db_enabled ? 1 : 0 + + name = "${var.prefix}-bq-bad-rows-topic" + + labels = var.labels +} + +resource "google_bigquery_dataset" "bigquery_db" { + count = var.bigquery_db_enabled ? 1 : 0 + + dataset_id = replace("${var.prefix}_pipeline_db", "-", "_") + location = var.region + + labels = var.labels +} + +resource "google_storage_bucket" "bq_loader_dead_letter_bucket" { + count = var.bigquery_db_enabled && var.bigquery_loader_dead_letter_bucket_deploy ? 1 : 0 + + name = var.bigquery_loader_dead_letter_bucket_name + location = var.region + force_destroy = true + + labels = var.labels +} + +locals { + bq_loader_dead_letter_bucket_name = coalesce( + join("", google_storage_bucket.bq_loader_dead_letter_bucket.*.name), + var.bigquery_loader_dead_letter_bucket_name, + ) +} + +module "bigquery_loader" { + # source = "snowplow-devops/bigquery-loader-pubsub-ce/google" + # version = "0.2.0" + + source = "git::https://github.com/snowplow-devops/terraform-google-bigquery-loader-pubsub-ce.git?ref=release/0.2.0" + + count = var.bigquery_db_enabled ? 1 : 0 + + name = "${var.prefix}-bq-loader" + + network = var.network + subnetwork = var.subnetwork + region = var.region + project_id = var.project_id + + ssh_ip_allowlist = var.ssh_ip_allowlist + ssh_key_pairs = var.ssh_key_pairs + + input_topic_name = module.enriched_topic.name + bad_rows_topic_name = join("", module.bq_bad_rows_topic.*.name) + gcs_dead_letter_bucket_name = local.bq_loader_dead_letter_bucket_name + bigquery_dataset_id = join("", google_bigquery_dataset.bigquery_db.*.dataset_id) + + # Linking in the custom Iglu Server here + custom_iglu_resolvers = local.custom_iglu_resolvers + + telemetry_enabled = var.telemetry_enabled + user_provided_id = var.user_provided_id + + labels = var.labels +} diff --git a/terraform/gcp/pipeline/default/target_postgres.tf b/terraform/gcp/pipeline/default/target_postgres.tf new file mode 100644 index 0000000..7f04fd2 --- /dev/null +++ b/terraform/gcp/pipeline/default/target_postgres.tf @@ -0,0 +1,95 @@ +module "postgres_db" { + # source = "snowplow-devops/cloud-sql/google" + # version = "0.3.0" + + source = "git::https://github.com/snowplow-devops/terraform-google-cloud-sql.git?ref=release/0.3.0" + + count = var.postgres_db_enabled ? 1 : 0 + + name = "${var.prefix}-postgres-db" + + region = var.region + db_name = var.postgres_db_name + db_username = var.postgres_db_username + db_password = var.postgres_db_password + + authorized_networks = var.postgres_db_authorized_networks + + tier = var.postgres_db_tier + + labels = var.labels +} + +module "postgres_loader_enriched" { + # source = "snowplow-devops/postgres-loader-pubsub-ce/google" + # version = "0.3.0" + + source = "git::https://github.com/snowplow-devops/terraform-google-postgres-loader-pubsub-ce.git?ref=release/0.3.0" + + count = var.postgres_db_enabled ? 1 : 0 + + name = "${var.prefix}-pg-loader-enriched" + + network = var.network + subnetwork = var.subnetwork + region = var.region + project_id = var.project_id + + ssh_ip_allowlist = var.ssh_ip_allowlist + ssh_key_pairs = var.ssh_key_pairs + + in_topic_name = module.enriched_topic.name + purpose = "ENRICHED_EVENTS" + schema_name = "atomic" + + db_instance_name = join("", module.postgres_db.*.connection_name) + db_port = join("", module.postgres_db.*.port) + db_name = var.postgres_db_name + db_username = var.postgres_db_username + db_password = var.postgres_db_password + + # Linking in the custom Iglu Server here + custom_iglu_resolvers = local.custom_iglu_resolvers + + telemetry_enabled = var.telemetry_enabled + user_provided_id = var.user_provided_id + + labels = var.labels +} + +module "postgres_loader_bad" { + # source = "snowplow-devops/postgres-loader-pubsub-ce/google" + # version = "0.3.0" + + source = "git::https://github.com/snowplow-devops/terraform-google-postgres-loader-pubsub-ce.git?ref=release/0.3.0" + + count = var.postgres_db_enabled ? 1 : 0 + + name = "${var.prefix}-pg-loader-bad" + + network = var.network + subnetwork = var.subnetwork + region = var.region + project_id = var.project_id + + ssh_ip_allowlist = var.ssh_ip_allowlist + ssh_key_pairs = var.ssh_key_pairs + + in_topic_name = module.bad_1_topic.name + purpose = "JSON" + schema_name = "atomic_bad" + + db_instance_name = join("", module.postgres_db.*.connection_name) + db_port = join("", module.postgres_db.*.port) + db_name = var.postgres_db_name + db_username = var.postgres_db_username + db_password = var.postgres_db_password + + # Linking in the custom Iglu Server here + custom_iglu_resolvers = local.custom_iglu_resolvers + + telemetry_enabled = var.telemetry_enabled + user_provided_id = var.user_provided_id + + labels = var.labels +} diff --git a/terraform/gcp/pipeline/default/postgres.terraform.tfvars b/terraform/gcp/pipeline/default/terraform.tfvars similarity index 83% rename from terraform/gcp/pipeline/default/postgres.terraform.tfvars rename to terraform/gcp/pipeline/default/terraform.tfvars index 7ca15d7..fe58814 100644 --- a/terraform/gcp/pipeline/default/postgres.terraform.tfvars +++ b/terraform/gcp/pipeline/default/terraform.tfvars @@ -8,14 +8,12 @@ project_id = "PROJECT_ID_TO_DEPLOY_INTO" # Where to deploy the infrastructure region = "REGION_TO_DEPLOY_INTO" -# --- Default Network # Update to the network you would like to deploy into # # Note: If you opt to use your own network then you will need to define a subnetwork to deploy into as well network = "default" subnetwork = "" -# --- SSH # Update this to your IP Address ssh_ip_allowlist = ["999.999.999.999/32"] # Generate a new SSH key locally with `ssh-keygen` @@ -27,15 +25,22 @@ ssh_key_pairs = [ } ] -# --- Iglu Server Configuration # Iglu Server DNS output from the Iglu Server stack -iglu_server_dns_name = "http://CHANGE-TO-MY-IGLU-IP" +iglu_server_dns_name = "http://" # Used for API actions on the Iglu Server # Change this to the same UUID from when you created the Iglu Server iglu_super_api_key = "00000000-0000-0000-0000-000000000000" -# --- Snowplow Postgres Loader -postgres_db_enabled = true +# Collector SSL Configuration (optional) +ssl_information = { + certificate_id = "" + enabled = false +} + +# --- TARGETS CONFIGURATION ZONE --- # + +# --- Target: PostgreSQL +postgres_db_enabled = false postgres_db_name = "snowplow" postgres_db_username = "snowplow" @@ -61,16 +66,20 @@ postgres_db_authorized_networks = [ # a sufficiently powerful database tier is important to not running out of connection slots postgres_db_tier = "db-g1-small" +# --- Target: BigQuery +bigquery_db_enabled = false + +# To use an existing bucket set this to false +bigquery_loader_dead_letter_bucket_deploy = true +# Must be globally unique so will need to be updated before applying +bigquery_loader_dead_letter_bucket_name = "sp-bq-loader-dead-letter" + +# --- ADVANCED CONFIGURATION ZONE --- # + # See for more information: https://registry.terraform.io/modules/snowplow-devops/collector-pubsub-ce/google/latest#telemetry # Telemetry principles: https://docs.snowplowanalytics.com/docs/open-source-quick-start/what-is-the-quick-start-for-open-source/telemetry-principles/ user_provided_id = "" telemetry_enabled = true -# --- SSL Configuration (optional) -ssl_information = { - certificate_id = "" - enabled = false -} - # --- Extra Labels to append to created resources (optional) labels = {} diff --git a/terraform/gcp/pipeline/default/versions.tf b/terraform/gcp/pipeline/default/versions.tf index 67f237f..81cb3d7 100644 --- a/terraform/gcp/pipeline/default/versions.tf +++ b/terraform/gcp/pipeline/default/versions.tf @@ -1,14 +1,14 @@ terraform { - required_version = "~> 1" + required_version = ">= 1.0.0" required_providers { google = { source = "hashicorp/google" - version = "~> 3.90.1" + version = ">= 3.90" } random = { source = "hashicorp/random" - version = "~> 3.1.0" + version = ">= 3" } } } diff --git a/terraform/gcp/pipeline/secure/README.md b/terraform/gcp/pipeline/secure/README.md index b466426..4316680 100644 --- a/terraform/gcp/pipeline/secure/README.md +++ b/terraform/gcp/pipeline/secure/README.md @@ -2,31 +2,31 @@ | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | ~> 1 | -| [google](#requirement\_google) | ~> 3.90.1 | -| [random](#requirement\_random) | ~> 3.1.0 | +| [terraform](#requirement\_terraform) | >= 1.0.0 | +| [google](#requirement\_google) | >= 3.90 | +| [random](#requirement\_random) | >= 3 | ## Providers | Name | Version | |------|---------| -| [google](#provider\_google) | ~> 3.90.1 | +| [google](#provider\_google) | >= 3.90 | ## Modules | Name | Source | Version | |------|--------|---------| -| [bad\_1\_topic](#module\_bad\_1\_topic) | snowplow-devops/pubsub-topic/google | 0.1.0 | -| [bad\_rows\_topic](#module\_bad\_rows\_topic) | snowplow-devops/pubsub-topic/google | 0.1.0 | -| [bigquery\_loader](#module\_bigquery\_loader) | snowplow-devops/bigquery-loader-pubsub-ce/google | 0.1.0 | -| [collector\_lb](#module\_collector\_lb) | snowplow-devops/lb/google | 0.1.0 | -| [collector\_pubsub](#module\_collector\_pubsub) | snowplow-devops/collector-pubsub-ce/google | 0.2.2 | -| [enrich\_pubsub](#module\_enrich\_pubsub) | snowplow-devops/enrich-pubsub-ce/google | 0.1.2 | -| [enriched\_topic](#module\_enriched\_topic) | snowplow-devops/pubsub-topic/google | 0.1.0 | -| [postgres\_db](#module\_postgres\_db) | snowplow-devops/cloud-sql/google | 0.1.1 | -| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.2.1 | -| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.2.1 | -| [raw\_topic](#module\_raw\_topic) | snowplow-devops/pubsub-topic/google | 0.1.0 | +| [bad\_1\_topic](#module\_bad\_1\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 | +| [bigquery\_loader](#module\_bigquery\_loader) | snowplow-devops/bigquery-loader-pubsub-ce/google | 0.2.0 | +| [bq\_bad\_rows\_topic](#module\_bq\_bad\_rows\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 | +| [collector\_lb](#module\_collector\_lb) | snowplow-devops/lb/google | 0.3.0 | +| [collector\_pubsub](#module\_collector\_pubsub) | snowplow-devops/collector-pubsub-ce/google | 0.4.0 | +| [enrich\_pubsub](#module\_enrich\_pubsub) | snowplow-devops/enrich-pubsub-ce/google | 0.2.0 | +| [enriched\_topic](#module\_enriched\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 | +| [postgres\_db](#module\_postgres\_db) | snowplow-devops/cloud-sql/google | 0.3.0 | +| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.3.0 | +| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.3.0 | +| [raw\_topic](#module\_raw\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 | ## Resources @@ -71,4 +71,4 @@ | [bq\_loader\_dead\_letter\_bucket\_name](#output\_bq\_loader\_dead\_letter\_bucket\_name) | The name of the GCS bucket for dead letter events emitted from the BigQuery loader | | [collector\_ip\_address](#output\_collector\_ip\_address) | The IP address for the Pipeline Collector | | [postgres\_db\_ip\_address](#output\_postgres\_db\_ip\_address) | The IP address of the database where your data is being streamed | -| [postgres\_db\_port](#output\_postgres\_db\_port) | The port of the database where your data is being streamed | \ No newline at end of file +| [postgres\_db\_port](#output\_postgres\_db\_port) | The port of the database where your data is being streamed | diff --git a/terraform/gcp/pipeline/secure/bigquery.terraform.tfvars b/terraform/gcp/pipeline/secure/bigquery.terraform.tfvars deleted file mode 100644 index 7331a03..0000000 --- a/terraform/gcp/pipeline/secure/bigquery.terraform.tfvars +++ /dev/null @@ -1,55 +0,0 @@ -# Will be prefixed to all resource names -# Use this to easily identify the resources created and provide entropy for subsequent environments -prefix = "sp" - -# The project to deploy the infrastructure into -project_id = "PROJECT_ID_TO_DEPLOY_INTO" - -# Where to deploy the infrastructure -region = "REGION_TO_DEPLOY_INTO" - -# --- Network -# NOTE: The network & sub-network configured must be configured with a Cloud NAT to allow the deployed Compute Engine instances to -# connect to the internet to download the required assets -network = "YOUR_NETWORK_HERE" -subnetwork = "YOUR_SUB_NETWORK_HERE" - -# --- SSH -# Update this to the internal IP of your Bastion Host -ssh_ip_allowlist = ["999.999.999.999/32"] -# Generate a new SSH key locally with `ssh-keygen` -# ssh-keygen -t rsa -b 4096 -ssh_key_pairs = [ - { - user_name = "snowplow" - public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQA0jSi9//bRsHW4M6czodTs6smCXsxZ0gijzth0aBmycE= snowplow@Snowplows-MacBook-Pro.local" - } -] - -# --- Iglu Server Configuration -# Iglu Server DNS output from the Iglu Server stack -iglu_server_dns_name = "http://CHANGE-TO-MY-IGLU-IP" -# Used for API actions on the Iglu Server -# Change this to the same UUID from when you created the Iglu Server -iglu_super_api_key = "00000000-0000-0000-0000-000000000000" - -# --- Snowplow BigQuery Loader -bigquery_db_enabled = true -# To use an existing bucket set this to false -bigquery_loader_dead_letter_bucket_deploy = true -# Must be globally unique so will need to be updated before applying -bigquery_loader_dead_letter_bucket_name = "sp-bq-loader-dead-letter" - -# See for more information: https://registry.terraform.io/modules/snowplow-devops/collector-pubsub-ce/google/latest#telemetry -# Telemetry principles: https://docs.snowplowanalytics.com/docs/open-source-quick-start/what-is-the-quick-start-for-open-source/telemetry-principles/ -user_provided_id = "" -telemetry_enabled = true - -# --- SSL Configuration (optional) -ssl_information = { - certificate_id = "" - enabled = false -} - -# --- Extra Labels to append to created resources (optional) -labels = {} diff --git a/terraform/gcp/pipeline/secure/main.tf b/terraform/gcp/pipeline/secure/main.tf index fc0084b..48887f1 100644 --- a/terraform/gcp/pipeline/secure/main.tf +++ b/terraform/gcp/pipeline/secure/main.tf @@ -18,7 +18,7 @@ provider "google" { # 1. Deploy PubSub Topics module "raw_topic" { source = "snowplow-devops/pubsub-topic/google" - version = "0.1.0" + version = "0.3.0" name = "${var.prefix}-raw-topic" @@ -27,7 +27,7 @@ module "raw_topic" { module "bad_1_topic" { source = "snowplow-devops/pubsub-topic/google" - version = "0.1.0" + version = "0.3.0" name = "${var.prefix}-bad-1-topic" @@ -36,7 +36,7 @@ module "bad_1_topic" { module "enriched_topic" { source = "snowplow-devops/pubsub-topic/google" - version = "0.1.0" + version = "0.3.0" name = "${var.prefix}-enriched-topic" @@ -46,7 +46,7 @@ module "enriched_topic" { # 2. Deploy Collector stack module "collector_pubsub" { source = "snowplow-devops/collector-pubsub-ce/google" - version = "0.2.2" + version = "0.4.0" name = "${var.prefix}-collector-server" @@ -71,7 +71,7 @@ module "collector_pubsub" { module "collector_lb" { source = "snowplow-devops/lb/google" - version = "0.1.0" + version = "0.3.0" name = "${var.prefix}-collector-lb" @@ -86,7 +86,7 @@ module "collector_lb" { # 3. Deploy Enrichment module "enrich_pubsub" { source = "snowplow-devops/enrich-pubsub-ce/google" - version = "0.1.2" + version = "0.2.0" name = "${var.prefix}-enrich-server" @@ -111,162 +111,3 @@ module "enrich_pubsub" { labels = var.labels } - -# 4. Deploy Postgres Loader -module "postgres_db" { - source = "snowplow-devops/cloud-sql/google" - version = "0.1.1" - - count = var.postgres_db_enabled ? 1 : 0 - - name = "${var.prefix}-postgres-db" - - region = var.region - db_name = var.postgres_db_name - db_username = var.postgres_db_username - db_password = var.postgres_db_password - - authorized_networks = var.postgres_db_authorized_networks - - tier = var.postgres_db_tier - - labels = var.labels -} - -module "postgres_loader_enriched" { - source = "snowplow-devops/postgres-loader-pubsub-ce/google" - version = "0.2.1" - - count = var.postgres_db_enabled ? 1 : 0 - - name = "${var.prefix}-pg-loader-enriched-server" - - network = var.network - subnetwork = var.subnetwork - region = var.region - project_id = var.project_id - - ssh_ip_allowlist = var.ssh_ip_allowlist - ssh_key_pairs = var.ssh_key_pairs - - in_topic_name = module.enriched_topic.name - purpose = "ENRICHED_EVENTS" - schema_name = "atomic" - - db_instance_name = join("", module.postgres_db.*.connection_name) - db_port = join("", module.postgres_db.*.port) - db_name = var.postgres_db_name - db_username = var.postgres_db_username - db_password = var.postgres_db_password - - # Linking in the custom Iglu Server here - custom_iglu_resolvers = local.custom_iglu_resolvers - - telemetry_enabled = var.telemetry_enabled - user_provided_id = var.user_provided_id - - labels = var.labels -} - -module "postgres_loader_bad" { - source = "snowplow-devops/postgres-loader-pubsub-ce/google" - version = "0.2.1" - - count = var.postgres_db_enabled ? 1 : 0 - - name = "${var.prefix}-pg-loader-bad-server" - - network = var.network - subnetwork = var.subnetwork - region = var.region - project_id = var.project_id - - ssh_ip_allowlist = var.ssh_ip_allowlist - ssh_key_pairs = var.ssh_key_pairs - - in_topic_name = module.bad_1_topic.name - purpose = "JSON" - schema_name = "atomic_bad" - - db_instance_name = join("", module.postgres_db.*.connection_name) - db_port = join("", module.postgres_db.*.port) - db_name = var.postgres_db_name - db_username = var.postgres_db_username - db_password = var.postgres_db_password - - # Linking in the custom Iglu Server here - custom_iglu_resolvers = local.custom_iglu_resolvers - - telemetry_enabled = var.telemetry_enabled - user_provided_id = var.user_provided_id - - labels = var.labels -} - -# 5. Deploy BigQuery Loader -module "bad_rows_topic" { - source = "snowplow-devops/pubsub-topic/google" - version = "0.1.0" - - count = var.bigquery_db_enabled ? 1 : 0 - - name = "${var.prefix}-bq-bad-rows-topic" - - labels = var.labels -} - -resource "google_bigquery_dataset" "bigquery_db" { - count = var.bigquery_db_enabled ? 1 : 0 - - dataset_id = replace("${var.prefix}_snowplow_db", "-", "_") - location = var.region - - labels = var.labels -} - -resource "google_storage_bucket" "bq_loader_dead_letter_bucket" { - count = var.bigquery_db_enabled && var.bigquery_loader_dead_letter_bucket_deploy ? 1 : 0 - - name = var.bigquery_loader_dead_letter_bucket_name - location = var.region - force_destroy = true - - labels = var.labels -} - -locals { - bq_loader_dead_letter_bucket_name = coalesce( - join("", google_storage_bucket.bq_loader_dead_letter_bucket.*.name), - var.bigquery_loader_dead_letter_bucket_name, - ) -} - -module "bigquery_loader" { - source = "snowplow-devops/bigquery-loader-pubsub-ce/google" - version = "0.1.0" - - count = var.bigquery_db_enabled ? 1 : 0 - - name = "${var.prefix}-bq-loader-server" - - network = var.network - subnetwork = var.subnetwork - region = var.region - project_id = var.project_id - - ssh_ip_allowlist = var.ssh_ip_allowlist - ssh_key_pairs = var.ssh_key_pairs - - input_topic_name = module.enriched_topic.name - bad_rows_topic_name = join("", module.bad_rows_topic.*.name) - gcs_dead_letter_bucket_name = local.bq_loader_dead_letter_bucket_name - bigquery_dataset_id = join("", google_bigquery_dataset.bigquery_db.*.dataset_id) - - # Linking in the custom Iglu Server here - custom_iglu_resolvers = local.custom_iglu_resolvers - - telemetry_enabled = var.telemetry_enabled - user_provided_id = var.user_provided_id - - labels = var.labels -} diff --git a/terraform/gcp/pipeline/secure/outputs.tf b/terraform/gcp/pipeline/secure/outputs.tf index 034f7a9..7c53128 100644 --- a/terraform/gcp/pipeline/secure/outputs.tf +++ b/terraform/gcp/pipeline/secure/outputs.tf @@ -25,5 +25,5 @@ output "bq_loader_dead_letter_bucket_name" { output "bq_loader_bad_rows_topic_name" { description = "The name of the topic for bad rows emitted from the BigQuery loader" - value = join("", module.bad_rows_topic.*.name) + value = join("", module.bq_bad_rows_topic.*.name) } diff --git a/terraform/gcp/pipeline/secure/target_bigquery.tf b/terraform/gcp/pipeline/secure/target_bigquery.tf new file mode 100644 index 0000000..5bbeea5 --- /dev/null +++ b/terraform/gcp/pipeline/secure/target_bigquery.tf @@ -0,0 +1,68 @@ +module "bq_bad_rows_topic" { + source = "snowplow-devops/pubsub-topic/google" + version = "0.3.0" + + count = var.bigquery_db_enabled ? 1 : 0 + + name = "${var.prefix}-bq-bad-rows-topic" + + labels = var.labels +} + +resource "google_bigquery_dataset" "bigquery_db" { + count = var.bigquery_db_enabled ? 1 : 0 + + dataset_id = replace("${var.prefix}_snowplow_db", "-", "_") + location = var.region + + labels = var.labels +} + +resource "google_storage_bucket" "bq_loader_dead_letter_bucket" { + count = var.bigquery_db_enabled && var.bigquery_loader_dead_letter_bucket_deploy ? 1 : 0 + + name = var.bigquery_loader_dead_letter_bucket_name + location = var.region + force_destroy = true + + labels = var.labels +} + +locals { + bq_loader_dead_letter_bucket_name = coalesce( + join("", google_storage_bucket.bq_loader_dead_letter_bucket.*.name), + var.bigquery_loader_dead_letter_bucket_name, + ) +} + +module "bigquery_loader" { + source = "snowplow-devops/bigquery-loader-pubsub-ce/google" + version = "0.2.0" + + count = var.bigquery_db_enabled ? 1 : 0 + + name = "${var.prefix}-bq-loader-server" + + network = var.network + subnetwork = var.subnetwork + region = var.region + project_id = var.project_id + + ssh_ip_allowlist = var.ssh_ip_allowlist + ssh_key_pairs = var.ssh_key_pairs + + input_topic_name = module.enriched_topic.name + bad_rows_topic_name = join("", module.bq_bad_rows_topic.*.name) + gcs_dead_letter_bucket_name = local.bq_loader_dead_letter_bucket_name + bigquery_dataset_id = join("", google_bigquery_dataset.bigquery_db.*.dataset_id) + + # Linking in the custom Iglu Server here + custom_iglu_resolvers = local.custom_iglu_resolvers + + telemetry_enabled = var.telemetry_enabled + user_provided_id = var.user_provided_id + + associate_public_ip_address = false + + labels = var.labels +} diff --git a/terraform/gcp/pipeline/secure/target_postgres.tf b/terraform/gcp/pipeline/secure/target_postgres.tf new file mode 100644 index 0000000..2232bd3 --- /dev/null +++ b/terraform/gcp/pipeline/secure/target_postgres.tf @@ -0,0 +1,91 @@ +module "postgres_db" { + source = "snowplow-devops/cloud-sql/google" + version = "0.3.0" + + count = var.postgres_db_enabled ? 1 : 0 + + name = "${var.prefix}-postgres-db" + + region = var.region + db_name = var.postgres_db_name + db_username = var.postgres_db_username + db_password = var.postgres_db_password + + authorized_networks = var.postgres_db_authorized_networks + + tier = var.postgres_db_tier + + labels = var.labels +} + +module "postgres_loader_enriched" { + source = "snowplow-devops/postgres-loader-pubsub-ce/google" + version = "0.3.0" + + count = var.postgres_db_enabled ? 1 : 0 + + name = "${var.prefix}-pg-loader-enriched-server" + + network = var.network + subnetwork = var.subnetwork + region = var.region + project_id = var.project_id + + ssh_ip_allowlist = var.ssh_ip_allowlist + ssh_key_pairs = var.ssh_key_pairs + + in_topic_name = module.enriched_topic.name + purpose = "ENRICHED_EVENTS" + schema_name = "atomic" + + db_instance_name = join("", module.postgres_db.*.connection_name) + db_port = join("", module.postgres_db.*.port) + db_name = var.postgres_db_name + db_username = var.postgres_db_username + db_password = var.postgres_db_password + + # Linking in the custom Iglu Server here + custom_iglu_resolvers = local.custom_iglu_resolvers + + telemetry_enabled = var.telemetry_enabled + user_provided_id = var.user_provided_id + + labels = var.labels +} + +module "postgres_loader_bad" { + source = "snowplow-devops/postgres-loader-pubsub-ce/google" + version = "0.3.0" + + count = var.postgres_db_enabled ? 1 : 0 + + name = "${var.prefix}-pg-loader-bad-server" + + network = var.network + subnetwork = var.subnetwork + region = var.region + project_id = var.project_id + + ssh_ip_allowlist = var.ssh_ip_allowlist + ssh_key_pairs = var.ssh_key_pairs + + in_topic_name = module.bad_1_topic.name + purpose = "JSON" + schema_name = "atomic_bad" + + db_instance_name = join("", module.postgres_db.*.connection_name) + db_port = join("", module.postgres_db.*.port) + db_name = var.postgres_db_name + db_username = var.postgres_db_username + db_password = var.postgres_db_password + + # Linking in the custom Iglu Server here + custom_iglu_resolvers = local.custom_iglu_resolvers + + telemetry_enabled = var.telemetry_enabled + user_provided_id = var.user_provided_id + + associate_public_ip_address = false + + labels = var.labels +} diff --git a/terraform/gcp/pipeline/secure/postgres.terraform.tfvars b/terraform/gcp/pipeline/secure/terraform.tfvars similarity index 83% rename from terraform/gcp/pipeline/secure/postgres.terraform.tfvars rename to terraform/gcp/pipeline/secure/terraform.tfvars index fc9a4a8..8267274 100644 --- a/terraform/gcp/pipeline/secure/postgres.terraform.tfvars +++ b/terraform/gcp/pipeline/secure/terraform.tfvars @@ -8,13 +8,11 @@ project_id = "PROJECT_ID_TO_DEPLOY_INTO" # Where to deploy the infrastructure region = "REGION_TO_DEPLOY_INTO" -# --- Network # NOTE: The network & sub-network configured must be configured with a Cloud NAT to allow the deployed Compute Engine instances to # connect to the internet to download the required assets network = "YOUR_NETWORK_HERE" subnetwork = "YOUR_SUB_NETWORK_HERE" -# --- SSH # Update this to the internal IP of your Bastion Host ssh_ip_allowlist = ["999.999.999.999/32"] # Generate a new SSH key locally with `ssh-keygen` @@ -26,15 +24,22 @@ ssh_key_pairs = [ } ] -# --- Iglu Server Configuration # Iglu Server DNS output from the Iglu Server stack -iglu_server_dns_name = "http://CHANGE-TO-MY-IGLU-IP" +iglu_server_dns_name = "http://" # Used for API actions on the Iglu Server # Change this to the same UUID from when you created the Iglu Server iglu_super_api_key = "00000000-0000-0000-0000-000000000000" -# --- Snowplow Postgres Loader -postgres_db_enabled = true +# Collector SSL Configuration (optional) +ssl_information = { + certificate_id = "" + enabled = false +} + +# --- TARGETS CONFIGURATION ZONE --- # + +# --- Target: PostgreSQL +postgres_db_enabled = false postgres_db_name = "snowplow" postgres_db_username = "snowplow" @@ -60,16 +65,20 @@ postgres_db_authorized_networks = [ # a sufficiently powerful database tier is important to not running out of connection slots postgres_db_tier = "db-g1-small" +# --- Target: BigQuery +bigquery_db_enabled = false + +# To use an existing bucket set this to false +bigquery_loader_dead_letter_bucket_deploy = true +# Must be globally unique so will need to be updated before applying +bigquery_loader_dead_letter_bucket_name = "sp-bq-loader-dead-letter" + +# --- ADVANCED CONFIGURATION ZONE --- # + # See for more information: https://registry.terraform.io/modules/snowplow-devops/collector-pubsub-ce/google/latest#telemetry # Telemetry principles: https://docs.snowplowanalytics.com/docs/open-source-quick-start/what-is-the-quick-start-for-open-source/telemetry-principles/ user_provided_id = "" telemetry_enabled = true -# --- SSL Configuration (optional) -ssl_information = { - certificate_id = "" - enabled = false -} - # --- Extra Labels to append to created resources (optional) labels = {} diff --git a/terraform/gcp/pipeline/secure/versions.tf b/terraform/gcp/pipeline/secure/versions.tf index 67f237f..81cb3d7 100644 --- a/terraform/gcp/pipeline/secure/versions.tf +++ b/terraform/gcp/pipeline/secure/versions.tf @@ -1,14 +1,14 @@ terraform { - required_version = "~> 1" + required_version = ">= 1.0.0" required_providers { google = { source = "hashicorp/google" - version = "~> 3.90.1" + version = ">= 3.90" } random = { source = "hashicorp/random" - version = "~> 3.1.0" + version = ">= 3" } } }