diff --git a/terraform/gcp/iglu_server/default/README.md b/terraform/gcp/iglu_server/default/README.md
index f176608..b21f7de 100644
--- a/terraform/gcp/iglu_server/default/README.md
+++ b/terraform/gcp/iglu_server/default/README.md
@@ -2,9 +2,9 @@
| Name | Version |
|------|---------|
-| [terraform](#requirement\_terraform) | ~> 1 |
-| [google](#requirement\_google) | ~> 3.90.1 |
-| [random](#requirement\_random) | ~> 3.1.0 |
+| [terraform](#requirement\_terraform) | >= 1.0.0 |
+| [google](#requirement\_google) | >= 3.90 |
+| [random](#requirement\_random) | >= 3 |
## Providers
@@ -14,9 +14,9 @@ No providers.
| Name | Source | Version |
|------|--------|---------|
-| [iglu\_db](#module\_iglu\_db) | snowplow-devops/cloud-sql/google | 0.1.1 |
-| [iglu\_lb](#module\_iglu\_lb) | snowplow-devops/lb/google | 0.1.0 |
-| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ce/google | 0.3.2 |
+| [iglu\_db](#module\_iglu\_db) | snowplow-devops/cloud-sql/google | 0.3.0 |
+| [iglu\_lb](#module\_iglu\_lb) | snowplow-devops/lb/google | 0.3.0 |
+| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ce/google | 0.4.0 |
## Resources
@@ -46,4 +46,4 @@ No resources.
| Name | Description |
|------|-------------|
-| [iglu\_server\_ip\_address](#output\_iglu\_server\_ip\_address) | The IP address for the Iglu Server |
\ No newline at end of file
+| [iglu\_server\_ip\_address](#output\_iglu\_server\_ip\_address) | The IP address for the Iglu Server |
diff --git a/terraform/gcp/iglu_server/default/main.tf b/terraform/gcp/iglu_server/default/main.tf
index 5e18395..301ded9 100644
--- a/terraform/gcp/iglu_server/default/main.tf
+++ b/terraform/gcp/iglu_server/default/main.tf
@@ -4,8 +4,10 @@ provider "google" {
}
module "iglu_db" {
- source = "snowplow-devops/cloud-sql/google"
- version = "0.1.1"
+ # source = "snowplow-devops/cloud-sql/google"
+ # version = "0.3.0"
+
+ source = "git::https://github.com/snowplow-devops/terraform-google-cloud-sql.git?ref=release/0.3.0"
name = "${var.prefix}-iglu-db"
@@ -18,8 +20,10 @@ module "iglu_db" {
}
module "iglu_server" {
- source = "snowplow-devops/iglu-server-ce/google"
- version = "0.3.2"
+ # source = "snowplow-devops/iglu-server-ce/google"
+ # version = "0.4.0"
+
+ source = "git::https://github.com/snowplow-devops/terraform-google-iglu-server-ce.git?ref=release/0.4.0"
name = "${var.prefix}-iglu-server"
@@ -46,8 +50,10 @@ module "iglu_server" {
}
module "iglu_lb" {
- source = "snowplow-devops/lb/google"
- version = "0.1.0"
+ # source = "snowplow-devops/lb/google"
+ # version = "0.3.0"
+
+ source = "git::https://github.com/snowplow-devops/terraform-google-lb.git?ref=release/0.3.0"
name = "${var.prefix}-iglu-lb"
diff --git a/terraform/gcp/iglu_server/default/terraform.tfvars b/terraform/gcp/iglu_server/default/terraform.tfvars
index f5f86a9..69ec6f3 100644
--- a/terraform/gcp/iglu_server/default/terraform.tfvars
+++ b/terraform/gcp/iglu_server/default/terraform.tfvars
@@ -31,7 +31,7 @@ ssh_key_pairs = [
iglu_db_name = "iglu"
iglu_db_username = "iglu"
# Change and keep this secret!
-iglu_db_password = "Hell0W0rld!"
+iglu_db_password = ""
# Used for API actions on the Iglu Server
# Change this to a new UUID and keep it secret!
diff --git a/terraform/gcp/iglu_server/default/versions.tf b/terraform/gcp/iglu_server/default/versions.tf
index 67f237f..81cb3d7 100644
--- a/terraform/gcp/iglu_server/default/versions.tf
+++ b/terraform/gcp/iglu_server/default/versions.tf
@@ -1,14 +1,14 @@
terraform {
- required_version = "~> 1"
+ required_version = ">= 1.0.0"
required_providers {
google = {
source = "hashicorp/google"
- version = "~> 3.90.1"
+ version = ">= 3.90"
}
random = {
source = "hashicorp/random"
- version = "~> 3.1.0"
+ version = ">= 3"
}
}
}
diff --git a/terraform/gcp/iglu_server/secure/README.md b/terraform/gcp/iglu_server/secure/README.md
index f176608..b21f7de 100644
--- a/terraform/gcp/iglu_server/secure/README.md
+++ b/terraform/gcp/iglu_server/secure/README.md
@@ -2,9 +2,9 @@
| Name | Version |
|------|---------|
-| [terraform](#requirement\_terraform) | ~> 1 |
-| [google](#requirement\_google) | ~> 3.90.1 |
-| [random](#requirement\_random) | ~> 3.1.0 |
+| [terraform](#requirement\_terraform) | >= 1.0.0 |
+| [google](#requirement\_google) | >= 3.90 |
+| [random](#requirement\_random) | >= 3 |
## Providers
@@ -14,9 +14,9 @@ No providers.
| Name | Source | Version |
|------|--------|---------|
-| [iglu\_db](#module\_iglu\_db) | snowplow-devops/cloud-sql/google | 0.1.1 |
-| [iglu\_lb](#module\_iglu\_lb) | snowplow-devops/lb/google | 0.1.0 |
-| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ce/google | 0.3.2 |
+| [iglu\_db](#module\_iglu\_db) | snowplow-devops/cloud-sql/google | 0.3.0 |
+| [iglu\_lb](#module\_iglu\_lb) | snowplow-devops/lb/google | 0.3.0 |
+| [iglu\_server](#module\_iglu\_server) | snowplow-devops/iglu-server-ce/google | 0.4.0 |
## Resources
@@ -46,4 +46,4 @@ No resources.
| Name | Description |
|------|-------------|
-| [iglu\_server\_ip\_address](#output\_iglu\_server\_ip\_address) | The IP address for the Iglu Server |
\ No newline at end of file
+| [iglu\_server\_ip\_address](#output\_iglu\_server\_ip\_address) | The IP address for the Iglu Server |
diff --git a/terraform/gcp/iglu_server/secure/main.tf b/terraform/gcp/iglu_server/secure/main.tf
index a634c9a..3255a14 100644
--- a/terraform/gcp/iglu_server/secure/main.tf
+++ b/terraform/gcp/iglu_server/secure/main.tf
@@ -5,7 +5,7 @@ provider "google" {
module "iglu_db" {
source = "snowplow-devops/cloud-sql/google"
- version = "0.1.1"
+ version = "0.3.0"
name = "${var.prefix}-iglu-db"
@@ -19,7 +19,7 @@ module "iglu_db" {
module "iglu_server" {
source = "snowplow-devops/iglu-server-ce/google"
- version = "0.3.2"
+ version = "0.4.0"
name = "${var.prefix}-iglu-server"
@@ -49,7 +49,7 @@ module "iglu_server" {
module "iglu_lb" {
source = "snowplow-devops/lb/google"
- version = "0.1.0"
+ version = "0.3.0"
name = "${var.prefix}-iglu-lb"
diff --git a/terraform/gcp/iglu_server/secure/terraform.tfvars b/terraform/gcp/iglu_server/secure/terraform.tfvars
index 9ac847f..04f31d6 100644
--- a/terraform/gcp/iglu_server/secure/terraform.tfvars
+++ b/terraform/gcp/iglu_server/secure/terraform.tfvars
@@ -30,7 +30,7 @@ ssh_key_pairs = [
iglu_db_name = "iglu"
iglu_db_username = "iglu"
# Change and keep this secret!
-iglu_db_password = "Hell0W0rld!"
+iglu_db_password = ""
# Used for API actions on the Iglu Server
# Change this to a new UUID and keep it secret!
diff --git a/terraform/gcp/iglu_server/secure/versions.tf b/terraform/gcp/iglu_server/secure/versions.tf
index 67f237f..81cb3d7 100644
--- a/terraform/gcp/iglu_server/secure/versions.tf
+++ b/terraform/gcp/iglu_server/secure/versions.tf
@@ -1,14 +1,14 @@
terraform {
- required_version = "~> 1"
+ required_version = ">= 1.0.0"
required_providers {
google = {
source = "hashicorp/google"
- version = "~> 3.90.1"
+ version = ">= 3.90"
}
random = {
source = "hashicorp/random"
- version = "~> 3.1.0"
+ version = ">= 3"
}
}
}
diff --git a/terraform/gcp/pipeline/default/README.md b/terraform/gcp/pipeline/default/README.md
index b466426..4316680 100644
--- a/terraform/gcp/pipeline/default/README.md
+++ b/terraform/gcp/pipeline/default/README.md
@@ -2,31 +2,31 @@
| Name | Version |
|------|---------|
-| [terraform](#requirement\_terraform) | ~> 1 |
-| [google](#requirement\_google) | ~> 3.90.1 |
-| [random](#requirement\_random) | ~> 3.1.0 |
+| [terraform](#requirement\_terraform) | >= 1.0.0 |
+| [google](#requirement\_google) | >= 3.90 |
+| [random](#requirement\_random) | >= 3 |
## Providers
| Name | Version |
|------|---------|
-| [google](#provider\_google) | ~> 3.90.1 |
+| [google](#provider\_google) | >= 3.90 |
## Modules
| Name | Source | Version |
|------|--------|---------|
-| [bad\_1\_topic](#module\_bad\_1\_topic) | snowplow-devops/pubsub-topic/google | 0.1.0 |
-| [bad\_rows\_topic](#module\_bad\_rows\_topic) | snowplow-devops/pubsub-topic/google | 0.1.0 |
-| [bigquery\_loader](#module\_bigquery\_loader) | snowplow-devops/bigquery-loader-pubsub-ce/google | 0.1.0 |
-| [collector\_lb](#module\_collector\_lb) | snowplow-devops/lb/google | 0.1.0 |
-| [collector\_pubsub](#module\_collector\_pubsub) | snowplow-devops/collector-pubsub-ce/google | 0.2.2 |
-| [enrich\_pubsub](#module\_enrich\_pubsub) | snowplow-devops/enrich-pubsub-ce/google | 0.1.2 |
-| [enriched\_topic](#module\_enriched\_topic) | snowplow-devops/pubsub-topic/google | 0.1.0 |
-| [postgres\_db](#module\_postgres\_db) | snowplow-devops/cloud-sql/google | 0.1.1 |
-| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.2.1 |
-| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.2.1 |
-| [raw\_topic](#module\_raw\_topic) | snowplow-devops/pubsub-topic/google | 0.1.0 |
+| [bad\_1\_topic](#module\_bad\_1\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 |
+| [bigquery\_loader](#module\_bigquery\_loader) | snowplow-devops/bigquery-loader-pubsub-ce/google | 0.2.0 |
+| [bq\_bad\_rows\_topic](#module\_bq\_bad\_rows\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 |
+| [collector\_lb](#module\_collector\_lb) | snowplow-devops/lb/google | 0.3.0 |
+| [collector\_pubsub](#module\_collector\_pubsub) | snowplow-devops/collector-pubsub-ce/google | 0.4.0 |
+| [enrich\_pubsub](#module\_enrich\_pubsub) | snowplow-devops/enrich-pubsub-ce/google | 0.2.0 |
+| [enriched\_topic](#module\_enriched\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 |
+| [postgres\_db](#module\_postgres\_db) | snowplow-devops/cloud-sql/google | 0.3.0 |
+| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.3.0 |
+| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.3.0 |
+| [raw\_topic](#module\_raw\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 |
## Resources
@@ -71,4 +71,4 @@
| [bq\_loader\_dead\_letter\_bucket\_name](#output\_bq\_loader\_dead\_letter\_bucket\_name) | The name of the GCS bucket for dead letter events emitted from the BigQuery loader |
| [collector\_ip\_address](#output\_collector\_ip\_address) | The IP address for the Pipeline Collector |
| [postgres\_db\_ip\_address](#output\_postgres\_db\_ip\_address) | The IP address of the database where your data is being streamed |
-| [postgres\_db\_port](#output\_postgres\_db\_port) | The port of the database where your data is being streamed |
\ No newline at end of file
+| [postgres\_db\_port](#output\_postgres\_db\_port) | The port of the database where your data is being streamed |
diff --git a/terraform/gcp/pipeline/default/bigquery.terraform.tfvars b/terraform/gcp/pipeline/default/bigquery.terraform.tfvars
deleted file mode 100644
index 979eec7..0000000
--- a/terraform/gcp/pipeline/default/bigquery.terraform.tfvars
+++ /dev/null
@@ -1,56 +0,0 @@
-# Will be prefixed to all resource names
-# Use this to easily identify the resources created and provide entropy for subsequent environments
-prefix = "sp"
-
-# The project to deploy the infrastructure into
-project_id = "PROJECT_ID_TO_DEPLOY_INTO"
-
-# Where to deploy the infrastructure
-region = "REGION_TO_DEPLOY_INTO"
-
-# --- Default Network
-# Update to the network you would like to deploy into
-#
-# Note: If you opt to use your own network then you will need to define a subnetwork to deploy into as well
-network = "default"
-subnetwork = ""
-
-# --- SSH
-# Update this to your IP Address
-ssh_ip_allowlist = ["999.999.999.999/32"]
-# Generate a new SSH key locally with `ssh-keygen`
-# ssh-keygen -t rsa -b 4096
-ssh_key_pairs = [
- {
- user_name = "snowplow"
- public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQA0jSi9//bRsHW4M6czodTs6smCXsxZ0gijzth0aBmycE= snowplow@Snowplows-MacBook-Pro.local"
- }
-]
-
-# --- Iglu Server Configuration
-# Iglu Server DNS output from the Iglu Server stack
-iglu_server_dns_name = "http://CHANGE-TO-MY-IGLU-IP"
-# Used for API actions on the Iglu Server
-# Change this to the same UUID from when you created the Iglu Server
-iglu_super_api_key = "00000000-0000-0000-0000-000000000000"
-
-# --- Snowplow BigQuery Loader
-bigquery_db_enabled = true
-# To use an existing bucket set this to false
-bigquery_loader_dead_letter_bucket_deploy = true
-# Must be globally unique so will need to be updated before applying
-bigquery_loader_dead_letter_bucket_name = "sp-bq-loader-dead-letter"
-
-# See for more information: https://registry.terraform.io/modules/snowplow-devops/collector-pubsub-ce/google/latest#telemetry
-# Telemetry principles: https://docs.snowplowanalytics.com/docs/open-source-quick-start/what-is-the-quick-start-for-open-source/telemetry-principles/
-user_provided_id = ""
-telemetry_enabled = true
-
-# --- SSL Configuration (optional)
-ssl_information = {
- certificate_id = ""
- enabled = false
-}
-
-# --- Extra Labels to append to created resources (optional)
-labels = {}
diff --git a/terraform/gcp/pipeline/default/main.tf b/terraform/gcp/pipeline/default/main.tf
index 9035efc..676e146 100644
--- a/terraform/gcp/pipeline/default/main.tf
+++ b/terraform/gcp/pipeline/default/main.tf
@@ -17,8 +17,10 @@ provider "google" {
# 1. Deploy PubSub Topics
module "raw_topic" {
- source = "snowplow-devops/pubsub-topic/google"
- version = "0.1.0"
+ # source = "snowplow-devops/pubsub-topic/google"
+ # version = "0.3.0"
+
+ source = "git::https://github.com/snowplow-devops/terraform-google-pubsub-topic.git?ref=release/0.3.0"
name = "${var.prefix}-raw-topic"
@@ -26,8 +28,10 @@ module "raw_topic" {
}
module "bad_1_topic" {
- source = "snowplow-devops/pubsub-topic/google"
- version = "0.1.0"
+ # source = "snowplow-devops/pubsub-topic/google"
+ # version = "0.3.0"
+
+ source = "git::https://github.com/snowplow-devops/terraform-google-pubsub-topic.git?ref=release/0.3.0"
name = "${var.prefix}-bad-1-topic"
@@ -35,8 +39,10 @@ module "bad_1_topic" {
}
module "enriched_topic" {
- source = "snowplow-devops/pubsub-topic/google"
- version = "0.1.0"
+ # source = "snowplow-devops/pubsub-topic/google"
+ # version = "0.3.0"
+
+ source = "git::https://github.com/snowplow-devops/terraform-google-pubsub-topic.git?ref=release/0.3.0"
name = "${var.prefix}-enriched-topic"
@@ -45,11 +51,15 @@ module "enriched_topic" {
# 2. Deploy Collector stack
module "collector_pubsub" {
- source = "snowplow-devops/collector-pubsub-ce/google"
- version = "0.2.2"
+ # source = "snowplow-devops/collector-pubsub-ce/google"
+ # version = "0.4.0"
+
+ source = "git::https://github.com/snowplow-devops/terraform-google-collector-pubsub-ce.git?ref=release/0.4.0"
name = "${var.prefix}-collector-server"
+ project_id = var.project_id
+
network = var.network
subnetwork = var.subnetwork
region = var.region
@@ -68,8 +78,10 @@ module "collector_pubsub" {
}
module "collector_lb" {
- source = "snowplow-devops/lb/google"
- version = "0.1.0"
+ # source = "snowplow-devops/lb/google"
+ # version = "0.3.0"
+
+ source = "git::https://github.com/snowplow-devops/terraform-google-lb.git?ref=release/0.3.0"
name = "${var.prefix}-collector-lb"
@@ -83,180 +95,25 @@ module "collector_lb" {
# 3. Deploy Enrichment
module "enrich_pubsub" {
- source = "snowplow-devops/enrich-pubsub-ce/google"
- version = "0.1.2"
-
- name = "${var.prefix}-enrich-server"
-
- network = var.network
- subnetwork = var.subnetwork
- region = var.region
-
- ssh_ip_allowlist = var.ssh_ip_allowlist
- ssh_key_pairs = var.ssh_key_pairs
-
- raw_topic_name = module.raw_topic.name
- good_topic_id = module.enriched_topic.id
- bad_topic_id = module.bad_1_topic.id
-
- # Linking in the custom Iglu Server here
- custom_iglu_resolvers = local.custom_iglu_resolvers
-
- telemetry_enabled = var.telemetry_enabled
- user_provided_id = var.user_provided_id
-
- labels = var.labels
-}
-
-# 4. Deploy Postgres Loader
-module "postgres_db" {
- source = "snowplow-devops/cloud-sql/google"
- version = "0.1.1"
-
- count = var.postgres_db_enabled ? 1 : 0
+ # source = "snowplow-devops/enrich-pubsub-ce/google"
+ # version = "0.2.0"
- name = "${var.prefix}-postgres-db"
+ source = "git::https://github.com/snowplow-devops/terraform-google-enrich-pubsub-ce.git?ref=release/0.2.0"
- region = var.region
- db_name = var.postgres_db_name
- db_username = var.postgres_db_username
- db_password = var.postgres_db_password
+ name = "${var.prefix}-enrich"
- authorized_networks = var.postgres_db_authorized_networks
-
- tier = var.postgres_db_tier
-
- labels = var.labels
-}
-
-module "postgres_loader_enriched" {
- source = "snowplow-devops/postgres-loader-pubsub-ce/google"
- version = "0.2.1"
-
- count = var.postgres_db_enabled ? 1 : 0
-
- name = "${var.prefix}-pg-loader-enriched-server"
-
- network = var.network
- subnetwork = var.subnetwork
- region = var.region
project_id = var.project_id
- ssh_ip_allowlist = var.ssh_ip_allowlist
- ssh_key_pairs = var.ssh_key_pairs
-
- in_topic_name = module.enriched_topic.name
- purpose = "ENRICHED_EVENTS"
- schema_name = "atomic"
-
- db_instance_name = join("", module.postgres_db.*.connection_name)
- db_port = join("", module.postgres_db.*.port)
- db_name = var.postgres_db_name
- db_username = var.postgres_db_username
- db_password = var.postgres_db_password
-
- # Linking in the custom Iglu Server here
- custom_iglu_resolvers = local.custom_iglu_resolvers
-
- telemetry_enabled = var.telemetry_enabled
- user_provided_id = var.user_provided_id
-
- labels = var.labels
-}
-
-module "postgres_loader_bad" {
- source = "snowplow-devops/postgres-loader-pubsub-ce/google"
- version = "0.2.1"
-
- count = var.postgres_db_enabled ? 1 : 0
-
- name = "${var.prefix}-pg-loader-bad-server"
-
network = var.network
subnetwork = var.subnetwork
region = var.region
- project_id = var.project_id
ssh_ip_allowlist = var.ssh_ip_allowlist
ssh_key_pairs = var.ssh_key_pairs
- in_topic_name = module.bad_1_topic.name
- purpose = "JSON"
- schema_name = "atomic_bad"
-
- db_instance_name = join("", module.postgres_db.*.connection_name)
- db_port = join("", module.postgres_db.*.port)
- db_name = var.postgres_db_name
- db_username = var.postgres_db_username
- db_password = var.postgres_db_password
-
- # Linking in the custom Iglu Server here
- custom_iglu_resolvers = local.custom_iglu_resolvers
-
- telemetry_enabled = var.telemetry_enabled
- user_provided_id = var.user_provided_id
-
- labels = var.labels
-}
-
-# 5. Deploy BigQuery Loader
-module "bad_rows_topic" {
- source = "snowplow-devops/pubsub-topic/google"
- version = "0.1.0"
-
- count = var.bigquery_db_enabled ? 1 : 0
-
- name = "${var.prefix}-bq-bad-rows-topic"
-
- labels = var.labels
-}
-
-resource "google_bigquery_dataset" "bigquery_db" {
- count = var.bigquery_db_enabled ? 1 : 0
-
- dataset_id = replace("${var.prefix}_pipeline_db", "-", "_")
- location = var.region
-
- labels = var.labels
-}
-
-resource "google_storage_bucket" "bq_loader_dead_letter_bucket" {
- count = var.bigquery_db_enabled && var.bigquery_loader_dead_letter_bucket_deploy ? 1 : 0
-
- name = var.bigquery_loader_dead_letter_bucket_name
- location = var.region
- force_destroy = true
-
- labels = var.labels
-}
-
-locals {
- bq_loader_dead_letter_bucket_name = coalesce(
- join("", google_storage_bucket.bq_loader_dead_letter_bucket.*.name),
- var.bigquery_loader_dead_letter_bucket_name,
- )
-}
-
-module "bigquery_loader" {
- source = "snowplow-devops/bigquery-loader-pubsub-ce/google"
- version = "0.1.0"
-
- count = var.bigquery_db_enabled ? 1 : 0
-
- name = "${var.prefix}-bq-loader-server"
-
- network = var.network
- subnetwork = var.subnetwork
- region = var.region
- project_id = var.project_id
-
- ssh_ip_allowlist = var.ssh_ip_allowlist
- ssh_key_pairs = var.ssh_key_pairs
-
- input_topic_name = module.enriched_topic.name
- bad_rows_topic_name = join("", module.bad_rows_topic.*.name)
- gcs_dead_letter_bucket_name = local.bq_loader_dead_letter_bucket_name
- bigquery_dataset_id = join("", google_bigquery_dataset.bigquery_db.*.dataset_id)
+ raw_topic_name = module.raw_topic.name
+ good_topic_id = module.enriched_topic.id
+ bad_topic_id = module.bad_1_topic.id
# Linking in the custom Iglu Server here
custom_iglu_resolvers = local.custom_iglu_resolvers
diff --git a/terraform/gcp/pipeline/default/outputs.tf b/terraform/gcp/pipeline/default/outputs.tf
index 034f7a9..7c53128 100644
--- a/terraform/gcp/pipeline/default/outputs.tf
+++ b/terraform/gcp/pipeline/default/outputs.tf
@@ -25,5 +25,5 @@ output "bq_loader_dead_letter_bucket_name" {
output "bq_loader_bad_rows_topic_name" {
description = "The name of the topic for bad rows emitted from the BigQuery loader"
- value = join("", module.bad_rows_topic.*.name)
+ value = join("", module.bq_bad_rows_topic.*.name)
}
diff --git a/terraform/gcp/pipeline/default/target_bigquery.tf b/terraform/gcp/pipeline/default/target_bigquery.tf
new file mode 100644
index 0000000..4af1ec3
--- /dev/null
+++ b/terraform/gcp/pipeline/default/target_bigquery.tf
@@ -0,0 +1,70 @@
+module "bq_bad_rows_topic" {
+ # source = "snowplow-devops/pubsub-topic/google"
+ # version = "0.3.0"
+
+ source = "git::https://github.com/snowplow-devops/terraform-google-pubsub-topic.git?ref=release/0.3.0"
+
+ count = var.bigquery_db_enabled ? 1 : 0
+
+ name = "${var.prefix}-bq-bad-rows-topic"
+
+ labels = var.labels
+}
+
+resource "google_bigquery_dataset" "bigquery_db" {
+ count = var.bigquery_db_enabled ? 1 : 0
+
+ dataset_id = replace("${var.prefix}_pipeline_db", "-", "_")
+ location = var.region
+
+ labels = var.labels
+}
+
+resource "google_storage_bucket" "bq_loader_dead_letter_bucket" {
+ count = var.bigquery_db_enabled && var.bigquery_loader_dead_letter_bucket_deploy ? 1 : 0
+
+ name = var.bigquery_loader_dead_letter_bucket_name
+ location = var.region
+ force_destroy = true
+
+ labels = var.labels
+}
+
+locals {
+ bq_loader_dead_letter_bucket_name = coalesce(
+ join("", google_storage_bucket.bq_loader_dead_letter_bucket.*.name),
+ var.bigquery_loader_dead_letter_bucket_name,
+ )
+}
+
+module "bigquery_loader" {
+ # source = "snowplow-devops/bigquery-loader-pubsub-ce/google"
+ # version = "0.2.0"
+
+ source = "git::https://github.com/snowplow-devops/terraform-google-bigquery-loader-pubsub-ce.git?ref=release/0.2.0"
+
+ count = var.bigquery_db_enabled ? 1 : 0
+
+ name = "${var.prefix}-bq-loader"
+
+ network = var.network
+ subnetwork = var.subnetwork
+ region = var.region
+ project_id = var.project_id
+
+ ssh_ip_allowlist = var.ssh_ip_allowlist
+ ssh_key_pairs = var.ssh_key_pairs
+
+ input_topic_name = module.enriched_topic.name
+ bad_rows_topic_name = join("", module.bq_bad_rows_topic.*.name)
+ gcs_dead_letter_bucket_name = local.bq_loader_dead_letter_bucket_name
+ bigquery_dataset_id = join("", google_bigquery_dataset.bigquery_db.*.dataset_id)
+
+ # Linking in the custom Iglu Server here
+ custom_iglu_resolvers = local.custom_iglu_resolvers
+
+ telemetry_enabled = var.telemetry_enabled
+ user_provided_id = var.user_provided_id
+
+ labels = var.labels
+}
diff --git a/terraform/gcp/pipeline/default/target_postgres.tf b/terraform/gcp/pipeline/default/target_postgres.tf
new file mode 100644
index 0000000..7f04fd2
--- /dev/null
+++ b/terraform/gcp/pipeline/default/target_postgres.tf
@@ -0,0 +1,95 @@
+module "postgres_db" {
+ # source = "snowplow-devops/cloud-sql/google"
+ # version = "0.3.0"
+
+ source = "git::https://github.com/snowplow-devops/terraform-google-cloud-sql.git?ref=release/0.3.0"
+
+ count = var.postgres_db_enabled ? 1 : 0
+
+ name = "${var.prefix}-postgres-db"
+
+ region = var.region
+ db_name = var.postgres_db_name
+ db_username = var.postgres_db_username
+ db_password = var.postgres_db_password
+
+ authorized_networks = var.postgres_db_authorized_networks
+
+ tier = var.postgres_db_tier
+
+ labels = var.labels
+}
+
+module "postgres_loader_enriched" {
+ # source = "snowplow-devops/postgres-loader-pubsub-ce/google"
+ # version = "0.3.0"
+
+ source = "git::https://github.com/snowplow-devops/terraform-google-postgres-loader-pubsub-ce.git?ref=release/0.3.0"
+
+ count = var.postgres_db_enabled ? 1 : 0
+
+ name = "${var.prefix}-pg-loader-enriched"
+
+ network = var.network
+ subnetwork = var.subnetwork
+ region = var.region
+ project_id = var.project_id
+
+ ssh_ip_allowlist = var.ssh_ip_allowlist
+ ssh_key_pairs = var.ssh_key_pairs
+
+ in_topic_name = module.enriched_topic.name
+ purpose = "ENRICHED_EVENTS"
+ schema_name = "atomic"
+
+ db_instance_name = join("", module.postgres_db.*.connection_name)
+ db_port = join("", module.postgres_db.*.port)
+ db_name = var.postgres_db_name
+ db_username = var.postgres_db_username
+ db_password = var.postgres_db_password
+
+ # Linking in the custom Iglu Server here
+ custom_iglu_resolvers = local.custom_iglu_resolvers
+
+ telemetry_enabled = var.telemetry_enabled
+ user_provided_id = var.user_provided_id
+
+ labels = var.labels
+}
+
+module "postgres_loader_bad" {
+ # source = "snowplow-devops/postgres-loader-pubsub-ce/google"
+ # version = "0.3.0"
+
+ source = "git::https://github.com/snowplow-devops/terraform-google-postgres-loader-pubsub-ce.git?ref=release/0.3.0"
+
+ count = var.postgres_db_enabled ? 1 : 0
+
+ name = "${var.prefix}-pg-loader-bad"
+
+ network = var.network
+ subnetwork = var.subnetwork
+ region = var.region
+ project_id = var.project_id
+
+ ssh_ip_allowlist = var.ssh_ip_allowlist
+ ssh_key_pairs = var.ssh_key_pairs
+
+ in_topic_name = module.bad_1_topic.name
+ purpose = "JSON"
+ schema_name = "atomic_bad"
+
+ db_instance_name = join("", module.postgres_db.*.connection_name)
+ db_port = join("", module.postgres_db.*.port)
+ db_name = var.postgres_db_name
+ db_username = var.postgres_db_username
+ db_password = var.postgres_db_password
+
+ # Linking in the custom Iglu Server here
+ custom_iglu_resolvers = local.custom_iglu_resolvers
+
+ telemetry_enabled = var.telemetry_enabled
+ user_provided_id = var.user_provided_id
+
+ labels = var.labels
+}
diff --git a/terraform/gcp/pipeline/default/postgres.terraform.tfvars b/terraform/gcp/pipeline/default/terraform.tfvars
similarity index 83%
rename from terraform/gcp/pipeline/default/postgres.terraform.tfvars
rename to terraform/gcp/pipeline/default/terraform.tfvars
index 7ca15d7..fe58814 100644
--- a/terraform/gcp/pipeline/default/postgres.terraform.tfvars
+++ b/terraform/gcp/pipeline/default/terraform.tfvars
@@ -8,14 +8,12 @@ project_id = "PROJECT_ID_TO_DEPLOY_INTO"
# Where to deploy the infrastructure
region = "REGION_TO_DEPLOY_INTO"
-# --- Default Network
# Update to the network you would like to deploy into
#
# Note: If you opt to use your own network then you will need to define a subnetwork to deploy into as well
network = "default"
subnetwork = ""
-# --- SSH
# Update this to your IP Address
ssh_ip_allowlist = ["999.999.999.999/32"]
# Generate a new SSH key locally with `ssh-keygen`
@@ -27,15 +25,22 @@ ssh_key_pairs = [
}
]
-# --- Iglu Server Configuration
# Iglu Server DNS output from the Iglu Server stack
-iglu_server_dns_name = "http://CHANGE-TO-MY-IGLU-IP"
+iglu_server_dns_name = "http://"
# Used for API actions on the Iglu Server
# Change this to the same UUID from when you created the Iglu Server
iglu_super_api_key = "00000000-0000-0000-0000-000000000000"
-# --- Snowplow Postgres Loader
-postgres_db_enabled = true
+# Collector SSL Configuration (optional)
+ssl_information = {
+ certificate_id = ""
+ enabled = false
+}
+
+# --- TARGETS CONFIGURATION ZONE --- #
+
+# --- Target: PostgreSQL
+postgres_db_enabled = false
postgres_db_name = "snowplow"
postgres_db_username = "snowplow"
@@ -61,16 +66,20 @@ postgres_db_authorized_networks = [
# a sufficiently powerful database tier is important to not running out of connection slots
postgres_db_tier = "db-g1-small"
+# --- Target: BigQuery
+bigquery_db_enabled = false
+
+# To use an existing bucket set this to false
+bigquery_loader_dead_letter_bucket_deploy = true
+# Must be globally unique so will need to be updated before applying
+bigquery_loader_dead_letter_bucket_name = "sp-bq-loader-dead-letter"
+
+# --- ADVANCED CONFIGURATION ZONE --- #
+
# See for more information: https://registry.terraform.io/modules/snowplow-devops/collector-pubsub-ce/google/latest#telemetry
# Telemetry principles: https://docs.snowplowanalytics.com/docs/open-source-quick-start/what-is-the-quick-start-for-open-source/telemetry-principles/
user_provided_id = ""
telemetry_enabled = true
-# --- SSL Configuration (optional)
-ssl_information = {
- certificate_id = ""
- enabled = false
-}
-
# --- Extra Labels to append to created resources (optional)
labels = {}
diff --git a/terraform/gcp/pipeline/default/versions.tf b/terraform/gcp/pipeline/default/versions.tf
index 67f237f..81cb3d7 100644
--- a/terraform/gcp/pipeline/default/versions.tf
+++ b/terraform/gcp/pipeline/default/versions.tf
@@ -1,14 +1,14 @@
terraform {
- required_version = "~> 1"
+ required_version = ">= 1.0.0"
required_providers {
google = {
source = "hashicorp/google"
- version = "~> 3.90.1"
+ version = ">= 3.90"
}
random = {
source = "hashicorp/random"
- version = "~> 3.1.0"
+ version = ">= 3"
}
}
}
diff --git a/terraform/gcp/pipeline/secure/README.md b/terraform/gcp/pipeline/secure/README.md
index b466426..4316680 100644
--- a/terraform/gcp/pipeline/secure/README.md
+++ b/terraform/gcp/pipeline/secure/README.md
@@ -2,31 +2,31 @@
| Name | Version |
|------|---------|
-| [terraform](#requirement\_terraform) | ~> 1 |
-| [google](#requirement\_google) | ~> 3.90.1 |
-| [random](#requirement\_random) | ~> 3.1.0 |
+| [terraform](#requirement\_terraform) | >= 1.0.0 |
+| [google](#requirement\_google) | >= 3.90 |
+| [random](#requirement\_random) | >= 3 |
## Providers
| Name | Version |
|------|---------|
-| [google](#provider\_google) | ~> 3.90.1 |
+| [google](#provider\_google) | >= 3.90 |
## Modules
| Name | Source | Version |
|------|--------|---------|
-| [bad\_1\_topic](#module\_bad\_1\_topic) | snowplow-devops/pubsub-topic/google | 0.1.0 |
-| [bad\_rows\_topic](#module\_bad\_rows\_topic) | snowplow-devops/pubsub-topic/google | 0.1.0 |
-| [bigquery\_loader](#module\_bigquery\_loader) | snowplow-devops/bigquery-loader-pubsub-ce/google | 0.1.0 |
-| [collector\_lb](#module\_collector\_lb) | snowplow-devops/lb/google | 0.1.0 |
-| [collector\_pubsub](#module\_collector\_pubsub) | snowplow-devops/collector-pubsub-ce/google | 0.2.2 |
-| [enrich\_pubsub](#module\_enrich\_pubsub) | snowplow-devops/enrich-pubsub-ce/google | 0.1.2 |
-| [enriched\_topic](#module\_enriched\_topic) | snowplow-devops/pubsub-topic/google | 0.1.0 |
-| [postgres\_db](#module\_postgres\_db) | snowplow-devops/cloud-sql/google | 0.1.1 |
-| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.2.1 |
-| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.2.1 |
-| [raw\_topic](#module\_raw\_topic) | snowplow-devops/pubsub-topic/google | 0.1.0 |
+| [bad\_1\_topic](#module\_bad\_1\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 |
+| [bigquery\_loader](#module\_bigquery\_loader) | snowplow-devops/bigquery-loader-pubsub-ce/google | 0.2.0 |
+| [bq\_bad\_rows\_topic](#module\_bq\_bad\_rows\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 |
+| [collector\_lb](#module\_collector\_lb) | snowplow-devops/lb/google | 0.3.0 |
+| [collector\_pubsub](#module\_collector\_pubsub) | snowplow-devops/collector-pubsub-ce/google | 0.4.0 |
+| [enrich\_pubsub](#module\_enrich\_pubsub) | snowplow-devops/enrich-pubsub-ce/google | 0.2.0 |
+| [enriched\_topic](#module\_enriched\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 |
+| [postgres\_db](#module\_postgres\_db) | snowplow-devops/cloud-sql/google | 0.3.0 |
+| [postgres\_loader\_bad](#module\_postgres\_loader\_bad) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.3.0 |
+| [postgres\_loader\_enriched](#module\_postgres\_loader\_enriched) | snowplow-devops/postgres-loader-pubsub-ce/google | 0.3.0 |
+| [raw\_topic](#module\_raw\_topic) | snowplow-devops/pubsub-topic/google | 0.3.0 |
## Resources
@@ -71,4 +71,4 @@
| [bq\_loader\_dead\_letter\_bucket\_name](#output\_bq\_loader\_dead\_letter\_bucket\_name) | The name of the GCS bucket for dead letter events emitted from the BigQuery loader |
| [collector\_ip\_address](#output\_collector\_ip\_address) | The IP address for the Pipeline Collector |
| [postgres\_db\_ip\_address](#output\_postgres\_db\_ip\_address) | The IP address of the database where your data is being streamed |
-| [postgres\_db\_port](#output\_postgres\_db\_port) | The port of the database where your data is being streamed |
\ No newline at end of file
+| [postgres\_db\_port](#output\_postgres\_db\_port) | The port of the database where your data is being streamed |
diff --git a/terraform/gcp/pipeline/secure/bigquery.terraform.tfvars b/terraform/gcp/pipeline/secure/bigquery.terraform.tfvars
deleted file mode 100644
index 7331a03..0000000
--- a/terraform/gcp/pipeline/secure/bigquery.terraform.tfvars
+++ /dev/null
@@ -1,55 +0,0 @@
-# Will be prefixed to all resource names
-# Use this to easily identify the resources created and provide entropy for subsequent environments
-prefix = "sp"
-
-# The project to deploy the infrastructure into
-project_id = "PROJECT_ID_TO_DEPLOY_INTO"
-
-# Where to deploy the infrastructure
-region = "REGION_TO_DEPLOY_INTO"
-
-# --- Network
-# NOTE: The network & sub-network configured must be configured with a Cloud NAT to allow the deployed Compute Engine instances to
-# connect to the internet to download the required assets
-network = "YOUR_NETWORK_HERE"
-subnetwork = "YOUR_SUB_NETWORK_HERE"
-
-# --- SSH
-# Update this to the internal IP of your Bastion Host
-ssh_ip_allowlist = ["999.999.999.999/32"]
-# Generate a new SSH key locally with `ssh-keygen`
-# ssh-keygen -t rsa -b 4096
-ssh_key_pairs = [
- {
- user_name = "snowplow"
- public_key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQA0jSi9//bRsHW4M6czodTs6smCXsxZ0gijzth0aBmycE= snowplow@Snowplows-MacBook-Pro.local"
- }
-]
-
-# --- Iglu Server Configuration
-# Iglu Server DNS output from the Iglu Server stack
-iglu_server_dns_name = "http://CHANGE-TO-MY-IGLU-IP"
-# Used for API actions on the Iglu Server
-# Change this to the same UUID from when you created the Iglu Server
-iglu_super_api_key = "00000000-0000-0000-0000-000000000000"
-
-# --- Snowplow BigQuery Loader
-bigquery_db_enabled = true
-# To use an existing bucket set this to false
-bigquery_loader_dead_letter_bucket_deploy = true
-# Must be globally unique so will need to be updated before applying
-bigquery_loader_dead_letter_bucket_name = "sp-bq-loader-dead-letter"
-
-# See for more information: https://registry.terraform.io/modules/snowplow-devops/collector-pubsub-ce/google/latest#telemetry
-# Telemetry principles: https://docs.snowplowanalytics.com/docs/open-source-quick-start/what-is-the-quick-start-for-open-source/telemetry-principles/
-user_provided_id = ""
-telemetry_enabled = true
-
-# --- SSL Configuration (optional)
-ssl_information = {
- certificate_id = ""
- enabled = false
-}
-
-# --- Extra Labels to append to created resources (optional)
-labels = {}
diff --git a/terraform/gcp/pipeline/secure/main.tf b/terraform/gcp/pipeline/secure/main.tf
index fc0084b..48887f1 100644
--- a/terraform/gcp/pipeline/secure/main.tf
+++ b/terraform/gcp/pipeline/secure/main.tf
@@ -18,7 +18,7 @@ provider "google" {
# 1. Deploy PubSub Topics
module "raw_topic" {
source = "snowplow-devops/pubsub-topic/google"
- version = "0.1.0"
+ version = "0.3.0"
name = "${var.prefix}-raw-topic"
@@ -27,7 +27,7 @@ module "raw_topic" {
module "bad_1_topic" {
source = "snowplow-devops/pubsub-topic/google"
- version = "0.1.0"
+ version = "0.3.0"
name = "${var.prefix}-bad-1-topic"
@@ -36,7 +36,7 @@ module "bad_1_topic" {
module "enriched_topic" {
source = "snowplow-devops/pubsub-topic/google"
- version = "0.1.0"
+ version = "0.3.0"
name = "${var.prefix}-enriched-topic"
@@ -46,7 +46,7 @@ module "enriched_topic" {
# 2. Deploy Collector stack
module "collector_pubsub" {
source = "snowplow-devops/collector-pubsub-ce/google"
- version = "0.2.2"
+ version = "0.4.0"
name = "${var.prefix}-collector-server"
@@ -71,7 +71,7 @@ module "collector_pubsub" {
module "collector_lb" {
source = "snowplow-devops/lb/google"
- version = "0.1.0"
+ version = "0.3.0"
name = "${var.prefix}-collector-lb"
@@ -86,7 +86,7 @@ module "collector_lb" {
# 3. Deploy Enrichment
module "enrich_pubsub" {
source = "snowplow-devops/enrich-pubsub-ce/google"
- version = "0.1.2"
+ version = "0.2.0"
name = "${var.prefix}-enrich-server"
@@ -111,162 +111,3 @@ module "enrich_pubsub" {
labels = var.labels
}
-
-# 4. Deploy Postgres Loader
-module "postgres_db" {
- source = "snowplow-devops/cloud-sql/google"
- version = "0.1.1"
-
- count = var.postgres_db_enabled ? 1 : 0
-
- name = "${var.prefix}-postgres-db"
-
- region = var.region
- db_name = var.postgres_db_name
- db_username = var.postgres_db_username
- db_password = var.postgres_db_password
-
- authorized_networks = var.postgres_db_authorized_networks
-
- tier = var.postgres_db_tier
-
- labels = var.labels
-}
-
-module "postgres_loader_enriched" {
- source = "snowplow-devops/postgres-loader-pubsub-ce/google"
- version = "0.2.1"
-
- count = var.postgres_db_enabled ? 1 : 0
-
- name = "${var.prefix}-pg-loader-enriched-server"
-
- network = var.network
- subnetwork = var.subnetwork
- region = var.region
- project_id = var.project_id
-
- ssh_ip_allowlist = var.ssh_ip_allowlist
- ssh_key_pairs = var.ssh_key_pairs
-
- in_topic_name = module.enriched_topic.name
- purpose = "ENRICHED_EVENTS"
- schema_name = "atomic"
-
- db_instance_name = join("", module.postgres_db.*.connection_name)
- db_port = join("", module.postgres_db.*.port)
- db_name = var.postgres_db_name
- db_username = var.postgres_db_username
- db_password = var.postgres_db_password
-
- # Linking in the custom Iglu Server here
- custom_iglu_resolvers = local.custom_iglu_resolvers
-
- telemetry_enabled = var.telemetry_enabled
- user_provided_id = var.user_provided_id
-
- labels = var.labels
-}
-
-module "postgres_loader_bad" {
- source = "snowplow-devops/postgres-loader-pubsub-ce/google"
- version = "0.2.1"
-
- count = var.postgres_db_enabled ? 1 : 0
-
- name = "${var.prefix}-pg-loader-bad-server"
-
- network = var.network
- subnetwork = var.subnetwork
- region = var.region
- project_id = var.project_id
-
- ssh_ip_allowlist = var.ssh_ip_allowlist
- ssh_key_pairs = var.ssh_key_pairs
-
- in_topic_name = module.bad_1_topic.name
- purpose = "JSON"
- schema_name = "atomic_bad"
-
- db_instance_name = join("", module.postgres_db.*.connection_name)
- db_port = join("", module.postgres_db.*.port)
- db_name = var.postgres_db_name
- db_username = var.postgres_db_username
- db_password = var.postgres_db_password
-
- # Linking in the custom Iglu Server here
- custom_iglu_resolvers = local.custom_iglu_resolvers
-
- telemetry_enabled = var.telemetry_enabled
- user_provided_id = var.user_provided_id
-
- labels = var.labels
-}
-
-# 5. Deploy BigQuery Loader
-module "bad_rows_topic" {
- source = "snowplow-devops/pubsub-topic/google"
- version = "0.1.0"
-
- count = var.bigquery_db_enabled ? 1 : 0
-
- name = "${var.prefix}-bq-bad-rows-topic"
-
- labels = var.labels
-}
-
-resource "google_bigquery_dataset" "bigquery_db" {
- count = var.bigquery_db_enabled ? 1 : 0
-
- dataset_id = replace("${var.prefix}_snowplow_db", "-", "_")
- location = var.region
-
- labels = var.labels
-}
-
-resource "google_storage_bucket" "bq_loader_dead_letter_bucket" {
- count = var.bigquery_db_enabled && var.bigquery_loader_dead_letter_bucket_deploy ? 1 : 0
-
- name = var.bigquery_loader_dead_letter_bucket_name
- location = var.region
- force_destroy = true
-
- labels = var.labels
-}
-
-locals {
- bq_loader_dead_letter_bucket_name = coalesce(
- join("", google_storage_bucket.bq_loader_dead_letter_bucket.*.name),
- var.bigquery_loader_dead_letter_bucket_name,
- )
-}
-
-module "bigquery_loader" {
- source = "snowplow-devops/bigquery-loader-pubsub-ce/google"
- version = "0.1.0"
-
- count = var.bigquery_db_enabled ? 1 : 0
-
- name = "${var.prefix}-bq-loader-server"
-
- network = var.network
- subnetwork = var.subnetwork
- region = var.region
- project_id = var.project_id
-
- ssh_ip_allowlist = var.ssh_ip_allowlist
- ssh_key_pairs = var.ssh_key_pairs
-
- input_topic_name = module.enriched_topic.name
- bad_rows_topic_name = join("", module.bad_rows_topic.*.name)
- gcs_dead_letter_bucket_name = local.bq_loader_dead_letter_bucket_name
- bigquery_dataset_id = join("", google_bigquery_dataset.bigquery_db.*.dataset_id)
-
- # Linking in the custom Iglu Server here
- custom_iglu_resolvers = local.custom_iglu_resolvers
-
- telemetry_enabled = var.telemetry_enabled
- user_provided_id = var.user_provided_id
-
- labels = var.labels
-}
diff --git a/terraform/gcp/pipeline/secure/outputs.tf b/terraform/gcp/pipeline/secure/outputs.tf
index 034f7a9..7c53128 100644
--- a/terraform/gcp/pipeline/secure/outputs.tf
+++ b/terraform/gcp/pipeline/secure/outputs.tf
@@ -25,5 +25,5 @@ output "bq_loader_dead_letter_bucket_name" {
output "bq_loader_bad_rows_topic_name" {
description = "The name of the topic for bad rows emitted from the BigQuery loader"
- value = join("", module.bad_rows_topic.*.name)
+ value = join("", module.bq_bad_rows_topic.*.name)
}
diff --git a/terraform/gcp/pipeline/secure/target_bigquery.tf b/terraform/gcp/pipeline/secure/target_bigquery.tf
new file mode 100644
index 0000000..5bbeea5
--- /dev/null
+++ b/terraform/gcp/pipeline/secure/target_bigquery.tf
@@ -0,0 +1,68 @@
+module "bq_bad_rows_topic" {
+ source = "snowplow-devops/pubsub-topic/google"
+ version = "0.3.0"
+
+ count = var.bigquery_db_enabled ? 1 : 0
+
+ name = "${var.prefix}-bq-bad-rows-topic"
+
+ labels = var.labels
+}
+
+resource "google_bigquery_dataset" "bigquery_db" {
+ count = var.bigquery_db_enabled ? 1 : 0
+
+ dataset_id = replace("${var.prefix}_snowplow_db", "-", "_")
+ location = var.region
+
+ labels = var.labels
+}
+
+resource "google_storage_bucket" "bq_loader_dead_letter_bucket" {
+ count = var.bigquery_db_enabled && var.bigquery_loader_dead_letter_bucket_deploy ? 1 : 0
+
+ name = var.bigquery_loader_dead_letter_bucket_name
+ location = var.region
+ force_destroy = true
+
+ labels = var.labels
+}
+
+locals {
+ bq_loader_dead_letter_bucket_name = coalesce(
+ join("", google_storage_bucket.bq_loader_dead_letter_bucket.*.name),
+ var.bigquery_loader_dead_letter_bucket_name,
+ )
+}
+
+module "bigquery_loader" {
+ source = "snowplow-devops/bigquery-loader-pubsub-ce/google"
+ version = "0.2.0"
+
+ count = var.bigquery_db_enabled ? 1 : 0
+
+ name = "${var.prefix}-bq-loader-server"
+
+ network = var.network
+ subnetwork = var.subnetwork
+ region = var.region
+ project_id = var.project_id
+
+ ssh_ip_allowlist = var.ssh_ip_allowlist
+ ssh_key_pairs = var.ssh_key_pairs
+
+ input_topic_name = module.enriched_topic.name
+ bad_rows_topic_name = join("", module.bq_bad_rows_topic.*.name)
+ gcs_dead_letter_bucket_name = local.bq_loader_dead_letter_bucket_name
+ bigquery_dataset_id = join("", google_bigquery_dataset.bigquery_db.*.dataset_id)
+
+ # Linking in the custom Iglu Server here
+ custom_iglu_resolvers = local.custom_iglu_resolvers
+
+ telemetry_enabled = var.telemetry_enabled
+ user_provided_id = var.user_provided_id
+
+ associate_public_ip_address = false
+
+ labels = var.labels
+}
diff --git a/terraform/gcp/pipeline/secure/target_postgres.tf b/terraform/gcp/pipeline/secure/target_postgres.tf
new file mode 100644
index 0000000..2232bd3
--- /dev/null
+++ b/terraform/gcp/pipeline/secure/target_postgres.tf
@@ -0,0 +1,91 @@
+module "postgres_db" {
+ source = "snowplow-devops/cloud-sql/google"
+ version = "0.3.0"
+
+ count = var.postgres_db_enabled ? 1 : 0
+
+ name = "${var.prefix}-postgres-db"
+
+ region = var.region
+ db_name = var.postgres_db_name
+ db_username = var.postgres_db_username
+ db_password = var.postgres_db_password
+
+ authorized_networks = var.postgres_db_authorized_networks
+
+ tier = var.postgres_db_tier
+
+ labels = var.labels
+}
+
+module "postgres_loader_enriched" {
+ source = "snowplow-devops/postgres-loader-pubsub-ce/google"
+ version = "0.3.0"
+
+ count = var.postgres_db_enabled ? 1 : 0
+
+ name = "${var.prefix}-pg-loader-enriched-server"
+
+ network = var.network
+ subnetwork = var.subnetwork
+ region = var.region
+ project_id = var.project_id
+
+ ssh_ip_allowlist = var.ssh_ip_allowlist
+ ssh_key_pairs = var.ssh_key_pairs
+
+ in_topic_name = module.enriched_topic.name
+ purpose = "ENRICHED_EVENTS"
+ schema_name = "atomic"
+
+ db_instance_name = join("", module.postgres_db.*.connection_name)
+ db_port = join("", module.postgres_db.*.port)
+ db_name = var.postgres_db_name
+ db_username = var.postgres_db_username
+ db_password = var.postgres_db_password
+
+ # Linking in the custom Iglu Server here
+ custom_iglu_resolvers = local.custom_iglu_resolvers
+
+ telemetry_enabled = var.telemetry_enabled
+ user_provided_id = var.user_provided_id
+
+ labels = var.labels
+}
+
+module "postgres_loader_bad" {
+ source = "snowplow-devops/postgres-loader-pubsub-ce/google"
+ version = "0.3.0"
+
+ count = var.postgres_db_enabled ? 1 : 0
+
+ name = "${var.prefix}-pg-loader-bad-server"
+
+ network = var.network
+ subnetwork = var.subnetwork
+ region = var.region
+ project_id = var.project_id
+
+ ssh_ip_allowlist = var.ssh_ip_allowlist
+ ssh_key_pairs = var.ssh_key_pairs
+
+ in_topic_name = module.bad_1_topic.name
+ purpose = "JSON"
+ schema_name = "atomic_bad"
+
+ db_instance_name = join("", module.postgres_db.*.connection_name)
+ db_port = join("", module.postgres_db.*.port)
+ db_name = var.postgres_db_name
+ db_username = var.postgres_db_username
+ db_password = var.postgres_db_password
+
+ # Linking in the custom Iglu Server here
+ custom_iglu_resolvers = local.custom_iglu_resolvers
+
+ telemetry_enabled = var.telemetry_enabled
+ user_provided_id = var.user_provided_id
+
+ associate_public_ip_address = false
+
+ labels = var.labels
+}
diff --git a/terraform/gcp/pipeline/secure/postgres.terraform.tfvars b/terraform/gcp/pipeline/secure/terraform.tfvars
similarity index 83%
rename from terraform/gcp/pipeline/secure/postgres.terraform.tfvars
rename to terraform/gcp/pipeline/secure/terraform.tfvars
index fc9a4a8..8267274 100644
--- a/terraform/gcp/pipeline/secure/postgres.terraform.tfvars
+++ b/terraform/gcp/pipeline/secure/terraform.tfvars
@@ -8,13 +8,11 @@ project_id = "PROJECT_ID_TO_DEPLOY_INTO"
# Where to deploy the infrastructure
region = "REGION_TO_DEPLOY_INTO"
-# --- Network
# NOTE: The network & sub-network configured must be configured with a Cloud NAT to allow the deployed Compute Engine instances to
# connect to the internet to download the required assets
network = "YOUR_NETWORK_HERE"
subnetwork = "YOUR_SUB_NETWORK_HERE"
-# --- SSH
# Update this to the internal IP of your Bastion Host
ssh_ip_allowlist = ["999.999.999.999/32"]
# Generate a new SSH key locally with `ssh-keygen`
@@ -26,15 +24,22 @@ ssh_key_pairs = [
}
]
-# --- Iglu Server Configuration
# Iglu Server DNS output from the Iglu Server stack
-iglu_server_dns_name = "http://CHANGE-TO-MY-IGLU-IP"
+iglu_server_dns_name = "http://"
# Used for API actions on the Iglu Server
# Change this to the same UUID from when you created the Iglu Server
iglu_super_api_key = "00000000-0000-0000-0000-000000000000"
-# --- Snowplow Postgres Loader
-postgres_db_enabled = true
+# Collector SSL Configuration (optional)
+ssl_information = {
+ certificate_id = ""
+ enabled = false
+}
+
+# --- TARGETS CONFIGURATION ZONE --- #
+
+# --- Target: PostgreSQL
+postgres_db_enabled = false
postgres_db_name = "snowplow"
postgres_db_username = "snowplow"
@@ -60,16 +65,20 @@ postgres_db_authorized_networks = [
# a sufficiently powerful database tier is important to not running out of connection slots
postgres_db_tier = "db-g1-small"
+# --- Target: BigQuery
+bigquery_db_enabled = false
+
+# To use an existing bucket set this to false
+bigquery_loader_dead_letter_bucket_deploy = true
+# Must be globally unique so will need to be updated before applying
+bigquery_loader_dead_letter_bucket_name = "sp-bq-loader-dead-letter"
+
+# --- ADVANCED CONFIGURATION ZONE --- #
+
# See for more information: https://registry.terraform.io/modules/snowplow-devops/collector-pubsub-ce/google/latest#telemetry
# Telemetry principles: https://docs.snowplowanalytics.com/docs/open-source-quick-start/what-is-the-quick-start-for-open-source/telemetry-principles/
user_provided_id = ""
telemetry_enabled = true
-# --- SSL Configuration (optional)
-ssl_information = {
- certificate_id = ""
- enabled = false
-}
-
# --- Extra Labels to append to created resources (optional)
labels = {}
diff --git a/terraform/gcp/pipeline/secure/versions.tf b/terraform/gcp/pipeline/secure/versions.tf
index 67f237f..81cb3d7 100644
--- a/terraform/gcp/pipeline/secure/versions.tf
+++ b/terraform/gcp/pipeline/secure/versions.tf
@@ -1,14 +1,14 @@
terraform {
- required_version = "~> 1"
+ required_version = ">= 1.0.0"
required_providers {
google = {
source = "hashicorp/google"
- version = "~> 3.90.1"
+ version = ">= 3.90"
}
random = {
source = "hashicorp/random"
- version = "~> 3.1.0"
+ version = ">= 3"
}
}
}