Release/0.2.0

CHANGELOG

@@ -1,3 +1,8 @@
+Version 0.2.0 (2023-07-26)
+--------------------------
+Leverage common service module (#13)
+Update to Snowplow Community Licence (#14)
+
 Version 0.1.4 (2022-05-13)
 --------------------------
 Ensure telemetry can be disabled without breaking configuration file (#9)

README.md

@@ -29,21 +29,21 @@ By default this module enables 5 enrichments which you can find in the `template
 ```hcl
 module "raw_topic" {
   source  = "snowplow-devops/pubsub-topic/google"
-  version = "0.1.0"
+  version = "0.3.0"
 
   name = "raw-topic"
 }
 
 module "bad_1_topic" {
   source  = "snowplow-devops/pubsub-topic/google"
-  version = "0.1.0"
+  version = "0.3.0"
 
   name = "bad-1-topic"
 }
 
 module "enriched_topic" {
   source  = "snowplow-devops/pubsub-topic/google"
-  version = "0.1.0"
+  version = "0.3.0"
 
   name = "enriched-topic"
 }
@@ -185,7 +185,7 @@ module "enrich_pubsub" {
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.15 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0.0 |
 | <a name="requirement_google"></a> [google](#requirement\_google) | >= 3.44.0 |
 
 ## Providers
@@ -198,24 +198,22 @@ module "enrich_pubsub" {
 
 | Name | Source | Version |
 |------|--------|---------|
-| <a name="module_telemetry"></a> [telemetry](#module\_telemetry) | snowplow-devops/telemetry/snowplow | 0.2.0 |
+| <a name="module_service"></a> [service](#module\_service) | snowplow-devops/service-ce/google | 0.1.0 |
+| <a name="module_telemetry"></a> [telemetry](#module\_telemetry) | snowplow-devops/telemetry/snowplow | 0.5.0 |
 
 ## Resources
 
 | Name | Type |
 |------|------|
 | [google_compute_firewall.egress](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource |
 | [google_compute_firewall.ingress_ssh](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_firewall) | resource |
-| [google_compute_instance_template.tpl](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template) | resource |
-| [google_compute_region_instance_group_manager.grp](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_region_instance_group_manager) | resource |
 | [google_project_iam_member.sa_logging_log_writer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
 | [google_project_iam_member.sa_pubsub_publisher](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
 | [google_project_iam_member.sa_pubsub_subscriber](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
 | [google_project_iam_member.sa_pubsub_viewer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
 | [google_project_iam_member.sa_storage_object_viewer](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/project_iam_member) | resource |
 | [google_pubsub_subscription.in](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/pubsub_subscription) | resource |
 | [google_service_account.sa](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/service_account) | resource |
-| [google_compute_image.ubuntu_20_04](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/compute_image) | data source |
 
 ## Inputs
 
@@ -225,8 +223,11 @@ module "enrich_pubsub" {
 | <a name="input_good_topic_id"></a> [good\_topic\_id](#input\_good\_topic\_id) | The id of the good pubsub topic that enrichment will insert data into | `string` | n/a | yes |
 | <a name="input_name"></a> [name](#input\_name) | A name which will be pre-pended to the resources created | `string` | n/a | yes |
 | <a name="input_network"></a> [network](#input\_network) | The name of the network to deploy within | `string` | n/a | yes |
+| <a name="input_project_id"></a> [project\_id](#input\_project\_id) | The project ID in which the stack is being deployed | `string` | n/a | yes |
 | <a name="input_raw_topic_name"></a> [raw\_topic\_name](#input\_raw\_topic\_name) | The name of the raw pubsub topic that enrichment will pull data from | `string` | n/a | yes |
 | <a name="input_region"></a> [region](#input\_region) | The name of the region to deploy within | `string` | n/a | yes |
+| <a name="input_app_version"></a> [app\_version](#input\_app\_version) | App version to use. This variable facilitates dev flow, the modules may not work with anything other than the default value. | `string` | `"3.8.0"` | no |
+| <a name="input_assets_update_period"></a> [assets\_update\_period](#input\_assets\_update\_period) | Period after which enrich assets should be checked for updates (e.g. MaxMind DB) | `string` | `"7 days"` | no |
 | <a name="input_associate_public_ip_address"></a> [associate\_public\_ip\_address](#input\_associate\_public\_ip\_address) | Whether to assign a public ip address to this instance; if false this instance must be behind a Cloud NAT to connect to the internet | `bool` | `true` | no |
 | <a name="input_custom_iglu_resolvers"></a> [custom\_iglu\_resolvers](#input\_custom\_iglu\_resolvers) | The custom Iglu Resolvers that will be used by Enrichment to resolve and validate events | <pre>list(object({<br>    name            = string<br>    priority        = number<br>    uri             = string<br>    api_key         = string<br>    vendor_prefixes = list(string)<br>  }))</pre> | `[]` | no |
 | <a name="input_custom_tcp_egress_port_list"></a> [custom\_tcp\_egress\_port\_list](#input\_custom\_tcp\_egress\_port\_list) | For opening up TCP ports to access other destinations not served over HTTP(s) (e.g. for SQL / API enrichments) | `list(string)` | `[]` | no |
@@ -248,6 +249,7 @@ module "enrich_pubsub" {
 | <a name="input_enrichment_weather_enrichment_config"></a> [enrichment\_weather\_enrichment\_config](#input\_enrichment\_weather\_enrichment\_config) | n/a | `string` | `""` | no |
 | <a name="input_enrichment_yauaa_enrichment_config"></a> [enrichment\_yauaa\_enrichment\_config](#input\_enrichment\_yauaa\_enrichment\_config) | n/a | `string` | `""` | no |
 | <a name="input_gcp_logs_enabled"></a> [gcp\_logs\_enabled](#input\_gcp\_logs\_enabled) | Whether application logs should be reported to GCP Logging | `bool` | `true` | no |
+| <a name="input_java_opts"></a> [java\_opts](#input\_java\_opts) | Custom JAVA Options | `string` | `"-XX:InitialRAMPercentage=75 -XX:MaxRAMPercentage=75"` | no |
 | <a name="input_labels"></a> [labels](#input\_labels) | The labels to append to this resource | `map(string)` | `{}` | no |
 | <a name="input_machine_type"></a> [machine\_type](#input\_machine\_type) | The machine type to use | `string` | `"e2-small"` | no |
 | <a name="input_ssh_block_project_keys"></a> [ssh\_block\_project\_keys](#input\_ssh\_block\_project\_keys) | Whether to block project wide SSH keys | `bool` | `true` | no |
@@ -269,10 +271,9 @@ module "enrich_pubsub" {
 
 # Copyright and license
 
-The Terraform Google Enrich PubSub on Compute Engine project is Copyright 2021-2021 Snowplow Analytics Ltd.
+The Terraform Google Enrich PubSub on Compute Engine project is Copyright 2021-present Snowplow Analytics Ltd.
 
-Licensed under the [Apache License, Version 2.0][license] (the "License");
-you may not use this software except in compliance with the License.
+Licensed under the [Snowplow Community License](https://docs.snowplow.io/community-license-1.0). _(If you are uncertain how it applies to your use case, check our answers to [frequently asked questions](https://docs.snowplow.io/docs/contributing/community-license-faq/).)_
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
@@ -286,8 +287,8 @@ limitations under the License.
 [ci]: https://github.com/snowplow-devops/terraform-google-enrich-pubsub-ce/actions?query=workflow%3Aci
 [ci-image]: https://github.com/snowplow-devops/terraform-google-enrich-pubsub-ce/workflows/ci/badge.svg
 
-[license]: https://www.apache.org/licenses/LICENSE-2.0
-[license-image]: https://img.shields.io/badge/license-Apache--2-blue.svg?style=flat
+[license]: https://docs.snowplow.io/docs/contributing/community-license-faq/
+[license-image]: https://img.shields.io/badge/license-Snowplow--Community-blue.svg?style=flat
 
 [registry]: https://registry.terraform.io/modules/snowplow-devops/enrich-pubsub-ce/google/latest
 [registry-image]: https://img.shields.io/static/v1?label=Terraform&message=Registry&color=7B42BC&logo=terraform

main.tf

@@ -1,9 +1,9 @@
 locals {
   module_name    = "enrich-pubsub-ce"
-  module_version = "0.1.4"
+  module_version = "0.2.0"
 
   app_name    = "enrich-pubsub"
-  app_version = "3.0.3"
+  app_version = var.app_version
 
   local_labels = {
     name           = var.name
@@ -21,7 +21,7 @@ locals {
 
 module "telemetry" {
   source  = "snowplow-devops/telemetry/snowplow"
-  version = "0.2.0"
+  version = "0.5.0"
 
   count = var.telemetry_enabled ? 1 : 0
 
@@ -34,11 +34,6 @@ module "telemetry" {
   module_version   = local.module_version
 }
 
-data "google_compute_image" "ubuntu_20_04" {
-  family  = "ubuntu-2004-lts"
-  project = "ubuntu-os-cloud"
-}
-
 # --- IAM: Service Account setup
 
 resource "google_service_account" "sa" {
@@ -47,28 +42,33 @@ resource "google_service_account" "sa" {
 }
 
 resource "google_project_iam_member" "sa_pubsub_viewer" {
-  role   = "roles/pubsub.viewer"
-  member = "serviceAccount:${google_service_account.sa.email}"
+  project = var.project_id
+  role    = "roles/pubsub.viewer"
+  member  = "serviceAccount:${google_service_account.sa.email}"
 }
 
 resource "google_project_iam_member" "sa_pubsub_subscriber" {
-  role   = "roles/pubsub.subscriber"
-  member = "serviceAccount:${google_service_account.sa.email}"
+  project = var.project_id
+  role    = "roles/pubsub.subscriber"
+  member  = "serviceAccount:${google_service_account.sa.email}"
 }
 
 resource "google_project_iam_member" "sa_pubsub_publisher" {
-  role   = "roles/pubsub.publisher"
-  member = "serviceAccount:${google_service_account.sa.email}"
+  project = var.project_id
+  role    = "roles/pubsub.publisher"
+  member  = "serviceAccount:${google_service_account.sa.email}"
 }
 
 resource "google_project_iam_member" "sa_logging_log_writer" {
-  role   = "roles/logging.logWriter"
-  member = "serviceAccount:${google_service_account.sa.email}"
+  project = var.project_id
+  role    = "roles/logging.logWriter"
+  member  = "serviceAccount:${google_service_account.sa.email}"
 }
 
 resource "google_project_iam_member" "sa_storage_object_viewer" {
-  role   = "roles/storage.objectViewer"
-  member = "serviceAccount:${google_service_account.sa.email}"
+  project = var.project_id
+  role    = "roles/storage.objectViewer"
+  member  = "serviceAccount:${google_service_account.sa.email}"
 }
 
 # --- CE: Firewall rules
@@ -126,12 +126,16 @@ locals {
     good_topic_id       = var.good_topic_id
     bad_topic_id        = var.bad_topic_id
 
-    disable           = !tobool(var.telemetry_enabled)
-    telemetry_url     = join("", module.telemetry.*.collector_uri)
-    user_provided_id  = var.user_provided_id
-    auto_generated_id = join("", module.telemetry.*.auto_generated_id)
-    module_name       = local.module_name
-    module_version    = local.module_version
+    assets_update_period = var.assets_update_period
+
+    telemetry_disable          = !var.telemetry_enabled
+    telemetry_collector_uri    = join("", module.telemetry.*.collector_uri)
+    telemetry_collector_port   = 443
+    telemetry_secure           = true
+    telemetry_user_provided_id = var.user_provided_id
+    telemetry_auto_gen_id      = join("", module.telemetry.*.auto_generated_id)
+    telemetry_module_name      = local.module_name
+    telemetry_module_version   = local.module_version
   })
 
   startup_script = templatefile("${path.module}/templates/startup-script.sh.tmpl", {
@@ -143,96 +147,29 @@ locals {
     telemetry_script = join("", module.telemetry.*.gcp_ubuntu_20_04_user_data)
 
     gcp_logs_enabled = var.gcp_logs_enabled
-  })
-
-  ssh_keys_metadata = <<EOF
-%{for v in var.ssh_key_pairs~}
-    ${v.user_name}:${v.public_key}
-%{endfor~}
-EOF
-}
-
-resource "google_compute_instance_template" "tpl" {
-  name_prefix = "${var.name}-"
-  description = "This template is used to create Enrich PubSub instances"
-
-  instance_description = var.name
-  machine_type         = var.machine_type
-
-  scheduling {
-    automatic_restart   = true
-    on_host_maintenance = "MIGRATE"
-  }
-
-  disk {
-    source_image = var.ubuntu_20_04_source_image == "" ? data.google_compute_image.ubuntu_20_04.self_link : var.ubuntu_20_04_source_image
-    auto_delete  = true
-    boot         = true
-    disk_type    = "pd-standard"
-    disk_size_gb = 10
-  }
-
-  # Note: Only one of either network or subnetwork can be supplied
-  #       https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/compute_instance_template#network_interface
-  network_interface {
-    network    = var.subnetwork == "" ? var.network : ""
-    subnetwork = var.subnetwork
-
-    dynamic "access_config" {
-      for_each = var.associate_public_ip_address ? [1] : []
-
-      content {
-        network_tier = "PREMIUM"
-      }
-    }
-  }
-
-  service_account {
-    email  = google_service_account.sa.email
-    scopes = ["cloud-platform"]
-  }
 
-  metadata_startup_script = local.startup_script
-
-  metadata = {
-    block-project-ssh-keys = var.ssh_block_project_keys
-
-    ssh-keys = local.ssh_keys_metadata
-  }
-
-  tags = [var.name]
-
-  labels = local.labels
-
-  lifecycle {
-    create_before_destroy = true
-  }
+    java_opts = var.java_opts
+  })
 }
 
-resource "google_compute_region_instance_group_manager" "grp" {
-  name = "${var.name}-grp"
-
-  base_instance_name = var.name
-  region             = var.region
-
-  target_size = var.target_size
-
-  version {
-    name              = "${local.app_name}-${local.app_version}"
-    instance_template = google_compute_instance_template.tpl.self_link
-  }
-
-  update_policy {
-    type                  = "PROACTIVE"
-    minimal_action        = "REPLACE"
-    max_unavailable_fixed = 3
-  }
-
-  wait_for_instances = true
-
-  timeouts {
-    create = "20m"
-    update = "20m"
-    delete = "30m"
-  }
+module "service" {
+  source  = "snowplow-devops/service-ce/google"
+  version = "0.1.0"
+
+  user_supplied_script        = local.startup_script
+  name                        = var.name
+  instance_group_version_name = "${local.app_name}-${local.app_version}"
+  labels                      = local.labels
+
+  region     = var.region
+  network    = var.network
+  subnetwork = var.subnetwork
+
+  ubuntu_20_04_source_image   = var.ubuntu_20_04_source_image
+  machine_type                = var.machine_type
+  target_size                 = var.target_size
+  ssh_block_project_keys      = var.ssh_block_project_keys
+  ssh_key_pairs               = var.ssh_key_pairs
+  service_account_email       = google_service_account.sa.email
+  associate_public_ip_address = var.associate_public_ip_address
 }

outputs.tf

@@ -1,14 +1,14 @@
 output "manager_id" {
-  value       = google_compute_region_instance_group_manager.grp.id
+  value       = module.service.manager_id
   description = "Identifier for the instance group manager"
 }
 
 output "manager_self_link" {
-  value       = google_compute_region_instance_group_manager.grp.self_link
+  value       = module.service.manager_self_link
   description = "The URL for the instance group manager"
 }
 
 output "instance_group_url" {
-  value       = google_compute_region_instance_group_manager.grp.instance_group
+  value       = module.service.instance_group_url
   description = "The full URL of the instance group created by the manager"
 }

templates/config.hocon.tmpl

@@ -14,26 +14,31 @@
       "topic": "${bad_topic_id}"
     }
   }
-  "assetsUpdatePeriod": "10080 minutes"
+
+  "assetsUpdatePeriod": "${assets_update_period}"
+
   "monitoring": {
     "metrics": {
       "stdout": {
         "period": "60 seconds"
       }
     }
   }
+
   "telemetry": {
-    "disable": ${disable}
+    "disable": ${telemetry_disable}
     "interval": 15 minutes
-    "method": POST
-    "collectorUri": "${telemetry_url}"
-    "collectorPort": 443
-    "secure": true
-    "userProvidedId": "${user_provided_id}"
-    "autoGeneratedId": "${auto_generated_id}"
-    "moduleName": "${module_name}"
-    "moduleVersion": "${module_version}"
+    "method": "POST"
+    "collectorUri": "${telemetry_collector_uri}"
+    "collectorPort": ${telemetry_collector_port}
+    "secure": ${telemetry_secure}
+    "userProvidedId": "${telemetry_user_provided_id}"
+    "autoGeneratedId": "${telemetry_auto_gen_id}"
+    "moduleName": "${telemetry_module_name}"
+    "moduleVersion": "${telemetry_module_version}"
+    "instanceId": $${INSTANCE_ID}
   }
+
   "featureFlags" : {
     "acceptInvalid": false
   }