More example and docs

distribution/ecs/README.md

@@ -38,10 +38,19 @@ you are not already using NAT Gateways in the AZs where Quickwit will be
 deployed, you should probably push the Quickwit image to ECR and use ECR
 interface VPC endpoints instead (~$0.01/hour/AZ).
 
+When using the default image, you will quickly run into the Docker Hub rate
+limiting. We recommand pushing the Quickwit image to ECR and configure that as
+`quickwit_image`. Note that the architecture of the image that you push to ECR
+must match the `quickwit_cpu_architecture` variable (`ARM64` by default).
+
 Sidecar container and custom logging configurations can be configured using the
 variables `sidecar_container_definitions`, `sidecar_container_dependencies`,
 `log_configuration`, `enable_cloudwatch_logging`. A more concrete example can be
-found in the `./example/sidecar.tf` file.
+found in the `./example/logging.tf` file.
+
+You can also use sidecars to inject additional secrets as files. This can be
+useful for configuring sources such as Kafka. See `./exaple/kafka.tf` for an
+example.
 
 ## Running the example stack
 
@@ -76,7 +85,7 @@ The successful `apply` command should output the IP of the bastion EC2 instance.
 You can port forward Quickwit's search UI using:
 
 ```bash
-ssh -L -N 7280:searcher.quickwit:7280 -i {your-private-key-file} ubuntu@{bastion_ip}
+ssh -N -L 7280:searcher.quickwit:7280 -i {your-private-key-file} ubuntu@{bastion_ip}
 ```
 
 To ingest some example dataset, log into the bastion:

distribution/ecs/example/kafka.tf

@@ -0,0 +1,58 @@
+# Example configuration for injecting SSL keys for securing a Kafka connection
+# You can then create a secured Kafka source along these lines:
+# 
+# version: 0.8
+# source_id: kafka-source
+# source_type: kafka
+# num_pipelines: 2
+# params:
+#   topic: your-topic
+#   client_params:
+#     bootstrap.servers: "your-kafka-broker.com"
+#     security.protocol: "SSL"
+#     ssl.ca.location: "/quickwit/keys/ca.pem"
+#     ssl.certificate.location: "/quickwit/keys/service.cert"
+#     ssl.key.location: "/quickwit/keys/service.key"
+
+
+locals {
+  ca_pem       = "echo \"$CA_PEM\" > /quickwit/cfg/ca.pem"
+  service_cert = "echo \"$SERVICE_CERT\" > /quickwit/cfg/service.cert"
+  service_key  = "echo \"$SERVICE_KEY\" > /quickwit/cfg/service.key"
+  example_kafka_sidecar_container_definitions = {
+    kafka_key_init = {
+      name                      = "kafka_key_init"
+      essential                 = false
+      image                     = "busybox"
+      command                   = ["sh", "-c", "${local.ca_pem} && ${local.service_cert} && ${local.service_key}"]
+      enable_cloudwatch_logging = true
+      mount_points = [
+        {
+          sourceVolume  = "quickwit-keys"
+          containerPath = "/quickwit/keys"
+        }
+      ]
+      secrets = [
+        {
+          name      = "CA_PEM"
+          valueFrom = "arn:aws:secretsmanager:eu-west-1:542709600413:secret:your_kafka_ca_pem"
+        },
+        {
+          name      = "SERVICE_CERT"
+          valueFrom = "arn:aws:secretsmanager:eu-west-1:542709600413:secret:your_kafka_service_cert"
+        },
+        {
+          name      = "SERVICE_KEY"
+          valueFrom = "arn:aws:secretsmanager:eu-west-1:542709600413:secret:your_kafka_service_key"
+        }
+      ]
+    }
+  }
+
+  example_kafka_sidecar_container_dependencies = [
+    {
+      condition     = "SUCCESS"
+      containerName = "kafka_key_init"
+    }
+  ]
+}

distribution/ecs/example/sidecar.tf → distribution/ecs/example/logging.tf

@@ -1,3 +1,5 @@
+# Example configuration for pushing ECS logs to Datadog
+
 locals {
   example_datadog_api_key_arn = "arn:aws:secretsmanager:eu-west-1:123456789012:secret:your-dd-api-key-secret"
   example_log_configuration = {

distribution/ecs/example/terraform.tf

@@ -17,6 +17,12 @@ provider "aws" {
   }
 }
 
+resource "aws_ecr_repository" "quickwit" {
+  name                 = "quickwit"
+  force_delete         = true
+  image_tag_mutability = "MUTABLE"
+}
+
 module "quickwit" {
   source                       = "../quickwit"
   vpc_id                       = module.vpc.vpc_id
@@ -27,7 +33,7 @@ module "quickwit" {
 
   # quickwit_index_s3_prefix  = "my-bucket/my-prefix"
   # quickwit_domain           = "quickwit"
-  # quickwit_image            = "quickwit/quickwit:latest"
+  quickwit_image = aws_ecr_repository.quickwit.repository_url
   # quickwit_cpu_architecture = "ARM64"
 
   # quickwit_indexer = {
@@ -63,10 +69,15 @@ module "quickwit" {
   #   multi_az       = false
   # }
 
+  ## Example logging configuration (see logging.tf)
   # sidecar_container_definitions  = local.example_sidecar_container_definitions
   # sidecar_container_dependencies = local.example_sidecar_container_dependencies
   # log_configuration              = local.example_log_configuration
-  # enable_cloudwatch_logging      = true
+  # enable_cloudwatch_logging      = false
+
+  ## Example Kafka key injection (see kafka.tf)
+  # sidecar_container_definitions  = local.example_kafka_sidecar_container_definitions
+  # sidecar_container_dependencies = local.example_kafka_sidecar_container_dependencies
 }
 
 

distribution/ecs/quickwit/service/ecs.tf

@@ -57,6 +57,11 @@ module "quickwit_service" {
         {
           sourceVolume  = "quickwit-data-vol"
           containerPath = local.quickwit_data_dir
+        },
+        # A volume that can be used to inject secrets as files.
+        {
+          sourceVolume  = "quickwit-keys"
+          containerPath = "/quickwit/keys"
         }
       ]
 
@@ -110,6 +115,9 @@ module "quickwit_service" {
   volume = [
     {
       name = "quickwit-data-vol"
+    },
+    {
+      name = "quickwit-keys"
     }
   ]