From ecda0b32ae9763cb19d4b904efb9e57a9f69e5a0 Mon Sep 17 00:00:00 2001 From: Remi Dettai Date: Wed, 3 Apr 2024 07:44:50 +0000 Subject: [PATCH] Address review comments --- distribution/ecs/README.md | 21 +++--- distribution/ecs/example/kafka.tf | 6 +- distribution/ecs/example/logging.tf | 94 -------------------------- distribution/ecs/example/terraform.tf | 10 +-- distribution/ecs/quickwit/variables.tf | 2 +- 5 files changed, 21 insertions(+), 112 deletions(-) delete mode 100644 distribution/ecs/example/logging.tf diff --git a/distribution/ecs/README.md b/distribution/ecs/README.md index 959dcc7a933..f1d84a7fc8b 100644 --- a/distribution/ecs/README.md +++ b/distribution/ecs/README.md @@ -31,7 +31,8 @@ file. Metastore database backups are disabled as restoring one would lead to inconsistencies with the index store on S3. To ensure high availability, you -should enable `rds_config.multi_az` instead. +should enable `rds_config.multi_az` instead. The module currently doesn't allow +using an externally provided metastore. Using NAT Gateways for the image registry is quite costly (~$0.05/hour/AZ). If you are not already using NAT Gateways in the AZs where Quickwit will be @@ -45,11 +46,11 @@ must match the `quickwit_cpu_architecture` variable (`ARM64` by default). Sidecar container and custom logging configurations can be configured using the variables `sidecar_container_definitions`, `sidecar_container_dependencies`, -`log_configuration`, `enable_cloudwatch_logging`. A more concrete example can be -found in the `./example/logging.tf` file. +`log_configuration`, `enable_cloudwatch_logging`. See [custom log +routing](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/using_firelens.html). -You can also use sidecars to inject additional secrets as files. This can be -useful for configuring sources such as Kafka. See `./exaple/kafka.tf` for an +You can use sidecars to inject additional secrets as files. This can be +useful for configuring sources such as Kafka. See `./example/kafka.tf` for an example. ## Running the example stack @@ -60,14 +61,14 @@ We provide an example of self contained deployment with an ad-hoc VPC. > and RDS) To make it easy to access your the Quickwit cluster, this stack includes a -bastion instance. Access is secured using an RSA key pair that you need to -provide (e.g generated with `ssh-keygen -t rsa`). +bastion instance. Access is secured using an SSH key pair that you need to +provide (e.g generated with `ssh-keygen -t ed25519`). In the `./example` directory create a `terraform.tfvars` file with the public key of your RSA key pair: ```terraform -bastion_public_key = "ssh-rsa ..." +bastion_public_key = "ssh-ed25519 ..." ``` > [!NOTE] You can skip the creation of the bastion by not specifying the @@ -105,8 +106,8 @@ wget https://quickwit-datasets-public.s3.amazonaws.com/hdfs-logs-multitenants-10 curl -X POST \ -H "content-type: application/json" \ --data-binary @hdfs-logs-multitenants-10000.json \ - http://indexer.quickwit:7280/api/v1/hdfs-logs/ingest + http://indexer.quickwit:7280/api/v1/hdfs-logs/ingest?commit=force ``` If your SSH tunnel to the searcher is still running, you should be able to see -the ingested data in the UI as soon as it is committed (~30 seconds). +the ingested data in the UI. diff --git a/distribution/ecs/example/kafka.tf b/distribution/ecs/example/kafka.tf index 5fe9bb23317..705e88f7e84 100644 --- a/distribution/ecs/example/kafka.tf +++ b/distribution/ecs/example/kafka.tf @@ -35,15 +35,15 @@ locals { secrets = [ { name = "CA_PEM" - valueFrom = "arn:aws:secretsmanager:eu-west-1:542709600413:secret:your_kafka_ca_pem" + valueFrom = "arn:aws:secretsmanager:eu-west-1:123456789:secret:your_kafka_ca_pem" }, { name = "SERVICE_CERT" - valueFrom = "arn:aws:secretsmanager:eu-west-1:542709600413:secret:your_kafka_service_cert" + valueFrom = "arn:aws:secretsmanager:eu-west-1:123456789:secret:your_kafka_service_cert" }, { name = "SERVICE_KEY" - valueFrom = "arn:aws:secretsmanager:eu-west-1:542709600413:secret:your_kafka_service_key" + valueFrom = "arn:aws:secretsmanager:eu-west-1:123456789:secret:your_kafka_service_key" } ] } diff --git a/distribution/ecs/example/logging.tf b/distribution/ecs/example/logging.tf deleted file mode 100644 index 10ef862b371..00000000000 --- a/distribution/ecs/example/logging.tf +++ /dev/null @@ -1,94 +0,0 @@ -# Example configuration for pushing ECS logs to Datadog - -locals { - example_datadog_api_key_arn = "arn:aws:secretsmanager:eu-west-1:123456789012:secret:your-dd-api-key-secret" - example_log_configuration = { - logDriver = "awsfirelens" - options = { - "Name" = "datadog" - "Host" = "http-intake.logs.datadoghq.eu" - "TLS" = "on" - "dd_service" = "quickwit" - "dd_source" = "quickwit" - "provider" = "ecs" - } - secretOptions = [ - { - "name" = "apikey" - "valueFrom" = local.example_datadog_api_key_arn - } - ] - } - example_sidecar_container_definitions = { - log_router = { - name = "log_router" - image = "public.ecr.aws/aws-observability/aws-for-fluent-bit:stable" - memory_reservation = 50, - enable_cloudwatch_logging = true - firelens_configuration = { - "type" = "fluentbit", - "options" = { - "enable-ecs-log-metadata" = "true" - } - }, - } - datadog_agent = { - name = "datadog-agent", - image = "public.ecr.aws/datadog/agent:latest", - port_mappings = [ - { - "containerPort" = 8126, - "hostPort" = 8126, - "protocol" = "tcp" - } - ], - start_timeout = 120 - readonly_root_filesystem = false - enable_cloudwatch_logging = true - environment = [ - { - name = "ECS_FARGATE", - value = "true" - }, - { - name = "DD_LOGS_ENABLED", - value = "true" - }, - { - name = "DD_LOGS_CONFIG_CONTAINER_COLLECT_ALL", - value = "true" - }, - { - name = "DD_SITE", - value = "datadoghq.eu" - } - ], - secrets = [ - { - name = "DD_API_KEY" - valueFrom = local.example_datadog_api_key_arn - } - ] - health_check = { - "command" = [ - "CMD-SHELL", - "agent health" - ], - "interval" = 30, - "timeout" = 5, - "retries" = 3 - } - } - } - - example_sidecar_container_dependencies = [ - { - condition = "START" - containerName = "log_router" - }, - { - condition = "HEALTHY" - containerName = "datadog-agent" - } - ] -} diff --git a/distribution/ecs/example/terraform.tf b/distribution/ecs/example/terraform.tf index a5b4a08c774..d2d0987de55 100644 --- a/distribution/ecs/example/terraform.tf +++ b/distribution/ecs/example/terraform.tf @@ -69,10 +69,12 @@ module "quickwit" { # multi_az = false # } - ## Example logging configuration (see logging.tf) - # sidecar_container_definitions = local.example_sidecar_container_definitions - # sidecar_container_dependencies = local.example_sidecar_container_dependencies - # log_configuration = local.example_log_configuration + ## Example logging configuration + # sidecar_container_definitions = { + # my_sidecar_container = see http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_ContainerDefinition.html + # } + # sidecar_container_dependencies = [{condition = "START", containerName = "my_sidecar_container"}] + # log_configuration = see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service#log_configuration # enable_cloudwatch_logging = false ## Example Kafka key injection (see kafka.tf) diff --git a/distribution/ecs/quickwit/variables.tf b/distribution/ecs/quickwit/variables.tf index 7cb4a887338..f3dfd058601 100644 --- a/distribution/ecs/quickwit/variables.tf +++ b/distribution/ecs/quickwit/variables.tf @@ -73,7 +73,7 @@ variable "quickwit_indexer" { description = "Indexer service sizing configurations" type = object({ desired_count = optional(number, 1) - memory = optional(number, 2048) + memory = optional(number, 4096) cpu = optional(number, 1024) }) default = {}