diff --git a/distribution/ecs/.gitignore b/distribution/ecs/.gitignore new file mode 100644 index 00000000000..b41a986bbb2 --- /dev/null +++ b/distribution/ecs/.gitignore @@ -0,0 +1,4 @@ +.terraform +terraform.tfstate* +.terraform.tfstate* +terraform.tfvars diff --git a/distribution/ecs/README.md b/distribution/ecs/README.md new file mode 100644 index 00000000000..f1d84a7fc8b --- /dev/null +++ b/distribution/ecs/README.md @@ -0,0 +1,113 @@ +# ECS deployment for quickwit + +## Run Quickwit in your infrastructure + +Create a Quickwit module using: + +```terraform +module "quickwit" { + source = "github.com/quickwit-oss/quickwit/distribution/ecs/quickwit" + + vpc_id = # VPC in which all resources will be created + subnet_ids = [...] # At least 2 private subnets must be specified + quickwit_ingress_cidr_blocks = [...] # List of CIDR blocks allowed to access to the Quickwit API +} +``` + +The Quickwit cluster is running on a private subnet. For ECS to pull the image: +- if using the default Docker Hub image `quickwit/quickwit`, the subnets +specified must be configured with a NAT Gateway (no public IPs are attached to +the tasks) +- if using an image hosted on ECR, a VPC endpoint for ECR can be used instead of +a NAT Gateway + + +## Module configurations + +To get the list of available configurations, check the `./quickwit/variables.tf` +file. + +### Tips + +Metastore database backups are disabled as restoring one would lead to +inconsistencies with the index store on S3. To ensure high availability, you +should enable `rds_config.multi_az` instead. The module currently doesn't allow +using an externally provided metastore. + +Using NAT Gateways for the image registry is quite costly (~$0.05/hour/AZ). If +you are not already using NAT Gateways in the AZs where Quickwit will be +deployed, you should probably push the Quickwit image to ECR and use ECR +interface VPC endpoints instead (~$0.01/hour/AZ). + +When using the default image, you will quickly run into the Docker Hub rate +limiting. We recommand pushing the Quickwit image to ECR and configure that as +`quickwit_image`. Note that the architecture of the image that you push to ECR +must match the `quickwit_cpu_architecture` variable (`ARM64` by default). + +Sidecar container and custom logging configurations can be configured using the +variables `sidecar_container_definitions`, `sidecar_container_dependencies`, +`log_configuration`, `enable_cloudwatch_logging`. See [custom log +routing](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/using_firelens.html). + +You can use sidecars to inject additional secrets as files. This can be +useful for configuring sources such as Kafka. See `./example/kafka.tf` for an +example. + +## Running the example stack + +We provide an example of self contained deployment with an ad-hoc VPC. + +> [!IMPORTANT] This stack costs ~$150/month to run (Fargate tasks, NAT Gateways +> and RDS) + +To make it easy to access your the Quickwit cluster, this stack includes a +bastion instance. Access is secured using an SSH key pair that you need to +provide (e.g generated with `ssh-keygen -t ed25519`). + +In the `./example` directory create a `terraform.tfvars` file with the public +key of your RSA key pair: + +```terraform +bastion_public_key = "ssh-ed25519 ..." +``` + +> [!NOTE] You can skip the creation of the bastion by not specifying the +> `bastion_public_key` variable, but that would make it hard to access and +> experiment with the created Quickwit cluster. + +In the same directory (`./example`) run: + +```bash +terraform init +terraform apply +``` + +The successful `apply` command should output the IP of the bastion EC2 instance. +You can port forward Quickwit's search UI using: + +```bash +ssh -N -L 7280:searcher.quickwit:7280 -i {your-private-key-file} ubuntu@{bastion_ip} +``` + +To ingest some example dataset, log into the bastion: + +```bash +ssh -i {your-private-key-file} ubuntu@{bastion_ip} + +# create the log index +wget https://raw.githubusercontent.com/quickwit-oss/quickwit/main/config/tutorials/hdfs-logs/index-config.yaml +curl -X POST \ + -H "content-type: application/yaml" \ + --data-binary @index-config.yaml \ + http://indexer.quickwit:7280/api/v1/indexes + +# import some data +wget https://quickwit-datasets-public.s3.amazonaws.com/hdfs-logs-multitenants-10000.json +curl -X POST \ + -H "content-type: application/json" \ + --data-binary @hdfs-logs-multitenants-10000.json \ + http://indexer.quickwit:7280/api/v1/hdfs-logs/ingest?commit=force +``` + +If your SSH tunnel to the searcher is still running, you should be able to see +the ingested data in the UI. diff --git a/distribution/ecs/example/.terraform.lock.hcl b/distribution/ecs/example/.terraform.lock.hcl new file mode 100644 index 00000000000..004446f6ca4 --- /dev/null +++ b/distribution/ecs/example/.terraform.lock.hcl @@ -0,0 +1,45 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "5.39.1" + constraints = ">= 4.66.1, >= 5.36.0, ~> 5.39.1" + hashes = [ + "h1:hQLlAd6O1LdQHy1GdWtgT5fcOlc3TWW+SaaFkpe+e8E=", + "zh:05c50a5d8edb3ba4ebc4eb6e0d0b5e319142f5983b27821710ed7d475d335bdc", + "zh:082986a5784dd21957e632371b289e549f051a4ea21d5c78c6d744c3537f03c5", + "zh:192ae622ba562eacc4921ed549a794506179233d724fdd15a4f147f3400724a0", + "zh:19a1d4637a62de90b0da174c0bf01000cd900488f7e8f709d8a37f082c59756b", + "zh:1d7689a8583515f1705972d7ce57ccfab96215b19905530d2c78c02dcfaff583", + "zh:22c446a21209a52ab74b4ba1ede0b220531e97ce479430047e493a2c45e1d8cb", + "zh:4154de82290ab4e9f81bac1ea62342de8b3b7a608f99258c190d4dd1c6663e47", + "zh:6bc4859ccdc54f28af9286b2fa090a31dcb345138d68c471510b737f6a052011", + "zh:73c69e000e0b321e78a4a12fef60d37285f2afec0ea7be9e06163d985101cb59", + "zh:890a3422f5e445b49bae30facf448d0ec9cd647e9155d0b685b5b39e9d331a94", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:9cd88bec0f5205df9032e3126d4e57edd1c5cc8d45cda25626882dafc485a3b0", + "zh:a3a8e3276d0fbf051bbafa192a2998b05745f2cf285ac8c36a9ad167a75c037f", + "zh:d47e4dcf4c0ad71b9a7c720be4f3a89f6786a82e77bbe8d950794562792a1da5", + "zh:f74e5b2af508c7de80a6ae5198df54a795eeba5058a0cd247828943f0c54f6e0", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.6.0" + constraints = ">= 3.1.0" + hashes = [ + "h1:R5Ucn26riKIEijcsiOMBR3uOAjuOMfI1x7XvH4P6B1w=", + "zh:03360ed3ecd31e8c5dac9c95fe0858be50f3e9a0d0c654b5e504109c2159287d", + "zh:1c67ac51254ba2a2bb53a25e8ae7e4d076103483f55f39b426ec55e47d1fe211", + "zh:24a17bba7f6d679538ff51b3a2f378cedadede97af8a1db7dad4fd8d6d50f829", + "zh:30ffb297ffd1633175d6545d37c2217e2cef9545a6e03946e514c59c0859b77d", + "zh:454ce4b3dbc73e6775f2f6605d45cee6e16c3872a2e66a2c97993d6e5cbd7055", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:91df0a9fab329aff2ff4cf26797592eb7a3a90b4a0c04d64ce186654e0cc6e17", + "zh:aa57384b85622a9f7bfb5d4512ca88e61f22a9cea9f30febaa4c98c68ff0dc21", + "zh:c4a3e329ba786ffb6f2b694e1fd41d413a7010f3a53c20b432325a94fa71e839", + "zh:e2699bc9116447f96c53d55f2a00570f982e6f9935038c3810603572693712d0", + "zh:e747c0fd5d7684e5bfad8aa0ca441903f15ae7a98a737ff6aca24ba223207e2c", + "zh:f1ca75f417ce490368f047b63ec09fd003711ae48487fba90b4aba2ccf71920e", + ] +} diff --git a/distribution/ecs/example/bastion.tf b/distribution/ecs/example/bastion.tf new file mode 100644 index 00000000000..e2f84723756 --- /dev/null +++ b/distribution/ecs/example/bastion.tf @@ -0,0 +1,65 @@ +variable "bastion_public_key" { + description = "The public key used to connect to the bastion host. If empty, no bastion is created." + default = "" +} + +output "bastion_ip" { + value = var.bastion_public_key != "" ? aws_instance.bastion[0].public_ip : null +} + +data "aws_ami" "ubuntu" { + most_recent = true + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + owners = ["099720109477"] # Canonical +} + +resource "aws_security_group" "allow_ssh" { + count = var.bastion_public_key != "" ? 1 : 0 + name = "qw_ecs_bastion_allow_ssh" + description = "Allow SSH inbound traffic from everywhere" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "aws_instance" "bastion" { + count = var.bastion_public_key != "" ? 1 : 0 + ami = data.aws_ami.ubuntu.id + instance_type = "t3.nano" + key_name = aws_key_pair.bastion_key[0].key_name + subnet_id = module.vpc.public_subnets[0] + associate_public_ip_address = true + vpc_security_group_ids = [aws_security_group.allow_ssh[0].id] + + tags = { + Name = "quickwit-ecs-bastion" + } +} + +resource "aws_key_pair" "bastion_key" { + count = var.bastion_public_key != "" ? 1 : 0 + key_name = "quickwit-ecs-bastion-key" + public_key = var.bastion_public_key +} diff --git a/distribution/ecs/example/kafka.tf b/distribution/ecs/example/kafka.tf new file mode 100644 index 00000000000..705e88f7e84 --- /dev/null +++ b/distribution/ecs/example/kafka.tf @@ -0,0 +1,58 @@ +# Example configuration for injecting SSL keys for securing a Kafka connection +# You can then create a secured Kafka source along these lines: +# +# version: 0.8 +# source_id: kafka-source +# source_type: kafka +# num_pipelines: 2 +# params: +# topic: your-topic +# client_params: +# bootstrap.servers: "your-kafka-broker.com" +# security.protocol: "SSL" +# ssl.ca.location: "/quickwit/keys/ca.pem" +# ssl.certificate.location: "/quickwit/keys/service.cert" +# ssl.key.location: "/quickwit/keys/service.key" + + +locals { + ca_pem = "echo \"$CA_PEM\" > /quickwit/cfg/ca.pem" + service_cert = "echo \"$SERVICE_CERT\" > /quickwit/cfg/service.cert" + service_key = "echo \"$SERVICE_KEY\" > /quickwit/cfg/service.key" + example_kafka_sidecar_container_definitions = { + kafka_key_init = { + name = "kafka_key_init" + essential = false + image = "busybox" + command = ["sh", "-c", "${local.ca_pem} && ${local.service_cert} && ${local.service_key}"] + enable_cloudwatch_logging = true + mount_points = [ + { + sourceVolume = "quickwit-keys" + containerPath = "/quickwit/keys" + } + ] + secrets = [ + { + name = "CA_PEM" + valueFrom = "arn:aws:secretsmanager:eu-west-1:123456789:secret:your_kafka_ca_pem" + }, + { + name = "SERVICE_CERT" + valueFrom = "arn:aws:secretsmanager:eu-west-1:123456789:secret:your_kafka_service_cert" + }, + { + name = "SERVICE_KEY" + valueFrom = "arn:aws:secretsmanager:eu-west-1:123456789:secret:your_kafka_service_key" + } + ] + } + } + + example_kafka_sidecar_container_dependencies = [ + { + condition = "SUCCESS" + containerName = "kafka_key_init" + } + ] +} diff --git a/distribution/ecs/example/terraform.tf b/distribution/ecs/example/terraform.tf new file mode 100644 index 00000000000..d2d0987de55 --- /dev/null +++ b/distribution/ecs/example/terraform.tf @@ -0,0 +1,92 @@ +terraform { + backend "local" {} + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.39.1" + } + } +} + +provider "aws" { + region = "eu-west-1" + default_tags { + tags = { + provisioner = "terraform" + } + } +} + +# resource "aws_ecr_repository" "quickwit" { +# name = "quickwit" +# force_delete = true +# image_tag_mutability = "MUTABLE" +# } + +module "quickwit" { + source = "../quickwit" + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + quickwit_ingress_cidr_blocks = [module.vpc.vpc_cidr_block] + + ## Optional configurations: + + # quickwit_index_s3_prefix = "my-bucket/my-prefix" + # quickwit_domain = "quickwit" + # quickwit_image = aws_ecr_repository.quickwit.repository_url + # quickwit_cpu_architecture = "ARM64" + + # quickwit_indexer = { + # desired_count = 1 + # memory = 2048 + # cpu = 1024 + # } + + # quickwit_metastore = { + # desired_count = 1 + # memory = 512 + # cpu = 256 + # } + + # quickwit_searcher = { + # desired_count = 1 + # memory = 2048 + # cpu = 1024 + # } + + # quickwit_control_plane = { + # memory = 512 + # cpu = 256 + # } + + # quickwit_janitor = { + # memory = 512 + # cpu = 256 + # } + + # rds_config = { + # instance_class = "db.t4g.micro" + # multi_az = false + # } + + ## Example logging configuration + # sidecar_container_definitions = { + # my_sidecar_container = see http://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_ContainerDefinition.html + # } + # sidecar_container_dependencies = [{condition = "START", containerName = "my_sidecar_container"}] + # log_configuration = see https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service#log_configuration + # enable_cloudwatch_logging = false + + ## Example Kafka key injection (see kafka.tf) + # sidecar_container_definitions = local.example_kafka_sidecar_container_definitions + # sidecar_container_dependencies = local.example_kafka_sidecar_container_dependencies +} + + +output "indexer_service_name" { + value = module.quickwit.indexer_service_name +} + +output "searcher_service_name" { + value = module.quickwit.searcher_service_name +} diff --git a/distribution/ecs/example/vpc.tf b/distribution/ecs/example/vpc.tf new file mode 100644 index 00000000000..9683fc59ea4 --- /dev/null +++ b/distribution/ecs/example/vpc.tf @@ -0,0 +1,13 @@ +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "5.5.3" + + name = "quickwit-ecs" + cidr = "10.0.0.0/16" + + azs = ["eu-west-1a", "eu-west-1b"] + private_subnets = ["10.0.1.0/24", "10.0.2.0/24"] + public_subnets = ["10.0.101.0/24", "10.0.102.0/24"] + + enable_nat_gateway = true +} diff --git a/distribution/ecs/quickwit/cluster.tf b/distribution/ecs/quickwit/cluster.tf new file mode 100644 index 00000000000..2f55b67ee5f --- /dev/null +++ b/distribution/ecs/quickwit/cluster.tf @@ -0,0 +1,18 @@ +module "ecs_cluster" { + source = "terraform-aws-modules/ecs/aws//modules/cluster" + version = "5.9.3" + + cluster_name = "quickwit-${local.module_id}" +} + +resource "aws_service_discovery_private_dns_namespace" "quickwit_internal" { + name = var.quickwit_domain + description = "Internal quickwit domain" + vpc = var.vpc_id +} + +resource "aws_security_group" "quickwit_cluster_member_sg" { + name = "quickwit-cluster-member-${local.module_id}" + description = "Security group for members of the Quickwit cluster" + vpc_id = var.vpc_id +} diff --git a/distribution/ecs/quickwit/configs.tf b/distribution/ecs/quickwit/configs.tf new file mode 100644 index 00000000000..8cf070ab440 --- /dev/null +++ b/distribution/ecs/quickwit/configs.tf @@ -0,0 +1,19 @@ +locals { + quickwit_peer_list = [ + "${aws_service_discovery_service.metastore.name}.${aws_service_discovery_private_dns_namespace.quickwit_internal.name}", + "${aws_service_discovery_service.control_plane.name}.${aws_service_discovery_private_dns_namespace.quickwit_internal.name}", + "${aws_service_discovery_service.janitor.name}.${aws_service_discovery_private_dns_namespace.quickwit_internal.name}", + "${aws_service_discovery_service.indexer.name}.${aws_service_discovery_private_dns_namespace.quickwit_internal.name}", + "${aws_service_discovery_service.searcher.name}.${aws_service_discovery_private_dns_namespace.quickwit_internal.name}", + ] + + # id to avoid conflicts when deploying this module multiple times (random by default) + module_id = var.module_id == "" ? random_id.module.hex : var.module_id + s3_id = var.module_id == "" ? random_id.module.hex : "${var.module_id}-${random_id.module.hex}" + + quickwit_index_s3_prefix = var.quickwit_index_s3_prefix == "" ? aws_s3_bucket.index[0].id : var.quickwit_index_s3_prefix +} + +resource "random_id" "module" { + byte_length = 3 +} diff --git a/distribution/ecs/quickwit/iam.tf b/distribution/ecs/quickwit/iam.tf new file mode 100644 index 00000000000..f9c4e9d7d5c --- /dev/null +++ b/distribution/ecs/quickwit/iam.tf @@ -0,0 +1,32 @@ +data "aws_iam_policy_document" "quickwit_task_permission" { + # Reference: https://quickwit.io/docs/guides/aws-setup#amazon-s3 + statement { + actions = [ + "s3:ListBucket", + "s3:ListObjects", + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ] + + resources = [ + "arn:aws:s3:::${local.quickwit_index_s3_prefix}*", + ] + } + + statement { + actions = [ + "ecs:Describe*", + "ecs:List*" + ] + + resources = ["*"] + } +} + +resource "aws_iam_policy" "quickwit" { + name = "quickwit-task-policy-${local.module_id}" + path = "/" + + policy = data.aws_iam_policy_document.quickwit_task_permission.json +} diff --git a/distribution/ecs/quickwit/outputs.tf b/distribution/ecs/quickwit/outputs.tf new file mode 100644 index 00000000000..3aa73c8432f --- /dev/null +++ b/distribution/ecs/quickwit/outputs.tf @@ -0,0 +1,7 @@ +output "indexer_service_name" { + value = "${aws_service_discovery_service.indexer.name}.${aws_service_discovery_private_dns_namespace.quickwit_internal.name}" +} + +output "searcher_service_name" { + value = "${aws_service_discovery_service.searcher.name}.${aws_service_discovery_private_dns_namespace.quickwit_internal.name}" +} diff --git a/distribution/ecs/quickwit/quickwit-control-plane.tf b/distribution/ecs/quickwit/quickwit-control-plane.tf new file mode 100644 index 00000000000..ec1e3ec1d25 --- /dev/null +++ b/distribution/ecs/quickwit/quickwit-control-plane.tf @@ -0,0 +1,37 @@ +module "quickwit_control_plane" { + source = "./service" + service_name = "control_plane" + service_discovery_registry_arn = aws_service_discovery_service.control_plane.arn + cluster_arn = module.ecs_cluster.arn + postgres_credential_arn = aws_ssm_parameter.postgres_credential.arn + quickwit_peer_list = local.quickwit_peer_list + s3_access_policy_arn = aws_iam_policy.quickwit.arn + module_id = local.module_id + quickwit_cluster_member_sg_id = aws_security_group.quickwit_cluster_member_sg.id + + subnet_ids = var.subnet_ids + ingress_cidr_blocks = var.quickwit_ingress_cidr_blocks + quickwit_image = var.quickwit_image + quickwit_cpu_architecture = var.quickwit_cpu_architecture + sidecar_container_definitions = var.sidecar_container_definitions + sidecar_container_dependencies = var.sidecar_container_dependencies + log_configuration = var.log_configuration + enable_cloudwatch_logging = var.enable_cloudwatch_logging + service_config = var.quickwit_control_plane + quickwit_index_s3_prefix = local.quickwit_index_s3_prefix +} + +resource "aws_service_discovery_service" "control_plane" { + name = "control-plane" + + dns_config { + namespace_id = aws_service_discovery_private_dns_namespace.quickwit_internal.id + + dns_records { + ttl = 10 + type = "A" + } + + routing_policy = "MULTIVALUE" + } +} diff --git a/distribution/ecs/quickwit/quickwit-indexer.tf b/distribution/ecs/quickwit/quickwit-indexer.tf new file mode 100644 index 00000000000..bd43a94b53e --- /dev/null +++ b/distribution/ecs/quickwit/quickwit-indexer.tf @@ -0,0 +1,37 @@ +module "quickwit_indexer" { + source = "./service" + service_name = "indexer" + service_discovery_registry_arn = aws_service_discovery_service.indexer.arn + cluster_arn = module.ecs_cluster.arn + postgres_credential_arn = aws_ssm_parameter.postgres_credential.arn + quickwit_peer_list = local.quickwit_peer_list + s3_access_policy_arn = aws_iam_policy.quickwit.arn + module_id = local.module_id + quickwit_cluster_member_sg_id = aws_security_group.quickwit_cluster_member_sg.id + + subnet_ids = var.subnet_ids + ingress_cidr_blocks = var.quickwit_ingress_cidr_blocks + quickwit_image = var.quickwit_image + quickwit_cpu_architecture = var.quickwit_cpu_architecture + sidecar_container_definitions = var.sidecar_container_definitions + sidecar_container_dependencies = var.sidecar_container_dependencies + log_configuration = var.log_configuration + enable_cloudwatch_logging = var.enable_cloudwatch_logging + service_config = var.quickwit_indexer + quickwit_index_s3_prefix = local.quickwit_index_s3_prefix +} + +resource "aws_service_discovery_service" "indexer" { + name = "indexer" + + dns_config { + namespace_id = aws_service_discovery_private_dns_namespace.quickwit_internal.id + + dns_records { + ttl = 10 + type = "A" + } + + routing_policy = "MULTIVALUE" + } +} diff --git a/distribution/ecs/quickwit/quickwit-janitor.tf b/distribution/ecs/quickwit/quickwit-janitor.tf new file mode 100644 index 00000000000..040b9693f4e --- /dev/null +++ b/distribution/ecs/quickwit/quickwit-janitor.tf @@ -0,0 +1,37 @@ +module "quickwit_janitor" { + source = "./service" + service_name = "janitor" + service_discovery_registry_arn = aws_service_discovery_service.janitor.arn + cluster_arn = module.ecs_cluster.arn + postgres_credential_arn = aws_ssm_parameter.postgres_credential.arn + quickwit_peer_list = local.quickwit_peer_list + s3_access_policy_arn = aws_iam_policy.quickwit.arn + module_id = local.module_id + quickwit_cluster_member_sg_id = aws_security_group.quickwit_cluster_member_sg.id + + subnet_ids = var.subnet_ids + ingress_cidr_blocks = var.quickwit_ingress_cidr_blocks + quickwit_image = var.quickwit_image + quickwit_cpu_architecture = var.quickwit_cpu_architecture + sidecar_container_definitions = var.sidecar_container_definitions + sidecar_container_dependencies = var.sidecar_container_dependencies + log_configuration = var.log_configuration + enable_cloudwatch_logging = var.enable_cloudwatch_logging + service_config = var.quickwit_janitor + quickwit_index_s3_prefix = local.quickwit_index_s3_prefix +} + +resource "aws_service_discovery_service" "janitor" { + name = "janitor" + + dns_config { + namespace_id = aws_service_discovery_private_dns_namespace.quickwit_internal.id + + dns_records { + ttl = 10 + type = "A" + } + + routing_policy = "MULTIVALUE" + } +} diff --git a/distribution/ecs/quickwit/quickwit-metastore.tf b/distribution/ecs/quickwit/quickwit-metastore.tf new file mode 100644 index 00000000000..0056b745091 --- /dev/null +++ b/distribution/ecs/quickwit/quickwit-metastore.tf @@ -0,0 +1,37 @@ +module "quickwit_metastore" { + source = "./service" + service_name = "metastore" + service_discovery_registry_arn = aws_service_discovery_service.metastore.arn + cluster_arn = module.ecs_cluster.arn + postgres_credential_arn = aws_ssm_parameter.postgres_credential.arn + quickwit_peer_list = local.quickwit_peer_list + s3_access_policy_arn = aws_iam_policy.quickwit.arn + module_id = local.module_id + quickwit_cluster_member_sg_id = aws_security_group.quickwit_cluster_member_sg.id + + subnet_ids = var.subnet_ids + ingress_cidr_blocks = var.quickwit_ingress_cidr_blocks + quickwit_image = var.quickwit_image + quickwit_cpu_architecture = var.quickwit_cpu_architecture + sidecar_container_definitions = var.sidecar_container_definitions + sidecar_container_dependencies = var.sidecar_container_dependencies + log_configuration = var.log_configuration + enable_cloudwatch_logging = var.enable_cloudwatch_logging + service_config = var.quickwit_metastore + quickwit_index_s3_prefix = local.quickwit_index_s3_prefix +} + +resource "aws_service_discovery_service" "metastore" { + name = "metastore" + + dns_config { + namespace_id = aws_service_discovery_private_dns_namespace.quickwit_internal.id + + dns_records { + ttl = 10 + type = "A" + } + + routing_policy = "MULTIVALUE" + } +} diff --git a/distribution/ecs/quickwit/quickwit-searcher.tf b/distribution/ecs/quickwit/quickwit-searcher.tf new file mode 100644 index 00000000000..dfe995f4ec1 --- /dev/null +++ b/distribution/ecs/quickwit/quickwit-searcher.tf @@ -0,0 +1,37 @@ +module "quickwit_searcher" { + source = "./service" + service_name = "searcher" + service_discovery_registry_arn = aws_service_discovery_service.searcher.arn + cluster_arn = module.ecs_cluster.arn + postgres_credential_arn = aws_ssm_parameter.postgres_credential.arn + quickwit_peer_list = local.quickwit_peer_list + s3_access_policy_arn = aws_iam_policy.quickwit.arn + module_id = local.module_id + quickwit_cluster_member_sg_id = aws_security_group.quickwit_cluster_member_sg.id + + subnet_ids = var.subnet_ids + ingress_cidr_blocks = var.quickwit_ingress_cidr_blocks + quickwit_image = var.quickwit_image + quickwit_cpu_architecture = var.quickwit_cpu_architecture + sidecar_container_definitions = var.sidecar_container_definitions + sidecar_container_dependencies = var.sidecar_container_dependencies + log_configuration = var.log_configuration + enable_cloudwatch_logging = var.enable_cloudwatch_logging + service_config = var.quickwit_searcher + quickwit_index_s3_prefix = local.quickwit_index_s3_prefix +} + +resource "aws_service_discovery_service" "searcher" { + name = "searcher" + + dns_config { + namespace_id = aws_service_discovery_private_dns_namespace.quickwit_internal.id + + dns_records { + ttl = 10 + type = "A" + } + + routing_policy = "MULTIVALUE" + } +} diff --git a/distribution/ecs/quickwit/rds.tf b/distribution/ecs/quickwit/rds.tf new file mode 100644 index 00000000000..b65c8e9deda --- /dev/null +++ b/distribution/ecs/quickwit/rds.tf @@ -0,0 +1,66 @@ +resource "random_password" "quickwit_db" { + length = 64 + special = false +} + +module "quickwit_db" { + source = "terraform-aws-modules/rds/aws" + version = "6.5.2" + + identifier = "quickwit-metastore-${local.module_id}" + + engine = "postgres" + engine_version = "16" + family = "postgres16" # DB parameter group + major_engine_version = "16" # DB option group + + instance_class = var.rds_config.instance_class + multi_az = var.rds_config.multi_az + allocated_storage = 5 + + db_name = "quickwit" + username = "quickwit" + password = random_password.quickwit_db.result + + port = "5432" + publicly_accessible = false + manage_master_user_password = false + iam_database_authentication_enabled = true + vpc_security_group_ids = [aws_security_group.quickwit_db.id] + db_subnet_group_name = aws_db_subnet_group.quickwit.name + + maintenance_window = "Mon:00:00-Mon:03:00" + + create_monitoring_role = true + monitoring_interval = "30" + monitoring_role_name = "RDSQuickwitMonitoringRole-${local.module_id}" + + deletion_protection = false + skip_final_snapshot = true +} + +resource "aws_security_group" "quickwit_db" { + name = "quickwit-db-${local.module_id}" + description = "Security group for the Quickwit Metastore DB" + vpc_id = var.vpc_id + + ingress { + description = "Connection from explicitly allowed resources" + from_port = 5432 + to_port = 5432 + protocol = "tcp" + security_groups = [aws_security_group.quickwit_cluster_member_sg.id] + } +} + +resource "aws_db_subnet_group" "quickwit" { + name = "quickwit-${local.module_id}" + description = "Quickwit metastore" + subnet_ids = var.subnet_ids +} + +resource "aws_ssm_parameter" "postgres_credential" { + name = "/quickwit/${local.module_id}/postgres" + type = "SecureString" + value = "postgres://${module.quickwit_db.db_instance_username}:${random_password.quickwit_db.result}@${module.quickwit_db.db_instance_address}:${module.quickwit_db.db_instance_port}/${module.quickwit_db.db_instance_name}" +} diff --git a/distribution/ecs/quickwit/s3.tf b/distribution/ecs/quickwit/s3.tf new file mode 100644 index 00000000000..9ffae0f00d5 --- /dev/null +++ b/distribution/ecs/quickwit/s3.tf @@ -0,0 +1,7 @@ +data "aws_caller_identity" "current" {} + +resource "aws_s3_bucket" "index" { + count = var.quickwit_index_s3_prefix == "" ? 1 : 0 + bucket = "quickwit-ecs-index-${data.aws_caller_identity.current.account_id}-${local.s3_id}" + force_destroy = true +} diff --git a/distribution/ecs/quickwit/service/config.tf b/distribution/ecs/quickwit/service/config.tf new file mode 100644 index 00000000000..c78a29620fe --- /dev/null +++ b/distribution/ecs/quickwit/service/config.tf @@ -0,0 +1,31 @@ +locals { + quickwit_data_dir = "/quickwit/qwdata" + + quickwit_common_environment = [ + { + name = "QW_PEER_SEEDS" + value = join(",", var.quickwit_peer_list) + }, + { + name = "NO_COLOR" + value = "true" + }, + { + name = "QW_CLUSTER_ID" + value = "ecs-${var.module_id}" + }, + { + name = "QW_LISTEN_ADDRESS" + value = "0.0.0.0" + }, + { + name = "QW_DATA_DIR" + value = local.quickwit_data_dir + }, + { + name = "QW_DEFAULT_INDEX_ROOT_URI" + value = "s3://${var.quickwit_index_s3_prefix}" + }, + ] + +} diff --git a/distribution/ecs/quickwit/service/ecs.tf b/distribution/ecs/quickwit/service/ecs.tf new file mode 100644 index 00000000000..2aa91a238e9 --- /dev/null +++ b/distribution/ecs/quickwit/service/ecs.tf @@ -0,0 +1,131 @@ +module "quickwit_service" { + source = "terraform-aws-modules/ecs/aws//modules/service" + version = "5.9.3" + + name = "quickwit-${var.service_name}-${var.module_id}" + cluster_arn = var.cluster_arn + + cpu = var.service_config.cpu + memory = var.service_config.memory + + container_definitions = merge(var.sidecar_container_definitions, { + quickwit = { + cpu = var.service_config.cpu + memory = var.service_config.memory + + essential = true + image = var.quickwit_image + enable_cloudwatch_logging = var.enable_cloudwatch_logging + + command = ["run"] + + environment = concat(local.quickwit_common_environment, [ + { + name = "QW_ENABLED_SERVICES" + value = var.service_name + } + ]) + + secrets = [ + { + name = "QW_METASTORE_URI" + valueFrom = var.postgres_credential_arn + } + ] + + port_mappings = [ + { + name = "rest" + containerPort = 7280 + protocol = "tcp" + }, + { + name = "grpc" + containerPort = 7281 + protocol = "tcp" + }, + { + name = "gossip" + containerPort = 7280 + protocol = "udp" + } + ] + + log_configuration = var.log_configuration + + mount_points = [ + { + sourceVolume = "quickwit-data-vol" + containerPath = local.quickwit_data_dir + }, + # A volume that can be used to inject secrets as files. + { + sourceVolume = "quickwit-keys" + containerPath = "/quickwit/keys" + } + ] + + dependencies = var.sidecar_container_dependencies + } + }) + + requires_compatibilities = ["FARGATE"] + runtime_platform = { + operating_system_family = "LINUX" + cpu_architecture = var.quickwit_cpu_architecture + } + + service_registries = { + registry_arn = var.service_discovery_registry_arn + container_name = "quickwit" + } + + subnet_ids = var.subnet_ids + security_group_rules = { + ingress_internal = { + type = "ingress" + from_port = 7280 + to_port = 7281 + protocol = "-1" + + source_security_group_id = var.quickwit_cluster_member_sg_id + } + ingress_external = { + type = "ingress" + from_port = 7280 + to_port = 7281 + protocol = "-1" + + cidr_blocks = var.ingress_cidr_blocks + } + egress_all = { + type = "egress" + from_port = 0 + to_port = 0 + protocol = "-1" + + cidr_blocks = ["0.0.0.0/0"] + } + } + security_group_ids = [var.quickwit_cluster_member_sg_id] + + enable_autoscaling = false + desired_count = var.service_config.desired_count + + volume = [ + { + name = "quickwit-data-vol" + }, + { + name = "quickwit-keys" + } + ] + + task_exec_ssm_param_arns = [ + var.postgres_credential_arn + ] + + tasks_iam_role_policies = { + s3_access = var.s3_access_policy_arn + } +} diff --git a/distribution/ecs/quickwit/service/variables.tf b/distribution/ecs/quickwit/service/variables.tf new file mode 100644 index 00000000000..e82e503341e --- /dev/null +++ b/distribution/ecs/quickwit/service/variables.tf @@ -0,0 +1,57 @@ +variable "service_name" { + description = "One of indexer, metastore, searcher, control_plane, janitor" +} + +variable "service_discovery_registry_arn" {} + +variable "sidecar_container_definitions" {} + +variable "sidecar_container_dependencies" { + type = list(object({ + containerName = string + condition = string + })) + default = [] +} + +variable "log_configuration" {} + +variable "enable_cloudwatch_logging" { + type = bool +} + +variable "cluster_arn" {} + +variable "ingress_cidr_blocks" { + type = list(string) +} + +variable "quickwit_cluster_member_sg_id" {} + +variable "subnet_ids" { + type = list(string) +} + +variable "postgres_credential_arn" {} + +variable "quickwit_image" {} + +variable "service_config" { + type = object({ + desired_count = optional(number, 1) + memory = number + cpu = number + }) +} + +variable "quickwit_index_s3_prefix" {} + +variable "quickwit_peer_list" { + type = list(string) +} + +variable "s3_access_policy_arn" {} + +variable "quickwit_cpu_architecture" {} + +variable "module_id" {} diff --git a/distribution/ecs/quickwit/variables.tf b/distribution/ecs/quickwit/variables.tf new file mode 100644 index 00000000000..f3dfd058601 --- /dev/null +++ b/distribution/ecs/quickwit/variables.tf @@ -0,0 +1,129 @@ +## REQUIRED VARIABLES + +variable "vpc_id" { + description = "VPC ID of the cluster" +} + +variable "subnet_ids" { + description = "Subnet(s) where quickwit will be deployed" + type = list(string) +} + + + +## OPTIONAL VARIABLES + +variable "module_id" { + description = "Identifier for the module, e.g the stage. If not specified, a random string is generated." + default = "" +} + +variable "quickwit_ingress_cidr_blocks" { + description = "CIDR blocks (private) that should have access to the Quickwit cluster" + type = list(string) + default = [] +} + + +variable "quickwit_index_s3_prefix" { + description = "S3 bucket name and prefix for the Quickwit data, e.g. my-bucket-name/my-prefix. Quickwit will only have access to this S3 location. Leave empty to create a new bucket." + default = "" +} + +variable "quickwit_domain" { + description = "Local domain for quickwit service discovery" + default = "quickwit" +} + +variable "quickwit_image" { + description = "Quickwit docker image" + default = "quickwit/quickwit:latest" +} + +variable "quickwit_cpu_architecture" { + description = "One of X86_64 / ARM64. Must match the arch of the provided image (var.quickwit_image)." + default = "ARM64" +} + +variable "sidecar_container_definitions" { + description = "Sidecar containers to be attached to Quickwit tasks" + default = {} +} + +variable "sidecar_container_dependencies" { + description = "Specify the Quickwit container's dependencies on sidecars" + type = list(object({ + containerName = string + condition = string + })) + default = [] +} + +variable "enable_cloudwatch_logging" { + description = "Cloudwatch logging for Quickwit tasks. Usually disabled when using a custom log configuration." + default = true +} + +variable "log_configuration" { + description = "Custom log configuraiton for Quickwit tasks" + default = {} +} + +variable "quickwit_indexer" { + description = "Indexer service sizing configurations" + type = object({ + desired_count = optional(number, 1) + memory = optional(number, 4096) + cpu = optional(number, 1024) + }) + default = {} +} + +variable "quickwit_metastore" { + description = "Metastore service sizing configurations" + type = object({ + desired_count = optional(number, 1) + memory = optional(number, 512) + cpu = optional(number, 256) + }) + default = {} +} + +variable "quickwit_searcher" { + description = "Searcher service sizing configurations" + type = object({ + desired_count = optional(number, 1) + memory = optional(number, 2048) + cpu = optional(number, 1024) + }) + default = {} +} + +variable "quickwit_control_plane" { + description = "Control plane service sizing configurations" + type = object({ + # only 1 task is necessary + memory = optional(number, 512) + cpu = optional(number, 256) + }) + default = {} +} + +variable "quickwit_janitor" { + description = "Janitor service sizing configurations" + type = object({ + # only 1 task is necessary + memory = optional(number, 512) + cpu = optional(number, 256) + }) + default = {} +} + +variable "rds_config" { + description = "Configurations of the metastore RDS database. Enable multi_az to ensure high availability." + type = object({ + instance_class = optional(string, "db.t4g.micro") + multi_az = optional(bool, false) + }) + default = {} +}