dynamic init file(s) configuration including IAM permissions

README.md

@@ -186,6 +186,7 @@ for example.
 | [aws_iam_policy.acm](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.cloudwatch_logs_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.enable_execute_command](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_policy.fluent_bit_config_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.otel](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_role.ecs_task_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role.task_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
@@ -194,6 +195,7 @@ for example.
 | [aws_iam_role_policy_attachment.appmesh](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.cloudwatch_logs_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.enable_execute_command](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.fluent_bit_config_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.otel](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_security_group_rule.trusted_egress_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource |
 | [aws_service_discovery_service.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/service_discovery_service) | resource |
@@ -205,6 +207,7 @@ for example.
 | [aws_iam_policy_document.cloudwatch_logs_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.ecs_task_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.enable_execute_command](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.fluent_bit_config_access](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.otel](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.task_execution_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_lb.public](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/lb) | data source |
@@ -258,7 +261,7 @@ for example.
 | <a name="input_ecr_repository_name"></a> [ecr\_repository\_name](#input\_ecr\_repository\_name) | Existing repo to register to use with this service module, e.g. creating deployment pipelines. | `string` | `""` | no |
 | <a name="input_efs_volumes"></a> [efs\_volumes](#input\_efs\_volumes) | Configuration block for EFS volumes. | `any` | `[]` | no |
 | <a name="input_enable_execute_command"></a> [enable\_execute\_command](#input\_enable\_execute\_command) | Specifies whether to enable Amazon ECS Exec for the tasks within the service. | `bool` | `false` | no |
-| <a name="input_firelens"></a> [firelens](#input\_firelens) | Configuration for optional custom log routing using FireLens over fluentbit sidecar. | <pre>object({<br>    container_name       = optional(string, "fluentbit")<br>    container_definition = optional(any, {})<br>    enabled              = optional(bool, false)<br>    opensearch_host      = optional(string, "")<br>  })</pre> | `{}` | no |
+| <a name="input_firelens"></a> [firelens](#input\_firelens) | Configuration for optional custom log routing using FireLens over fluentbit sidecar. | <pre>object({<br>    container_name       = optional(string, "fluentbit")<br>    container_definition = optional(any, {})<br>    enabled              = optional(bool, false)<br>    init_config_files    = optional(list(string), [])<br>    opensearch_host      = optional(string, "")<br>  })</pre> | `{}` | no |
 | <a name="input_force_new_deployment"></a> [force\_new\_deployment](#input\_force\_new\_deployment) | Enable to force a new task deployment of the service. This can be used to update tasks to use a newer Docker image with same image/tag combination (e.g. myimage:latest), roll Fargate tasks onto a newer platform version, or immediately deploy ordered\_placement\_strategy and placement\_constraints updates. | `bool` | `false` | no |
 | <a name="input_health_check_grace_period_seconds"></a> [health\_check\_grace\_period\_seconds](#input\_health\_check\_grace\_period\_seconds) | Seconds to ignore failing load balancer health checks on newly instantiated tasks to prevent premature shutdown, up to 2147483647. Only valid for services configured to use load balancers. | `number` | `0` | no |
 | <a name="input_https_listener_rules"></a> [https\_listener\_rules](#input\_https\_listener\_rules) | A list of maps describing the Listener Rules for this ALB. Required key/values: actions, conditions. Optional key/values: priority, https\_listener\_index (default to https\_listeners[count.index]) | `any` | `[]` | no |

fluentbit.tf

@@ -1,32 +1,30 @@
 locals {
+  // additional init config files from S3 or files inside a custom image
+  // which are added to the FluentBit container as environment variables, see
+  // https://github.com/aws/aws-for-fluent-bit/tree/develop/use_cases/init-process-for-fluent-bit
+  init_config_files = [
+    for idx, file_or_arn in var.firelens.init_config_files : {
+      name  = format("aws_fluent_bit_init_%s", idx)
+      value = file_or_arn
+    }
+  ]
+
+  // additional init config files ARNs from S3 to be used in an IAM policy for the task role
+  s3_init_file_arns   = [for conf in local.init_config_files : conf.value if can(regex("^arn:.*:s3:", conf.value))]
+  s3_init_bucket_arns = distinct([for arn in local.s3_init_file_arns : split("/", arn)[0]])
+
   // optional FluentBit container for log aggregation
   fluentbit_container_defaults = {
     name                   = var.firelens.container_name
     image                  = "${data.aws_caller_identity.current.account_id}.dkr.ecr.${data.aws_region.current.name}.amazonaws.com/ecr-public/aws-observability/aws-for-fluent-bit:init-2.32.0.20240122"
+    environment            = concat([{ name = "FLB_LOG_LEVEL", value = "error" }], local.init_config_files)
     essential              = true
     mountPoints            = []
     portMappings           = []
     readonlyRootFilesystem = false
     user                   = startswith(upper(var.operating_system_family), "WINDOWS") ? null : "0:1337"
     volumesFrom            = []
 
-    environment = [
-      // Valid values are: debug, info and error, default if missing: info
-      { name = "FLB_LOG_LEVEL", value = "error" },
-      {
-        "name" : "aws_fluent_bit_init_s3_1",
-        "value" : "arn:aws:s3:::config-${data.aws_caller_identity.current.account_id}-${data.aws_region.current.name}/ecs/fluent-bit/service-custom.conf"
-      },
-      {
-        "name" : "aws_fluent_bit_init_s3_2",
-        "value" : "arn:aws:s3:::config-${data.aws_caller_identity.current.account_id}-${data.aws_region.current.name}/ecs/fluent-bit/filters-custom.conf"
-      },
-      {
-        "name" : "aws_fluent_bit_init_s3_3",
-        "value" : "arn:aws:s3:::config-${data.aws_caller_identity.current.account_id}-${data.aws_region.current.name}/ecs/fluent-bit/parsers-custom.conf"
-      },
-    ],
-
     # https://github.com/aws-samples/amazon-ecs-firelens-examples/tree/mainline/examples/fluent-bit/health-check
     healthCheck = {
       retries = 3
@@ -70,32 +68,42 @@ module "fluentbit_container_definition" {
 }
 
 data "aws_iam_policy_document" "fluent_bit_config_access" {
-  count = var.firelens.enabled && var.task_role_arn == "" ? 1 : 0
-
-  statement {
-    actions = [
-      "s3:GetObject",
-      "s3:GetBucketLocation"
-    ]
-
-    resources = [
-      "arn:aws:s3:::config-${data.aws_caller_identity.current.account_id}-${data.aws_region.current.name}/ecs/fluent-bit/*",
-      "arn:aws:s3:::config-${data.aws_caller_identity.current.account_id}-${data.aws_region.current.name}",
-    ]
+  count = var.firelens.enabled && var.task_role_arn == "" && length(local.s3_init_file_arns) > 0 ? 1 : 0
+
+  // allow reading the init config files from S3
+  dynamic "statement" {
+    for_each = [true]
+
+    content {
+      effect    = "Allow"
+      actions   = ["s3:GetObject"]
+      resources = local.s3_init_file_arns
+    }
+  }
+
+  // allow listing the S3 buckets containing the init config files
+  dynamic "statement" {
+    for_each = [true]
+
+    content {
+      effect    = "Allow"
+      actions   = ["s3:GetBucketLocation"]
+      resources = local.s3_init_bucket_arns
+    }
   }
 }
 
 resource "aws_iam_policy" "fluent_bit_config_access" {
-  count = var.firelens.enabled && var.task_role_arn == "" ? 1 : 0
+  count = var.firelens.enabled && var.task_role_arn == "" && length(local.s3_init_file_arns) > 0 ? 1 : 0
 
   name   = "fluent-bit-config-access-${var.service_name}-${data.aws_region.current.name}"
   path   = "/ecs/task-role/"
   policy = data.aws_iam_policy_document.fluent_bit_config_access[count.index].json
 }
 
 resource "aws_iam_role_policy_attachment" "fluent_bit_config_access" {
-  count = var.firelens.enabled && var.task_role_arn == "" ? 1 : 0
+  count = var.firelens.enabled && var.task_role_arn == "" && length(local.s3_init_file_arns) > 0 ? 1 : 0
 
   role       = aws_iam_role.ecs_task_role[count.index].name
   policy_arn = aws_iam_policy.fluent_bit_config_access[count.index].arn
-}
+}

variables.tf

@@ -329,6 +329,7 @@ variable "firelens" {
     container_name       = optional(string, "fluentbit")
     container_definition = optional(any, {})
     enabled              = optional(bool, false)
+    init_config_files    = optional(list(string), [])
     opensearch_host      = optional(string, "")
   })
 }