diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/config/WorkerConfigsProvider.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/config/WorkerConfigsProvider.java index 9f4282e9176..dccab9a6810 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/config/WorkerConfigsProvider.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/config/WorkerConfigsProvider.java @@ -187,7 +187,10 @@ record WorkerConfigsDefaults(WorkerEnvironment workerEnvironment, record KubeResourceKey(String variant, ResourceType type, ResourceSubType subType) {} public WorkerConfigsProvider(final List kubeResourceConfigs, final WorkerConfigsDefaults defaults) { - this.kubeResourceKeyPattern = Pattern.compile(String.format("^((?[a-z]+)-)?(?%s)(-(?%s))?$", + // In the variant name, we do not support uppercase. This is because micronaut normalizes uppercases + // with dashes (CamelCase becomes camel-case) which is confusing because the variant name no longer + // matches the config file. + this.kubeResourceKeyPattern = Pattern.compile(String.format("^((?[a-z0-9]+)-)?(?%s)(-(?%s))?$", String.join("|", Arrays.stream(ResourceType.values()).map(ResourceType::toString).toList()), String.join("|", Arrays.stream(ResourceSubType.values()).map(ResourceSubType::toString).toList())), Pattern.CASE_INSENSITIVE); diff --git a/airbyte-workers/src/main/resources/application.yml b/airbyte-workers/src/main/resources/application.yml index f9760459168..87735d63b45 100644 --- a/airbyte-workers/src/main/resources/application.yml +++ b/airbyte-workers/src/main/resources/application.yml @@ -156,16 +156,80 @@ airbyte: cpu-request: ${REPLICATION_ORCHESTRATOR_CPU_REQUEST:} memory-limit: ${REPLICATION_ORCHESTRATOR_MEMORY_LIMIT:} memory-request: ${REPLICATION_ORCHESTRATOR_MEMORY_REQUEST:} + + # Generally low resource containers + destination-stderr: + cpu-limit: 0.5 + cpu-request: 0.1 + memory-limit: 50Mi + memory-request: 25Mi + destination-stdout: + cpu-limit: 0.5 + cpu-request: 0.1 + memory-limit: 50Mi + memory-request: 25Mi + heartbeat: + cpu-limit: 0.2 + cpu-request: 0.05 + memory-limit: 50Mi + memory-request: 25Mi + source-stderr: + cpu-limit: 0.5 + cpu-request: 0.1 + memory-limit: 50Mi + memory-request: 25Mi + # Default base sync values + destination: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 2Gi + memory-request: 1Gi + destination-stdin: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 50Mi + memory-request: 25Mi + orchestrator: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 2Gi + memory-request: 2Gi source: - cpu-limit: ${SOURCE_CONTAINER_CPU_LIMIT:} - cpu-request: ${SOURCE_CONTAINER_CPU_REQUEST:0.5} - memory-limit: ${SOURCE_CONTAINER_MEMORY_LIMIT:} - memory-request: ${SOURCE_CONTAINER_MEMORY_REQUEST:} + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 2Gi + memory-request: 1Gi + source-stdout: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 50Mi + memory-request: 25Mi + # Database syncs resource override source-database: - cpu-limit: ${SOURCE_DATABASE_CONTAINER_CPU_LIMIT:} - cpu-request: ${SOURCE_DATABASE_CONTAINER_CPU_REQUEST:1} - memory-limit: ${SOURCE_DATABASE_CONTAINER_MEMORY_LIMIT:} - memory-request: ${SOURCE_DATABASE_CONTAINER_MEMORY_REQUEST:} + cpu-limit: 2 + cpu-request: 1 + memory-limit: 2Gi + memory-request: 1Gi + source-stdout-database: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 50Mi + memory-request: 25Mi + orchestrator-database: + cpu-limit: 2 + cpu-request: 1 + memory-limit: 2Gi + memory-request: 2Gi + destination-database: + cpu-limit: 2 + cpu-request: 1 + memory-limit: 2Gi + memory-request: 1Gi + destination-stdin-database: + cpu-limit: 2 + cpu-request: 0.5 + memory-limit: 50Mi + memory-request: 25Mi # base is the proposed default syncs values # Generally low resource containers @@ -244,28 +308,13 @@ airbyte: memory-request: 25Mi # Currently in testing for APIs - lowrss--heartbeat: # delete once base becomes the default - cpu-limit: 0.5 - cpu-request: 0.05 - memory-limit: 50Mi - memory-request: 25Mi - lowrss--destination-stderr: # delete once base becomes the default - cpu-limit: 0.5 - cpu-request: 0.1 - memory-limit: 50Mi - memory-request: 25Mi - lowrss--source-stderr: # delete once base becomes the default - cpu-limit: 0.5 - cpu-request: 0.1 - memory-limit: 50Mi - memory-request: 25Mi lowrss--source-api: - cpu-limit: 0.5 + cpu-limit: 1 cpu-request: 0.2 memory-limit: 2Gi memory-request: 1Gi lowrss--source-stdout-api: - cpu-limit: 0.5 + cpu-limit: 1 cpu-request: 0.2 memory-limit: 50Mi memory-request: 25Mi @@ -280,13 +329,35 @@ airbyte: memory-limit: 2Gi memory-request: 1Gi lowrss--destination-stdin-api: + cpu-limit: 1 + cpu-request: 0.2 + memory-limit: 50Mi + memory-request: 25Mi + + # Previous version of lowrss + lowrss2--source-api: + cpu-limit: 0.5 + cpu-request: 0.2 + memory-limit: 2Gi + memory-request: 1Gi + lowrss2--source-stdout-api: cpu-limit: 0.5 cpu-request: 0.2 memory-limit: 50Mi memory-request: 25Mi - lowrss--destination-stdout-api: # delete once base becomes the default + lowrss2--orchestrator-api: + cpu-limit: 1 + cpu-request: 0.5 + memory-limit: 2Gi + memory-request: 2Gi + lowrss2--destination-api: + cpu-limit: 1 + cpu-request: 0.2 + memory-limit: 2Gi + memory-request: 1Gi + lowrss2-destination-stdin-api: cpu-limit: 0.5 - cpu-request: 0.1 + cpu-request: 0.2 memory-limit: 50Mi memory-request: 25Mi verylowrss--source: diff --git a/airbyte-workers/src/test/resources/application-config-test.yaml b/airbyte-workers/src/test/resources/application-config-test.yaml index cfba78e1f18..1e09bf89046 100644 --- a/airbyte-workers/src/test/resources/application-config-test.yaml +++ b/airbyte-workers/src/test/resources/application-config-test.yaml @@ -20,12 +20,16 @@ airbyte: memory-request: ${SOMETHING_NOT_THERE:} source: cpu-request: 0.5 + cpu-limit: "" # This unsets the value from the inherited test file to test defaults source-database: cpu-request: 1 + cpu-limit: "" # This unsets the value from the inherited test file to test defaults orchestrator-api: cpu-request: 11 destination-api: cpu-request: 12 + variantwithnumb3rz-source: # Making sure we support numbers + cpu-request: 13 incompletevariant-source-api: cpu-request: 42 micronauttest-source: