diff --git a/.envrc b/.envrc index ef336ba2a..d39526413 100644 --- a/.envrc +++ b/.envrc @@ -1,3 +1,15 @@ +#!/usr/bin/env bash + +# Export all: +# - (should be) .gitignored +# - (potentially) secret environment variables +# - from dotenv-formatted files w/names starting w/`.env` +DOTENV_FILES="$(find . -maxdepth 1 -type f -name '.env*' -and -not -name '.envrc')" +for file in ${DOTENV_FILES}; do + dotenv "${file}" +done +export DOTENV_FILES + if command -v nix-shell &>/dev/null; then use flake fi diff --git a/docs/persistence/kafka/explanations/tiered-storage.md b/docs/persistence/kafka/explanations/tiered-storage.md index 2a78b9db1..4413f3256 100644 --- a/docs/persistence/kafka/explanations/tiered-storage.md +++ b/docs/persistence/kafka/explanations/tiered-storage.md @@ -12,6 +12,8 @@ The ratio of local to remote can be configured per topic. When using remote storage, cluster maintenance will be more efficient as less data needs to be transferred between brokers. With less data locally we can run smaller sized brokers. +See [the how-to documentation for tiered storage](../how-to/tiered-storage.md) for how to enable tiered storage for your kafka topic. + ## Local vs. remote data retention When using tiered storage you need to decide how much retention you need to keep locally, for low latency retrieval, and what can be stored remotely. diff --git a/docs/persistence/kafka/how-to/tiered-storage.md b/docs/persistence/kafka/how-to/tiered-storage.md new file mode 100644 index 000000000..bec0d14c7 --- /dev/null +++ b/docs/persistence/kafka/how-to/tiered-storage.md @@ -0,0 +1,40 @@ +--- +tags: [how-to, kafka, tiered-storage] +--- + +# How to turn on Tiered Storage for your Kafka topic + +This guide shows you how to enable tiered storage for your Kafka topic. + +## Prerequisites + +You need to own an existing topic in nais. +Check [Create a Kafka topic](create.md) for how to create a topic. + +## Enable tiered storage by specifying local retention on a topic + +???+ note "Git diff showing how to enable tiered storage in a 'nai example_topic.yaml' file" + + ```diff + diff --git a/example_topic.yaml b/example_topic.yaml + index 230ec001..cbc01774 100644 + --- a/example_topic.yaml + +++ b/example_topic.yaml + @@ -8,3 +8,8 @@ labels: + spec: + kafka: + # ... other configuration ... + + config: + + # Must be a value smaller than `retentionHours` + + localRetentionHours: X + + # Must be a value smaller than `retentionBytes` + + localRetentionBytes: Y + ``` + +## Apply the new configuration +=== "Automatically" + Add the file to your application repository and deploy with [NAIS github action](../../../build/how-to/build-and-deploy.md). +=== "Manually" + ```bash + kubectl apply -f ./example_topic.yaml --namespace= --context= + ``` diff --git a/pyproject.toml b/pyproject.toml index 5ebd96578..6326444a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,7 @@ name = "doc" version = "0.1.0" description = "nais developer documentation" authors = [] +package-mode = false [tool.poetry.dependencies] python = "^3.11"