From 8158fadfd5f6d32fd7f0f8e437b49b4423f629c9 Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Wed, 16 Aug 2023 11:55:38 +0200 Subject: [PATCH] feat: update resolver to newest omegaconf requirements --- Dockerfile | 6 +- docs/pages/inventory/omegaconf.md | 254 ++++++++++++++++++++++++++++++ kapitan/omegaconf_inv.py | 2 +- kapitan/resolvers.py | 16 +- 4 files changed, 267 insertions(+), 11 deletions(-) create mode 100644 docs/pages/inventory/omegaconf.md diff --git a/Dockerfile b/Dockerfile index b555844fe..86ea72017 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the virtualenv for Kapitan -FROM python:3.8-slim AS python-builder +FROM python:3.8-slim-bullseye AS python-builder ARG TARGETARCH @@ -16,7 +16,9 @@ ENV PATH="/opt/venv/bin:${PATH}" RUN apt-get update \ && apt-get install --no-install-recommends -y \ curl \ - build-essential + build-essential \ + git \ + default-jre # Install Go (for go-jsonnet) RUN curl -fsSL -o go.tar.gz https://go.dev/dl/go1.17.3.linux-${TARGETARCH}.tar.gz \ diff --git a/docs/pages/inventory/omegaconf.md b/docs/pages/inventory/omegaconf.md new file mode 100644 index 000000000..fd481ca7d --- /dev/null +++ b/docs/pages/inventory/omegaconf.md @@ -0,0 +1,254 @@ +## OmegaConf + +With version `0.33.0` we are introducing a new inventory backend as an alternative to reclass. +Here are all of the differences to using reclass, new features and some instructions, how to migrate your inventory. + +!!! warning + + OmegaConf is currently in experimental mode. If you encouter unexpected errors or bugs, please let us know and create an [issue](https://github.com/kapicorp/kapitan/issues/new/choose). + +### Differences to reclass + +#### Supported: +* compose-node-name +* key overwrite prefix '`~`' +* interpolations +* relative class names +* init class +* nested interpolations +* escaped interpolations + +#### Not (yet) supported +* exports +* inventory queries +* interpolation to yaml-keys containing '`.`' (the delimiter itself) + +#### Syntax changes + +OmegaConf uses the default yaml-path notation using dots (`.`) as key delimiter. + +### New Functionalities + +All of [OmegaConfs native functionalities](https://omegaconf.readthedocs.io/en/2.3_branch/grammar.html#the-omegaconf-grammar) are supported. + +#### General +* relative interpolation +* list accessing +* mandatory values +* Resolvers and Custom Resolvers + +#### Resolvers + +Resolvers are the main benefits of using OmegaConf. +You can define any behavior with a python function that gets executed with the given input parameters. + +We provide some basic resolvers: + +* OmegaConf + * `oc.env`: access a environment variable + * `oc.select`: provide a default value for an interpolation + * `oc.dict.keys`: get the keys of a dictionary object as a list + * `oc.dict.values`: get the values of dictionary object as a list +* Utilities + * `key`: get the name of the nodes key + * `fullkey`: get the full name of the key + * `parentkey`: get the name of the nodes parent key + * `relpath`: takes an absolute path and convert it to relative + * `tag`: creates an escaped interpolation +* Casting and Merging + * `dict`: cast a dict inside a list into the actual dict + * `list`: put a dict inside a list with that dict + * `merge`: merge objects + +#### Usage + +Using a resolver is as simple as an interpolation: `yamlkey: ${resolver:input}` + +#### Custom Resolver + +You can write your own resolvers and are able to achieve any behavior you want. + +In your inventory you have to create a file `resolvers.py`. +You should define a function `pass_resolvers()` that returns a dictionary with the resolvers name and the respective function pointer. + +Now you can start writing python functions with your custom . + +#### Example + +```python +# inventory/resolvers.py + +def concat(input1, input2): + return input1 + input2 + +def add_ten(input): + assert isinstance(input, int) + return input + 10 + +def default_value(): + return "DEFAULT" + +def split_dots(input: str): + return input.split(".") + +# required function +def pass_resolvers(): + return {"concat": concat, "plus_ten": add_ten, "default": default_value, "split", split_dots} +``` + +If we render a file the result would be: + +```yaml +string: ${concat:Hello, World} # --> Hello World +int: ${plus_ten:90} # --> 100 +default: ${default:} # --> DEFAULT +list: ${split:hello.world} # --> yaml list [hello, world] +``` + +### Access the feature + +To access the feature you have to use a kapitan version >=0.33.0. + +Use the flag `--omegaconf` in your command to indicate to use OmegaConf as backend. To specify that you want to use it everytime, add this to your `.kapitan` file: +```yaml +inventory_backend: + omegaconf: true +``` + +If this is your first time running you have to specify `--migrate` to adapt to OmegaConfs syntax. + +!!! danger + + Please backup your inventory, if you're running `--migrate` or make sure you are able to revert the changes using source control. + Also check your inventory if it contains some yaml errors like duplicate keys or wrong yaml types. The command will not change anything if some errors occur. + +The migration consists of the following steps: +* replacing the delimiter '`:`' with '`.`' in interpolations +* replacing meta interpolations '`_reclass_`' to '`_meta_`' +* replacing escaped interpolations `\${content}` to resolver `${tag:content}` + +### Examples + +One important usecase with this is the definition of default values and overwriting them with specific target/component values. + + +```yaml +# inventory/classes/templates/deployment.yml +parameters: + + # define default values using the 'relpath' resolver + deployment: + + namespace: ${target_name} + component_name: \${parentkey:} # hack to get the components name + + labels: + app.kubernetes.io/name: ${relpath:deployment.namespace} # gets resolved relatively + + image: ??? # OmegaConf mandatory value (has to be set in target) + pull_policy: Always + image_pull_secrets: + - name: default-secret + + service_port: 8080 # default value + + service: + type: ClusterIP + selector: + app: ${target_name} + ports: + http: + service_port: ${relpath:deployment.service_port} # allows us to overwrite this in another key + +``` + +```yaml +# inventory/targets/example.yml +classes: + - templates.deployment + +parameters: + + target_name: ${_meta_.name.short} + + components: + # merge each component with a deployment + backend: ${merge:${deployment},${backend}} + keycloak: ${merge:${deployment},${keycloak}} + keycloak-copy: ${merge:${keycloak},${keycloak-copy}} # merge with another component to specify even more + + # components config (would be in their own classes) + + # backend config + backend: + image: backend:latest + + # keycloak config + keycloak: + namespace: example1 + image: keycloak:latest + + env: + [...] + + # keycloak-copy config (inherits env and namespace from keycloak) + keycloak-copy: + namespace: example2 +``` + +This would generate the following components definition: + +```yaml +components: + backend: + component_name: deployment + image: backend:latest + image_pull_secrets: + - name: default-secret + labels: + app.kubernetes.io/name: example + namespace: example + ports: + http: + service_port: 8080 + pull_policy: Always + service: + selector: + app: example + type: ClusterIP + service_port: 8080 + keycloak: + component_name: deployment + image: keycloak:latest + image_pull_secrets: + - name: default-secret + labels: + app.kubernetes.io/name: example1 + namespace: example1 + ports: + http: + service_port: 8080 + pull_policy: Always + service: + selector: + app: example + type: ClusterIP + service_port: 8080 + keycloak-copy: + component_name: deployment + image: keycloak:latest + image_pull_secrets: + - name: default-secret + labels: + app.kubernetes.io/name: example1 + namespace: example2 + ports: + http: + service_port: 8080 + pull_policy: Always + service: + selector: + app: example + type: ClusterIP + service_port: 8080 +``` \ No newline at end of file diff --git a/kapitan/omegaconf_inv.py b/kapitan/omegaconf_inv.py index b48e9034f..db13690d7 100644 --- a/kapitan/omegaconf_inv.py +++ b/kapitan/omegaconf_inv.py @@ -153,7 +153,7 @@ def load_target( target_config_parameters["_reclass_"] = _meta_ # legacy # resolve references / interpolate values - OmegaConf.resolve(target_config_parameters) + OmegaConf.resolve(target_config_parameters, False) target_config["parameters"] = OmegaConf.to_object(target_config_parameters) # obtain target name to insert in inv dict diff --git a/kapitan/resolvers.py b/kapitan/resolvers.py index c8fb9483b..a098bdc47 100644 --- a/kapitan/resolvers.py +++ b/kapitan/resolvers.py @@ -85,10 +85,10 @@ def helm_dep(name: str, source: str): """kapitan template for a helm chart dependency""" return { "type": "helm", - "output_path": f"components/charts/${{parameters.{name}.chart_name}}/${{parameters.{name}.chart_version}}/${{parameters.{name}.application_version}}", + "output_path": f"components/charts/${{{name}.chart_name}}/${{{name}.chart_version}}/${{{name}.application_version}}", "source": source, - "version": f"${{parameters.{name}.chart_version}}", - "chart_name": f"${{parameters.{name}.chart_name}}", + "version": f"${{{name}.chart_version}}", + "chart_name": f"${{{name}.chart_name}}", } @@ -97,15 +97,15 @@ def helm_input(name: str): return { "input_type": "helm", "input_paths": [ - f"components/charts/${{parameters.{name}.chart_name}}/${{parameters.{name}.chart_version}}/${{parameters.{name}.application_version}}" + f"components/charts/${{{name}.chart_name}}/${{{name}.chart_version}}/${{{name}.application_version}}" ], - "output_path": f"k8s/${{parameters.{name}.namespace}}", + "output_path": f"k8s/${{{name}.namespace}}", "helm_params": { - "namespace": f"${{parameters.{name}.namespace}}", - "name": f"${{parameters.{name}.chart_name}}", + "namespace": f"${{{name}.namespace}}", + "name": f"${{{name}.chart_name}}", "output_file": f"{name}.yml", }, - "helm_values": f"${{parameters.{name}.helm_values}}", + "helm_values": f"\\${{{name}.helm_values}}", # \\ used for delaying the resolving of helm values }