From a0ea50bf362c697456a1cef3c461bc6eecca982d Mon Sep 17 00:00:00 2001 From: Matteo Voges Date: Wed, 9 Aug 2023 18:30:25 +0200 Subject: [PATCH] feat(omegaconf): fix escaped interpolations --- kapitan/inventory.py | 24 +++++----- kapitan/omegaconf_inv.py | 6 +-- kapitan/resolvers.py | 17 ++++---- kapitan/resources.py | 6 +-- poetry.lock | 94 ++++++++++++++++++++++++++++++---------- pyproject.toml | 2 +- 6 files changed, 99 insertions(+), 50 deletions(-) diff --git a/kapitan/inventory.py b/kapitan/inventory.py index b32774d26..041f53c6b 100644 --- a/kapitan/inventory.py +++ b/kapitan/inventory.py @@ -113,6 +113,8 @@ def migrate(input: str): def migrate_dir(path: str): """migrates all .yml/.yaml files in the given path to omegaconfs syntax""" + # TODO: write migrations to temp dir and copy only if suceeded + for root, subdirs, files in os.walk(path): for file in files: file = os.path.join(root, file) @@ -140,22 +142,22 @@ def migrate_file(file: str): @staticmethod def migrate_str(content: str): - # replace colons in tags and change _reclass_ to _meta_ + + # TODO: dont migrate custom resolvers + # TODO: migrate interpolations with '.' in the keyname + + # search for interpolation pattern updated_content = regex.sub( r"(? None: OmegaConf.register_new_resolver("relpath", relpath) # yaml object utility functions - OmegaConf.register_new_resolver("tag", escape_tag) + OmegaConf.register_new_resolver("access", access_key_with_dots) OmegaConf.register_new_resolver("merge", merge) OmegaConf.register_new_resolver("dict", to_dict) OmegaConf.register_new_resolver("list", to_list) diff --git a/kapitan/resources.py b/kapitan/resources.py index 557099bc1..20926903a 100644 --- a/kapitan/resources.py +++ b/kapitan/resources.py @@ -301,7 +301,7 @@ def generate_inventory(args): except Exception as e: if not isinstance(e, KapitanError): logger.exception("\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") - sys.exit(1) + logger.error(e) def get_inventory(inventory_path, ignore_class_notfound=False, targets=[]): @@ -335,8 +335,8 @@ def get_inventory(inventory_path, ignore_class_notfound=False, targets=[]): try: inventory = inventory_backend.inventory() except Exception as e: - if not args.migrate: - logger.warning("Make sure to migrate your inventory using --migrate") + # if hasattr(args, "omegaconf") and args.omegaconf and not args.migrate: + # logger.warning("Make sure to migrate your inventory using --migrate if you are using omegaconf the first time") raise InventoryError(e) cached.inv = inventory diff --git a/poetry.lock b/poetry.lock index feaeb0f75..444ca68d6 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "addict" version = "2.4.0" description = "Addict is a dictionary whose items can be set using both attribute and item syntax." +category = "main" optional = false python-versions = "*" files = [ @@ -26,6 +27,7 @@ files = [ name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -44,6 +46,7 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "azure-common" version = "1.1.28" description = "Microsoft Azure Client Library for Python (Common)" +category = "main" optional = false python-versions = "*" files = [ @@ -55,6 +58,7 @@ files = [ name = "azure-core" version = "1.26.4" description = "Microsoft Azure Core Library for Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -74,6 +78,7 @@ aio = ["aiohttp (>=3.0)"] name = "azure-identity" version = "1.12.0" description = "Microsoft Azure Identity Library for Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -92,6 +97,7 @@ six = ">=1.12.0" name = "azure-keyvault-keys" version = "4.8.0" description = "Microsoft Azure Key Vault Keys Client Library for Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -110,6 +116,7 @@ typing-extensions = ">=4.0.1" name = "boto3" version = "1.26.121" description = "The AWS SDK for Python" +category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -129,6 +136,7 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] name = "botocore" version = "1.29.121" description = "Low-level, data-driven core of boto 3." +category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -148,6 +156,7 @@ crt = ["awscrt (==0.16.9)"] name = "cachetools" version = "5.3.0" description = "Extensible memoizing collections and decorators" +category = "main" optional = false python-versions = "~=3.7" files = [ @@ -159,6 +168,7 @@ files = [ name = "certifi" version = "2023.7.22" description = "Python package for providing Mozilla's CA Bundle." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -170,6 +180,7 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." +category = "main" optional = false python-versions = "*" files = [ @@ -246,6 +257,7 @@ pycparser = "*" name = "charset-normalizer" version = "3.1.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -330,6 +342,7 @@ files = [ name = "cryptography" version = "41.0.2" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -375,6 +388,7 @@ test-randomorder = ["pytest-randomly"] name = "docker" version = "5.0.3" description = "A Python library for the Docker Engine API." +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -395,6 +409,7 @@ tls = ["cryptography (>=3.4.7)", "idna (>=2.0.0)", "pyOpenSSL (>=17.5.0)"] name = "gitdb" version = "4.0.10" description = "Git Object Database" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -409,6 +424,7 @@ smmap = ">=3.0.1,<6" name = "gitpython" version = "3.1.31" description = "GitPython is a Python library used to interact with Git repositories" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -423,6 +439,7 @@ gitdb = ">=4.0.1,<5" name = "gojsonnet" version = "0.17.0" description = "Python bindings for Jsonnet - The data templating language" +category = "main" optional = true python-versions = "*" files = [ @@ -433,6 +450,7 @@ files = [ name = "google-api-core" version = "2.11.0" description = "Google API client core library" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -455,6 +473,7 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0dev)"] name = "google-api-python-client" version = "2.86.0" description = "Google API Client Library for Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -463,7 +482,7 @@ files = [ ] [package.dependencies] -google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-api-core = ">=1.31.5,<2.0.0 || >2.3.0,<3.0.0dev" google-auth = ">=1.19.0,<3.0.0dev" google-auth-httplib2 = ">=0.1.0" httplib2 = ">=0.15.0,<1dev" @@ -473,6 +492,7 @@ uritemplate = ">=3.0.1,<5" name = "google-auth" version = "2.17.3" description = "Google Authentication Library" +category = "main" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*" files = [ @@ -497,6 +517,7 @@ requests = ["requests (>=2.20.0,<3.0.0dev)"] name = "google-auth-httplib2" version = "0.1.0" description = "Google Authentication Library: httplib2 transport" +category = "main" optional = false python-versions = "*" files = [ @@ -513,6 +534,7 @@ six = "*" name = "googleapis-common-protos" version = "1.59.0" description = "Common protobufs used in Google APIs" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -530,6 +552,7 @@ grpc = ["grpcio (>=1.44.0,<2.0.0dev)"] name = "httplib2" version = "0.22.0" description = "A comprehensive HTTP client library." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -544,6 +567,7 @@ pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0 name = "hvac" version = "1.0.2" description = "HashiCorp Vault API client" +category = "main" optional = false python-versions = ">=3.6.2,<4.0.0" files = [ @@ -559,6 +583,7 @@ requests = ">=2.27.1,<3.0.0" name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -570,6 +595,7 @@ files = [ name = "importlib-metadata" version = "6.6.0" description = "Read metadata from Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -589,6 +615,7 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "importlib-resources" version = "5.12.0" description = "Read resources from Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -607,6 +634,7 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "isodate" version = "0.6.1" description = "An ISO 8601 date/time/duration parser and formatter" +category = "main" optional = false python-versions = "*" files = [ @@ -621,6 +649,7 @@ six = "*" name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -638,6 +667,7 @@ i18n = ["Babel (>=2.7)"] name = "jmespath" version = "1.0.1" description = "JSON Matching Expressions" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -649,6 +679,7 @@ files = [ name = "jsonnet" version = "0.18.0" description = "Python bindings for Jsonnet - The data templating language" +category = "main" optional = false python-versions = "*" files = [ @@ -659,6 +690,7 @@ files = [ name = "jsonschema" version = "4.17.3" description = "An implementation of JSON Schema validation for Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -680,6 +712,7 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- name = "kadet" version = "0.2.2" description = "Easily define and reuse complex Python objects that serialize into JSON or YAML." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -697,6 +730,7 @@ typeguard = ">=2.12.1" name = "markupsafe" version = "2.1.2" description = "Safely add untrusted strings to HTML/XML markup." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -756,6 +790,7 @@ files = [ name = "msal" version = "1.22.0" description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." +category = "main" optional = false python-versions = "*" files = [ @@ -775,6 +810,7 @@ broker = ["pymsalruntime (>=0.13.2,<0.14)"] name = "msal-extensions" version = "1.0.0" description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." +category = "main" optional = false python-versions = "*" files = [ @@ -805,14 +841,15 @@ PyYAML = ">=5.1.0" [package.source] type = "git" -url = "https://github.com/omry/omegaconf.git" -reference = "HEAD" -resolved_reference = "7dae67e4a3869b584fd6d6408b2f916c176755db" +url = "https://github.com/neXenio/omegaconf.git" +reference = "try-escape-interpolations" +resolved_reference = "5cb3689a95db045df09b081a2247c1efc90fab80" [[package]] name = "packaging" version = "23.1" description = "Core utilities for Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -824,6 +861,7 @@ files = [ name = "pathspec" version = "0.11.1" description = "Utility library for gitignore style pattern matching of file paths." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -835,6 +873,7 @@ files = [ name = "pkgutil-resolve-name" version = "1.3.10" description = "Resolve a name to an object." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -846,6 +885,7 @@ files = [ name = "portalocker" version = "2.7.0" description = "Wraps the portalocker recipe for easy usage" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -865,6 +905,7 @@ tests = ["pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-mypy (>=0.8.0)", "p name = "protobuf" version = "4.22.3" description = "" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -914,6 +955,7 @@ pyasn1 = ">=0.4.6,<0.6.0" name = "pycparser" version = "2.21" description = "C parser in Python" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -925,6 +967,7 @@ files = [ name = "pydantic" version = "1.10.7" description = "Data validation and settings management using python type hints" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -988,6 +1031,7 @@ files = [ name = "pyjwt" version = "2.6.0" description = "JSON Web Token implementation in Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1008,6 +1052,7 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] name = "pyparsing" version = "3.0.9" description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "main" optional = false python-versions = ">=3.6.8" files = [ @@ -1022,6 +1067,7 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "pyrsistent" version = "0.19.3" description = "Persistent/Functional/Immutable data structures" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1058,6 +1104,7 @@ files = [ name = "python-box" version = "6.0.2" description = "Advanced Python dictionaries with dot notation access" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1089,6 +1136,7 @@ yaml = ["ruamel.yaml (>=0.17)"] name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -1103,6 +1151,7 @@ six = ">=1.5" name = "python-gnupg" version = "0.4.9" description = "A wrapper for the Gnu Privacy Guard (GPG or GnuPG)" +category = "main" optional = false python-versions = "*" files = [ @@ -1114,6 +1163,7 @@ files = [ name = "python-magic" version = "0.4.27" description = "File type identification using libmagic" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1125,6 +1175,7 @@ files = [ name = "pywin32" version = "227" description = "Python for Window Extensions" +category = "main" optional = true python-versions = "*" files = [ @@ -1146,6 +1197,7 @@ files = [ name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1293,6 +1345,7 @@ files = [ name = "requests" version = "2.31.0" description = "Python HTTP for Humans." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1314,6 +1367,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "rsa" version = "4.9" description = "Pure-Python RSA implementation" +category = "main" optional = false python-versions = ">=3.6,<4" files = [ @@ -1328,6 +1382,7 @@ pyasn1 = ">=0.1.3" name = "s3transfer" version = "0.6.0" description = "An Amazon S3 Transfer Manager" +category = "main" optional = false python-versions = ">= 3.7" files = [ @@ -1341,26 +1396,11 @@ botocore = ">=1.12.36,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] -[[package]] -name = "setuptools" -version = "67.6.0" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -optional = false -python-versions = ">=3.7" -files = [ - {file = "setuptools-67.6.0-py3-none-any.whl", hash = "sha256:b78aaa36f6b90a074c1fa651168723acbf45d14cb1196b6f02c0fd07f17623b2"}, - {file = "setuptools-67.6.0.tar.gz", hash = "sha256:2ee892cd5f29f3373097f5a814697e397cf3ce313616df0af11231e2ad118077"}, -] - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - [[package]] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1372,6 +1412,7 @@ files = [ name = "smmap" version = "5.0.0" description = "A pure Python implementation of a sliding window memory map manager" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1383,6 +1424,7 @@ files = [ name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1394,6 +1436,7 @@ files = [ name = "typeguard" version = "3.0.2" description = "Run-time type checker for Python" +category = "main" optional = false python-versions = ">=3.7.4" files = [ @@ -1413,6 +1456,7 @@ test = ["mypy (>=0.991)", "pytest (>=7)"] name = "typing-extensions" version = "4.5.0" description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1424,6 +1468,7 @@ files = [ name = "uritemplate" version = "4.1.1" description = "Implementation of RFC 6570 URI Templates" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1435,6 +1480,7 @@ files = [ name = "urllib3" version = "1.26.15" description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -1451,6 +1497,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "websocket-client" version = "1.5.1" description = "WebSocket client for Python with low level API options" +category = "main" optional = true python-versions = ">=3.7" files = [ @@ -1467,6 +1514,7 @@ test = ["websockets"] name = "yamllint" version = "1.31.0" description = "A linter for YAML files." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1485,6 +1533,7 @@ dev = ["doc8", "flake8", "flake8-import-order", "rstcheck[sphinx]", "sphinx"] name = "zipp" version = "3.15.0" description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1502,6 +1551,5 @@ test = ["docker"] [metadata] lock-version = "2.0" - python-versions = "^3.8" -content-hash = "e8c1a9fdd874ef3e43ed6e11e70445e23c2a03c2b5a2245bf9419077ee81daaa" \ No newline at end of file +content-hash = "557dc45857272e18b1627e920173e760fd30e1a544c3041a09239568b193285c" diff --git a/pyproject.toml b/pyproject.toml index 6e22f2127..1902a2655 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ jsonschema = "^4.17.3" kadet = "^0.2.2" python-gnupg = "^0.4.7" pyyaml = "^6.0" -omegaconf = { git = "https://github.com/neXenio/omegaconf.git", branch = "fix-logging-redundant-keys" } +omegaconf = { git = "https://github.com/neXenio/omegaconf.git", branch = "try-escape-interpolations" } requests = "^2.28.2" six = "^1.16.0" toml = "^0.10.2"