From cb2c6c23ffc8e207f2ff3f8f690f7991a376c573 Mon Sep 17 00:00:00 2001 From: Christian Siegel Date: Mon, 9 May 2022 20:03:27 +0200 Subject: [PATCH] feat: support JSONPath (#182) Use library `jsonpath-ng` to replace own but limited JSONPath implementation. Especially the filter expressions allow more complex replacements within lists. Example: `backend.env[?name=='TEST'].value` Resolves #181. --- docs/commands/deploy.md | 18 +++++----- docs/includes/preview-configuration.md | 2 +- gitopscli/commands/deploy.py | 2 +- gitopscli/io_api/yaml_util.py | 46 ++++++++++-------------- setup.py | 1 + tests/commands/test_deploy.py | 2 +- tests/io_api/test_yaml_util.py | 48 ++++++++++++++++++++++---- 7 files changed, 75 insertions(+), 44 deletions(-) diff --git a/docs/commands/deploy.md b/docs/commands/deploy.md index 0421440d..3f3241de 100644 --- a/docs/commands/deploy.md +++ b/docs/commands/deploy.md @@ -15,10 +15,10 @@ backend: tag: 1.0.0 # <- and this one env: - name: TEST - value: foo # <- and even one in a list + value: foo # <- and this one in a list, selected via sibling value 'TEST' ``` -With the following command GitOps CLI will update both values to `1.1.0` on the default branch. +With the following command GitOps CLI will update all values on the default branch. ```bash gitopscli deploy \ @@ -30,9 +30,11 @@ gitopscli deploy \ --organisation "deployment" \ --repository-name "myapp-non-prod" \ --file "example/values.yaml" \ - --values "{frontend.tag: 1.1.0, backend.tag: 1.1.0, 'backend.env.[0].value': bar}" + --values "{frontend.tag: 1.1.0, backend.tag: 1.1.0, 'backend.env[?name==''TEST''].value': bar}" ``` +You could also use the list index to replace the latter (`my-app.env.[0].value`). For more details on the underlying *JSONPath* syntax, please refer to the [documenatation of the used library *jsonpath-ng*](https://github.com/h2non/jsonpath-ng#jsonpath-syntax). + ### Number Of Commits Note that by default GitOps CLI will create a separate commit for every value change: @@ -42,7 +44,7 @@ commit 0dcaa136b4c5249576bb1f40b942bff6ac718144 Author: GitOpsCLI Date: Thu Mar 12 15:30:32 2020 +0100 - changed 'backend.env.[0].value' to 'bar' in example/values.yaml + changed 'backend.env[?name=='TEST'].value' to 'bar' in example/values.yaml commit d98913ad8fecf571d5f8c3635f8070b05c43a9ca Author: GitOpsCLI @@ -64,11 +66,11 @@ commit 3b96839e90c35b8decf89f34a65ab6d66c8bab28 Author: GitOpsCLI Date: Thu Mar 12 15:30:00 2020 +0100 - updated 2 values in example/values.yaml + updated 3 values in example/values.yaml frontend.tag: '1.1.0' backend.tag: '1.1.0' - backend.env.[0].value: 'bar' + 'backend.env[?name==''TEST''].value': 'bar' ``` ### Specific Commit Message @@ -88,7 +90,7 @@ gitopscli deploy \ --repository-name "myapp-non-prod" \ --commit-message "test commit message" \ --file "example/values.yaml" \ - --values "{frontend.tag: 1.1.0, backend.tag: 1.1.0, 'backend.env.[0].value': bar}" + --values "{frontend.tag: 1.1.0, backend.tag: 1.1.0, 'backend.env[?name==''TEST''].value': bar}" ``` This will end up in one single commit with your specified commit-message. @@ -107,7 +109,7 @@ gitopscli deploy \ --organisation "deployment" \ --repository-name "myapp-non-prod" \ --file "example/values.yaml" \ - --values "{frontend.tag: 1.1.0, backend.tag: 1.1.0, 'backend.env.[0].value': bar}" \ + --values "{frontend.tag: 1.1.0, backend.tag: 1.1.0, 'backend.env[?name==''TEST''].value': bar}" \ --create-pr \ --auto-merge ``` diff --git a/docs/includes/preview-configuration.md b/docs/includes/preview-configuration.md index 6802760a..40898972 100644 --- a/docs/includes/preview-configuration.md +++ b/docs/includes/preview-configuration.md @@ -23,7 +23,7 @@ Make sure that your *app repository* contains a `.gitops.config.yaml` file. This 1. find repository, branch, and folder containing the template 2. templates for host and namespace name -3. replace values in template files +3. replace values in template files (see [`deploy` command](/gitopscli/commands/deploy/) for details on the key syntax) 4. find repository and branch where the preview should be created (i.e. your *deployment config repository*) 5. message templates used to comment your pull request diff --git a/gitopscli/commands/deploy.py b/gitopscli/commands/deploy.py index 691d243c..9aca11a8 100644 --- a/gitopscli/commands/deploy.py +++ b/gitopscli/commands/deploy.py @@ -70,7 +70,7 @@ def __update_values(self, git_repo: GitRepo) -> Dict[str, Any]: except YAMLException as ex: raise GitOpsException(f"Error loading file: {args.file}") from ex except KeyError as ex: - raise GitOpsException(f"Key '{key}' not found in file: {args.file}") from ex + raise GitOpsException(str(ex)) from ex if not updated_value: logging.info("Yaml property %s already up-to-date", key) diff --git a/gitopscli/io_api/yaml_util.py b/gitopscli/io_api/yaml_util.py index 7e4c4acc..b809cec4 100644 --- a/gitopscli/io_api/yaml_util.py +++ b/gitopscli/io_api/yaml_util.py @@ -1,10 +1,8 @@ -import re from io import StringIO from typing import Any from ruamel.yaml import YAML, YAMLError - - -_ARRAY_KEY_SEGMENT_PATTERN = re.compile(r"\[(\d+)\]") +from jsonpath_ng.exceptions import JSONPathError +from jsonpath_ng.ext import parse YAML_INSTANCE = YAML() YAML_INSTANCE.preserve_quotes = True # type: ignore @@ -41,30 +39,24 @@ def yaml_dump(yaml: Any) -> str: def update_yaml_file(file_path: str, key: str, value: Any) -> bool: + if not key: + raise KeyError("Empty key!") content = yaml_file_load(file_path) - - key_segments = key.split(".") if key else [] - current_key_segments = [] - parent_item = content - for current_key_segment in key_segments: - current_key_segments.append(current_key_segment) - current_key = ".".join(current_key_segments) - is_array = _ARRAY_KEY_SEGMENT_PATTERN.match(current_key_segment) - if is_array: - current_array_index = int(is_array.group(1)) - if not isinstance(parent_item, list) or current_array_index >= len(parent_item): - raise KeyError(f"Key '{current_key}' not found in YAML!") - else: - if not isinstance(parent_item, dict) or current_key_segment not in parent_item: - raise KeyError(f"Key '{current_key}' not found in YAML!") - if current_key == key: - if parent_item[current_array_index if is_array else current_key_segment] == value: - return False # nothing to update - parent_item[current_array_index if is_array else current_key_segment] = value - yaml_file_dump(content, file_path) - return True - parent_item = parent_item[current_array_index if is_array else current_key_segment] - raise KeyError(f"Empty key!") + try: + jsonpath_expr = parse(key) + except JSONPathError as ex: + raise KeyError(f"Key '{key}' is invalid JSONPath expression: {ex}!") from ex + matches = jsonpath_expr.find(content) + if not matches: + raise KeyError(f"Key '{key}' not found in YAML!") + if all(match.value == value for match in matches): + return False # nothing to update + try: + jsonpath_expr.update(content, value) + except TypeError as ex: + raise KeyError(f"Key '{key}' cannot be updated: {ex}!") from ex + yaml_file_dump(content, file_path) + return True def merge_yaml_element(file_path: str, element_path: str, desired_value: Any) -> None: diff --git a/setup.py b/setup.py index 8a756068..ec938dc4 100644 --- a/setup.py +++ b/setup.py @@ -9,6 +9,7 @@ install_requires=[ "GitPython==3.0.6", "ruamel.yaml==0.16.5", + "jsonpath-ng==1.5.3", "atlassian-python-api==1.14.5", "PyGithub==1.53", "python-gitlab==2.6.0", diff --git a/tests/commands/test_deploy.py b/tests/commands/test_deploy.py index c0acfa3f..99408dff 100644 --- a/tests/commands/test_deploy.py +++ b/tests/commands/test_deploy.py @@ -405,7 +405,7 @@ def test_key_not_found(self): ) with pytest.raises(GitOpsException) as ex: DeployCommand(args).execute() - self.assertEqual(str(ex.value), "Key 'a.b.c' not found in file: test/file.yml") + self.assertEqual(str(ex.value), "'dummy key error'") assert self.mock_manager.method_calls == [ call.GitRepoApiFactory.create(args, "ORGA", "REPO"), diff --git a/tests/io_api/test_yaml_util.py b/tests/io_api/test_yaml_util.py index b44c766f..8bea0fe6 100644 --- a/tests/io_api/test_yaml_util.py +++ b/tests/io_api/test_yaml_util.py @@ -16,6 +16,8 @@ class YamlUtilTest(unittest.TestCase): + maxDiff = None + @classmethod def setUpClass(cls): cls.tmp_dir = f"/tmp/gitopscli-test-{uuid.uuid4()}" @@ -117,7 +119,15 @@ def test_update_yaml_file(self): g: 4 # comment 6 - [hello, world] # comment 7 - foo: # comment 8 - bar # comment 9""" + bar # comment 9 + - list: # comment 10 + - key: k1 # comment 11 + value: v1 # comment 12 + - key: k2 # comment 13 + value: v2 # comment 14 + - {key: k3+4, value: v3} # comment 15 + - key: k3+4 # comment 16 + value: v4 # comment 17""" ) self.assertTrue(update_yaml_file(test_file, "a.b.c", "2")) @@ -132,6 +142,11 @@ def test_update_yaml_file(self): self.assertTrue(update_yaml_file(test_file, "a.e.[2]", "replaced object")) self.assertFalse(update_yaml_file(test_file, "a.e.[2]", "replaced object")) # already updated + self.assertTrue(update_yaml_file(test_file, "a.e.[*].list[?key=='k3+4'].value", "replaced v3 and v4")) + self.assertFalse( + update_yaml_file(test_file, "a.e.[*].list[?key=='k3+4'].value", "replaced v3 and v4") + ) # already updated + expected = """\ a: # comment 1 # comment 2 @@ -144,17 +159,25 @@ def test_update_yaml_file(self): g: 42 # comment 6 - [hello, tester] # comment 7 - replaced object + - list: # comment 10 + - key: k1 # comment 11 + value: v1 # comment 12 + - key: k2 # comment 13 + value: v2 # comment 14 + - {key: k3+4, value: replaced v3 and v4} # comment 15 + - key: k3+4 # comment 16 + value: replaced v3 and v4 # comment 17 """ actual = self._read_file(test_file) self.assertEqual(expected, actual) with pytest.raises(KeyError) as ex: update_yaml_file(test_file, "x.y", "foo") - self.assertEqual("\"Key 'x' not found in YAML!\"", str(ex.value)) + self.assertEqual("\"Key 'x.y' not found in YAML!\"", str(ex.value)) with pytest.raises(KeyError) as ex: update_yaml_file(test_file, "[42].y", "foo") - self.assertEqual("\"Key '[42]' not found in YAML!\"", str(ex.value)) + self.assertEqual("\"Key '[42].y' not found in YAML!\"", str(ex.value)) with pytest.raises(KeyError) as ex: update_yaml_file(test_file, "a.x", "foo") @@ -165,12 +188,25 @@ def test_update_yaml_file(self): self.assertEqual("\"Key 'a.[42]' not found in YAML!\"", str(ex.value)) with pytest.raises(KeyError) as ex: - update_yaml_file(test_file, "a.e.[3]", "foo") - self.assertEqual("\"Key 'a.e.[3]' not found in YAML!\"", str(ex.value)) + update_yaml_file(test_file, "a.e.[100]", "foo") + self.assertEqual("\"Key 'a.e.[100]' not found in YAML!\"", str(ex.value)) + + with pytest.raises(KeyError) as ex: + update_yaml_file(test_file, "a.e.[*].list[?key=='foo'].value", "foo") + self.assertEqual("\"Key 'a.e.[*].list[?key=='foo'].value' not found in YAML!\"", str(ex.value)) with pytest.raises(KeyError) as ex: update_yaml_file(test_file, "a.e.[2].[2]", "foo") - self.assertEqual("\"Key 'a.e.[2].[2]' not found in YAML!\"", str(ex.value)) + self.assertEqual( + "\"Key 'a.e.[2].[2]' cannot be updated: 'str' object does not support item assignment!\"", str(ex.value) + ) + + with pytest.raises(KeyError) as ex: + update_yaml_file(test_file, "invalid JSONPath", "foo") + self.assertEqual( + "\"Key 'invalid JSONPath' is invalid JSONPath expression: Parse error at 1:8 near token JSONPath (ID)!\"", + str(ex.value), + ) with pytest.raises(KeyError) as ex: update_yaml_file(test_file, "", "foo")