From 98884a5d8f2922a88b3b2e8e25a303dbe080cf1d Mon Sep 17 00:00:00 2001 From: makrelas <35281518+makrelas@users.noreply.github.com> Date: Sat, 24 Dec 2022 16:19:33 +0100 Subject: [PATCH] refactor: change of a sync_apps.py logic (#190) Extend sync-apps with custom config file in tenant application directories / Refactor sync-apps --- .gitignore | 1 + Dockerfile | 2 +- docs/commands/sync-apps.md | 16 +++ gitopscli/appconfig_api/__init__.py | 0 gitopscli/appconfig_api/app_tenant_config.py | 132 +++++++++++++++++ gitopscli/appconfig_api/root_repo.py | 83 +++++++++++ gitopscli/commands/sync_apps.py | 134 ++++------------- requirements-test.txt | 2 +- tests/commands/test_sync_apps.py | 142 +++++++++++-------- 9 files changed, 343 insertions(+), 169 deletions(-) create mode 100644 gitopscli/appconfig_api/__init__.py create mode 100644 gitopscli/appconfig_api/app_tenant_config.py create mode 100644 gitopscli/appconfig_api/root_repo.py diff --git a/.gitignore b/.gitignore index 5dcaa083..95d752a9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ testrepo/ *.iml .eggs/ site/ +build/ .mypy_cache \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f11095bf..5eb8fd1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ FROM base AS dev WORKDIR /workdir RUN apk add --no-cache gcc linux-headers musl-dev make RUN python -m venv /opt/venv -RUN pip install --upgrade pip +RUN python -m pip install --upgrade pip # ========= FROM dev AS deps diff --git a/docs/commands/sync-apps.md b/docs/commands/sync-apps.md index 4b77a849..2e57ff21 100644 --- a/docs/commands/sync-apps.md +++ b/docs/commands/sync-apps.md @@ -28,6 +28,22 @@ root-config-repo/ └── bootstrap └── values.yaml ``` +### app specific values +app specific values may be set using a .config.yaml file directly in the app directory. gitopscli will process these values and add them under customAppConfig parameter of application +**tenantrepo.git/app1/app_value_file.yaml** +```yaml +customvalue: test +``` +**rootrepo.git/apps/tenantrepo.yaml** +```yaml +config: + repository: https://tenantrepo.git + applications: + app1: + customAppConfig: + customvalue: test + app2: {} +``` **bootstrap/values.yaml** ```yaml diff --git a/gitopscli/appconfig_api/__init__.py b/gitopscli/appconfig_api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gitopscli/appconfig_api/app_tenant_config.py b/gitopscli/appconfig_api/app_tenant_config.py new file mode 100644 index 00000000..f3a16edb --- /dev/null +++ b/gitopscli/appconfig_api/app_tenant_config.py @@ -0,0 +1,132 @@ +import logging +from dataclasses import dataclass, field +import os +from typing import Any + +from gitopscli.git_api import GitRepo +from gitopscli.io_api.yaml_util import yaml_load, yaml_file_load + +from gitopscli.gitops_exception import GitOpsException + + +@dataclass +class AppTenantConfig: + yaml: dict[str, dict[str, Any]] + tenant_config: dict[str, dict[str, Any]] = field(default_factory=dict) + repo_url: str = "" + file_path: str = "" + dirty: bool = False + + def __post_init__(self) -> None: + if "config" in self.yaml: + self.tenant_config = self.yaml["config"] + else: + self.tenant_config = self.yaml + if "repository" not in self.tenant_config: + raise GitOpsException("Cannot find key 'repository' in " + self.file_path) + self.repo_url = str(self.tenant_config["repository"]) + + def list_apps(self) -> dict[str, dict[str, Any]]: + return dict(self.tenant_config["applications"]) + + def merge_applications(self, desired_tenant_config: "AppTenantConfig") -> None: + desired_apps = desired_tenant_config.list_apps() + self.__delete_removed_applications(desired_apps) + self.__add_new_applications(desired_apps) + self.__update_custom_app_config(desired_apps) + + def __update_custom_app_config(self, desired_apps: dict[str, dict[str, Any]]) -> None: + for desired_app_name, desired_app_value in desired_apps.items(): + if desired_app_name in self.list_apps(): + existing_application_value = self.list_apps()[desired_app_name] + if "customAppConfig" not in desired_app_value: + if existing_application_value and "customAppConfig" in existing_application_value: + logging.info( + "Removing customAppConfig in for %s in %s applications", + existing_application_value, + self.file_path, + ) + del existing_application_value["customAppConfig"] + self.__set_dirty() + else: + if ( + "customAppConfig" not in existing_application_value + or existing_application_value["customAppConfig"] != desired_app_value["customAppConfig"] + ): + logging.info( + "Updating customAppConfig in for %s in %s applications", + existing_application_value, + self.file_path, + ) + existing_application_value["customAppConfig"] = desired_app_value["customAppConfig"] + self.__set_dirty() + + def __add_new_applications(self, desired_apps: dict[str, Any]) -> None: + for desired_app_name, desired_app_value in desired_apps.items(): + if desired_app_name not in self.list_apps().keys(): + logging.info("Adding % in %s applications", desired_app_name, self.file_path) + self.tenant_config["applications"][desired_app_name] = desired_app_value + self.__set_dirty() + + def __delete_removed_applications(self, desired_apps: dict[str, Any]) -> None: + for current_app in self.list_apps().keys(): + if current_app not in desired_apps.keys(): + logging.info("Removing %s from %s applications", current_app, self.file_path) + del self.tenant_config["applications"][current_app] + self.__set_dirty() + + def __set_dirty(self) -> None: + self.dirty = True + + +def __generate_config_from_tenant_repo( + tenant_repo: GitRepo, +) -> Any: # TODO: supposed to be ruamel object than Any pylint: disable=fixme + tenant_app_dirs = __get_all_tenant_applications_dirs(tenant_repo) + tenant_config_template = """ + config: + repository: {} + applications: {{}} + """.format( + tenant_repo.get_clone_url() + ) + yaml = yaml_load(tenant_config_template) + for app_dir in tenant_app_dirs: + tenant_application_template = """ + {}: {{}} + """.format( + app_dir + ) + tenant_applications_yaml = yaml_load(tenant_application_template) + # dict path hardcoded as object generated will always be in v2 or later + yaml["config"]["applications"].update(tenant_applications_yaml) + custom_app_config = __get_custom_config(app_dir, tenant_repo) + if custom_app_config: + yaml["config"]["applications"][app_dir]["customAppConfig"] = custom_app_config + return yaml + + +def __get_all_tenant_applications_dirs(tenant_repo: GitRepo) -> set[str]: + repo_dir = tenant_repo.get_full_file_path(".") + applist = { + name + for name in os.listdir(repo_dir) + if os.path.isdir(os.path.join(repo_dir, name)) and not name.startswith(".") + } + return applist + + +def __get_custom_config(appname: str, tenant_config_git_repo: GitRepo) -> Any: + custom_config_path = tenant_config_git_repo.get_full_file_path(f"{appname}/.config.yaml") + if os.path.exists(custom_config_path): + custom_config_content = yaml_file_load(custom_config_path) + return custom_config_content + return dict() + + +def create_app_tenant_config_from_repo( + tenant_repo: GitRepo, +) -> "AppTenantConfig": + tenant_repo.clone() + tenant_config_yaml = __generate_config_from_tenant_repo(tenant_repo) + return AppTenantConfig(yaml=tenant_config_yaml) diff --git a/gitopscli/appconfig_api/root_repo.py b/gitopscli/appconfig_api/root_repo.py new file mode 100644 index 00000000..ae581965 --- /dev/null +++ b/gitopscli/appconfig_api/root_repo.py @@ -0,0 +1,83 @@ +from dataclasses import dataclass +from typing import List, Any, Optional + +from gitopscli.git_api import GitRepo +from gitopscli.io_api.yaml_util import yaml_file_load + +from gitopscli.appconfig_api.app_tenant_config import AppTenantConfig +from gitopscli.gitops_exception import GitOpsException + + +@dataclass +class RootRepo: + tenants: dict[str, AppTenantConfig] + + def list_tenants(self) -> list[str]: + return list(self.tenants.keys()) + + def get_tenant_by_repo_url(self, repo_url: str) -> Optional[AppTenantConfig]: + for tenant in self.tenants.values(): + if tenant.repo_url == repo_url: + return tenant + return None + + def get_all_applications(self) -> list[str]: + apps: list[str] = list() + for tenant in self.tenants.values(): + apps.extend(tenant.list_apps().keys()) + return apps + + def validate_tenant(self, tenant_config: AppTenantConfig) -> None: + apps_from_other_tenants: list[str] = list() + for tenant in self.tenants.values(): + if tenant.repo_url != tenant_config.repo_url: + apps_from_other_tenants.extend(tenant.list_apps().keys()) + for app_name in tenant_config.list_apps().keys(): + if app_name in apps_from_other_tenants: + raise GitOpsException(f"Application '{app_name}' already exists in a different repository") + + +def __load_tenants_from_bootstrap_values(root_repo: GitRepo) -> dict[str, AppTenantConfig]: + boostrap_tenant_list = __get_bootstrap_tenant_list(root_repo) + tenants = dict() + for bootstrap_tenant in boostrap_tenant_list: + try: + tenant_name = bootstrap_tenant["name"] + absolute_tenant_file_path = root_repo.get_full_file_path("apps/" + tenant_name + ".yaml") + yaml = yaml_file_load(absolute_tenant_file_path) + tenants[tenant_name] = AppTenantConfig( + yaml=yaml, + file_path=absolute_tenant_file_path, + ) + except FileNotFoundError as ex: + raise GitOpsException(f"File '{absolute_tenant_file_path}' not found in root repository.") from ex + return tenants + + +def __get_bootstrap_tenant_list(root_repo: GitRepo) -> List[Any]: + root_repo.clone() + try: + boostrap_values_path = root_repo.get_full_file_path("bootstrap/values.yaml") + bootstrap_yaml = yaml_file_load(boostrap_values_path) + except FileNotFoundError as ex: + raise GitOpsException("File 'bootstrap/values.yaml' not found in root repository.") from ex + bootstrap_tenants = [] + if "bootstrap" in bootstrap_yaml: + bootstrap_tenants = list(bootstrap_yaml["bootstrap"]) + if "config" in bootstrap_yaml and "bootstrap" in bootstrap_yaml["config"]: + bootstrap_tenants = list(bootstrap_yaml["config"]["bootstrap"]) + __validate_bootstrap_tenants(bootstrap_tenants) + return bootstrap_tenants + + +def __validate_bootstrap_tenants(bootstrap_entries: Optional[List[Any]]) -> None: + if not bootstrap_entries: + raise GitOpsException("Cannot find key 'bootstrap' or 'config.bootstrap' in 'bootstrap/values.yaml'") + for bootstrap_entry in bootstrap_entries: + if "name" not in bootstrap_entry: + raise GitOpsException("Every bootstrap entry must have a 'name' property.") + + +def create_root_repo(root_repo: GitRepo) -> "RootRepo": + root_repo_tenants = __load_tenants_from_bootstrap_values(root_repo) + return RootRepo(root_repo_tenants) diff --git a/gitopscli/commands/sync_apps.py b/gitopscli/commands/sync_apps.py index 100811c4..5b63b852 100644 --- a/gitopscli/commands/sync_apps.py +++ b/gitopscli/commands/sync_apps.py @@ -1,11 +1,11 @@ import logging -import os from dataclasses import dataclass -from typing import Any, Set, Tuple from gitopscli.git_api import GitApiConfig, GitRepo, GitRepoApiFactory -from gitopscli.io_api.yaml_util import merge_yaml_element, yaml_file_load from gitopscli.gitops_exception import GitOpsException -from .command import Command +from gitopscli.io_api.yaml_util import yaml_file_dump +from gitopscli.commands.command import Command +from gitopscli.appconfig_api.app_tenant_config import create_app_tenant_config_from_repo +from gitopscli.appconfig_api.root_repo import create_root_repo class SyncAppsCommand(Command): @@ -35,83 +35,29 @@ def _sync_apps_command(args: SyncAppsCommand.Args) -> None: __sync_apps(team_config_git_repo, root_config_git_repo, args.git_user, args.git_email) -def __sync_apps(team_config_git_repo: GitRepo, root_config_git_repo: GitRepo, git_user: str, git_email: str) -> None: - logging.info("Team config repository: %s", team_config_git_repo.get_clone_url()) - logging.info("Root config repository: %s", root_config_git_repo.get_clone_url()) - - repo_apps = __get_repo_apps(team_config_git_repo) - logging.info("Found %s app(s) in apps repository: %s", len(repo_apps), ", ".join(repo_apps)) - - logging.info("Searching apps repository in root repository's 'apps/' directory...") - ( - apps_config_file, - apps_config_file_name, - current_repo_apps, - apps_from_other_repos, - found_apps_path, - ) = __find_apps_config_from_repo(team_config_git_repo, root_config_git_repo) - - if current_repo_apps == repo_apps: - logging.info("Root repository already up-to-date. I'm done here.") - return - - __check_if_app_already_exists(repo_apps, apps_from_other_repos) - - logging.info("Sync applications in root repository's %s.", apps_config_file_name) - merge_yaml_element(apps_config_file, found_apps_path, {repo_app: {} for repo_app in repo_apps}) - __commit_and_push(team_config_git_repo, root_config_git_repo, git_user, git_email, apps_config_file_name) - - -def __find_apps_config_from_repo( - team_config_git_repo: GitRepo, root_config_git_repo: GitRepo -) -> Tuple[str, str, Set[str], Set[str], str]: - apps_from_other_repos: Set[str] = set() # Set for all entries in .applications from each config repository - found_app_config_file = None - found_app_config_file_name = None - found_apps_path = "applications" - found_app_config_apps: Set[str] = set() - bootstrap_entries = __get_bootstrap_entries(root_config_git_repo) - team_config_git_repo_clone_url = team_config_git_repo.get_clone_url() - for bootstrap_entry in bootstrap_entries: - if "name" not in bootstrap_entry: - raise GitOpsException("Every bootstrap entry must have a 'name' property.") - app_file_name = "apps/" + bootstrap_entry["name"] + ".yaml" - logging.info("Analyzing %s in root repository", app_file_name) - app_config_file = root_config_git_repo.get_full_file_path(app_file_name) - try: - app_config_content = yaml_file_load(app_config_file) - except FileNotFoundError as ex: - raise GitOpsException(f"File '{app_file_name}' not found in root repository.") from ex - if "config" in app_config_content: - app_config_content = app_config_content["config"] - found_apps_path = "config.applications" - if "repository" not in app_config_content: - raise GitOpsException(f"Cannot find key 'repository' in '{app_file_name}'") - if app_config_content["repository"] == team_config_git_repo_clone_url: - logging.info("Found apps repository in %s", app_file_name) - found_app_config_file = app_config_file - found_app_config_file_name = app_file_name - found_app_config_apps = __get_applications_from_app_config(app_config_content) - else: - apps_from_other_repos.update(__get_applications_from_app_config(app_config_content)) - - if found_app_config_file is None or found_app_config_file_name is None: +# TODO: BETTER NAMES FOR STUFF HERE pylint: disable=fixme +def __sync_apps(tenant_git_repo: GitRepo, root_git_repo: GitRepo, git_user: str, git_email: str) -> None: + logging.info("Team config repository: %s", tenant_git_repo.get_clone_url()) + logging.info("Root config repository: %s", root_git_repo.get_clone_url()) + root_repo = create_root_repo(root_repo=root_git_repo) + root_repo_tenant = root_repo.get_tenant_by_repo_url(tenant_git_repo.get_clone_url()) + if root_repo_tenant is None: raise GitOpsException("Couldn't find config file for apps repository in root repository's 'apps/' directory") - - return ( - found_app_config_file, - found_app_config_file_name, - found_app_config_apps, - apps_from_other_repos, - found_apps_path, + tenant_from_repo = create_app_tenant_config_from_repo(tenant_repo=tenant_git_repo) + logging.info( + "Found %s app(s) in apps repository: %s", + len(tenant_from_repo.list_apps().keys()), + ", ".join(tenant_from_repo.list_apps().keys()), ) - - -def __get_applications_from_app_config(app_config: Any) -> Set[str]: - apps = [] - if "applications" in app_config and app_config["applications"] is not None: - apps += app_config["applications"].keys() - return set(apps) + root_repo.validate_tenant(tenant_from_repo) + root_repo_tenant.merge_applications(tenant_from_repo) + if root_repo_tenant.dirty: + logging.info("Appling changes to: %s", root_repo_tenant.file_path) + yaml_file_dump(root_repo_tenant.yaml, root_repo_tenant.file_path) + logging.info("Commiting and pushing changes to %s", root_git_repo.get_clone_url()) + __commit_and_push(tenant_git_repo, root_git_repo, git_user, git_email, root_repo_tenant.file_path) + else: + logging.info("No changes applied to %s", root_repo_tenant.file_path) def __commit_and_push( @@ -120,33 +66,3 @@ def __commit_and_push( author = team_config_git_repo.get_author_from_last_commit() root_config_git_repo.commit(git_user, git_email, f"{author} updated " + app_file_name) root_config_git_repo.push() - - -def __get_bootstrap_entries(root_config_git_repo: GitRepo) -> Any: - root_config_git_repo.clone() - bootstrap_values_file = root_config_git_repo.get_full_file_path("bootstrap/values.yaml") - try: - bootstrap_yaml = yaml_file_load(bootstrap_values_file) - except FileNotFoundError as ex: - raise GitOpsException("File 'bootstrap/values.yaml' not found in root repository.") from ex - if "bootstrap" in bootstrap_yaml: - return bootstrap_yaml["bootstrap"] - if "config" in bootstrap_yaml and "bootstrap" in bootstrap_yaml["config"]: - return bootstrap_yaml["config"]["bootstrap"] - raise GitOpsException("Cannot find key 'bootstrap' or 'config.bootstrap' in 'bootstrap/values.yaml'") - - -def __get_repo_apps(team_config_git_repo: GitRepo) -> Set[str]: - team_config_git_repo.clone() - repo_dir = team_config_git_repo.get_full_file_path(".") - return { - name - for name in os.listdir(repo_dir) - if os.path.isdir(os.path.join(repo_dir, name)) and not name.startswith(".") - } - - -def __check_if_app_already_exists(apps_dirs: Set[str], apps_from_other_repos: Set[str]) -> None: - for app_key in apps_dirs: - if app_key in apps_from_other_repos: - raise GitOpsException(f"Application '{app_key}' already exists in a different repository") diff --git a/requirements-test.txt b/requirements-test.txt index c5901131..b10860c9 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -3,5 +3,5 @@ coverage==5.3 pylint==2.7.1 pytest==7.1.3 mypy==0.982 -typeguard==2.10.0 +typeguard==2.13.3 pre-commit==2.20.0 diff --git a/tests/commands/test_sync_apps.py b/tests/commands/test_sync_apps.py index 9dcd3573..38e67545 100644 --- a/tests/commands/test_sync_apps.py +++ b/tests/commands/test_sync_apps.py @@ -1,11 +1,14 @@ +import posixpath import logging import os import unittest -from unittest.mock import call +from unittest.mock import call, patch + from gitopscli.git_api import GitProvider, GitRepo, GitRepoApi, GitRepoApiFactory from gitopscli.commands.sync_apps import SyncAppsCommand -from gitopscli.io_api.yaml_util import merge_yaml_element, yaml_file_load +from gitopscli.io_api.yaml_util import yaml_file_load, yaml_file_dump from gitopscli.gitops_exception import GitOpsException +from ruamel.yaml.compat import ordereddict from .mock_mixin import MockMixin ARGS = SyncAppsCommand.Args( @@ -26,10 +29,15 @@ class SyncAppsCommandTest(MockMixin, unittest.TestCase): def setUp(self): self.init_mock_manager(SyncAppsCommand) - self.os_mock = self.monkey_patch(os) + patcher = patch("gitopscli.appconfig_api.app_tenant_config.os", spec_set=os) + self.addCleanup(patcher.stop) + self.os_mock = patcher.start() + self.mock_manager.attach_mock(self.os_mock, "os") + self.os_mock.path.isdir.return_value = True - self.os_mock.path.join.side_effect = os.path.join + self.os_mock.path.join.side_effect = posixpath.join # tests are designed to emulate posix env self.os_mock.listdir.return_value = ["my-app"] + self.os_mock.path.exists.return_value = False self.logging_mock = self.monkey_patch(logging) self.logging_mock.info.return_value = None @@ -40,7 +48,7 @@ def setUp(self): self.team_config_git_repo_mock = self.create_mock(GitRepo, "GitRepo_team") self.team_config_git_repo_mock.__enter__.return_value = self.team_config_git_repo_mock self.team_config_git_repo_mock.__exit__.return_value = False - self.team_config_git_repo_mock.get_clone_url.return_value = "https://team.config.repo.git" + self.team_config_git_repo_mock.get_clone_url.return_value = "https://repository.url/team/team-non-prod.git" self.team_config_git_repo_mock.clone.return_value = None self.team_config_git_repo_mock.get_full_file_path.side_effect = lambda x: f"/tmp/team-config-repo/{x}" self.team_config_git_repo_mock.get_author_from_last_commit.return_value = "author" @@ -49,7 +57,7 @@ def setUp(self): self.root_config_git_repo_mock.__enter__.return_value = self.root_config_git_repo_mock self.root_config_git_repo_mock.__exit__.return_value = False self.root_config_git_repo_mock.get_full_file_path.side_effect = lambda x: f"/tmp/root-config-repo/{x}" - self.root_config_git_repo_mock.get_clone_url.return_value = "https://root.config.repo.git" + self.root_config_git_repo_mock.get_clone_url.return_value = "https://repository.url/root/root-config.git" self.root_config_git_repo_mock.clone.return_value = None self.root_config_git_repo_mock.commit.return_value = None self.root_config_git_repo_mock.push.return_value = None @@ -66,22 +74,30 @@ def setUp(self): id(self.root_config_git_repo_api_mock): self.root_config_git_repo_mock, }[id(api)] - self.yaml_file_load_mock = self.monkey_patch(yaml_file_load) + patcher = patch("gitopscli.appconfig_api.root_repo.yaml_file_load", spec_set=yaml_file_load) + self.addCleanup(patcher.stop) + self.yaml_file_load_mock = patcher.start() + self.mock_manager.attach_mock(self.yaml_file_load_mock, "yaml_file_load") + + # self.yaml_file_load_mock = self.monkey_patch(yaml_file_load) self.yaml_file_load_mock.side_effect = lambda file_path: { "/tmp/root-config-repo/bootstrap/values.yaml": { "bootstrap": [{"name": "team-non-prod"}, {"name": "other-team-non-prod"}], }, "/tmp/root-config-repo/apps/team-non-prod.yaml": { - "config": {"repository": "https://team.config.repo.git", "applications": {"some-other-app-1": None}} + "config": { + "repository": "https://repository.url/team/team-non-prod.git", + "applications": {"some-other-app-1": {}}, + } }, "/tmp/root-config-repo/apps/other-team-non-prod.yaml": { - "repository": "https://other-team.config.repo.git", - "applications": {"some-other-app-2": None}, + "repository": "https://repository.url/other-team/other-team-non-prod.git", + "applications": {"some-other-app-2": {}}, }, }[file_path] - self.merge_yaml_element_mock = self.monkey_patch(merge_yaml_element) - self.merge_yaml_element_mock.return_value = None + self.yaml_file_dump_mock = self.monkey_patch(yaml_file_dump) + self.yaml_file_dump_mock.return_value = None self.seal_mocks() @@ -93,33 +109,42 @@ def test_sync_apps_happy_flow(self): call.GitRepo(self.team_config_git_repo_api_mock), call.GitRepo(self.root_config_git_repo_api_mock), call.GitRepo_team.get_clone_url(), - call.logging.info("Team config repository: %s", "https://team.config.repo.git"), + call.logging.info("Team config repository: %s", "https://repository.url/team/team-non-prod.git"), call.GitRepo_root.get_clone_url(), - call.logging.info("Root config repository: %s", "https://root.config.repo.git"), - call.GitRepo_team.clone(), - call.GitRepo_team.get_full_file_path("."), - call.os.listdir("/tmp/team-config-repo/."), - call.os.path.join("/tmp/team-config-repo/.", "my-app"), - call.os.path.isdir("/tmp/team-config-repo/./my-app"), - call.logging.info("Found %s app(s) in apps repository: %s", 1, "my-app"), - call.logging.info("Searching apps repository in root repository's 'apps/' directory..."), + call.logging.info("Root config repository: %s", "https://repository.url/root/root-config.git"), call.GitRepo_root.clone(), call.GitRepo_root.get_full_file_path("bootstrap/values.yaml"), call.yaml_file_load("/tmp/root-config-repo/bootstrap/values.yaml"), - call.GitRepo_team.get_clone_url(), - call.logging.info("Analyzing %s in root repository", "apps/team-non-prod.yaml"), call.GitRepo_root.get_full_file_path("apps/team-non-prod.yaml"), call.yaml_file_load("/tmp/root-config-repo/apps/team-non-prod.yaml"), - call.logging.info("Found apps repository in %s", "apps/team-non-prod.yaml"), - call.logging.info("Analyzing %s in root repository", "apps/other-team-non-prod.yaml"), call.GitRepo_root.get_full_file_path("apps/other-team-non-prod.yaml"), call.yaml_file_load("/tmp/root-config-repo/apps/other-team-non-prod.yaml"), - call.logging.info("Sync applications in root repository's %s.", "apps/team-non-prod.yaml"), - call.merge_yaml_element( - "/tmp/root-config-repo/apps/team-non-prod.yaml", "config.applications", {"my-app": {}} + call.GitRepo_team.get_clone_url(), + call.GitRepo_team.clone(), + call.GitRepo_team.get_full_file_path("."), + call.os.listdir("/tmp/team-config-repo/."), + call.os.path.join("/tmp/team-config-repo/.", "my-app"), + call.os.path.isdir("/tmp/team-config-repo/./my-app"), + call.GitRepo_team.get_clone_url(), + call.GitRepo_team.get_full_file_path("my-app/.config.yaml"), + call.os.path.exists("/tmp/team-config-repo/my-app/.config.yaml"), + call.logging.info("Found %s app(s) in apps repository: %s", 1, "my-app"), + call.logging.info("Appling changes to: %s", "/tmp/root-config-repo/apps/team-non-prod.yaml"), + call.yaml_file_dump( + { + "config": { + "repository": "https://repository.url/team/team-non-prod.git", + "applications": {"my-app": ordereddict()}, + } + }, + "/tmp/root-config-repo/apps/team-non-prod.yaml", ), + call.GitRepo_root.get_clone_url(), + call.logging.info("Commiting and pushing changes to %s", "https://repository.url/root/root-config.git"), call.GitRepo_team.get_author_from_last_commit(), - call.GitRepo_root.commit("GIT_USER", "GIT_EMAIL", "author updated apps/team-non-prod.yaml"), + call.GitRepo_root.commit( + "GIT_USER", "GIT_EMAIL", "author updated /tmp/root-config-repo/apps/team-non-prod.yaml" + ), call.GitRepo_root.push(), ] @@ -129,11 +154,11 @@ def test_sync_apps_already_up_to_date(self): "bootstrap": [{"name": "team-non-prod"}, {"name": "other-team-non-prod"}], }, "/tmp/root-config-repo/apps/team-non-prod.yaml": { - "repository": "https://team.config.repo.git", - "applications": {"my-app": None}, # my-app already exists + "repository": "https://repository.url/team/team-non-prod.git", + "applications": {"my-app": {}}, # my-app already exists }, "/tmp/root-config-repo/apps/other-team-non-prod.yaml": { - "repository": "https://other-team.config.repo.git", + "repository": "https://repository.url/other-team/other-team-non-prod.git", "applications": {}, }, }[file_path] @@ -145,28 +170,27 @@ def test_sync_apps_already_up_to_date(self): call.GitRepo(self.team_config_git_repo_api_mock), call.GitRepo(self.root_config_git_repo_api_mock), call.GitRepo_team.get_clone_url(), - call.logging.info("Team config repository: %s", "https://team.config.repo.git"), + call.logging.info("Team config repository: %s", "https://repository.url/team/team-non-prod.git"), call.GitRepo_root.get_clone_url(), - call.logging.info("Root config repository: %s", "https://root.config.repo.git"), - call.GitRepo_team.clone(), - call.GitRepo_team.get_full_file_path("."), - call.os.listdir("/tmp/team-config-repo/."), - call.os.path.join("/tmp/team-config-repo/.", "my-app"), - call.os.path.isdir("/tmp/team-config-repo/./my-app"), - call.logging.info("Found %s app(s) in apps repository: %s", 1, "my-app"), - call.logging.info("Searching apps repository in root repository's 'apps/' directory..."), + call.logging.info("Root config repository: %s", "https://repository.url/root/root-config.git"), call.GitRepo_root.clone(), call.GitRepo_root.get_full_file_path("bootstrap/values.yaml"), call.yaml_file_load("/tmp/root-config-repo/bootstrap/values.yaml"), - call.GitRepo_team.get_clone_url(), - call.logging.info("Analyzing %s in root repository", "apps/team-non-prod.yaml"), call.GitRepo_root.get_full_file_path("apps/team-non-prod.yaml"), call.yaml_file_load("/tmp/root-config-repo/apps/team-non-prod.yaml"), - call.logging.info("Found apps repository in %s", "apps/team-non-prod.yaml"), - call.logging.info("Analyzing %s in root repository", "apps/other-team-non-prod.yaml"), call.GitRepo_root.get_full_file_path("apps/other-team-non-prod.yaml"), call.yaml_file_load("/tmp/root-config-repo/apps/other-team-non-prod.yaml"), - call.logging.info("Root repository already up-to-date. I'm done here."), + call.GitRepo_team.get_clone_url(), + call.GitRepo_team.clone(), + call.GitRepo_team.get_full_file_path("."), + call.os.listdir("/tmp/team-config-repo/."), + call.os.path.join("/tmp/team-config-repo/.", "my-app"), + call.os.path.isdir("/tmp/team-config-repo/./my-app"), + call.GitRepo_team.get_clone_url(), + call.GitRepo_team.get_full_file_path("my-app/.config.yaml"), + call.os.path.exists("/tmp/team-config-repo/my-app/.config.yaml"), + call.logging.info("Found %s app(s) in apps repository: %s", 1, "my-app"), + call.logging.info("No changes applied to %s", "/tmp/root-config-repo/apps/team-non-prod.yaml"), ] def test_sync_apps_bootstrap_chart(self): @@ -177,17 +201,17 @@ def test_sync_apps_bootstrap_chart(self): } }, "/tmp/root-config-repo/apps/team-non-prod.yaml": { - "repository": "https://team.config.repo.git", - "applications": {"my-app": None}, # my-app already exists + "repository": "https://repository.url/team/team-non-prod.git", + "applications": {"my-app": {}}, # my-app already exists }, "/tmp/root-config-repo/apps/other-team-non-prod.yaml": { - "repository": "https://other-team.config.repo.git", + "repository": "https://repository.url/team/other-team-non-prod.git", "applications": {}, }, }[file_path] try: SyncAppsCommand(ARGS).execute() - except GitOpsException: + except GitOpsException as ex: self.fail("'config.bootstrap' should be read correctly'") def test_sync_apps_bootstrap_yaml_not_found(self): @@ -251,13 +275,15 @@ def file_load_mock_side_effect(file_path): SyncAppsCommand(ARGS).execute() self.fail() except GitOpsException as ex: - self.assertEqual("File 'apps/team-non-prod.yaml' not found in root repository.", str(ex)) + self.assertEqual( + "File '/tmp/root-config-repo/apps/team-non-prod.yaml' not found in root repository.", str(ex) + ) def test_sync_apps_missing_repository_element_in_team_yaml(self): self.yaml_file_load_mock.side_effect = lambda file_path: { "/tmp/root-config-repo/bootstrap/values.yaml": {"bootstrap": [{"name": "team-non-prod"}]}, "/tmp/root-config-repo/apps/team-non-prod.yaml": { - # missing: "repository": "https://team.config.repo.git", + # missing: "repository": "https://repository.url/team/team-non-prod.git", "applications": {}, }, }[file_path] @@ -266,13 +292,13 @@ def test_sync_apps_missing_repository_element_in_team_yaml(self): SyncAppsCommand(ARGS).execute() self.fail() except GitOpsException as ex: - self.assertEqual("Cannot find key 'repository' in 'apps/team-non-prod.yaml'", str(ex)) + self.assertEqual("Cannot find key 'repository' in /tmp/root-config-repo/apps/team-non-prod.yaml", str(ex)) def test_sync_apps_undefined_team_repo(self): self.yaml_file_load_mock.side_effect = lambda file_path: { "/tmp/root-config-repo/bootstrap/values.yaml": {"bootstrap": [{"name": "other-team-non-prod"}]}, "/tmp/root-config-repo/apps/other-team-non-prod.yaml": { - "repository": "https://other-team.config.repo.git", # there is no repo matching the command's team repo + "repository": "https://repository.url/other-team/other-team-non-prod.git", # there is no repo matching the command's team repo "applications": {}, }, }[file_path] @@ -291,12 +317,12 @@ def test_sync_apps_app_name_collission(self): "bootstrap": [{"name": "team-non-prod"}, {"name": "other-team-non-prod"}], }, "/tmp/root-config-repo/apps/team-non-prod.yaml": { - "repository": "https://team.config.repo.git", - "applications": {"some-other-app-1": None}, + "repository": "https://repository.url/team/team-non-prod.git", + "applications": {"some-other-app-1": {}}, }, "/tmp/root-config-repo/apps/other-team-non-prod.yaml": { - "repository": "https://other-team.config.repo.git", - "applications": {"my-app": None}, # the other-team already has an app named "my-app" + "repository": "https://repository.url/other-team/other-team-non-prod.git", + "applications": {"my-app": {}}, # the other-team already has an app named "my-app" }, }[file_path]