Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor: prepare support for optional inventory backends #1003

Closed
wants to merge 4 commits into from
Closed

refactor: prepare support for optional inventory backends #1003

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
785bf88
refactor: rename inventory function
MatteoVoges May 3, 2023
bc5bec9
Merge branch 'master' into prepare-pluggable-inventory
MatteoVoges Jul 5, 2023
be6bf34
Merge branch 'master' into prepare-pluggable-inventory
MatteoVoges Jul 5, 2023
b9ab5b6
Merge branch 'master' into prepare-pluggable-inventory
MatteoVoges Sep 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions kapitan/refs/cmd_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from kapitan.refs.secrets.gpg import GPGSecret, lookup_fingerprints
from kapitan.refs.secrets.vaultkv import VaultSecret
from kapitan.refs.secrets.vaulttransit import VaultTransit
from kapitan.resources import inventory_reclass
from kapitan.resources import get_inventory
from kapitan.utils import fatal_error, search_target_token_paths

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -65,7 +65,7 @@ def ref_write(args, ref_controller):
type_name, token_path = token_name.split(":")
recipients = [dict((("name", name),)) for name in args.recipients]
if args.target_name:
inv = inventory_reclass(args.inventory_path)
inv = get_inventory(args.inventory_path)
kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"]
if "secrets" not in kap_inv_params:
raise KapitanError(
Expand Down Expand Up @@ -95,7 +95,7 @@ def ref_write(args, ref_controller):
type_name, token_path = token_name.split(":")
key = args.key
if args.target_name:
inv = inventory_reclass(args.inventory_path)
inv = get_inventory(args.inventory_path)
kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"]
if "secrets" not in kap_inv_params:
raise KapitanError(
Expand Down Expand Up @@ -123,7 +123,7 @@ def ref_write(args, ref_controller):
type_name, token_path = token_name.split(":")
key = args.key
if args.target_name:
inv = inventory_reclass(args.inventory_path)
inv = get_inventory(args.inventory_path)
kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"]
if "secrets" not in kap_inv_params:
raise KapitanError(
Expand Down Expand Up @@ -152,7 +152,7 @@ def ref_write(args, ref_controller):
type_name, token_path = token_name.split(":")
key = args.key
if args.target_name:
inv = inventory_reclass(args.inventory_path)
inv = get_inventory(args.inventory_path)
kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"]
if "secrets" not in kap_inv_params:
raise KapitanError(
Expand Down Expand Up @@ -196,7 +196,7 @@ def ref_write(args, ref_controller):
vault_params = {}
encoding = "original"
if args.target_name:
inv = inventory_reclass(args.inventory_path)
inv = get_inventory(args.inventory_path)
kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"]
if "secrets" not in kap_inv_params:
raise KapitanError(
Expand Down Expand Up @@ -230,7 +230,7 @@ def ref_write(args, ref_controller):
_data = data.encode()
vault_params = {}
if args.target_name:
inv = inventory_reclass(args.inventory_path)
inv = get_inventory(args.inventory_path)
kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"]
if "secrets" not in kap_inv_params:
raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name))
Expand Down Expand Up @@ -302,7 +302,7 @@ def secret_update(args, ref_controller):
for name in args.recipients
]
if args.target_name:
inv = inventory_reclass(args.inventory_path)
inv = get_inventory(args.inventory_path)
kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"]
if "secrets" not in kap_inv_params:
raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name))
Expand Down Expand Up @@ -330,7 +330,7 @@ def secret_update(args, ref_controller):
elif token_name.startswith("gkms:"):
key = args.key
if args.target_name:
inv = inventory_reclass(args.inventory_path)
inv = get_inventory(args.inventory_path)
kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"]
if "secrets" not in kap_inv_params:
raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name))
Expand All @@ -356,7 +356,7 @@ def secret_update(args, ref_controller):
elif token_name.startswith("azkms:"):
key = args.key
if args.target_name:
inv = inventory_reclass(args.inventory_path)
inv = get_inventory(args.inventory_path)
kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"]
if "secrets" not in kap_inv_params:
raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name))
Expand All @@ -382,7 +382,7 @@ def secret_update(args, ref_controller):
elif token_name.startswith("awskms:"):
key = args.key
if args.target_name:
inv = inventory_reclass(args.inventory_path)
inv = get_inventory(args.inventory_path)
kap_inv_params = inv["nodes"][args.target_name]["parameters"]["kapitan"]
if "secrets" not in kap_inv_params:
raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name))
Expand Down Expand Up @@ -439,7 +439,7 @@ def secret_update_validate(args, ref_controller):
"Validate and/or update target secrets"
# update gpg recipients/gkms/awskms key for all secrets in secrets_path
# use --refs-path to set scanning path
inv = inventory_reclass(args.inventory_path)
inv = get_inventory(args.inventory_path)
targets = set(inv["nodes"].keys())
secrets_path = os.path.abspath(args.refs_path)
target_token_paths = search_target_token_paths(secrets_path, targets)
Expand Down
33 changes: 22 additions & 11 deletions kapitan/resources.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,14 +282,14 @@ def inventory(search_paths, target, inventory_path=None):
raise InventoryError(f"Inventory not found in search paths: {search_paths}")

if target is None:
return inventory_reclass(full_inv_path)["nodes"]
return get_inventory(full_inv_path)["nodes"]

return inventory_reclass(full_inv_path)["nodes"][target]
return get_inventory(full_inv_path)["nodes"][target]


def generate_inventory(args):
try:
inv = inventory_reclass(args.inventory_path)
inv = get_inventory(args.inventory_path)
if args.target_name != "":
inv = inv["nodes"][args.target_name]
if args.pattern != "":
Expand All @@ -306,6 +306,23 @@ def generate_inventory(args):
sys.exit(1)


def get_inventory(inventory_path, ignore_class_notfound=False):
"""
generic inventory function that makes inventory backend pluggable
default backend is reclass
"""

# if inventory is already cached theres nothing to do
if cached.inv:
return cached.inv

logger.debug("Using reclass as inventory backend")
inv = inventory_reclass(inventory_path, ignore_class_notfound)

cached.inv = inv
return inv


def inventory_reclass(inventory_path, ignore_class_notfound=False):
"""
Runs a reclass inventory in inventory_path
Expand All @@ -316,12 +333,8 @@ def inventory_reclass(inventory_path, ignore_class_notfound=False):

Does not throw errors if a class is not found while --fetch flag is enabled
"""
# if inventory is already cached theres nothing to do
if cached.inv:
return cached.inv

# set default values initially
reclass_config = reclass_config_defaults = {
reclass_config = {
"storage_type": "yaml_fs",
"inventory_base_uri": inventory_path,
"nodes_uri": "targets",
Expand Down Expand Up @@ -360,12 +373,10 @@ def inventory_reclass(inventory_path, ignore_class_notfound=False):
class_mappings = reclass_config.get("class_mappings") # this defaults to None (disabled)
_reclass = reclass.core.Core(storage, class_mappings, reclass.settings.Settings(reclass_config))

cached.inv = _reclass.inventory()
return _reclass.inventory()
except ReclassException as e:
if isinstance(e, NotFoundError):
logger.error("Inventory reclass error: inventory not found")
else:
logger.error("Inventory reclass error: %s", e.message)
raise InventoryError(e.message)

return cached.inv
10 changes: 5 additions & 5 deletions kapitan/targets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
from kapitan.inputs.kadet import Kadet
from kapitan.inputs.remove import Remove
from kapitan.remoteinventory.fetch import fetch_inventories, list_sources
from kapitan.resources import inventory_reclass
from kapitan.resources import get_inventory
from kapitan.utils import dictionary_hash, directory_hash, hashable_lru_cache
from kapitan.validator.kubernetes_validator import KubernetesManifestValidator

Expand Down Expand Up @@ -241,7 +241,7 @@ def generate_inv_cache_hashes(inventory_path, targets, cache_paths):
...
}
"""
inv = inventory_reclass(inventory_path)
inv = get_inventory(inventory_path)
cached.inv_cache = {}
cached.inv_cache["inventory"] = {}
cached.inv_cache["folder"] = {}
Expand Down Expand Up @@ -295,7 +295,7 @@ def generate_inv_cache_hashes(inventory_path, targets, cache_paths):
def changed_targets(inventory_path, output_path):
"""returns a list of targets that have changed since last compilation"""
targets = []
inv = inventory_reclass(inventory_path)
inv = get_inventory(inventory_path)

saved_inv_cache = None
saved_inv_cache_path = os.path.join(output_path, "compiled/.kapitan_cache")
Expand Down Expand Up @@ -388,7 +388,7 @@ def save_inv_cache(compile_path, targets):
def load_target_inventory(inventory_path, targets, ignore_class_notfound=False):
"""returns a list of target objects from the inventory"""
target_objs = []
inv = inventory_reclass(inventory_path, ignore_class_notfound)
inv = get_inventory(inventory_path, ignore_class_notfound)

# if '-t' is set on compile, only loop through selected targets
if targets:
Expand Down Expand Up @@ -431,7 +431,7 @@ def search_targets(inventory_path, targets, labels):
)

targets_found = []
inv = inventory_reclass(inventory_path)
inv = get_inventory(inventory_path)

for target_name in inv["nodes"]:
matched_all_labels = False
Expand Down