[POC] feat: experiment with pyoxidizer

Dockerfile.pyoxidizer

@@ -0,0 +1,40 @@
+# Generate requirements.txt from Poetry config.
+FROM python:slim as requirements
+
+WORKDIR /code
+
+RUN pip install poetry==1.4.0
+COPY pyproject.toml poetry.lock /code/
+RUN poetry export --output requirements.txt
+
+# Compile binary using Pyoxidizer.
+FROM messense/rust-musl-cross:x86_64-musl as compiler
+
+ARG TARGET="x86_64-unknown-linux-musl"
+# ARG TARGET="aarch64-apple-darwin"
+
+WORKDIR /opt/pyoxidizer
+
+ADD https://github.com/indygreg/PyOxidizer/releases/download/pyoxidizer%2F0.24.0/pyoxidizer-0.24.0-${TARGET}.tar.gz /opt/pyoxidizer/pyoxidizer-0.24.0-${TARGET}.tar.gz
+RUN tar xzvf /opt/pyoxidizer/pyoxidizer-0.24.0-${TARGET}.tar.gz
+RUN cp -v /opt/pyoxidizer/pyoxidizer-0.24.0-${TARGET}/pyoxidizer /usr/bin/pyoxidizer && chmod +x /usr/bin/pyoxidizer
+
+RUN rustup target add ${TARGET}
+
+WORKDIR /code
+
+COPY . /code
+
+COPY --from=requirements /code/requirements.txt /code/requirements.txt
+RUN pyoxidizer build --release --verbose --target-triple ${TARGET}
+RUN ls -lahk /code/build/$TARGET/debug/exe/
+
+# Final image.
+FROM scratch
+
+ARG TARGET="x86_64-unknown-linux-musl"
+# ARG TARGET="aarch64-apple-darwin"
+
+COPY --from=compiler /code/build/${TARGET}/debug/exe/kapitan /kapitan
+
+CMD ["/kapitan"] 

Makefile

@@ -37,6 +37,13 @@ endif
 package:
 	python3 setup.py sdist bdist_wheel
 
+generate_requirements:
+	poetry export -o requirements.txt
+
+.PHONY: build_binary
+build_binary: generate_requirements
+	pyoxidizer build --release
+
 .PHONY: clean
 clean:
 	rm -rf dist/ build/ kapitan.egg-info/ bindist/

kapitan/__init__.py

@@ -35,4 +35,4 @@ def setup_logging(name=None, level=logging.INFO, force=False):
     setup_logging()
 
 # Adding reclass to PYTHONPATH
-sys.path.insert(0, os.path.dirname(__file__) + "/reclass")
+sys.path.insert(0, os.path.dirname("/Users/jacek/Code/kapitan_fork/kapitan") + "/reclass")

kapitan/cli.py

@@ -32,6 +32,13 @@
 logger = logging.getLogger(__name__)
 
 
+# Patch missing sys.argv[0] which is None for some reason when using PyOxidizer
+# Typer fails on this because it tries to emit the filename
+if sys.argv[0] is None:
+    sys.argv[0] = sys.executable
+    print(f"Patched sys.argv to {sys.argv}")
+
+
 def print_deprecated_secrets_msg(args):
     logger.error("Secrets have been renamed to refs, please refer to: '$ kapitan refs --help'")
     sys.exit(1)

kapitan/resources.py

@@ -20,7 +20,8 @@
 import jsonschema
 import kapitan.cached as cached
 import yaml
-from kapitan import __file__ as kapitan_install_path
+
+# from kapitan import __file__ as kapitan_install_path
 from kapitan.errors import CompileError, InventoryError, KapitanError
 from kapitan.utils import PrettyDumper, deep_get, flatten_dict, render_jinja2_file, sha256_string
 
@@ -30,6 +31,8 @@
 
 logger = logging.getLogger(__name__)
 
+kapitan_install_path = "/Users/jacek/Code/kapitan_fork/kapitan"
+
 try:
     from yaml import CSafeLoader as YamlLoader
 except ImportError:

pyoxidizer.bzl

@@ -0,0 +1,126 @@
+# Below packages are using __file__ and need to be placed into a lib directory.
+PACKAGES_TO_ADD = [
+    "boto3",
+    "boto",
+    "git",
+    "gitdb",
+    "googleapiclient",
+    "httplib2",
+    "idlelib",
+    "jinja2",
+    "jsonschema",
+]
+
+def handle_resource(policy, resource):
+    # Based on solution from https://github.com/indygreg/PyOxidizer/issues/436#issuecomment-939408632
+    # and https://pyoxidizer.readthedocs.io/en/stable/pyoxidizer_packaging_resources.html#using-callbacks-to-influence-resource-attributes
+    name = ""
+    if type(resource) in ("PythonPackageResource", "PythonPackageDistributionResource"):
+        name = resource.package
+    elif type(resource) == "PythonModuleSource":
+        name = resource.name
+    elif type(resource) == "File":
+        name = resource.path
+
+    if any([True for pkg in PACKAGES_TO_ADD if pkg in name]):
+        resource.add_location = "filesystem-relative:lib"
+    else:
+        resource.add_location = "in-memory"
+
+def make_dist():
+    return default_python_distribution( python_version = "3.9" )
+
+def make_packaging_policy(dist):
+    policy = dist.make_python_packaging_policy()
+
+    # Register a function to be called when resources are created.
+    policy.register_resource_callback(handle_resource)
+
+    # Controls whether `File` instances are emitted by the file scanner.
+    policy.file_scanner_emit_files = True
+
+    # Try to add resources to in-memory first. If that fails, add them to a
+    # "lib" directory relative to the built executable.
+    policy.resources_location = "in-memory"
+    policy.resources_location_fallback = "filesystem-relative:lib"
+
+    # Enable support for non-classified "file" resources to be added to
+    # resource collections.
+    policy.allow_files = True
+
+    # Controls the `add_include` attribute of `File` resources.
+    policy.include_file_resources = False
+
+    # Toggle whether Python module source code for modules in the Python
+    # distribution's standard library are included.
+    policy.include_distribution_sources = True
+
+    # Controls the `add_include` attribute of `PythonModuleSource` not in
+    # the standard library.
+    policy.include_non_distribution_sources = True
+
+    # # Toggle whether files associated with tests are included.
+    policy.include_test = False
+
+    return policy
+
+def make_config(dist):
+    python_config = dist.make_python_interpreter_config()
+
+    # Set initial value for `sys.path`. If the string `$ORIGIN` exists in
+    # a value, it will be expanded to the directory of the built executable.
+    python_config.module_search_paths = ["$ORIGIN/lib"]
+
+    # Enable the stdlib path-based importer.
+    python_config.filesystem_importer = True
+
+    # Evaluate a string as Python code when the interpreter starts.
+    python_config.run_command = "import kapitan.cli; kapitan.cli.main()"
+
+    return python_config
+
+def make_exe(dist, policy, config):
+    # Produce a PythonExecutable from a Python distribution, embedded
+    # resources, and other options. The returned object represents the
+    # standalone executable that will be built.
+    exe = dist.to_python_executable(
+        name="kapitan",
+        packaging_policy=policy,
+        config=config,
+    )
+
+    # Invoke `pip install` using a requirements file and add the collected resources
+    # to our binary.
+    exe.add_python_resources(exe.pip_install(["-r", "requirements.txt"]))
+
+    # Read Python files from a local directory and add them to our embedded
+    # context, taking just the resources belonging to the `foo` and `bar`
+    # Python packages.
+    exe.add_python_resources(exe.read_package_root(
+        path=".",
+        packages=["kapitan"],
+    ))
+    exe.add_python_resources(exe.read_package_root(
+        path="kapitan/reclass",
+        packages=["reclass"],
+    ))
+
+    return exe
+
+def make_embedded_resources(exe):
+    return exe.to_embedded_resources()
+
+def make_install(exe, resources):
+    files = FileManifest()
+    files.add_python_resource(".", exe)
+
+    return files
+
+register_target("dist", make_dist)
+register_target("policy", make_packaging_policy, depends=["dist"])
+register_target("config", make_config, depends=["dist"])
+register_target("exe", make_exe, depends=["dist", "policy", "config"])
+register_target("resources", make_embedded_resources, depends=["exe"], default_build_script=True)
+register_target("install", make_install, depends=["exe", "resources"], default=True)
+
+resolve_targets()

pyproject.toml

@@ -47,7 +47,7 @@ gitpython = "^3.1.30"
 google-api-python-client = "^2.15.0"
 hvac = "^0.11.0"
 jinja2 = "^3.0.1"
-jsonnet = "^0.18.0"
+
 jsonschema = "^4.17.3"
 kadet = "^0.2.2"
 python-gnupg = "^0.4.7"