Skip to content

Commit

Permalink
Clean up keystone deployment fixture
Browse files Browse the repository at this point in the history
  • Loading branch information
@addyess
addyess committed Jun 13, 2024
1 parent fcbc4db commit 5d1a1c7
Show file tree
Hide file tree
Showing 2 changed files with 94 additions and 58 deletions.
46 changes: 37 additions & 9 deletions jobs/integration/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@
from contextlib import contextmanager
from typing import Mapping, Any, Union, Sequence

import jinja2
from juju.unit import Unit
from juju.model import Model
from juju.controller import Controller
from juju.machine import Machine
from juju.errors import JujuError
Expand Down Expand Up @@ -817,19 +819,28 @@ async def kubectl(model, *args: str, check=True, **kwargs) -> JujuRunResult:
return await juju_run(control_plane, c, check)


async def _kubectl_doc(document, model, action, **kwds):
async def _kubectl_doc(document: str, model, action, **kwds):
if action not in ["apply", "delete"]:
raise ValueError(f"Invalid action {action}")

control_plane = model.applications["kubernetes-control-plane"].units[0]
remote_path = f"/tmp/{document.name}"
await scp_to(
document,
control_plane,
remote_path,
None,
model.info.uuid,
)
with TemporaryDirectory(dir=Path.home() / ".local" / "share" / "juju") as tmpdir:
if isinstance(document, Path):
local_path = Path(tmpdir) / document.name
remote_path = f"/tmp/{document.name}"
elif isinstance(document, str):
local_path = Path(tmpdir) / "source"
remote_path = f"/tmp/{Path(tmpdir).name}"
local_path.write_text(document)
else:
raise ValueError(f"Invalid document type {type(document)}")
await scp_to(
local_path,
control_plane,
remote_path,
None,
model.info.uuid,
)
cmd = f"{action} -f {remote_path}"
return await kubectl(model, cmd, **kwds)

Expand Down Expand Up @@ -904,3 +915,20 @@ async def get_svc_ingress(model, svc_name, timeout=2 * 60):
raise TimeoutError(
f"Timed out waiting for {svc_name} to have an ingress address"
)


def render(path: os.PathLike, context: dict) -> Path:
"""Render a jinja2 template with the given context."""
source = Path(__file__).parent / path
template = jinja2.Template(source.read_text())
return template.render(context)


async def render_and_apply(*resources: os.PathLike, context: dict, model: Model):
await asyncio.gather(*(kubectl_apply(render(r, context), model) for r in resources))


async def render_and_delete(*resources: os.PathLike, context: dict, model: Model):
await asyncio.gather(
*(kubectl_delete(render(r, context), model) for r in resources)
)
106 changes: 57 additions & 49 deletions jobs/integration/validation.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@

import backoff
import ipaddress
import jinja2
import json
import os
import requests
Expand Down Expand Up @@ -42,6 +41,9 @@
kubectl,
kubectl_apply,
kubectl_delete,
render,
render_and_apply,
render_and_delete,
juju_run,
juju_run_action,
machine_reboot,
Expand Down Expand Up @@ -1340,6 +1342,16 @@ async def get_webhook_server_entry_count():
async def any_keystone(model, apps_by_charm, tools):
keystone_apps = apps_by_charm("keystone")

if not (client := model.applications.get("keystone-client")):
client = await model.deploy(
"ubuntu",
channel="stable",
application_name="keystone-client",
series=tools.series,
num_units=1,
config={},
)

if len(keystone_apps) > 1:
pytest.fail(f"More than one keystone app available {','.join(keystone_apps)}")
elif len(keystone_apps) == 1:
Expand All @@ -1350,7 +1362,9 @@ async def any_keystone(model, apps_by_charm, tools):
action = await juju_run(keystone_main, "leader-get admin_passwd")
admin_password = action.stdout.strip()

yield SimpleNamespace(app=keystone, admin_password=admin_password)
yield SimpleNamespace(
app=keystone, admin_password=admin_password, client=client.units[0]
)
else:
# No keystone available, add/setup one
admin_password = "testpw"
Expand Down Expand Up @@ -1385,7 +1399,10 @@ async def any_keystone(model, apps_by_charm, tools):
await model.integrate(f"{db.name}:db-router", f"{db_router.name}:db-router")
await tools.juju_wait()

yield SimpleNamespace(app=keystone, admin_password=admin_password)
yield SimpleNamespace(
app=keystone, admin_password=admin_password, client=client.units[0]
)
await tools.juju_wait()

# cleanup
await model.applications[keystone.name].destroy()
Expand Down Expand Up @@ -1439,21 +1456,26 @@ async def leader_read(unit: Unit, path: str):


@pytest.fixture()
async def keystone_deployment(model, apps_by_charm, any_keystone, tools, tmp_path):
async def keystone_deployment(
model, apps_by_charm, any_keystone, tools, tmp_path: Path
):
keystone: Application = any_keystone.app
control_plane: Application = model.applications["kubernetes-control-plane"]
control_plane_unit = control_plane.units[0]
kubeapi_loadbalancer: Application = model.applications["kubeapi-load-balancer"]

control_plane_unit = control_plane.units[0]
server_crt = await leader_read(control_plane_unit, "/root/cdk/server.crt")
server_key = await leader_read(control_plane_unit, "/root/cdk/server.key")

# assemble key variables to populate the keystone deployment templates
context = dict(
# keystone endpoint data
keystone_server_url=f"http://{keystone.units[0].public_address}:5000/v3",
keystone_server_ca=await load_keystone_ca(model, apps_by_charm, keystone),
# keystone auth endpoint data
keystone_auth_crt=cert_encode(server_crt),
keystone_auth_key=cert_encode(server_key),
keystone_auth_service_ip="",
keystone_auth_service_ip="", # filled in after service starts
# keystone login data
keystone_user="admin",
keystone_password=any_keystone.admin_password,
Expand All @@ -1462,6 +1484,7 @@ async def keystone_deployment(model, apps_by_charm, any_keystone, tools, tmp_pat
# kube api data
kubernetes_api_server=f"https://{kubeapi_loadbalancer.units[0].public_address}",
)
# render and apply the keystone deployment templates
resources = [
"templates/keystone/keystone-deployment.yaml",
"templates/keystone/keystone-policy-configmap.yaml",
Expand All @@ -1470,52 +1493,23 @@ async def keystone_deployment(model, apps_by_charm, any_keystone, tools, tmp_pat
"templates/keystone/keystone-service.yaml",
]

def _render(resource):
source = Path(__file__).parent / resource
template = jinja2.Template(source.read_text())
rendered = tmp_path / source.name
rendered.write_text(template.render(context))
return rendered

async def _render_and_apply(resource):
await kubectl_apply(_render(resource), model)

async def _render_and_delete(resource):
await kubectl_delete(_render(resource), model)

await asyncio.gather(*(_render_and_apply(resource) for resource in resources))
await render_and_apply(*resources, context=context, model=model)

# find the service ip of the keystone auth service
while context["keystone_auth_service_ip"] == "":
svc_ip = ""
while not svc_ip:
auth_svc = await find_entities(
control_plane_unit, "svc", ["k8s-keystone-auth-service"], "-n kube-system"
)
if auth_svc and (svc_ip := auth_svc[0]["spec"]["clusterIP"]):
context["keystone_auth_service_ip"] = svc_ip

keystone_webhook_endpoint = f"https://{svc_ip}:8443/webhook"
webhook_config = _render("templates/keystone/keystone-apiserver-webhook.yaml")
svc_ip = auth_svc[0]["spec"]["clusterIP"]
context["keystone_auth_service_ip"] = svc_ip

# setups the keystone-client with a kubeconfig
original_config = await control_plane.get_config()
await control_plane.set_config(
{
"authorization-webhook-config-file": webhook_config.read_text(),
"authorization-mode": "Node,Webhook,RBAC",
"authn-webhook-endpoint": keystone_webhook_endpoint,
}
kubeconfig = tmp_path / "config"
kubeconfig.write_text(
render("templates/keystone/keystone-kubeconfig.yaml", context=context)
)
if not (keystone_client := model.applications.get("keystone-client")):
keystone_client = await model.deploy(
"ubuntu",
application_name="keystone-client",
series=tools.series,
config={},
num_units=1,
channel="stable",
)
await tools.juju_wait()
any_keystone.client = keystone_client.units[0]
kubeconfig = _render("templates/keystone/keystone-kubeconfig.yaml")
await scp_to(
kubeconfig,
any_keystone.client,
Expand All @@ -1527,18 +1521,32 @@ async def _render_and_delete(resource):
await juju_run(
any_keystone.client,
(
f'snap install kubectl --channel={original_config["channel"]["value"]} --classic;'
f'snap install client-keystone-auth --channel={original_config["channel"]["value"]};'
"mkdir -p /root/.kube;"
"mv /home/ubuntu/config /root/.kube/config;"
f'snap install kubectl --channel={original_config["channel"]["value"]} --classic;\n'
f'snap install client-keystone-auth --channel={original_config["channel"]["value"]};\n'
"mkdir -p /root/.kube;\n"
"mv /home/ubuntu/config /root/.kube/config;\n"
"chown -R root:root /root/.kube;"
),
)

# configure the kube-apiserver to use the keystone auth webhook
webhook_config = render(
"templates/keystone/keystone-apiserver-webhook.yaml", context=context
)
await control_plane.set_config(
{
"authorization-webhook-config-file": webhook_config,
"authorization-mode": "Node,Webhook,RBAC",
"authn-webhook-endpoint": f"https://{svc_ip}:8443/webhook",
}
)
await tools.juju_wait()
yield any_keystone

# cleanup
await render_and_delete(*resources, context=context, model=model)
await control_plane.set_config(original_config)
await tools.juju_wait()
await asyncio.gather(*(_render_and_delete(resource) for resource in resources))


@pytest.mark.usefixtures("ceph_apps")
Expand Down

0 comments on commit 5d1a1c7

Please sign in to comment.