Refactor multicluster conftest & settings

Signed-off-by: averevki <[email protected]>

config/settings.local.yaml.tpl

@@ -1,9 +1,5 @@
 #default:
 #  tester: "someuser"                          # Optional: name of the user, who is running the tests, defaults to whoami/uid
-#  cluster:                                    # Primary cluster where tests should run
-#    api_url: "https://api.kubernetes.com"     # Optional: Kubernetes API URL, if None it will use Kubernetes that you are logged in
-#    token: "KUADRANT_RULEZ"                   # Optional: Kubernetes Token, if None it will Kubernetes that you are logged in
-#    kubeconfig_path: "~/.kube/config"         # Optional: Kubeconfig to use, if None the default one is used
 #  kuadrantctl: kuadrantctl
 #  tools:
 #    project: "tools"                          # Optional: Kubernetes project, where external tools are located
@@ -44,7 +40,14 @@
 #      metrics_service_name: ""                    # controller metrics service name for already deployed Authorino
 #  default_exposer: "kubernetes"                   # Force Exposer typem options: 'openshift', 'kind', 'kubernetes'
 #  control_plane:
-#    additional_clusters: []                       # List of additional clusters for Multicluster testing, see 'cluster' option for more details
+#    cluster:                                      # Primary cluster where tests should run
+#        api_url: "https://api.kubernetes.com"     # Optional: Kubernetes API URL, if None it will use Kubernetes that you are logged in
+#        token: "KUADRANT_RULEZ"                   # Optional: Kubernetes Token, if None it will Kubernetes that you are logged in
+#        kubeconfig_path: "~/.kube/config"         # Optional: Kubeconfig to use, if None the default one is used
+#    cluster2:                                     # Second cluster for the multicluster tests
+#        api_url: "https://api.kubernetes2.com"
+#        token: "KUADRANT_RULEZ"
+#        kubeconfig_path: "~/.kube/config2"
 #    managedzone: aws-mz                           # Name of the ManagedZone resource
 #    issuer:                                       # Issuer object for testing TLSPolicy
 #      name: "selfsigned-cluster-issuer"           # Name of Issuer CR

testsuite/capabilities.py

@@ -3,7 +3,6 @@
 import functools
 
 from openshift_client import selector
-from weakget import weakget
 
 from testsuite.config import settings
 
@@ -12,14 +11,14 @@
 def has_kuadrant():
     """Returns True, if Kuadrant deployment is present and should be used"""
     project = settings["service_protection"]["system_project"]
-    clusters = weakget(settings)["control_plane"]["additional_clusters"] % []
-    clusters.append(settings["cluster"])
+    clusters = [settings["control_plane"]["cluster"]]
+    if cluster2 := settings["control_plane"]["cluster2"]:
+        clusters.append(cluster2)
     for cluster in clusters:
         system_project = cluster.change_project(project)
-        # Try if Kuadrant is deployed
         if not system_project.connected:
             return False, f"Cluster {cluster.api_url} is not connected, or namespace {project} does not exist"
-        system_project = cluster.change_project(project)
+
         with system_project.context:
             if selector("kuadrant").count_existing() == 0:
                 return False, f"Cluster {cluster.api_url} does not have Kuadrant resource in project {project}"

testsuite/config/exposer.py

@@ -9,7 +9,7 @@
 def load(obj, env=None, silent=True, key=None, filename=None):
     """Selects proper Exposes class"""
     if "default_exposer" not in obj or not obj["default_exposer"]:
-        client = obj["cluster"]
+        client = obj["control_plane"]["cluster"]
         if "route.openshift.io/v1" in client.do_action("api-versions").out():
             obj["default_exposer"] = EXPOSERS["openshift"]
         else:

testsuite/config/openshift_loader.py

@@ -15,18 +15,19 @@ def inject_client(obj, base_client, path):
 # pylint: disable=unused-argument, too-many-locals
 def load(obj, env=None, silent=True, key=None, filename=None):
     """Creates all KubernetesClients"""
-    section = obj.setdefault("cluster", {})
+    control_plane = obj.setdefault("control_plane", {})
+
+    cluster = control_plane.setdefault("cluster", {})
     client = KubernetesClient(
-        section.get("project"), section.get("api_url"), section.get("token"), section.get("kubeconfig_path")
+        cluster.get("project"), cluster.get("api_url"), cluster.get("token"), cluster.get("kubeconfig_path")
     )
-    obj["cluster"] = client
+    obj["control_plane"]["cluster"] = client
 
     tools = None
     if "tools" in obj and "project" in obj["tools"]:
         tools = client.change_project(obj["tools"]["project"])
     obj["tools"] = tools
 
-    control_plane = obj.setdefault("control_plane", {})
     clients = []
     clusters = control_plane.setdefault("additional_clusters", [])
     for value in clusters:
@@ -37,3 +38,8 @@ def load(obj, env=None, silent=True, key=None, filename=None):
         )
     if len(clients) > 0:
         control_plane["additional_clusters"] = clients
+
+    if cluster2 := control_plane.setdefault("cluster2", {}):
+        obj["control_plane"]["cluster2"] = KubernetesClient(
+            cluster2.get("project"), cluster2.get("api_url"), cluster2.get("token"), cluster2.get("kubeconfig_path")
+        )

testsuite/tests/conftest.py

@@ -239,7 +239,7 @@ def module_label(label):
 def cluster(testconfig):
     """Kubernetes client for the primary namespace"""
     project = testconfig["service_protection"]["project"]
-    client = testconfig["cluster"].change_project(testconfig["service_protection"]["project"])
+    client = testconfig["control_plane"]["cluster"].change_project(testconfig["service_protection"]["project"])
     if not client.connected:
         pytest.fail(f"You are not logged into Kubernetes or the {project} namespace doesn't exist")
     return client

testsuite/tests/multicluster/conftest.py

@@ -1,29 +1,31 @@
 """Conftest for Multicluster tests"""
 
 from importlib import resources
-from typing import TypeVar
 
 import pytest
 from openshift_client import selector, OpenShiftPythonException
 
 from testsuite.backend.httpbin import Httpbin
 from testsuite.certificates import Certificate
-from testsuite.gateway import Exposer, Gateway, CustomReference, Hostname
+from testsuite.gateway import Exposer, CustomReference, Hostname
 from testsuite.gateway.gateway_api.gateway import KuadrantGateway
 from testsuite.gateway.gateway_api.hostname import DNSPolicyExposer
 from testsuite.gateway.gateway_api.route import HTTPRoute
-from testsuite.kubernetes.client import KubernetesClient
-from testsuite.kuadrant.policy import Policy
 from testsuite.kuadrant.policy.dns import DNSPolicy
 from testsuite.kuadrant.policy.tls import TLSPolicy
 
 
-AnyPolicy = TypeVar("AnyPolicy", bound=Policy)
-
+@pytest.fixture(scope="session")
+def cluster2(testconfig):
+    """Kubernetes client for the primary namespace"""
+    if not testconfig["control_plane"]["cluster2"]:
+        pytest.skip("Second cluster is not configured properly")
 
-def generate_policies(clusters: list[KubernetesClient], policy: AnyPolicy) -> dict[KubernetesClient, AnyPolicy]:
-    """Copy policies for each cluster"""
-    return {cluster: policy.__class__(policy.as_dict(), context=cluster.context) for cluster in clusters}
+    project = testconfig["service_protection"]["project"]
+    client = testconfig["control_plane"]["cluster2"].change_project(project)
+    if not client.connected:
+        pytest.fail(f"You are not logged into the second cluster or the {project} namespace doesn't exist")
+    return client
 
 
 @pytest.fixture(scope="module")
@@ -43,126 +45,122 @@ def cluster_issuer(testconfig, cluster, skip_or_fail):
     )
 
 
-@pytest.fixture(scope="session")
-def clusters(testconfig, cluster, skip_or_fail) -> list[KubernetesClient]:
-    """Returns list of all clusters on which to run Multicluster tests"""
-    additional_clusters = testconfig["control_plane"]["additional_clusters"]
-    if len(additional_clusters) == 0:
-        skip_or_fail("Only one cluster was provided for multi-cluster tests")
-    return [
-        cluster,
-        *(cluster.change_project(testconfig["service_protection"]["project"]) for cluster in additional_clusters),
-    ]
+@pytest.fixture(scope="module")
+def hostname(gateway, exposer, blame) -> Hostname:
+    """Exposed Hostname object"""
+    return exposer.expose_hostname(blame("hostname"), gateway)
+
+
+@pytest.fixture(scope="module")
+def exposer(request, cluster) -> Exposer:
+    """Expose using DNSPolicy"""
+    exposer = DNSPolicyExposer(cluster)
+    request.addfinalizer(exposer.delete)
+    exposer.commit()
+    return exposer
+
+
+@pytest.fixture(scope="module")
+def base_domain(exposer):
+    """Returns preconfigured base domain"""
+    return exposer.base_domain
+
+
+@pytest.fixture(scope="module")
+def wildcard_domain(base_domain):
+    """
+    Wildcard domain for the exposer
+    """
+    return f"*.{base_domain}"
 
 
 @pytest.fixture(scope="session")
-def backends(request, clusters, blame, label, testconfig) -> dict[KubernetesClient, Httpbin]:
+def backends(request, cluster, cluster2, blame, label, testconfig) -> list[Httpbin]:
     """Deploys Backend to each Kubernetes cluster"""
-    backends = {}
+    backends = []
     name = blame("httpbin")
     image = testconfig["httpbin"]["image"]
-    for cluster in clusters:
-        httpbin = Httpbin(cluster, name, label, image)
+    for client in [cluster, cluster2]:
+        httpbin = Httpbin(client, name, label, image)
         request.addfinalizer(httpbin.delete)
         httpbin.commit()
-        backends[cluster] = httpbin
+        backends.append(httpbin)
     return backends
 
 
 @pytest.fixture(scope="module")
-def gateways(request, clusters, blame, label, wildcard_domain) -> dict[KubernetesClient, Gateway]:
-    """Deploys Gateway to each Kubernetes cluster"""
-    gateways = {}
-    name = blame("gw")
-    for cluster in clusters:
-        gw = KuadrantGateway.create_instance(cluster, name, wildcard_domain, {"app": label}, tls=True)
-        request.addfinalizer(gw.delete)
-        gw.commit()
-        gateways[cluster] = gw
-    for gateway in gateways.values():
-        gateway.wait_for_ready()
-    return gateways
-
-
-@pytest.fixture(scope="module")
-def routes(request, gateways, blame, hostname, backends, module_label) -> dict[KubernetesClient, HTTPRoute]:
-    """Deploys HttpRoute to each Kubernetes cluster"""
-    routes = {}
+def routes(request, gateway, gateway2, blame, hostname, backends, module_label) -> list[HTTPRoute]:
+    """Deploys HttpRoute for each gateway"""
+    routes = []
     name = blame("route")
-    for client, gateway in gateways.items():
-        route = HTTPRoute.create_instance(gateway.cluster, name, gateway, {"app": module_label})
+    for i, gateway_ in enumerate([gateway, gateway2]):
+        route = HTTPRoute.create_instance(gateway_.cluster, name, gateway_, {"app": module_label})
         route.add_hostname(hostname.hostname)
-        route.add_backend(backends[client])
+        route.add_backend(backends[i])
         request.addfinalizer(route.delete)
         route.commit()
-        routes[client] = route
+        routes.append(route)
     return routes
 
 
 @pytest.fixture(scope="module")
-def hostname(gateways, cluster, exposer, blame) -> Hostname:
-    """Exposed Hostname object"""
-    hostname = exposer.expose_hostname(blame("hostname"), gateways[cluster])
-    return hostname
-
-
-@pytest.fixture(scope="module")
-def exposer(request, cluster) -> Exposer:
-    """Expose using DNSPolicy"""
-    exposer = DNSPolicyExposer(cluster)
-    request.addfinalizer(exposer.delete)
-    exposer.commit()
-    return exposer
+def gateway(request, cluster, blame, label, wildcard_domain):
+    """Deploys Gateway to first Kubernetes cluster"""
+    gw = KuadrantGateway.create_instance(cluster, blame("gw"), wildcard_domain, {"app": label}, tls=True)
+    request.addfinalizer(gw.delete)
+    gw.commit()
+    gw.wait_for_ready()
+    return gw
 
 
 @pytest.fixture(scope="module")
-def base_domain(exposer):
-    """Returns preconfigured base domain"""
-    return exposer.base_domain
+def gateway2(request, cluster2, blame, label, wildcard_domain):
+    """Deploys Gateway to second Kubernetes cluster"""
+    gw = KuadrantGateway.create_instance(cluster2, blame("gw"), wildcard_domain, {"app": label}, tls=True)
+    request.addfinalizer(gw.delete)
+    gw.commit()
+    gw.wait_for_ready()
+    return gw
 
 
 @pytest.fixture(scope="module")
-def wildcard_domain(base_domain):
-    """
-    Wildcard domain for the exposer
-    """
-    return f"*.{base_domain}"
+def dns_policy(blame, cluster, gateway, module_label):
+    """DNSPolicy for the first cluster"""
+    return DNSPolicy.create_instance(cluster, blame("dns"), gateway, labels={"app": module_label})
 
 
 @pytest.fixture(scope="module")
-def dns_policy(blame, cluster, gateways, module_label):
-    """DNSPolicy fixture"""
-    policy = DNSPolicy.create_instance(cluster, blame("dns"), gateways[cluster], labels={"app": module_label})
-    return policy
+def dns_policy2(blame, cluster2, gateway2, module_label):
+    """DNSPolicy for the second cluster"""
+    return DNSPolicy.create_instance(cluster2, blame("dns"), gateway2, labels={"app": module_label})
 
 
 @pytest.fixture(scope="module")
-def tls_policy(blame, cluster, gateways, module_label, cluster_issuer):
-    """TLSPolicy fixture"""
-    policy = TLSPolicy.create_instance(
+def tls_policy(blame, cluster, gateway, module_label, cluster_issuer):
+    """TLSPolicy for the first cluster"""
+    return TLSPolicy.create_instance(
         cluster,
         blame("tls"),
-        parent=gateways[cluster],
+        parent=gateway,
         issuer=cluster_issuer,
         labels={"app": module_label},
     )
-    return policy
 
 
 @pytest.fixture(scope="module")
-def dns_policies(clusters, dns_policy) -> dict[KubernetesClient, DNSPolicy]:
-    """Creates DNSPolicy for each Kubernetes cluster"""
-    return generate_policies(clusters, dns_policy)
-
-
-@pytest.fixture(scope="module")
-def tls_policies(clusters, tls_policy) -> dict[KubernetesClient, TLSPolicy]:
-    """Creates TLSPolicy for each Kubernetes cluster"""
-    return generate_policies(clusters, tls_policy)
+def tls_policy2(blame, cluster2, gateway2, module_label, cluster_issuer):
+    """TLSPolicy for the second cluster"""
+    return TLSPolicy.create_instance(
+        cluster2,
+        blame("tls"),
+        parent=gateway2,
+        issuer=cluster_issuer,
+        labels={"app": module_label},
+    )
 
 
 @pytest.fixture(scope="module")
-def client(hostname, gateways):  # pylint: disable=unused-argument
+def client(hostname, gateway, gateway2):  # pylint: disable=unused-argument
     """Returns httpx client to be used for requests"""
     root_cert = resources.files("testsuite.resources").joinpath("letsencrypt-stg-root-x1.pem").read_text()
     client = hostname.client(verify=Certificate(certificate=root_cert, chain=root_cert, key=""))
@@ -171,9 +169,9 @@ def client(hostname, gateways):  # pylint: disable=unused-argument
 
 
 @pytest.fixture(scope="module", autouse=True)
-def commit(request, routes, dns_policies, tls_policies):  # pylint: disable=unused-argument
+def commit(request, routes, dns_policy, dns_policy2, tls_policy, tls_policy2):  # pylint: disable=unused-argument
     """Commits all policies before tests"""
-    components = [*dns_policies.values(), *tls_policies.values()]
+    components = [dns_policy, dns_policy2, tls_policy, tls_policy2]
     for component in components:
         request.addfinalizer(component.delete)
         component.commit()

testsuite/tests/multicluster/test_multicluster_dns.py → testsuite/tests/multicluster/test_simple_strategy.py

@@ -6,13 +6,13 @@
 pytestmark = [pytest.mark.multicluster]
 
 
-def test_gateway_readiness(gateways):
+def test_gateway_readiness(gateway, gateway2):
     """Tests whether the Gateway was successfully placed by having its IP address assigned"""
-    for client, gateway in gateways.items():
-        assert gateway.is_ready(), f"Gateway {gateway.name()} on a server {client.api_url} did not get ready"
+    assert gateway.is_ready(), "Gateway on the first cluster did not get ready in time"
+    assert gateway2.is_ready(), "Gateway on the second cluster did not get ready in time"
 
 
-def test_multicluster_dns(client, hostname, gateways):
+def test_simple_strategy(client, hostname, gateway, gateway2):
     """
     Tests DNS/TLS across multiple clusters
     - Checks that all Gateways will get ready
@@ -24,6 +24,6 @@ def test_multicluster_dns(client, hostname, gateways):
     assert not result.has_cert_verify_error(), result.error
     assert result.status_code == 200
 
-    ips = {gateway.external_ip().split(":")[0] for gateway in gateways.values()}
+    ips = {gateway.external_ip().split(":")[0], gateway2.external_ip().split(":")[0]}
     dns_ips = {ip.address for ip in dns.resolver.resolve(hostname.hostname)}
     assert ips == dns_ips, f"Expected IPs and actual IP mismatch, got {dns_ips}, expected {ips}"

testsuite/tests/singlecluster/authorino/operator/clusterwide/conftest.py

@@ -29,9 +29,9 @@ def route2(request, gateway, blame, hostname2):
 
 
 @pytest.fixture(scope="module")
-def authorization2(route2, blame, cluster2, label, oidc_provider):
+def authorization2(route2, blame, second_namespace, label, oidc_provider):
     """Second valid hostname"""
-    auth = AuthConfig.create_instance(cluster2, blame("ac"), route2, labels={"testRun": label})
+    auth = AuthConfig.create_instance(second_namespace, blame("ac"), route2, labels={"testRun": label})
     auth.identity.add_oidc("default", oidc_provider.well_known["issuer"])
     return auth