diff --git a/ci/e2e-hub/Dockerfile b/ci/e2e-hub/Dockerfile new file mode 100644 index 000000000..13e2b3a92 --- /dev/null +++ b/ci/e2e-hub/Dockerfile @@ -0,0 +1,8 @@ +# Create a kmod image that mocks the load of a kmod. +# The e2e-hub test only checks that the kmod image is successfully pulled by the Spoke; it does not verify that the +# kmod is actually loaded. + +ARG DTK_AUTO +FROM $DTK_AUTO + +RUN ln -s /usr/bin/true /bin/modprobe diff --git a/ci/e2e-hub/kustomization.yaml b/ci/e2e-hub/kustomization.yaml new file mode 100644 index 000000000..f4c83fd17 --- /dev/null +++ b/ci/e2e-hub/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: openshift-kmm-hub + +resources: + - ../sign-key-certs + - policy.yaml + - placementrule.yaml + - placementbinding.yaml + - managedclustermodule.yaml + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: + - name: mod-example + files: [dockerfile=Dockerfile] diff --git a/ci/e2e-hub/managedclustermodule.yaml b/ci/e2e-hub/managedclustermodule.yaml new file mode 100644 index 000000000..0cf27d1b1 --- /dev/null +++ b/ci/e2e-hub/managedclustermodule.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: hub.kmm.sigs.x-k8s.io/v1beta1 +kind: ManagedClusterModule +metadata: + name: mod-example +spec: + spokeNamespace: openshift-kmm + selector: + name: local-cluster + moduleSpec: + moduleLoader: + container: + modprobe: + moduleName: mod-example + imagePullPolicy: Always + kernelMappings: + - regexp: '^.+$' + containerImage: image-registry.openshift-image-registry.svc:5000/$MOD_NAMESPACE/$MOD_NAME:$KERNEL_FULL_VERSION + registryTLS: + insecure: true + build: + baseImageRegistryTLS: + insecure: true + dockerfileConfigMap: + name: mod-example + sign: + keySecret: + name: kmm-kmod-signing-key + certSecret: + name: kmm-kmod-signing-cert + unsignedImageRegistryTLS: + insecure: true + filesToSign: + - /bin/getconf + selector: + node-role.kubernetes.io/worker: "" diff --git a/ci/e2e-hub/placementbinding.yaml b/ci/e2e-hub/placementbinding.yaml new file mode 100644 index 000000000..940928ca8 --- /dev/null +++ b/ci/e2e-hub/placementbinding.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: policy.open-cluster-management.io/v1 +kind: PlacementBinding +metadata: + name: bind-klusterlet-kmm-all-clusters +placementRef: + apiGroup: apps.open-cluster-management.io + kind: PlacementRule + name: all-managed-clusters +subjects: + - apiGroup: policy.open-cluster-management.io + kind: Policy + name: allow-klusterlet-deploy-kmm-modules diff --git a/ci/e2e-hub/placementrule.yaml b/ci/e2e-hub/placementrule.yaml new file mode 100644 index 000000000..9dde90aca --- /dev/null +++ b/ci/e2e-hub/placementrule.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: apps.open-cluster-management.io/v1 +kind: PlacementRule +metadata: + name: all-managed-clusters +spec: + clusterSelector: + matchExpressions: [] diff --git a/ci/e2e-hub/policy.yaml b/ci/e2e-hub/policy.yaml new file mode 100644 index 000000000..d0edba740 --- /dev/null +++ b/ci/e2e-hub/policy.yaml @@ -0,0 +1,40 @@ +apiVersion: policy.open-cluster-management.io/v1 +kind: Policy +metadata: + name: allow-klusterlet-deploy-kmm-modules +spec: + remediationAction: enforce + disabled: false + policy-templates: + - objectDefinition: + apiVersion: policy.open-cluster-management.io/v1 + kind: ConfigurationPolicy + metadata: + name: klusterlet-deploy-modules + spec: + severity: high + object-templates: + - complianceType: mustonlyhave + objectDefinition: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: kmm-module-manager + rules: + - apiGroups: [kmm.sigs.x-k8s.io] + resources: [modules] + verbs: [create, delete, get, list, patch, update, watch] + - complianceType: mustonlyhave + objectDefinition: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: klusterlet-kmm + subjects: + - kind: ServiceAccount + name: klusterlet-work-sa + namespace: open-cluster-management-agent + roleRef: + kind: ClusterRole + name: kmm-module-manager + apiGroup: rbac.authorization.k8s.io diff --git a/ci/e2e/Dockerfile b/ci/e2e/Dockerfile new file mode 100644 index 000000000..640c7f80f --- /dev/null +++ b/ci/e2e/Dockerfile @@ -0,0 +1,25 @@ +ARG DTK_AUTO + +FROM ${DTK_AUTO} as builder + +ARG KERNEL_VERSION + +WORKDIR /usr/src + +RUN grep super-secret-value /run/secrets/build-secret/ci-build-secret + +RUN git clone https://github.com/rh-ecosystem-edge/kernel-module-management.git + +WORKDIR /usr/src/kernel-module-management/ci/kmm-kmod + +RUN KERNEL_SRC_DIR=/lib/modules/${KERNEL_VERSION}/build make all + +FROM registry.redhat.io/ubi9/ubi-minimal + +ARG KERNEL_VERSION + +RUN ["microdnf", "install", "-y", "kmod"] + +COPY --from=builder /usr/src/kernel-module-management/ci/kmm-kmod/kmm_ci_a.ko /opt/lib/modules/${KERNEL_VERSION}/ +COPY --from=builder /usr/src/kernel-module-management/ci/kmm-kmod/kmm_ci_b.ko /opt/lib/modules/${KERNEL_VERSION}/ +RUN depmod -b /opt ${KERNEL_VERSION} diff --git a/ci/e2e/kustomization.yaml b/ci/e2e/kustomization.yaml new file mode 100644 index 000000000..e6fca9068 --- /dev/null +++ b/ci/e2e/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../sign-key-certs + - module.yaml + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: + - name: kmm-kmod-dockerfile + files: [dockerfile=Dockerfile] + +secretGenerator: + - name: build-secret + literals: [ci-build-secret=super-secret-value] diff --git a/ci/module-kmm-ci-build-sign.yaml b/ci/e2e/module.yaml similarity index 100% rename from ci/module-kmm-ci-build-sign.yaml rename to ci/e2e/module.yaml diff --git a/ci/install-ci-hub/kustomization.yaml b/ci/install-ci-hub/kustomization.yaml deleted file mode 100644 index f1e38781a..000000000 --- a/ci/install-ci-hub/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namespace: openshift-kmm - -resources: -- ../../config/default-hub diff --git a/ci/kmm-kmod-dockerfile.yaml b/ci/kmm-kmod-dockerfile.yaml deleted file mode 100644 index 69121ccff..000000000 --- a/ci/kmm-kmod-dockerfile.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: kmm-kmod-dockerfile -data: - dockerfile: | - - ARG DTK_AUTO - - FROM ${DTK_AUTO} as builder - - ARG KERNEL_VERSION - - WORKDIR /usr/src - - RUN grep super-secret-value /run/secrets/build-secret/ci-build-secret - - RUN git clone https://github.com/rh-ecosystem-edge/kernel-module-management.git - - WORKDIR /usr/src/kernel-module-management/ci/kmm-kmod - - RUN KERNEL_SRC_DIR=/lib/modules/${KERNEL_VERSION}/build make all - - FROM registry.redhat.io/ubi9/ubi-minimal - - ARG KERNEL_VERSION - - RUN ["microdnf", "install", "-y", "kmod"] - - COPY --from=builder /usr/src/kernel-module-management/ci/kmm-kmod/kmm_ci_a.ko /opt/lib/modules/${KERNEL_VERSION}/ - COPY --from=builder /usr/src/kernel-module-management/ci/kmm-kmod/kmm_ci_b.ko /opt/lib/modules/${KERNEL_VERSION}/ - RUN depmod -b /opt ${KERNEL_VERSION} diff --git a/ci/managedclustermodule-kmm-ci-build-sign.yaml b/ci/managedclustermodule-kmm-ci-build-sign.yaml deleted file mode 100644 index 70ee57d5d..000000000 --- a/ci/managedclustermodule-kmm-ci-build-sign.yaml +++ /dev/null @@ -1,208 +0,0 @@ -apiVersion: policy.open-cluster-management.io/v1 -kind: Policy -metadata: - name: allow-klusterlet-deploy-kmm-modules -spec: - remediationAction: enforce - disabled: false - policy-templates: - - objectDefinition: - apiVersion: policy.open-cluster-management.io/v1 - kind: ConfigurationPolicy - metadata: - name: klusterlet-deploy-modules - spec: - severity: high - object-templates: - - complianceType: mustonlyhave - objectDefinition: - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: kmm-module-manager - rules: - - apiGroups: [kmm.sigs.x-k8s.io] - resources: [modules] - verbs: [create, delete, get, list, patch, update, watch] - - complianceType: mustonlyhave - objectDefinition: - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - name: klusterlet-kmm - subjects: - - kind: ServiceAccount - name: klusterlet-work-sa - namespace: open-cluster-management-agent - roleRef: - kind: ClusterRole - name: kmm-module-manager - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps.open-cluster-management.io/v1 -kind: PlacementRule -metadata: - name: all-managed-clusters -spec: - clusterSelector: - matchExpressions: [] ---- -apiVersion: policy.open-cluster-management.io/v1 -kind: PlacementBinding -metadata: - name: bind-klusterlet-kmm-all-clusters -placementRef: - apiGroup: apps.open-cluster-management.io - kind: PlacementRule - name: all-managed-clusters -subjects: - - apiGroup: policy.open-cluster-management.io - kind: Policy - name: allow-klusterlet-deploy-kmm-modules ---- -apiVersion: v1 -kind: Secret -metadata: - name: fake-modprobe-signing-key-pub - namespace: openshift-kmm -type: Opaque -stringData: - cert: | - -----BEGIN CERTIFICATE----- - MIIE8DCCAtigAwIBAgIUSRZJsM/2jzb7fm1PdqB/bjUOAtEwDQYJKoZIhvcNAQEN - BQAwEjEQMA4GA1UEAwwHTW9kdWxlczAgFw0yMzAxMjQxMDM3MzNaGA8yMTIyMTIz - MTEwMzczM1owEjEQMA4GA1UEAwwHTW9kdWxlczCCAiIwDQYJKoZIhvcNAQEBBQAD - ggIPADCCAgoCggIBAImPO0SLpA7l3/tOv/+ZwUm+o7W8c2BtDjmoVjLfWP0nwaV1 - mWJOOxPMfwPwykCUL7s84E+FMNjcd0jZ5xPZD/uYBg3H1JUTfkFhaBDFc9EG+Sbn - HJYt9BBRTeH0pF2wbII7lU2Da7/qIIW9TZ5aH31RsMoPw57DTTMoJkUfp07YKr3a - PgY4Fjjf4haaLSBxzgv+WS88af7NSPFeuSVEGNCyI6Zu56LRJ7s/BlZrD1Q7Y7tD - GlASvK5s2GuNu+i9LD34yEgYK0EqBEdz82FDbic5uKecjvu1Wg+yg32tmQJYTHK9 - joaPe6lvnUmIS1jlfk3qk8Tx44L0vdLeBf+ZMSYrugxvKq/Le7N5TTaKaHxd5pNz - GyHgvh3CN5SdroLAR+36gSvcBzM1uvAzsCRAYKRPult9z7GrLu8cnU7GVXBLIthP - P8mm26aBSYQGycJsUox8/MpEi5X/v1IYTIzpwevG7gruH8cfzvnVD90dwmE4D2nT - o5a9VmDaxbgreXLjCTihguSLqbMcH8l7OGRB1g8E2NgQrYNko9tDaYxP53fQhuZM - KS9zmTcC/EAqOym8f+j20y0/uxTCX5VCBAZzgn2ZvRY6shDXs5zZcFiujRWwzlA5 - 6fpjqpz5eI24GDJ039VXHQ8Gs14h67HgnrLwQSqX/Ml1en+OKkQAViC9FfVXAgMB - AAGjPDA6MAwGA1UdEwEB/wQCMAAwCwYDVR0PBAQDAgeAMB0GA1UdDgQWBBQeRCk5 - b8GGuuc8QWV9PmZb3o638jANBgkqhkiG9w0BAQ0FAAOCAgEATtaTgC84GB5JVaIv - bfS8I68eEk/kCr9MShW7LGUgahUMvsEKZHj4zyXXPAz9udDkx4YE8be/dnz35evG - MwZ5vTb0uncJ+uhBQz1uITyS33cO/GQXXb6wSNZpBNg+VwN1c7L1iQ47VzFiqUeM - rVGblHC0/V95zbL/VeooXuRP/t0VHHxdM1AaNH21+JUGBVAq9BonRH1fx+AZptFg - kkmbefdquS12C1g6phlVnPakKVvlLe8kj3hsnaF7mRQJhKBxicxkJf7gIgbiH34v - SXADqWoD/WDGwHL2u40rEuShEA0lCodRAwKB0lXTFpHMyAFw30zOyXHbOpZyQC3m - BIY69QlIyKXknpQT2bvKeIAPOCsYD7ATt/hMf0aa94INBCBswa9gg/eX8+xQQIJn - DNIFbNtjmbuaFyDt1nT9xmeB2JPKkrpOx5B/BBNJGwK0fAuJa2623fGFuM95z2ug - sCrOlEiodxmMEVPZbCW084dHb3qoy9nTKrCcDQEp/s/5UVkjFe9BHaxlZZQbEGO9 - 2WQEuovQEGSHGGKuGeKrxJ2l5vccHFb6h8mXZbhNCCxom68lTYK7/0DGxPYsS0Vs - C9QOeeHlnKABV9wMzXbFiUccqjBzc8W/j1QKHD9mxFfVk6V/QmgVsViGPcTGnIVJ - 8Fxc2/LOrAZzIL/OFXqZHUy6hW0= - -----END CERTIFICATE----- ---- -apiVersion: v1 -kind: Secret -metadata: - name: fake-modprobe-signing-key - namespace: openshift-kmm -type: Opaque -stringData: - key: | - -----BEGIN PRIVATE KEY----- - MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQCJjztEi6QO5d/7 - Tr//mcFJvqO1vHNgbQ45qFYy31j9J8GldZliTjsTzH8D8MpAlC+7POBPhTDY3HdI - 2ecT2Q/7mAYNx9SVE35BYWgQxXPRBvkm5xyWLfQQUU3h9KRdsGyCO5VNg2u/6iCF - vU2eWh99UbDKD8Oew00zKCZFH6dO2Cq92j4GOBY43+IWmi0gcc4L/lkvPGn+zUjx - XrklRBjQsiOmbuei0Se7PwZWaw9UO2O7QxpQEryubNhrjbvovSw9+MhIGCtBKgRH - c/NhQ24nObinnI77tVoPsoN9rZkCWExyvY6Gj3upb51JiEtY5X5N6pPE8eOC9L3S - 3gX/mTEmK7oMbyqvy3uzeU02imh8XeaTcxsh4L4dwjeUna6CwEft+oEr3AczNbrw - M7AkQGCkT7pbfc+xqy7vHJ1OxlVwSyLYTz/JptumgUmEBsnCbFKMfPzKRIuV/79S - GEyM6cHrxu4K7h/HH8751Q/dHcJhOA9p06OWvVZg2sW4K3ly4wk4oYLki6mzHB/J - ezhkQdYPBNjYEK2DZKPbQ2mMT+d30IbmTCkvc5k3AvxAKjspvH/o9tMtP7sUwl+V - QgQGc4J9mb0WOrIQ17Oc2XBYro0VsM5QOen6Y6qc+XiNuBgydN/VVx0PBrNeIeux - 4J6y8EEql/zJdXp/jipEAFYgvRX1VwIDAQABAoICAAu8O2sqmpFmzQfsfPa6UEIR - UHgVnqYwR5Flqgzm/vmB5cRO1woK8IFM5d0ZMrf1rGjF1cES9MrbbBW7+Tm5P+LP - 4lxUmu8o3ZGIVM9xeOLxPFo0Rz0MqJCnzwQi0c7oWSl3GdKmDpAt0h52uuNyftM7 - H+DtHVwy/ygJGdK105knWeYec0ImkMbDWs9g0c69xr1BMqyT89r3+mwVD6h9Oby8 - JcxyE7LRMxmHU+wVcBZYhraYD+ua3Zvnz0LFsa02TkrvKW71GOOTSRArdSzsCr6v - 7u/lrFEO9KL4ML2NDMulasgzIbCbFwqqafm29e6cjXzqMTcRB77JUFRn+o0Grama - 9o0HR2UH1L/dpob01s1BLOrQT7Z9nwjj7xcTWLeMhrvwRQZ0Gthag6fsyimnt3If - 7SLs6y/4XvLepauu0wk6dVJq7gJjj3O8ZdHjYKobN04PmFHvIaKHdJCZlH9DZD3Q - Dh6qH9duet52BOIaEjRVqq2zP9Ug2rK0gjKxNpprxGkaZL4NcRNXvNSWoP5ClhO8 - 0XszqsKxpYu4SmAya4Yf2cdxT03YNmmy0CwM/m1k4lkLaLBtHJ4qGRqWWnZ4K+Ld - 6UotTcCpufTp/9XqbGa1XS28MOhhoTz8xUCKsEKjomMlAX8D6Jpl3fTty5rN+PBv - KD5it2DatATeniyMPqihAoIBAQC7PLsJ3omdtyB1sE5g6A2FICIu+SpXi7GM+XNA - c278YhAonB47I7wbYgU4So1b5D48Ml7NafD2b2YLrHFNiFSPvVlUMUdIXDlL1wDA - zDJDuuK7rWbXsIwpEnWFM3lx6VAGvQZsmruGLBgsTS3GBw4DSCuBeXDpk+4p8GXc - ih+FZiz+5IYHQ0QuIlTRh3mKsjByJj/Tb+pwacFiE8UlVlVREPkFKhvZYItAz2xR - ehTW2EDqalpOo29AJ9/2309nFWlCsJsASImZpqi2rKniZFyG2Yyuhh3AmUBD61cC - rDRGgKyy0bxJpP9XK1H0MvwRL1ZcUbPoPtvSq8lcNJb1YjXnAoIBAQC8FAGYYJom - LVnOhtrpo4CRVpVKbWN9CgOs+pfAarONw4EZasQ5fGAxSWAttrSF1niDA0E7ISro - iblA5t5F/VIPQTEhOWXc7ORPVno+gHLQYttkzRj/zwMDFc4NnzImr9kMEKiNfjKN - B+RpX9LI3At2ZEdvoKzetffVGgo4Ki4IHOSWTwtP6956OvrM4BVc9+XzHeuoboc3 - KGkCV//QsrfXPYxukZc4ClZyAafh+0Kf+kLSl3GPYQgmFIL/XLjCaqakPWyvrZ3L - O2YmTTALeB0JugNyiyrdJO5cKagVCTakdRntBZZZQYwN/2NedDQTJGZ5olBYW9ke - CQ1/3XGtLncRAoIBAQCkyMgb4XzgDyFEMoxogRIVDbjH+oYnUkxKs+eY237cJDnw - Y6W/iq2ss9HAPuy16ORU+kexJQHrAROgRn3ZLJivgIezr0L7Dh9m/awuAghiz4Fo - hy2xujFAnHxNZIYHmLzTkVysDGnppbkIlEwqsl7EyKlxY73Z9FVxF5+uPumK7+e9 - 1MWYcrz4lUJsvXxccoJsk/ksTpzXnluRw7CIMbkYhYWD5BFqeKtVuLet5XGdu+k1 - 5a/g8bmhjvZvQQ2SvadMCD4CO9TOI3zseH1zP7ym+aP0nRhmPvnuTJrhQUkom2pi - XnAFqG8I+pk///cT8MNisI1TYq74dDPdMHODNpIVAoIBAQCWjjVQs1PyZ69SYiVv - jhDnRzYu5C4V9CRJHjjIMSIWw3/extbZZbTi/2fccDtGQA8sYTigisZ8a3M+fxFD - WauMxpT7kaT3GzMQR5uH2O8P3XLe2EfAzwgMOjI+OUe+LFpFDDjBO1pgBwdTSv/D - BlKurUt6ElcP2P7ROg0gNLOwjuB1akUV7yyjD/+gbWjsiQ0sdrEknYXkI5cb5zHC - 8K21wXFfOXChzbSxgp3PioEk6wL+7XhcXEJXqWfL33+PMiy/oOQWJZcBD0WQN43U - QIyOMrd7w+fGK0AbysTgQgXZ1FuynxxhytTAh8YV4FOnVyYpDEPg13WQU5LY3n2X - MnDhAoIBAQChbx/vHwgNP8yfKDmIRmN94DuXnYXj+7peKJJm7xr1laf1kTDOne2T - x1BOHctLhwHlEXt2YjU0nkJ3ownpPNiCuGmt+IoDapwyksCMziiZvuom4+jINU3z - btS7MEKsjJO6OmTXyLNVaGWp1AOIdyyO6yQ4iSnXv7/ndi+Rdrdh6USeUgV9oxmi - 5qc2Peh7SXRkE5uof9SxVSoR8ieHkHAjHXacddjhccGQOQ7j098fX6tWjPuD52pQ - EeGNdPWeKdxrQp2VjCkownXnU+lW4v/yo0oJNRdc+DDdvd5ERK3jGFNrpyo3YksF - HAjEMoifkadfTUWDzUaCKwUMCUdBvyb8 - -----END PRIVATE KEY----- ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: mod-example - namespace: openshift-kmm -data: - dockerfile: | - ARG DTK_AUTO - FROM $DTK_AUTO - - RUN ln -s /usr/bin/true /bin/modprobe ---- -apiVersion: hub.kmm.sigs.x-k8s.io/v1beta1 -kind: ManagedClusterModule -metadata: - name: mod-example -spec: - spokeNamespace: openshift-kmm - selector: - name: local-cluster - moduleSpec: - moduleLoader: - container: - modprobe: - moduleName: mod-example - imagePullPolicy: Always - kernelMappings: - - regexp: '^.+$' - containerImage: image-registry.openshift-image-registry.svc:5000/$MOD_NAMESPACE/$MOD_NAME:$KERNEL_FULL_VERSION - registryTLS: - insecure: true - build: - baseImageRegistryTLS: - insecure: true - dockerfileConfigMap: - name: mod-example - sign: - keySecret: - name: fake-modprobe-signing-key - certSecret: - name: fake-modprobe-signing-key-pub - unsignedImageRegistryTLS: - insecure: true - filesToSign: - - /bin/getconf - selector: - node-role.kubernetes.io/worker: "" diff --git a/ci/prow/e2e-hub-spoke-incluster-build b/ci/prow/e2e-hub-spoke-incluster-build index 6b5e8f6b0..18bd7b0bd 100755 --- a/ci/prow/e2e-hub-spoke-incluster-build +++ b/ci/prow/e2e-hub-spoke-incluster-build @@ -8,6 +8,7 @@ readonly SKIP_ACM_INSTALLATION="${SKIP_ACM_INSTALLATION:-false}" export CLUSTER_NAME=local-cluster export OPERATOR_NAMESPACE=openshift-kmm +export HUB_OPERATOR_NAMESPACE=openshift-kmm-hub export ACM_NAMESPACE=advanced-cluster-management @@ -37,8 +38,8 @@ if [ "$SKIP_MAKE_DEPLOY_HUB" == true ]; then echo "Skipping KMM-Hub deployment" else echo "Deploy KMMO-Hub..." - make deploy-hub KUSTOMIZE_CONFIG_HUB_DEFAULT=ci/install-ci-hub - oc wait --for=condition=Available --timeout=1m deployment/kmm-operator-hub-controller deployment/kmm-operator-hub-webhook-server -n ${OPERATOR_NAMESPACE} + make deploy-hub + oc wait --for=condition=Available --timeout=1m deployment/kmm-operator-hub-controller deployment/kmm-operator-hub-webhook-server -n ${HUB_OPERATOR_NAMESPACE} fi if [ "$SKIP_MAKE_DEPLOY" == true ]; then @@ -49,18 +50,18 @@ else oc wait --for=condition=Available --timeout=1m deployment/kmm-operator-controller deployment/kmm-operator-webhook-server -n ${OPERATOR_NAMESPACE} fi -# Apply the ManagedClusterModule -oc apply -f ci/managedclustermodule-kmm-ci-build-sign.yaml +# Apply resources +oc apply -k ci/e2e-hub # Wait for the build build to be created and completed -timeout 1m bash -c 'until oc -n ${OPERATOR_NAMESPACE} get builds -o json | jq -er ".items[].metadata.name | select(.? | match(\"build\"))"; do sleep 1; done' -export build_build=$(oc -n ${OPERATOR_NAMESPACE} get builds -o json | jq -r '.items[].metadata.name | select(.? | match("build"))') -oc -n ${OPERATOR_NAMESPACE} wait --for=condition=complete --timeout=3m build/${build_build} +timeout 1m bash -c 'until oc -n ${HUB_OPERATOR_NAMESPACE} get builds -o json | jq -er ".items[].metadata.name | select(.? | match(\"build\"))"; do sleep 1; done' +export build_build=$(oc -n ${HUB_OPERATOR_NAMESPACE} get builds -o json | jq -r '.items[].metadata.name | select(.? | match("build"))') +oc -n ${HUB_OPERATOR_NAMESPACE} wait --for=condition=complete --timeout=3m build/${build_build} # Wait for the sign build to be created and completed -timeout 1m bash -c 'until oc -n ${OPERATOR_NAMESPACE} get builds -o json | jq -er ".items[].metadata.name | select(.? | match(\"sign\"))"; do sleep 1; done' -export sign_build=$(oc -n ${OPERATOR_NAMESPACE} get builds -o json | jq -r '.items[].metadata.name | select(.? | match("sign"))') -oc -n ${OPERATOR_NAMESPACE} wait --for=condition=complete --timeout=1m build/${sign_build} +timeout 1m bash -c 'until oc -n ${HUB_OPERATOR_NAMESPACE} get builds -o json | jq -er ".items[].metadata.name | select(.? | match(\"sign\"))"; do sleep 1; done' +export sign_build=$(oc -n ${HUB_OPERATOR_NAMESPACE} get builds -o json | jq -r '.items[].metadata.name | select(.? | match("sign"))') +oc -n ${HUB_OPERATOR_NAMESPACE} wait --for=condition=complete --timeout=1m build/${sign_build} # Waiting for the manifestwork to be created timeout 1m bash -c 'until oc -n ${CLUSTER_NAME} get manifestwork/mod-example; do sleep 1; done' diff --git a/ci/prow/e2e-incluster-build b/ci/prow/e2e-incluster-build index 343c07aa4..89fda2cc4 100755 --- a/ci/prow/e2e-incluster-build +++ b/ci/prow/e2e-incluster-build @@ -38,19 +38,10 @@ oc debug "node/${NODE}" -- chroot host/ sudo modprobe dummy echo "Verify that dummy is loaded" oc debug "node/${NODE}" -- chroot host/ lsmod | grep dummy -echo "Create a build secret..." -oc create secret generic build-secret --from-literal=ci-build-secret=super-secret-value - -echo "Add a configmap that contain the kernel module build dockerfile..." -oc apply -f ci/kmm-kmod-dockerfile.yaml - -echo "Add Secrets containing the signing key and certificate..." -oc apply -f ci/secret_kmm-kmod-signing.yaml - oc wait --for=condition=Available deployment/kmm-operator-controller deployment/kmm-operator-webhook-server -n openshift-kmm -echo "Add an kmm-ci Module that contains a valid mapping..." -oc apply -f ci/module-kmm-ci-build-sign.yaml +echo "Add resources" +oc apply -k ci/e2e # Wait for the build pod to be created. `kubectl wait` doesn't support such option, # see https://github.com/kubernetes/kubernetes/issues/83242. @@ -75,7 +66,7 @@ oc debug "node/${NODE}" -- chroot host/ lsmod | grep kmm_ci_b check_module_not_loaded "dummy" echo "Remove the Module..." -oc delete -f ci/module-kmm-ci-build-sign.yaml --wait=false +oc delete modules.kmm.sigs.x-k8s.io/kmm-ci --wait=false echo "Check that the module gets unloaded from the node..." timeout 1m bash -c 'until ! oc debug node/${NODE} -- chroot host/ lsmod | grep kmm_ci_a; do sleep 3; done' diff --git a/ci/secret_kmm-kmod-signing.yaml b/ci/secret_kmm-kmod-signing.yaml deleted file mode 100644 index 97ebe73a1..000000000 --- a/ci/secret_kmm-kmod-signing.yaml +++ /dev/null @@ -1,97 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: kmm-kmod-signing-cert -type: Opaque -stringData: - cert: | - -----BEGIN CERTIFICATE----- - MIIE8DCCAtigAwIBAgIUSRZJsM/2jzb7fm1PdqB/bjUOAtEwDQYJKoZIhvcNAQEN - BQAwEjEQMA4GA1UEAwwHTW9kdWxlczAgFw0yMzAxMjQxMDM3MzNaGA8yMTIyMTIz - MTEwMzczM1owEjEQMA4GA1UEAwwHTW9kdWxlczCCAiIwDQYJKoZIhvcNAQEBBQAD - ggIPADCCAgoCggIBAImPO0SLpA7l3/tOv/+ZwUm+o7W8c2BtDjmoVjLfWP0nwaV1 - mWJOOxPMfwPwykCUL7s84E+FMNjcd0jZ5xPZD/uYBg3H1JUTfkFhaBDFc9EG+Sbn - HJYt9BBRTeH0pF2wbII7lU2Da7/qIIW9TZ5aH31RsMoPw57DTTMoJkUfp07YKr3a - PgY4Fjjf4haaLSBxzgv+WS88af7NSPFeuSVEGNCyI6Zu56LRJ7s/BlZrD1Q7Y7tD - GlASvK5s2GuNu+i9LD34yEgYK0EqBEdz82FDbic5uKecjvu1Wg+yg32tmQJYTHK9 - joaPe6lvnUmIS1jlfk3qk8Tx44L0vdLeBf+ZMSYrugxvKq/Le7N5TTaKaHxd5pNz - GyHgvh3CN5SdroLAR+36gSvcBzM1uvAzsCRAYKRPult9z7GrLu8cnU7GVXBLIthP - P8mm26aBSYQGycJsUox8/MpEi5X/v1IYTIzpwevG7gruH8cfzvnVD90dwmE4D2nT - o5a9VmDaxbgreXLjCTihguSLqbMcH8l7OGRB1g8E2NgQrYNko9tDaYxP53fQhuZM - KS9zmTcC/EAqOym8f+j20y0/uxTCX5VCBAZzgn2ZvRY6shDXs5zZcFiujRWwzlA5 - 6fpjqpz5eI24GDJ039VXHQ8Gs14h67HgnrLwQSqX/Ml1en+OKkQAViC9FfVXAgMB - AAGjPDA6MAwGA1UdEwEB/wQCMAAwCwYDVR0PBAQDAgeAMB0GA1UdDgQWBBQeRCk5 - b8GGuuc8QWV9PmZb3o638jANBgkqhkiG9w0BAQ0FAAOCAgEATtaTgC84GB5JVaIv - bfS8I68eEk/kCr9MShW7LGUgahUMvsEKZHj4zyXXPAz9udDkx4YE8be/dnz35evG - MwZ5vTb0uncJ+uhBQz1uITyS33cO/GQXXb6wSNZpBNg+VwN1c7L1iQ47VzFiqUeM - rVGblHC0/V95zbL/VeooXuRP/t0VHHxdM1AaNH21+JUGBVAq9BonRH1fx+AZptFg - kkmbefdquS12C1g6phlVnPakKVvlLe8kj3hsnaF7mRQJhKBxicxkJf7gIgbiH34v - SXADqWoD/WDGwHL2u40rEuShEA0lCodRAwKB0lXTFpHMyAFw30zOyXHbOpZyQC3m - BIY69QlIyKXknpQT2bvKeIAPOCsYD7ATt/hMf0aa94INBCBswa9gg/eX8+xQQIJn - DNIFbNtjmbuaFyDt1nT9xmeB2JPKkrpOx5B/BBNJGwK0fAuJa2623fGFuM95z2ug - sCrOlEiodxmMEVPZbCW084dHb3qoy9nTKrCcDQEp/s/5UVkjFe9BHaxlZZQbEGO9 - 2WQEuovQEGSHGGKuGeKrxJ2l5vccHFb6h8mXZbhNCCxom68lTYK7/0DGxPYsS0Vs - C9QOeeHlnKABV9wMzXbFiUccqjBzc8W/j1QKHD9mxFfVk6V/QmgVsViGPcTGnIVJ - 8Fxc2/LOrAZzIL/OFXqZHUy6hW0= - -----END CERTIFICATE----- ---- -apiVersion: v1 -kind: Secret -metadata: - name: kmm-kmod-signing-key -type: Opaque -stringData: - key: | - -----BEGIN PRIVATE KEY----- - MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQCJjztEi6QO5d/7 - Tr//mcFJvqO1vHNgbQ45qFYy31j9J8GldZliTjsTzH8D8MpAlC+7POBPhTDY3HdI - 2ecT2Q/7mAYNx9SVE35BYWgQxXPRBvkm5xyWLfQQUU3h9KRdsGyCO5VNg2u/6iCF - vU2eWh99UbDKD8Oew00zKCZFH6dO2Cq92j4GOBY43+IWmi0gcc4L/lkvPGn+zUjx - XrklRBjQsiOmbuei0Se7PwZWaw9UO2O7QxpQEryubNhrjbvovSw9+MhIGCtBKgRH - c/NhQ24nObinnI77tVoPsoN9rZkCWExyvY6Gj3upb51JiEtY5X5N6pPE8eOC9L3S - 3gX/mTEmK7oMbyqvy3uzeU02imh8XeaTcxsh4L4dwjeUna6CwEft+oEr3AczNbrw - M7AkQGCkT7pbfc+xqy7vHJ1OxlVwSyLYTz/JptumgUmEBsnCbFKMfPzKRIuV/79S - GEyM6cHrxu4K7h/HH8751Q/dHcJhOA9p06OWvVZg2sW4K3ly4wk4oYLki6mzHB/J - ezhkQdYPBNjYEK2DZKPbQ2mMT+d30IbmTCkvc5k3AvxAKjspvH/o9tMtP7sUwl+V - QgQGc4J9mb0WOrIQ17Oc2XBYro0VsM5QOen6Y6qc+XiNuBgydN/VVx0PBrNeIeux - 4J6y8EEql/zJdXp/jipEAFYgvRX1VwIDAQABAoICAAu8O2sqmpFmzQfsfPa6UEIR - UHgVnqYwR5Flqgzm/vmB5cRO1woK8IFM5d0ZMrf1rGjF1cES9MrbbBW7+Tm5P+LP - 4lxUmu8o3ZGIVM9xeOLxPFo0Rz0MqJCnzwQi0c7oWSl3GdKmDpAt0h52uuNyftM7 - H+DtHVwy/ygJGdK105knWeYec0ImkMbDWs9g0c69xr1BMqyT89r3+mwVD6h9Oby8 - JcxyE7LRMxmHU+wVcBZYhraYD+ua3Zvnz0LFsa02TkrvKW71GOOTSRArdSzsCr6v - 7u/lrFEO9KL4ML2NDMulasgzIbCbFwqqafm29e6cjXzqMTcRB77JUFRn+o0Grama - 9o0HR2UH1L/dpob01s1BLOrQT7Z9nwjj7xcTWLeMhrvwRQZ0Gthag6fsyimnt3If - 7SLs6y/4XvLepauu0wk6dVJq7gJjj3O8ZdHjYKobN04PmFHvIaKHdJCZlH9DZD3Q - Dh6qH9duet52BOIaEjRVqq2zP9Ug2rK0gjKxNpprxGkaZL4NcRNXvNSWoP5ClhO8 - 0XszqsKxpYu4SmAya4Yf2cdxT03YNmmy0CwM/m1k4lkLaLBtHJ4qGRqWWnZ4K+Ld - 6UotTcCpufTp/9XqbGa1XS28MOhhoTz8xUCKsEKjomMlAX8D6Jpl3fTty5rN+PBv - KD5it2DatATeniyMPqihAoIBAQC7PLsJ3omdtyB1sE5g6A2FICIu+SpXi7GM+XNA - c278YhAonB47I7wbYgU4So1b5D48Ml7NafD2b2YLrHFNiFSPvVlUMUdIXDlL1wDA - zDJDuuK7rWbXsIwpEnWFM3lx6VAGvQZsmruGLBgsTS3GBw4DSCuBeXDpk+4p8GXc - ih+FZiz+5IYHQ0QuIlTRh3mKsjByJj/Tb+pwacFiE8UlVlVREPkFKhvZYItAz2xR - ehTW2EDqalpOo29AJ9/2309nFWlCsJsASImZpqi2rKniZFyG2Yyuhh3AmUBD61cC - rDRGgKyy0bxJpP9XK1H0MvwRL1ZcUbPoPtvSq8lcNJb1YjXnAoIBAQC8FAGYYJom - LVnOhtrpo4CRVpVKbWN9CgOs+pfAarONw4EZasQ5fGAxSWAttrSF1niDA0E7ISro - iblA5t5F/VIPQTEhOWXc7ORPVno+gHLQYttkzRj/zwMDFc4NnzImr9kMEKiNfjKN - B+RpX9LI3At2ZEdvoKzetffVGgo4Ki4IHOSWTwtP6956OvrM4BVc9+XzHeuoboc3 - KGkCV//QsrfXPYxukZc4ClZyAafh+0Kf+kLSl3GPYQgmFIL/XLjCaqakPWyvrZ3L - O2YmTTALeB0JugNyiyrdJO5cKagVCTakdRntBZZZQYwN/2NedDQTJGZ5olBYW9ke - CQ1/3XGtLncRAoIBAQCkyMgb4XzgDyFEMoxogRIVDbjH+oYnUkxKs+eY237cJDnw - Y6W/iq2ss9HAPuy16ORU+kexJQHrAROgRn3ZLJivgIezr0L7Dh9m/awuAghiz4Fo - hy2xujFAnHxNZIYHmLzTkVysDGnppbkIlEwqsl7EyKlxY73Z9FVxF5+uPumK7+e9 - 1MWYcrz4lUJsvXxccoJsk/ksTpzXnluRw7CIMbkYhYWD5BFqeKtVuLet5XGdu+k1 - 5a/g8bmhjvZvQQ2SvadMCD4CO9TOI3zseH1zP7ym+aP0nRhmPvnuTJrhQUkom2pi - XnAFqG8I+pk///cT8MNisI1TYq74dDPdMHODNpIVAoIBAQCWjjVQs1PyZ69SYiVv - jhDnRzYu5C4V9CRJHjjIMSIWw3/extbZZbTi/2fccDtGQA8sYTigisZ8a3M+fxFD - WauMxpT7kaT3GzMQR5uH2O8P3XLe2EfAzwgMOjI+OUe+LFpFDDjBO1pgBwdTSv/D - BlKurUt6ElcP2P7ROg0gNLOwjuB1akUV7yyjD/+gbWjsiQ0sdrEknYXkI5cb5zHC - 8K21wXFfOXChzbSxgp3PioEk6wL+7XhcXEJXqWfL33+PMiy/oOQWJZcBD0WQN43U - QIyOMrd7w+fGK0AbysTgQgXZ1FuynxxhytTAh8YV4FOnVyYpDEPg13WQU5LY3n2X - MnDhAoIBAQChbx/vHwgNP8yfKDmIRmN94DuXnYXj+7peKJJm7xr1laf1kTDOne2T - x1BOHctLhwHlEXt2YjU0nkJ3ownpPNiCuGmt+IoDapwyksCMziiZvuom4+jINU3z - btS7MEKsjJO6OmTXyLNVaGWp1AOIdyyO6yQ4iSnXv7/ndi+Rdrdh6USeUgV9oxmi - 5qc2Peh7SXRkE5uof9SxVSoR8ieHkHAjHXacddjhccGQOQ7j098fX6tWjPuD52pQ - EeGNdPWeKdxrQp2VjCkownXnU+lW4v/yo0oJNRdc+DDdvd5ERK3jGFNrpyo3YksF - HAjEMoifkadfTUWDzUaCKwUMCUdBvyb8 - -----END PRIVATE KEY----- diff --git a/ci/sign-key-certs/kustomization.yaml b/ci/sign-key-certs/kustomization.yaml new file mode 100644 index 000000000..610755582 --- /dev/null +++ b/ci/sign-key-certs/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +generatorOptions: + disableNameSuffixHash: true + +secretGenerator: + - name: kmm-kmod-signing-cert + files: [cert=signing-cert.pem] + - name: kmm-kmod-signing-key + files: [key=signing-key.pem] diff --git a/ci/sign-key-certs/signing-cert.pem b/ci/sign-key-certs/signing-cert.pem new file mode 100644 index 000000000..6939f9178 --- /dev/null +++ b/ci/sign-key-certs/signing-cert.pem @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE8DCCAtigAwIBAgIUSRZJsM/2jzb7fm1PdqB/bjUOAtEwDQYJKoZIhvcNAQEN +BQAwEjEQMA4GA1UEAwwHTW9kdWxlczAgFw0yMzAxMjQxMDM3MzNaGA8yMTIyMTIz +MTEwMzczM1owEjEQMA4GA1UEAwwHTW9kdWxlczCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBAImPO0SLpA7l3/tOv/+ZwUm+o7W8c2BtDjmoVjLfWP0nwaV1 +mWJOOxPMfwPwykCUL7s84E+FMNjcd0jZ5xPZD/uYBg3H1JUTfkFhaBDFc9EG+Sbn +HJYt9BBRTeH0pF2wbII7lU2Da7/qIIW9TZ5aH31RsMoPw57DTTMoJkUfp07YKr3a +PgY4Fjjf4haaLSBxzgv+WS88af7NSPFeuSVEGNCyI6Zu56LRJ7s/BlZrD1Q7Y7tD +GlASvK5s2GuNu+i9LD34yEgYK0EqBEdz82FDbic5uKecjvu1Wg+yg32tmQJYTHK9 +joaPe6lvnUmIS1jlfk3qk8Tx44L0vdLeBf+ZMSYrugxvKq/Le7N5TTaKaHxd5pNz +GyHgvh3CN5SdroLAR+36gSvcBzM1uvAzsCRAYKRPult9z7GrLu8cnU7GVXBLIthP +P8mm26aBSYQGycJsUox8/MpEi5X/v1IYTIzpwevG7gruH8cfzvnVD90dwmE4D2nT +o5a9VmDaxbgreXLjCTihguSLqbMcH8l7OGRB1g8E2NgQrYNko9tDaYxP53fQhuZM +KS9zmTcC/EAqOym8f+j20y0/uxTCX5VCBAZzgn2ZvRY6shDXs5zZcFiujRWwzlA5 +6fpjqpz5eI24GDJ039VXHQ8Gs14h67HgnrLwQSqX/Ml1en+OKkQAViC9FfVXAgMB +AAGjPDA6MAwGA1UdEwEB/wQCMAAwCwYDVR0PBAQDAgeAMB0GA1UdDgQWBBQeRCk5 +b8GGuuc8QWV9PmZb3o638jANBgkqhkiG9w0BAQ0FAAOCAgEATtaTgC84GB5JVaIv +bfS8I68eEk/kCr9MShW7LGUgahUMvsEKZHj4zyXXPAz9udDkx4YE8be/dnz35evG +MwZ5vTb0uncJ+uhBQz1uITyS33cO/GQXXb6wSNZpBNg+VwN1c7L1iQ47VzFiqUeM +rVGblHC0/V95zbL/VeooXuRP/t0VHHxdM1AaNH21+JUGBVAq9BonRH1fx+AZptFg +kkmbefdquS12C1g6phlVnPakKVvlLe8kj3hsnaF7mRQJhKBxicxkJf7gIgbiH34v +SXADqWoD/WDGwHL2u40rEuShEA0lCodRAwKB0lXTFpHMyAFw30zOyXHbOpZyQC3m +BIY69QlIyKXknpQT2bvKeIAPOCsYD7ATt/hMf0aa94INBCBswa9gg/eX8+xQQIJn +DNIFbNtjmbuaFyDt1nT9xmeB2JPKkrpOx5B/BBNJGwK0fAuJa2623fGFuM95z2ug +sCrOlEiodxmMEVPZbCW084dHb3qoy9nTKrCcDQEp/s/5UVkjFe9BHaxlZZQbEGO9 +2WQEuovQEGSHGGKuGeKrxJ2l5vccHFb6h8mXZbhNCCxom68lTYK7/0DGxPYsS0Vs +C9QOeeHlnKABV9wMzXbFiUccqjBzc8W/j1QKHD9mxFfVk6V/QmgVsViGPcTGnIVJ +8Fxc2/LOrAZzIL/OFXqZHUy6hW0= +-----END CERTIFICATE----- diff --git a/ci/sign-key-certs/signing-key.pem b/ci/sign-key-certs/signing-key.pem new file mode 100644 index 000000000..3107ae2cd --- /dev/null +++ b/ci/sign-key-certs/signing-key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQCJjztEi6QO5d/7 +Tr//mcFJvqO1vHNgbQ45qFYy31j9J8GldZliTjsTzH8D8MpAlC+7POBPhTDY3HdI +2ecT2Q/7mAYNx9SVE35BYWgQxXPRBvkm5xyWLfQQUU3h9KRdsGyCO5VNg2u/6iCF +vU2eWh99UbDKD8Oew00zKCZFH6dO2Cq92j4GOBY43+IWmi0gcc4L/lkvPGn+zUjx +XrklRBjQsiOmbuei0Se7PwZWaw9UO2O7QxpQEryubNhrjbvovSw9+MhIGCtBKgRH +c/NhQ24nObinnI77tVoPsoN9rZkCWExyvY6Gj3upb51JiEtY5X5N6pPE8eOC9L3S +3gX/mTEmK7oMbyqvy3uzeU02imh8XeaTcxsh4L4dwjeUna6CwEft+oEr3AczNbrw +M7AkQGCkT7pbfc+xqy7vHJ1OxlVwSyLYTz/JptumgUmEBsnCbFKMfPzKRIuV/79S +GEyM6cHrxu4K7h/HH8751Q/dHcJhOA9p06OWvVZg2sW4K3ly4wk4oYLki6mzHB/J +ezhkQdYPBNjYEK2DZKPbQ2mMT+d30IbmTCkvc5k3AvxAKjspvH/o9tMtP7sUwl+V +QgQGc4J9mb0WOrIQ17Oc2XBYro0VsM5QOen6Y6qc+XiNuBgydN/VVx0PBrNeIeux +4J6y8EEql/zJdXp/jipEAFYgvRX1VwIDAQABAoICAAu8O2sqmpFmzQfsfPa6UEIR +UHgVnqYwR5Flqgzm/vmB5cRO1woK8IFM5d0ZMrf1rGjF1cES9MrbbBW7+Tm5P+LP +4lxUmu8o3ZGIVM9xeOLxPFo0Rz0MqJCnzwQi0c7oWSl3GdKmDpAt0h52uuNyftM7 +H+DtHVwy/ygJGdK105knWeYec0ImkMbDWs9g0c69xr1BMqyT89r3+mwVD6h9Oby8 +JcxyE7LRMxmHU+wVcBZYhraYD+ua3Zvnz0LFsa02TkrvKW71GOOTSRArdSzsCr6v +7u/lrFEO9KL4ML2NDMulasgzIbCbFwqqafm29e6cjXzqMTcRB77JUFRn+o0Grama +9o0HR2UH1L/dpob01s1BLOrQT7Z9nwjj7xcTWLeMhrvwRQZ0Gthag6fsyimnt3If +7SLs6y/4XvLepauu0wk6dVJq7gJjj3O8ZdHjYKobN04PmFHvIaKHdJCZlH9DZD3Q +Dh6qH9duet52BOIaEjRVqq2zP9Ug2rK0gjKxNpprxGkaZL4NcRNXvNSWoP5ClhO8 +0XszqsKxpYu4SmAya4Yf2cdxT03YNmmy0CwM/m1k4lkLaLBtHJ4qGRqWWnZ4K+Ld +6UotTcCpufTp/9XqbGa1XS28MOhhoTz8xUCKsEKjomMlAX8D6Jpl3fTty5rN+PBv +KD5it2DatATeniyMPqihAoIBAQC7PLsJ3omdtyB1sE5g6A2FICIu+SpXi7GM+XNA +c278YhAonB47I7wbYgU4So1b5D48Ml7NafD2b2YLrHFNiFSPvVlUMUdIXDlL1wDA +zDJDuuK7rWbXsIwpEnWFM3lx6VAGvQZsmruGLBgsTS3GBw4DSCuBeXDpk+4p8GXc +ih+FZiz+5IYHQ0QuIlTRh3mKsjByJj/Tb+pwacFiE8UlVlVREPkFKhvZYItAz2xR +ehTW2EDqalpOo29AJ9/2309nFWlCsJsASImZpqi2rKniZFyG2Yyuhh3AmUBD61cC +rDRGgKyy0bxJpP9XK1H0MvwRL1ZcUbPoPtvSq8lcNJb1YjXnAoIBAQC8FAGYYJom +LVnOhtrpo4CRVpVKbWN9CgOs+pfAarONw4EZasQ5fGAxSWAttrSF1niDA0E7ISro +iblA5t5F/VIPQTEhOWXc7ORPVno+gHLQYttkzRj/zwMDFc4NnzImr9kMEKiNfjKN +B+RpX9LI3At2ZEdvoKzetffVGgo4Ki4IHOSWTwtP6956OvrM4BVc9+XzHeuoboc3 +KGkCV//QsrfXPYxukZc4ClZyAafh+0Kf+kLSl3GPYQgmFIL/XLjCaqakPWyvrZ3L +O2YmTTALeB0JugNyiyrdJO5cKagVCTakdRntBZZZQYwN/2NedDQTJGZ5olBYW9ke +CQ1/3XGtLncRAoIBAQCkyMgb4XzgDyFEMoxogRIVDbjH+oYnUkxKs+eY237cJDnw +Y6W/iq2ss9HAPuy16ORU+kexJQHrAROgRn3ZLJivgIezr0L7Dh9m/awuAghiz4Fo +hy2xujFAnHxNZIYHmLzTkVysDGnppbkIlEwqsl7EyKlxY73Z9FVxF5+uPumK7+e9 +1MWYcrz4lUJsvXxccoJsk/ksTpzXnluRw7CIMbkYhYWD5BFqeKtVuLet5XGdu+k1 +5a/g8bmhjvZvQQ2SvadMCD4CO9TOI3zseH1zP7ym+aP0nRhmPvnuTJrhQUkom2pi +XnAFqG8I+pk///cT8MNisI1TYq74dDPdMHODNpIVAoIBAQCWjjVQs1PyZ69SYiVv +jhDnRzYu5C4V9CRJHjjIMSIWw3/extbZZbTi/2fccDtGQA8sYTigisZ8a3M+fxFD +WauMxpT7kaT3GzMQR5uH2O8P3XLe2EfAzwgMOjI+OUe+LFpFDDjBO1pgBwdTSv/D +BlKurUt6ElcP2P7ROg0gNLOwjuB1akUV7yyjD/+gbWjsiQ0sdrEknYXkI5cb5zHC +8K21wXFfOXChzbSxgp3PioEk6wL+7XhcXEJXqWfL33+PMiy/oOQWJZcBD0WQN43U +QIyOMrd7w+fGK0AbysTgQgXZ1FuynxxhytTAh8YV4FOnVyYpDEPg13WQU5LY3n2X +MnDhAoIBAQChbx/vHwgNP8yfKDmIRmN94DuXnYXj+7peKJJm7xr1laf1kTDOne2T +x1BOHctLhwHlEXt2YjU0nkJ3ownpPNiCuGmt+IoDapwyksCMziiZvuom4+jINU3z +btS7MEKsjJO6OmTXyLNVaGWp1AOIdyyO6yQ4iSnXv7/ndi+Rdrdh6USeUgV9oxmi +5qc2Peh7SXRkE5uof9SxVSoR8ieHkHAjHXacddjhccGQOQ7j098fX6tWjPuD52pQ +EeGNdPWeKdxrQp2VjCkownXnU+lW4v/yo0oJNRdc+DDdvd5ERK3jGFNrpyo3YksF +HAjEMoifkadfTUWDzUaCKwUMCUdBvyb8 +-----END PRIVATE KEY-----