enable private-link for rosa (openshift#43430)

update update update update update update update update
sebsoto · Sep 19, 2023 · 1990b66 · 1990b66
1 parent 3b17ea0
commit 1990b66
Show file tree

Hide file tree

Showing 11 changed files with 212 additions and 47 deletions.
diff --git a/.../openshift/openshift-tests-private/openshift-openshift-tests-private-master__rosacli.yaml b/.../openshift/openshift-tests-private/openshift-openshift-tests-private-master__rosacli.yaml
@@ -47,6 +47,14 @@ tests:
       CHANNEL_GROUP: stable
       ROSA_LOGIN_ENV: staging
     workflow: rosa-aws-sts-advanced
+- as: aws-rosa-private-link-f7
+  cron: 24 2 4,11,18,26 * *
+  steps:
+    cluster_profile: aws-qe
+    env:
+      CHANNEL_GROUP: stable
+      ROSA_LOGIN_ENV: staging
+    workflow: rosa-aws-sts-private-link
 - as: aws-rosa-hypershift-advanced-f3
   cron: 32 0 3,6,9,12,14,18,21,24,27,30 * *
   steps:

diff --git a/...openshift/openshift-tests-private/openshift-openshift-tests-private-master-periodics.yaml b/...openshift/openshift-tests-private/openshift-openshift-tests-private-master-periodics.yaml
@@ -183,6 +183,98 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build05
+  cron: 24 2 4,11,18,26 * *
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: master
+    org: openshift
+    repo: openshift-tests-private
+  labels:
+    ci-operator.openshift.io/cloud: aws
+    ci-operator.openshift.io/cloud-cluster-profile: aws-qe
+    ci-operator.openshift.io/variant: rosacli
+    ci.openshift.io/generator: prowgen
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-openshift-tests-private-master-rosacli-aws-rosa-private-link-f7
+  reporter_config:
+    slack:
+      channel: '#ocm-qe-prow-ci-jobs'
+      job_states_to_report:
+      - failure
+      - error
+      - success
+      report_template: '{{if eq .Status.State "success"}} :rainbow: Job *{{.Spec.Job}}*
+        ended with *{{.Status.State}}*. <{{.Status.URL}}|View logs> :rainbow: {{else}}
+        :volcano: Job *{{.Spec.Job}}* ended with *{{.Status.State}}*. <{{.Status.URL}}|View
+        logs> :volcano: {{end}}'
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --oauth-token-path=/usr/local/github-credentials/oauth
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --secret-dir=/usr/local/aws-rosa-private-link-f7-cluster-profile
+      - --target=aws-rosa-private-link-f7
+      - --variant=rosacli
+      command:
+      - ci-operator
+      image: ci-operator:latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /usr/local/aws-rosa-private-link-f7-cluster-profile
+        name: cluster-profile
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /usr/local/github-credentials
+        name: github-credentials-openshift-ci-robot-private-git-cloner
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: cluster-profile
+      secret:
+        secretName: cluster-secrets-aws-qe
+    - name: github-credentials-openshift-ci-robot-private-git-cloner
+      secret:
+        secretName: github-credentials-openshift-ci-robot-private-git-cloner
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build05
   cron: 41 2 2,5,7,11,14,17,20,23,26,29 * *

diff --git a/...r/step-registry/rosa/aws/sts/proxy/OWNERS → ...registry/rosa/aws/sts/private-link/OWNERS b/...r/step-registry/rosa/aws/sts/proxy/OWNERS → ...registry/rosa/aws/sts/private-link/OWNERS
diff --git a/...rosa-aws-sts-proxy-workflow.metadata.json → ...s-sts-private-link-workflow.metadata.json b/...rosa-aws-sts-proxy-workflow.metadata.json → ...s-sts-private-link-workflow.metadata.json
@@ -1,5 +1,5 @@
 {
-	"path": "rosa/aws/sts/proxy/rosa-aws-sts-proxy-workflow.yaml",
+	"path": "rosa/aws/sts/private-link/rosa-aws-sts-private-link-workflow.yaml",
 	"owners": {
 		"approvers": [
 			"yasun1",

diff --git a/ci-operator/step-registry/rosa/aws/sts/private-link/rosa-aws-sts-private-link-workflow.yaml b/ci-operator/step-registry/rosa/aws/sts/private-link/rosa-aws-sts-private-link-workflow.yaml
@@ -0,0 +1,32 @@
+workflow:
+  as: rosa-aws-sts-private-link
+  steps:
+    env:
+      HOSTED_CP: "false"
+      STS: "true"
+      MULTI_AZ: "false"
+      COMPUTE_MACHINE_TYPE: "m5.xlarge"
+      REPLICAS: "2"
+      PRIVATE: "true"
+      PRIVATE_LINK: "true"
+      BYO_OIDC: "true"
+      OIDC_CONFIG_MANAGED: "true"
+      ZONES_COUNT: "1"
+    pre:
+    - ref: aws-provision-vpc-shared
+    - ref: aws-provision-tags-for-byo-vpc-ocm-pre
+    - chain: aws-provision-bastionhost
+    - ref: proxy-config-generate
+    - chain: rosa-sts-oidc-config-create
+    - chain: rosa-cluster-provision
+    - ref: aws-provision-tags-for-byo-vpc
+    - ref: osd-ccs-conf-idp-htpasswd-multi-users
+    - ref: rosa-cluster-wait-ready-nodes
+    post:
+    - chain: rosa-cluster-deprovision
+    - chain: rosa-sts-oidc-config-delete
+    - ref: aws-deprovision-s3buckets
+    - ref: aws-deprovision-stacks
+  documentation: |-
+    This workflow installs a single AZ rosa sts cluster configured to use private-link. The cluster is set with htpasswd idp, and the login informations are stored under $SHARED_DIR/api.login.
+    After finish testing, the cluster will be deprovsioned.
diff --git a/ci-operator/step-registry/rosa/aws/sts/proxy/rosa-aws-sts-proxy-workflow.yaml b/ci-operator/step-registry/rosa/aws/sts/proxy/rosa-aws-sts-proxy-workflow.yaml
diff --git a/ci-operator/step-registry/rosa/cluster/provision/rosa-cluster-provision-commands.sh b/ci-operator/step-registry/rosa/cluster/provision/rosa-cluster-provision-commands.sh
@@ -131,6 +131,17 @@ if [[ "$MULTI_AZ" == "true" ]]; then
   MULTI_AZ_SWITCH="--multi-az"
 fi
 
+COMPUTER_NODE_ZONES_SWITCH=""
+if [[ ! -z "$AVAILABILITY_ZONES" ]]; then
+  AVAILABILITY_ZONES=$(echo $AVAILABILITY_ZONES | sed -E "s|(\w+)|${CLOUD_PROVIDER_REGION}&|g")
+  COMPUTER_NODE_ZONES_SWITCH="--availability-zones ${AVAILABILITY_ZONES}"
+fi
+
+COMPUTER_NODE_DISK_SIZE_SWITCH=""
+if [[ ! -z "$WORKER_DISK_SIZE" ]]; then
+  COMPUTER_NODE_DISK_SIZE_SWITCH="--worker-disk-size ${WORKER_DISK_SIZE}"
+fi
+
 AUDIT_LOG_SWITCH=""
 if [[ "$ENABLE_AUDIT_LOG" == "true" ]]; then
   iam_role_arn=$(head -n 1 ${SHARED_DIR}/iam_role_arn)
@@ -354,6 +365,8 @@ ${DISABLE_SCP_CHECKS_SWITCH} \
 ${DEFAULT_MP_LABELS_SWITCH} \
 ${STORAGE_ENCRYPTION_SWITCH} \
 ${AUDIT_LOG_SWITCH} \
+${COMPUTER_NODE_ZONES_SWITCH} \
+${COMPUTER_NODE_DISK_SIZE_SWITCH} \
 ${DRY_RUN_SWITCH}
 "
 
@@ -389,6 +402,8 @@ rosa create cluster -y \
                     ${DEFAULT_MP_LABELS_SWITCH} \
                     ${STORAGE_ENCRYPTION_SWITCH} \
                     ${AUDIT_LOG_SWITCH} \
+                    ${COMPUTER_NODE_ZONES_SWITCH} \
+                    ${COMPUTER_NODE_DISK_SIZE_SWITCH} \
                     ${DRY_RUN_SWITCH} \
                     > "${CLUSTER_INFO}"
 

diff --git a/ci-operator/step-registry/rosa/cluster/provision/rosa-cluster-provision-ref.yaml b/ci-operator/step-registry/rosa/cluster/provision/rosa-cluster-provision-ref.yaml
@@ -21,15 +21,24 @@ ref:
   - name: STS
     default: "true"
     documentation: If the cluster is deployed with AWS Security Token Service (STS) instead of IAM credentials, it is a STS cluster.
+  - name: HOSTED_CP
+    default: "false"
+    documentation: Enable the use of hosted control planes (HyperShift).
   - name: COMPUTE_MACHINE_TYPE
     default: ""
     documentation: The instance size for compute nodes. If not specified, a default will be chosen appropriate for your cluster_profile.
+  - name: WORKER_DISK_SIZE
+    default: ""
+    documentation: Machine pool root disk size with a **unit suffix** like GiB or TiB, e.g. 200GiB.
   - name: REGION
     default: ""
     documentation: Use a specific AWS region, overriding the LEASED_RESOURCE environment variable in the cluster_profile.
   - name: MULTI_AZ
     default: "false"
     documentation: Set to 'true' if you want to deploy a cluster across muiltiple availability zones.
+  - name: AVAILABILITY_ZONES
+    default: ""
+    documentation: The availability zones to use when installing a non-BYOVPC cluster. Format should be a comma-separated list, etc. 'a,b'.
   - name: REPLICAS
     default: "2"
     documentation: Number of compute nodes to provision. Single zone clusters need at least 2 nodes, multizone clusters need at least 3 nodes. It is only valid while enable_autoscaling is false.
@@ -42,6 +51,9 @@ ref:
   - name: MAX_REPLICAS
     default: "6"
     documentation: The max number of the compute nodes. It is valid when the enable_autoscaling is true. The value should be a multiple of three for multizone clusters. The value must not less that the min_replica.
+  - name: DEFAULT_MACHINE_POOL_LABELS
+    default: ""
+    documentation: Labels for the default machine pool. Format should be a comma-separated list of 'key=value'.
   - name: OPENSHIFT_VERSION
     default: ""
     documentation: The openshift version for rosa to install (e.g. "4.10.12"). Specify a major/minor (e.g. "4.10") to get the latest version from that stream.
@@ -51,51 +63,45 @@ ref:
   - name: EC_BUILD
     default: "false"
     documentation: Set to 'true' to choose the engineer candidate openshift version.
-  - name: EC2_METADATA_HTTP_TOKENS
-    default: "optional"
-    documentation: Configure the use of IMDSv2 for ec2 instances, the supported values are [optional, required].
+  - name: ENABLE_BYOVPC
+    default: "false"
+    documentation: Use the customized VPC to install the cluster.
   - name: ETCD_ENCRYPTION
     default: "false"
     documentation: Add etcd encryption. By default etcd data is encrypted at rest. This option configures etcd encryption on top of existing storage encryption.
   - name: STORAGE_ENCRYPTION
     default: "false"
     documentation: Add storage encryption to encrypt EBS instance volumes with the KMS key.
-  - name: DISABLE_WORKLOAD_MONITORING
-    default: "false"
-    documentation: Enables you to monitor your own projects in isolation from Red Hat Site Reliability Engineer (SRE) platform metrics.
-  - name: FIPS
-    default: "false"
-    documentation: Create cluster that uses FIPS Validated / Modules in Process cryptographic libraries.
   - name: PRIVATE
     default: "false"
     documentation: Restrict master API endpoint and application routes to direct, private connectivity.
   - name: PRIVATE_LINK
     default: "false"
     documentation: Provides private connectivity between VPCs, AWS services, and your on-premises networks, without exposing your traffic to the public internet.
-  - name: CLUSTER_TAGS
-    default: ""
-    documentation: Apply user defined tags to all resources created by ROSA in AWS. Tags are comma separated example - 'foo:bar,bar:baz', The default value is "prowci:${CLUSTER_NAME}".
-  - name: DEFAULT_MACHINE_POOL_LABELS
-    default: ""
-    documentation: Labels for the default machine pool. Format should be a comma-separated list of 'key=value'.
   - name: ENABLE_PROXY
     default: "false"
     documentation: Use proxy to create HTTP/HTTPs connections outside the cluster.
-  - name: ENABLE_BYOVPC
+  - name: BYO_OIDC
     default: "false"
-    documentation: Use the customized VPC to install the cluster.
+    documentation: Use the customized OIDC Config and operator-roles to install the cluster.
   - name: ENABLE_AUDIT_LOG
     default: "false"
     documentation: Enable Forwarding audit logs to AWS CloudWatch.
+  - name: EC2_METADATA_HTTP_TOKENS
+    default: "optional"
+    documentation: Configure the use of IMDSv2 for ec2 instances, the supported values are [optional, required].
+  - name: DISABLE_WORKLOAD_MONITORING
+    default: "false"
+    documentation: Enables you to monitor your own projects in isolation from Red Hat Site Reliability Engineer (SRE) platform metrics.
   - name: DISABLE_SCP_CHECKS
     default: "false"
     documentation: Indicates if cloud permission checks are disabled when attempting installation of the cluster.
-  - name: HOSTED_CP
-    default: "false"
-    documentation: Enable the use of hosted control planes (HyperShift).
-  - name: BYO_OIDC
+  - name: FIPS
     default: "false"
-    documentation: Use the customized OIDC Config and operator-roles to install the cluster.
+    documentation: Create cluster that uses FIPS Validated / Modules in Process cryptographic libraries.
+  - name: CLUSTER_TAGS
+    default: ""
+    documentation: Apply user defined tags to all resources created by ROSA in AWS. Tags are comma separated example - 'foo:bar,bar:baz', The default value is "prowci:${CLUSTER_NAME}".
   - name: ENABLE_SECTOR
     default: "false"
     documentation: Enable the sector to specify the provision shard to provision the hosted control planes (HyperShift) cluster.

diff --git a/...tor/step-registry/rosa/cluster/wait-ready/nodes/rosa-cluster-wait-ready-nodes-commands.sh b/...tor/step-registry/rosa/cluster/wait-ready/nodes/rosa-cluster-wait-ready-nodes-commands.sh
@@ -10,6 +10,18 @@ log(){
     echo -e "\033[1m$(date "+%d-%m-%YT%H:%M:%S") " "${*}\033[0m"
 }
 
+function set_proxy () {
+    if test -s "${SHARED_DIR}/proxy-conf.sh" ; then
+        echo "setting the proxy"
+        # cat "${SHARED_DIR}/proxy-conf.sh"
+        echo "source ${SHARED_DIR}/proxy-conf.sh"
+        source "${SHARED_DIR}/proxy-conf.sh"
+    else
+        echo "no proxy setting."
+    fi
+}
+set_proxy
+
 # Display only node details 
 function listNodeDetails() {
     echo "List node details"

diff --git a/...-registry/rosa/cluster/wait-ready/operators/rosa-cluster-wait-ready-operators-commands.sh b/...-registry/rosa/cluster/wait-ready/operators/rosa-cluster-wait-ready-operators-commands.sh
@@ -6,6 +6,18 @@ set -o pipefail
 
 trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM
 
+function set_proxy () {
+    if test -s "${SHARED_DIR}/proxy-conf.sh" ; then
+        echo "setting the proxy"
+        # cat "${SHARED_DIR}/proxy-conf.sh"
+        echo "source ${SHARED_DIR}/proxy-conf.sh"
+        source "${SHARED_DIR}/proxy-conf.sh"
+    else
+        echo "no proxy setting."
+    fi
+}
+set_proxy
+
 # Even the cluster is shown ready on ocm side, and the cluster operators are available, some of the cluster operators are
 # still progressing. The ocp e2e test scenarios requires PROGRESSING=False for each cluster operator.
 echo "Wait for cluster operators' progressing ready..."

diff --git a/ci-operator/step-registry/rosa/conf/idp/htpasswd/rosa-conf-idp-htpasswd-commands.sh b/ci-operator/step-registry/rosa/conf/idp/htpasswd/rosa-conf-idp-htpasswd-commands.sh
@@ -8,6 +8,17 @@ trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wa
 
 CLUSTER_ID=$(cat "${SHARED_DIR}/cluster-id")
 
+function set_proxy () {
+    if test -s "${SHARED_DIR}/proxy-conf.sh" ; then
+        echo "setting the proxy"
+        # cat "${SHARED_DIR}/proxy-conf.sh"
+        echo "source ${SHARED_DIR}/proxy-conf.sh"
+        source "${SHARED_DIR}/proxy-conf.sh"
+    else
+        echo "no proxy setting."
+    fi
+}
+
 # Configure aws
 CLOUD_PROVIDER_REGION=${LEASED_RESOURCE}
 AWSCRED="${CLUSTER_PROFILE_DIR}/.awscred"
@@ -66,6 +77,7 @@ echo "oc login ${API_URL} -u ${IDP_USER} -p ${IDP_PASSWD} --insecure-skip-tls-ve
 rosa grant user cluster-admin --user=${IDP_USER} --cluster=${CLUSTER_ID}
 
 echo "Waiting for idp ready..."
+set_proxy
 IDP_LOGIN_LOG="${ARTIFACT_DIR}/htpasswd_login.log"
 start_time=$(date +"%s")
 while true; do