Allow passing of install task variables to downstream tasks of a dag

dags/openshift_nightlies/scripts/install/ocm_gcp.sh

@@ -103,6 +103,16 @@ postinstall(){
     gcloud compute firewall-rules create ${NETWORK_NAME}-hostnet --network ${NETWORK_NAME} --priority 105 --description 'scale-ci allow tcp,udp hostnetwork tests' --rules tcp:32768-60999,udp:32768-60999 --action allow
 }
 
+display_install_data(){
+    IFS='@'
+    # Read the xcom pushed by install task and store it into an array based on '@' as delimiter 
+    read -ra newarr <<< "$Install_vars"
+    clustername="${newarr[0]}"
+    installuuid="${newarr[1]}"
+    echo "Cluster Name = ${clustername}"
+    echo "Install UUID = ${installuuid}"
+}
+
 cleanup(){
     ocm delete cluster $(_get_cluster_id ${CLUSTER_NAME})
     ocm logout
@@ -144,8 +154,10 @@ if [[ "$operation" == "install" ]]; then
         printf "INFO: Cluster ${CLUSTER_NAME} already installed but not ready, exiting..."
 	exit 1
     fi
+    echo "${CLUSTER_NAME}@${UUID}"
 
 elif [[ "$operation" == "cleanup" ]]; then
     printf "Running Cleanup Steps"
+    display_install_data
     cleanup
 fi

dags/openshift_nightlies/scripts/install/rosa.sh

@@ -338,6 +338,16 @@ EOF
     return 0
 }
 
+display_install_data(){
+    IFS='@'
+    # Read the xcom pushed by install task and store it into an array based on '@' as delimiter 
+    read -ra newarr <<< "$Install_vars"
+    clustername="${newarr[0]}"
+    installuuid="${newarr[1]}"
+    echo "Cluster Name = ${clustername}"
+    echo "Install UUID = ${installuuid}"
+}
+
 cleanup(){
     if [[ $INSTALL_METHOD == "osd" ]]; then
         ocm delete cluster $(_get_cluster_id ${CLUSTER_NAME})
@@ -377,9 +387,11 @@ if [[ "$operation" == "install" ]]; then
         printf "INFO: Cluster ${CLUSTER_NAME} already installed but not ready, exiting..."
 	    exit 1
     fi
+    echo "${CLUSTER_NAME}@${UUID}"
 
 elif [[ "$operation" == "cleanup" ]]; then
     printf "Running Cleanup Steps"
+    display_install_data
     cleanup
     index_metadata
     rosa logout

dags/openshift_nightlies/scripts/run_benchmark.sh

@@ -70,6 +70,19 @@ EOF
         exit 1
     fi
 }
+
+display_install_data(){    
+    if [[ $PLATFORM == "rosa" || $PLATFORM == "rogcp" ]]; then
+        IFS='@'
+        # Read the xcom pushed by install task and store it into an array based on '@' as delimiter 
+        read -ra newarr <<< "$Install_vars"
+        clustername="${newarr[0]}"
+        installuuid="${newarr[1]}"
+        echo "Cluster Name = ${clustername}"
+        echo "Install UUID = ${installuuid}"
+    fi
+}
+
 export UUID=$(uuidgen | head -c8)-$AIRFLOW_CTX_TASK_ID-$(date '+%Y%m%d')
 echo "############################################"
 echo "# Benchmark UUID: ${UUID}"
@@ -80,6 +93,7 @@ if [[ $PLATFORM == "baremetal" ]]; then
     run_baremetal_benchmark
     echo $UUID
 else
+    display_install_data
     setup
     cd /home/airflow/workspace/e2e-benchmarking/workloads/$workload
 

dags/openshift_nightlies/tasks/benchmarks/e2e.py

@@ -112,7 +112,7 @@ def _add_indexer(self, benchmark):
         benchmark >> indexer 
 
     def _get_benchmark(self, benchmark):
-        env = {**self.env, **benchmark.get('env', {}), **{"ES_SERVER": var_loader.get_secret('elasticsearch'), "KUBEADMIN_PASSWORD": environ.get("KUBEADMIN_PASSWORD", "")}}
+        env = {**self.env, "Install_vars": '{{ ti.xcom_pull(task_ids="install")}}', **benchmark.get('env', {}), **{"ES_SERVER": var_loader.get_secret('elasticsearch'), "KUBEADMIN_PASSWORD": environ.get("KUBEADMIN_PASSWORD", "")}}
         # Fetch variables from a secret with the name <DAG_NAME>-<TASK_NAME>
         task_variables = var_loader.get_secret(f"{self.dag.dag_id}-{benchmark['name']}", True, False)
         env.update(task_variables)

dags/openshift_nightlies/tasks/install/rogcp/rogcp.py

@@ -26,6 +26,7 @@ def __init__(self, dag, config: DagConfig, release: OpenshiftRelease):
     def _get_task(self, operation="install", trigger_rule="all_success"):
         self._setup_task(operation=operation)
         command=f"{constants.root_dag_dir}/scripts/install/ocm_gcp.sh -v {self.release.version} -j /tmp/{self.release_name}-{operation}-task.json -o {operation}"
+        env={ **self.env , "Install_vars": '{{ ti.xcom_pull(task_ids="install")}}'}
         return BashOperator(
             task_id=f"{operation}",
             depends_on_past=False,
@@ -34,5 +35,5 @@ def _get_task(self, operation="install", trigger_rule="all_success"):
             dag=self.dag,
             trigger_rule=trigger_rule,
             executor_config=self.exec_config,
-            env=self.env
+            env=env
         )

dags/openshift_nightlies/tasks/install/rosa/rosa.py

@@ -26,7 +26,7 @@ def __init__(self, dag, config: DagConfig, release: OpenshiftRelease):
     def _get_task(self, operation="install", trigger_rule="all_success"):
         self._setup_task(operation=operation)
         command=f"{constants.root_dag_dir}/scripts/install/rosa.sh -v {self.release.version} -j /tmp/{self.release_name}-{operation}-task.json -o {operation}"
-
+        env={ **self.env , "Install_vars": '{{ ti.xcom_pull(task_ids="install")}}'}
         return BashOperator(
             task_id=f"{operation}",
             depends_on_past=False,
@@ -35,5 +35,5 @@ def _get_task(self, operation="install", trigger_rule="all_success"):
             dag=self.dag,
             trigger_rule=trigger_rule,
             executor_config=self.exec_config,
-            env=self.env
+            env=env
         )

dags/openshift_nightlies/tasks/utils/final_dag_status.py

@@ -8,6 +8,7 @@
 
 
 def final_status(**kwargs):
+    ti = kwargs['ti']
     failed_tasks=[]
     for task_instance in kwargs['dag_run'].get_task_instances():
         if "index" in task_instance.task_id: