Merge pull request #581 from jgu17/v1_ucp_status_check_during_update

[1.0] Enhance UCP status check for the ucp service update use case

playbooks/roles/airship-deploy-ucp/tasks/main.yml

@@ -246,33 +246,80 @@
 
 # TODO(aagate): Add a changed_when: to help idempotency
 - name: Deploy Airship UCP ... quick coffee break, maybe?
-  command: 'kubectl exec {{ armada_pod_name }} -n ucp -- armada apply /armada/{{ socok8s_site_name }}-ucp.yaml --wait --target-manifest ucp-bootstrap'
+  command: 'kubectl exec {{ armada_pod_name }} -n ucp -- armada apply /armada/{{ socok8s_site_name }}-ucp.yaml --target-manifest ucp-bootstrap'
   tags:
     - install
     - skip_ansible_lint
 
-# TODO(aagate): Add a changed_when: to help idempotency
-- name: Wait until Armada api pod is deployed
-  command: 'kubectl get pod -l application=armada,component=api -n {{ ucp_namespace_name }} -o jsonpath="{.items[0].metadata.name}"'
-  register: armada_results
-  until: armada_results.stdout.find('armada-api-') == 0
-  retries: 180
-  delay: 10
-  tags:
-    - skip_ansible_lint
+- block:
+    - name: Wait until Keystone api pod is deployed
+      command: 'kubectl get pod -l application=keystone,component=api -n {{ ucp_namespace_name }} -o jsonpath="{.items[0].metadata.name}"'
+      register: keystone_results
+      until: keystone_results.stdout.find('keystone-api-') == 0
+      retries: 240
+      delay: 10
+      changed_when: False
 
-- name: Set armada api pod name
-  set_fact: armada_api_pod_name={{ armada_results.stdout }}
+    - name: Set Keystone api pod name
+      set_fact: keystone_api_pod_name={{ keystone_results.stdout }}
 
-- debug:
-    msg: "armada-api pod found: {{ armada_api_pod_name }}"
+    - name: Wait until Keystone api becomes ready
+      command: "kubectl get pod {{ keystone_api_pod_name }} -n {{ ucp_namespace_name }} -o jsonpath='{.status.containerStatuses[].ready}'"
+      register: keystone_api_pod_status
+      until: keystone_api_pod_status.stdout == "true"
+      retries: 120
+      delay: 10
+      changed_when: False
 
-# TODO(aagate): Add a changed_when: to help idempotency
-- name: Wait until Airship api becomes ready
-  command: "kubectl get pod {{ armada_api_pod_name }} -n {{ ucp_namespace_name }} -o jsonpath='{.status.containerStatuses[].ready}'"
-  register: armada_api_pod_status
-  until: armada_api_pod_status.stdout == "true"
-  retries: 60
-  delay: 10
-  tags:
-    - skip_ansible_lint
+    - name: Wait until Shipyard api pod is deployed
+      command: 'kubectl get pod -l application=shipyard,component=api -n {{ ucp_namespace_name }} -o jsonpath="{.items[0].metadata.name}"'
+      register: shipyard_results
+      until: shipyard_results.stdout.find('shipyard-api-') == 0
+      retries: 240
+      delay: 10
+      changed_when: False
+
+    - name: Set shipyard api pod name
+      set_fact: shipyard_api_pod_name={{ shipyard_results.stdout }}
+
+    - name: Wait until Shipyard api becomes ready
+      command: "kubectl get pod {{ shipyard_api_pod_name }} -n {{ ucp_namespace_name }} -o jsonpath='{.status.containerStatuses[].ready}'"
+      register: shipyard_api_pod_status
+      until: shipyard_api_pod_status.stdout == "true"
+      retries: 120
+      delay: 10
+      changed_when: False
+
+    - name: Wait until Armada api pod is deployed
+      command: 'kubectl get pod -l application=armada,component=api -n {{ ucp_namespace_name }} -o jsonpath="{.items[0].metadata.name}"'
+      register: armada_results
+      until: armada_results.stdout.find('armada-api-') == 0
+      retries: 240
+      delay: 10
+      changed_when: False
+
+    - name: Set armada api pod name
+      set_fact: armada_api_pod_name={{ armada_results.stdout }}
+
+    - name: Wait until Armada api becomes ready
+      command: "kubectl get pod {{ armada_api_pod_name }} -n {{ ucp_namespace_name }} -o jsonpath='{.status.containerStatuses[].ready}'"
+      register: armada_api_pod_status
+      until: armada_api_pod_status.stdout == "true"
+      retries: 120
+      delay: 10
+      changed_when: False
+
+    # Wait until all pods are in ready or completed state. In case of site update,
+    # some pods can be redeployed while the api pods are ready.
+    - name: Wait until all pods to be ready
+      command: "{{ upstream_repos_clone_folder }}/openstack/openstack-helm/tools/deployment/common/wait-for-pods.sh {{ ucp_namespace_name }} 3600"
+
+  rescue:
+    - name: List all ucp pods
+      command: "kubectl get pods -n {{ ucp_namespace_name }}"
+      register: list_ucp_pods
+
+    - name: "Airship UCP services are not ready, all {{ ucp_namespace_name }} pods have been listed for debugging"
+      debug:
+        var: list_ucp_pods.stdout_lines
+      failed_when: true