Make provision less sensitive to quay hiccups (#385)

* Make provision less sensitive to quay hiccups Add retry on docker pull, because quay might have connection hiccups, that a simple retry solves instead of failing the whole provision. Signed-off-by: Or Shoval <[email protected]> * Update K8s-1.17 / 1.18 provider hashes Signed-off-by: Or Shoval <[email protected]>
kubevirt · Jul 6, 2020 · 3ad11b7 · 3ad11b7
1 parent 796d629
commit 3ad11b7
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 20 deletions.
diff --git a/cluster-provision/k8s/1.17/provision.sh b/cluster-provision/k8s/1.17/provision.sh
@@ -2,6 +2,23 @@
 
 set -ex
 
+function docker_pull_retry() {
+    retry=0
+    maxRetries=5
+    retryAfterSeconds=3
+    until [ ${retry} -ge ${maxRetries} ]; do
+        docker pull $@ && break
+        retry=$((${retry} + 1))
+        echo "Retrying ${FUNCNAME} [${retry}/${maxRetries}] in ${retryAfterSeconds}(s)"
+        sleep ${retryAfterSeconds}
+    done
+
+    if [ ${retry} -ge ${maxRetries} ]; then
+        echo "${FUNCNAME} Failed after ${maxRetries} attempts!"
+        exit 1
+    fi
+}
+
 kubeadmn_patches_path="/provision/kubeadm-patches"
 
 # Need to have the latest kernel
@@ -317,23 +334,23 @@ chmod -R 777 /var/local/kubevirt-storage/local-volume
 chcon -R unconfined_u:object_r:svirt_sandbox_file_t:s0 /mnt/local-storage/
 
 # Pre pull fluentd image used in logging
-docker pull fluent/fluentd:v1.2-debian
-docker pull fluent/fluentd-kubernetes-daemonset:v1.2-debian-syslog
+docker_pull_retry fluent/fluentd:v1.2-debian
+docker_pull_retry fluent/fluentd-kubernetes-daemonset:v1.2-debian-syslog
 
 # Pre pull images used in Ceph CSI
-docker pull quay.io/k8scsi/csi-attacher:v1.0.1
-docker pull quay.io/k8scsi/csi-provisioner:v1.0.1
-docker pull quay.io/k8scsi/csi-snapshotter:v1.0.1
-docker pull quay.io/cephcsi/rbdplugin:v1.0.0
-docker pull quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
+docker_pull_retry quay.io/k8scsi/csi-attacher:v1.0.1
+docker_pull_retry quay.io/k8scsi/csi-provisioner:v1.0.1
+docker_pull_retry quay.io/k8scsi/csi-snapshotter:v1.0.1
+docker_pull_retry quay.io/cephcsi/rbdplugin:v1.0.0
+docker_pull_retry quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
 
 # Pre pull cluster network addons operator images and store manifests
 # so we can use them at cluster-up
 cp -rf /tmp/cnao/ /opt/
-for i in $(grep -A 2 "IMAGE" /opt/cnao/operator.yaml |grep value | awk '{print $2}'); do docker pull $i; done
+for i in $(grep -A 2 "IMAGE" /opt/cnao/operator.yaml | grep value | awk '{print $2}'); do docker_pull_retry $i; done
 
 # Pre pull local-volume-provisioner
-grep -A 2 "IMAGE" /tmp/local-volume.yaml | grep value | awk '{print $2}' | xargs docker pull
+for i in $(grep -A 2 "IMAGE" /provision/local-volume.yaml | grep value | awk -F\" '{print $2}'); do docker_pull_retry $i; done
 
 # Create a properly labelled tmp directory for testing
 mkdir -p /provision/kubevirt.io/tests

diff --git a/cluster-provision/k8s/1.18/provision.sh b/cluster-provision/k8s/1.18/provision.sh
@@ -2,6 +2,23 @@
 
 set -ex
 
+function docker_pull_retry() {
+    retry=0
+    maxRetries=5
+    retryAfterSeconds=3
+    until [ ${retry} -ge ${maxRetries} ]; do
+        docker pull $@ && break
+        retry=$((${retry} + 1))
+        echo "Retrying ${FUNCNAME} [${retry}/${maxRetries}] in ${retryAfterSeconds}(s)"
+        sleep ${retryAfterSeconds}
+    done
+
+    if [ ${retry} -ge ${maxRetries} ]; then
+        echo "${FUNCNAME} Failed after ${maxRetries} attempts!"
+        exit 1
+    fi
+}
+
 kubeadmn_patches_path="/provision/kubeadm-patches"
 
 # Need to have the latest kernel
@@ -317,23 +334,23 @@ chmod -R 777 /var/local/kubevirt-storage/local-volume
 chcon -R unconfined_u:object_r:svirt_sandbox_file_t:s0 /mnt/local-storage/
 
 # Pre pull fluentd image used in logging
-docker pull fluent/fluentd:v1.2-debian
-docker pull fluent/fluentd-kubernetes-daemonset:v1.2-debian-syslog
+docker_pull_retry fluent/fluentd:v1.2-debian
+docker_pull_retry fluent/fluentd-kubernetes-daemonset:v1.2-debian-syslog
 
 # Pre pull images used in Ceph CSI
-docker pull quay.io/k8scsi/csi-attacher:v1.0.1
-docker pull quay.io/k8scsi/csi-provisioner:v1.0.1
-docker pull quay.io/k8scsi/csi-snapshotter:v1.0.1
-docker pull quay.io/cephcsi/rbdplugin:v1.0.0
-docker pull quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
+docker_pull_retry quay.io/k8scsi/csi-attacher:v1.0.1
+docker_pull_retry quay.io/k8scsi/csi-provisioner:v1.0.1
+docker_pull_retry quay.io/k8scsi/csi-snapshotter:v1.0.1
+docker_pull_retry quay.io/cephcsi/rbdplugin:v1.0.0
+docker_pull_retry quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
 
 # Pre pull cluster network addons operator images and store manifests
 # so we can use them at cluster-up
 cp -rf /tmp/cnao/ /opt/
-for i in $(grep -A 2 "IMAGE" /opt/cnao/operator.yaml |grep value | awk '{print $2}'); do docker pull $i; done
+for i in $(grep -A 2 "IMAGE" /opt/cnao/operator.yaml | grep value | awk '{print $2}'); do docker_pull_retry $i; done
 
 # Pre pull local-volume-provisioner
-grep -A 2 "IMAGE" /tmp/local-volume.yaml | grep value | awk '{print $2}' | xargs docker pull
+for i in $(grep -A 2 "IMAGE" /provision/local-volume.yaml | grep value | awk -F\" '{print $2}'); do docker_pull_retry $i; done
 
 # Create a properly labelled tmp directory for testing
 mkdir -p /provision/kubevirt.io/tests

diff --git a/cluster-up/cluster/images.sh b/cluster-up/cluster/images.sh
@@ -6,8 +6,8 @@ declare -A IMAGES
 IMAGES[gocli]="gocli@sha256:220f55f6b1bcb3975d535948d335bd0e6b6297149a3eba1a4c14cad9ac80f80d"
 if [ -z $KUBEVIRTCI_PROVISION_CHECK ]; then
     IMAGES[k8s-fedora-1.17.0]="k8s-fedora-1.17.0@sha256:aebf67b8b1b499c721f4d98a7ab9542c680553a14cbc144d1fa701fe611f3c0d"
-    IMAGES[k8s-1.18]="k8s-1.18@sha256:c41e3d9adb756b60e1fbce2ffd774c66c99fdde7ee337460c472ee92868e579e"
-    IMAGES[k8s-1.17]="k8s-1.17@sha256:7506c49307944d2bbbf6486bc9c9aa4496694d3284143a51ee87333c1a0ab7f0"
+    IMAGES[k8s-1.18]="k8s-1.18@sha256:91ea0c4e10a356fe315049e4e56801ce62b73b5b5d901e6e0eccbcc171070674"
+    IMAGES[k8s-1.17]="k8s-1.17@sha256:17fec14930cae00bda913cb690c46b9738dbc9b06b11b858a58369d7f9419688"
     IMAGES[k8s-1.16]="k8s-1.16@sha256:1e153fb62c9a30ce6bc6ddc1af21bb28a56b780ec93ab15e113d729bf664469a"
     IMAGES[k8s-1.15]="k8s-1.15@sha256:c58cb9d79968590f24e070bc2517088d44fa2f83ba73e989a7e0f690ad08460b"
     IMAGES[k8s-1.14]="k8s-1.14@sha256:46e449b292dcb420f0944cac0a7a5c667c6f19bba2a4192737380e8e77f27ed0"