From 6856db88f91867abd7b9afced134efc45817d17d Mon Sep 17 00:00:00 2001
From: subhamkrai <srai@redhat.com>
Date: Tue, 25 Jan 2022 18:07:52 +0530
Subject: [PATCH] ci: add ci to test intree migration to ceph-csi

This commit add CI to test intree migration to Ceph-csi

Signed-off-by: subhamkrai <srai@redhat.com>
---
 .github/workflows/flex-migration-test.yaml    |  68 +++++-----
 .github/workflows/in-tree-migtation.yaml      |  57 +++++++++
 .../setup-cluster-resources/action.yaml       |   6 +-
 tests/script/github_action_helper.sh          | 117 +++++++++++++++++-
 4 files changed, 208 insertions(+), 40 deletions(-)
 create mode 100644 .github/workflows/in-tree-migtation.yaml

diff --git a/.github/workflows/flex-migration-test.yaml b/.github/workflows/flex-migration-test.yaml
index 8a9beba..4dc5b94 100644
--- a/.github/workflows/flex-migration-test.yaml
+++ b/.github/workflows/flex-migration-test.yaml
@@ -11,41 +11,47 @@ jobs:
   Flex-to-CSI:
     runs-on: ubuntu-latest
     steps:
-    - name: checkout
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
+      - name: checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
-    - name: setup cluster resources
-      uses: ./.github/workflows/setup-cluster-resources
-      with:
-        github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: setup cluster resources
+        uses: ./.github/workflows/setup-cluster-resources
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
 
-    - name: Test migration of all PVC
-      run: tests/script/github_action_helper.sh test_flex_migration_for_all_pvc
+      - name: setup flex
+        run: tests/script/github_action_helper.sh deploy_rook_ceph_with_flex
 
-    - name: setup tmate session for debugging when event is PR
-      if: failure()
-      uses: mxschmitt/action-tmate@v3
-      timeout-minutes: 300
+      - name: Test migration of all PVC
+        run: tests/script/github_action_helper.sh test_flex_migration_for_all_pvc
+
+      - name: setup tmate session for debugging when event is PR
+        if: failure()
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 300
 
   Flex-Mirgate-single-PVC:
     runs-on: ubuntu-latest
     steps:
-    - name: checkout
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-
-    - name: setup cluster resources
-      uses: ./.github/workflows/setup-cluster-resources
-      with:
-        github-token: ${{ secrets.GITHUB_TOKEN }}
-
-    - name: Test migration of all PVC
-      run: tests/script/github_action_helper.sh test_flex_migration_for_single_pvc
-
-    - name: setup tmate session for debugging when event is PR
-      if: failure()
-      uses: mxschmitt/action-tmate@v3
-      timeout-minutes: 300
+      - name: checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: setup cluster resources
+        uses: ./.github/workflows/setup-cluster-resources
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: setup flex
+        run: tests/script/github_action_helper.sh deploy_rook_ceph_with_flex
+
+      - name: Test migration of all PVC
+        run: tests/script/github_action_helper.sh test_flex_migration_for_single_pvc
+
+      - name: setup tmate session for debugging when event is PR
+        if: failure()
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 300
diff --git a/.github/workflows/in-tree-migtation.yaml b/.github/workflows/in-tree-migtation.yaml
new file mode 100644
index 0000000..20f2ef1
--- /dev/null
+++ b/.github/workflows/in-tree-migtation.yaml
@@ -0,0 +1,57 @@
+name: Migrate-to-In-tree
+on:
+  pull_request:
+
+defaults:
+  run:
+    # reference: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
+    shell: bash --noprofile --norc -eo pipefail -x {0}
+
+jobs:
+  In-tree-to-CSI:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: setup cluster resources
+        uses: ./.github/workflows/setup-cluster-resources
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: setup In-tree
+        run: tests/script/github_action_helper.sh deploy_rook_ceph_with_intree
+
+      - name: Test migration of all PVC
+        run: tests/script/github_action_helper.sh test_intree_migration_for_all_pvc
+
+      - name: setup tmate session for debugging when event is PR
+        if: failure()
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 300
+
+  In-tree-Mirgate-single-PVC:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: setup cluster resources
+        uses: ./.github/workflows/setup-cluster-resources
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: setup In-tree
+        run: tests/script/github_action_helper.sh deploy_rook_ceph_with_intree
+
+      - name: Test migration of all PVC
+        run: tests/script/github_action_helper.sh test_intree_migration_for_single_pvc
+
+      - name: setup tmate session for debugging when event is PR
+        if: failure()
+        uses: mxschmitt/action-tmate@v3
+        timeout-minutes: 300
diff --git a/.github/workflows/setup-cluster-resources/action.yaml b/.github/workflows/setup-cluster-resources/action.yaml
index 717b44a..8d2f553 100644
--- a/.github/workflows/setup-cluster-resources/action.yaml
+++ b/.github/workflows/setup-cluster-resources/action.yaml
@@ -18,7 +18,7 @@ runs:
 
     - name: print k8s cluster status
       shell: bash --noprofile --norc -eo pipefail -x {0}
-      run:  |
+      run: |
         minikube status
         kubectl get nodes
 
@@ -31,7 +31,3 @@ runs:
       run: |
         sudo wget -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_amd64
         sudo chmod +x /usr/local/bin/yq
-
-    - name: setup flex
-      shell: bash --noprofile --norc -eo pipefail -x {0}
-      run: tests/script/github_action_helper.sh deploy_rook_ceph_with_flex
diff --git a/tests/script/github_action_helper.sh b/tests/script/github_action_helper.sh
index 783df42..5c1f9bd 100755
--- a/tests/script/github_action_helper.sh
+++ b/tests/script/github_action_helper.sh
@@ -26,8 +26,8 @@ deploy_rook_ceph_with_flex() {
   kubectl create -f manifests/migrator.yaml
   # wait_for_pod_to_be_ready_state check for osd pod to in ready state
   wait_for_osd_pod_to_be_ready_state
-  # wait_for_pod_to_be_ready_state check for toolbox pod to in ready state
-  wait_for_toolboxpod_to_be_ready_state
+  # wait_for_migrator_pod_to_be_ready_state check for migrator pod to in ready state
+  wait_for_migrator_pod_to_be_ready_state
   kubectl create -f https://raw.githubusercontent.com/rook/rook/release-1.7/cluster/examples/kubernetes/ceph/flex/storageclass-test.yaml
   kubectl create -f https://raw.githubusercontent.com/rook/rook/release-1.7/cluster/examples/kubernetes/ceph/csi/rbd/pvc.yaml
   # creating sample application pod, writing some data into pod and deletes the pod
@@ -72,6 +72,11 @@ test_flex_migration_for_single_pvc(){
   MIGRATION_POD=$(kubectl -n rook-ceph get pod -l app=rook-ceph-migrator -o jsonpath='{.items[*].metadata.name}')
   kubectl -n rook-ceph cp pv-migrator "$MIGRATION_POD":/root/
   kubectl -n rook-ceph exec -it "$MIGRATION_POD" -- sh -c "cd root/ && ./pv-migrator --pvc=rbd-pvc --pvc-ns=default --destination-sc=csi-rook-ceph-block"
+  exit_code_of_last_command=$?
+  if [ $exit_code_of_last_command -ne 0 ]; then
+    echo "Exit code migration command is non-zero $exit_code_of_last_command. Migration failed"
+    exit 1
+  fi
   kubectl create -f https://raw.githubusercontent.com/rook/rook/release-1.7/cluster/examples/kubernetes/ceph/csi/rbd/pod.yaml
   wait_for_sample_pod_to_be_ready_state
   verify_file_data_and_file_data
@@ -103,8 +108,8 @@ wait_for_osd_pod_to_be_ready_state() {
 EOF
 }
 
-# wait_for_pod_to_be_ready_state check for osd pod to in ready state
-wait_for_toolboxpod_to_be_ready_state() {
+# wait_for_migrator_pod_to_be_ready_state check for migrator pod to in ready state
+wait_for_migrator_pod_to_be_ready_state() {
   timeout 200 bash <<-'EOF'
     until [ $(kubectl get pod -l app=rook-ceph-migrator -n rook-ceph -o jsonpath='{.items[*].metadata.name}' -o custom-columns=READY:status.containerStatuses[*].ready | grep -c true) -eq 1 ]; do
       echo "waiting for the toolbox pods to be in ready state"
@@ -122,6 +127,110 @@ wait_for_sample_pod_to_be_ready_state() {
 EOF
 }
 
+deploy_rook_ceph_with_intree() {
+  kubectl create -f https://raw.githubusercontent.com/rook/rook/release-1.7/cluster/examples/kubernetes/ceph/common.yaml
+  kubectl create -f https://raw.githubusercontent.com/rook/rook/release-1.7/cluster/examples/kubernetes/ceph/crds.yaml
+  kubectl create -f https://raw.githubusercontent.com/rook/rook/release-1.7/cluster/examples/kubernetes/ceph/operator.yaml
+  wget https://raw.githubusercontent.com/rook/rook/release-1.7/cluster/examples/kubernetes/ceph/cluster-test.yaml
+  sed -i "s|#deviceFilter:|deviceFilter: $(lsblk|awk '/14G/ {print $1}'| head -1)|g" cluster-test.yaml
+  kubectl create -f cluster-test.yaml
+  wait_for_osd_pod_to_be_ready_state
+  kubectl create -f manifests/migrator.yaml
+  wait_for_toolboxpod_to_be_ready_state
+  kubectl create -f https://raw.githubusercontent.com/rook/rook/release-1.7/cluster/examples/kubernetes/ceph/pool-test.yaml
+
+  kubectl patch storageclass standard -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"false"}}}'
+  sudo sed -i 's/image: k8s.gcr.io\/kube-controller-manager:v1.22.2/image: gcr.io\/google_containers\/hyperkube:v1.16.3/g' /etc/kubernetes/manifests/kube-controller-manager.yaml
+  kubectl create -f manifests/migrator.yaml
+  # wait_for_pod_to_be_ready_state check for migrator pod to in ready state
+  wait_for_migrator_pod_to_be_ready_state
+  MIGRATION_POD=$(kubectl -n rook-ceph get pod -l app=rook-ceph-migrator -o jsonpath='{.items[*].metadata.name}')
+  ADMIN_KEY=$(kubectl -n rook-ceph exec "$MIGRATION_POD" -- /bin/bash -c "ceph auth get-key client.admin")
+  AKEY=$(echo "$ADMIN_KEY"|base64)
+  cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ceph-secret
+  namespace: kube-system
+data:
+  key: ${AKEY}
+type: kubernetes.io/rbd
+EOF
+  kubectl -n kube-system get secret ceph-secret
+  kubectl -n rook-ceph exec "$MIGRATION_POD" -- /bin/bash -c "ceph auth get-or-create client.replicapool mon 'allow r' osd 'allow class-read object_prefix rbd_children, allow rwx pool=kube' -o ceph.client.replicapool.keyring"
+  USER_KEY=$(kubectl -n rook-ceph exec "$MIGRATION_POD" -- /bin/bash -c "ceph auth get-key client.replicapool")
+  UKEY=$(echo "$USER_KEY"|base64)
+  cat <<EOF | kubectl create -f -
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ceph-user-secret
+  namespace: default
+data:
+  key: ${UKEY}
+type: kubernetes.io/rbd
+EOF
+  kubectl get secret ceph-user-secret
+  MON_STAT=$(kubectl -n rook-ceph exec "$MIGRATION_POD" -- /bin/bash -c "ceph mon stat")
+  MON_IP=$(echo "$MON_STAT" | awk -F "v1:" '{print $2}' | cut -d/ -f1)
+  cat <<EOF | kubectl create -f -
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: dynamic
+  annotations:
+     storageclass.beta.kubernetes.io/is-default-class: "true"
+provisioner: kubernetes.io/rbd
+parameters:
+  monitors: ${MON_IP}
+  adminId: admin
+  adminSecretName: ceph-secret
+  adminSecretNamespace: kube-system
+  pool: replicapool
+  userId: replicapool
+  userSecretName: ceph-user-secret
+EOF
+cat <<EOF | kubectl create -f -
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: ceph-claim
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+EOF
+
+  create_csi_resources
+}
+
+test_intree_migration_for_all_pvc(){
+  go build -o pv-migrator
+  MIGRATION_POD=$(kubectl -n rook-ceph get pod -l app=rook-ceph-migrator -o jsonpath='{.items[*].metadata.name}')
+  kubectl -n rook-ceph cp pv-migrator "$MIGRATION_POD":/root/
+  kubectl -n rook-ceph exec -it "$MIGRATION_POD" -- sh -c "cd root/ && ./pv-migrator --source-sc=dynamic --destination-sc=csi-rook-ceph-block"
+  exit_code_of_last_command=$?
+  if [ $exit_code_of_last_command -ne 0 ]; then
+    echo "Exit code migration command is non-zero $exit_code_of_last_command. Migration failed"
+    exit 1
+  fi
+}
+
+test_intree_migration_for_single_pvc(){
+  go build -o pv-migrator
+  MIGRATION_POD=$(kubectl -n rook-ceph get pod -l app=rook-ceph-migrator -o jsonpath='{.items[*].metadata.name}')
+  kubectl -n rook-ceph cp pv-migrator "$MIGRATION_POD":/root/
+  kubectl -n rook-ceph exec -it "$MIGRATION_POD" -- sh -c "cd root/ && ./pv-migrator --pvc=ceph-claim --pvc-ns=default --destination-sc=csi-rook-ceph-block"
+  exit_code_of_last_command=$?
+  if [ $exit_code_of_last_command -ne 0 ]; then
+    echo "Exit code migration command is non-zero $exit_code_of_last_command. Migration failed"
+    exit 1
+  fi
+}
+
 ########
 # MAIN #
 ########