Merge pull request #264 from CloudSlang/ci-env

Add CI Matrix for various envs

ci-env/cleanup_env.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# print droplets
+# cat < "droplets_${CIRCLE_BUILD_NUM}.txt"
+
+DROPLET_ID_ACC=$(cat < "droplets_${CIRCLE_BUILD_NUM}.txt")
+# echo "DROPLET_ID_ACC: $DROPLET_ID_ACC"
+
+for DROPLET_ID in ${DROPLET_ID_ACC}
+do
+  CURL_OUTPUT=$(curl -i -s -L -X DELETE -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \
+  "https://api.digitalocean.com/v2/droplets/$DROPLET_ID")
+  # echo "CURL_OUTPUT: $CURL_OUTPUT"
+
+  STATUS_CODE=$(echo "$CURL_OUTPUT" | grep "Status" | awk '{print $2}')
+  echo "STATUS_CODE: $STATUS_CODE"
+
+  if [ "$STATUS_CODE" = "204" ]
+  then
+    echo "Droplet($DROPLET_ID) deleted successfully"
+  else
+    echo "Problem occurred: destroying droplet($DROPLET_ID) - status code: $STATUS_CODE"
+  fi
+done

ci-env/cloud-config.yaml

@@ -0,0 +1,15 @@
+#cloud-config
+
+coreos:
+  etcd:
+    discovery: <discovery_url>
+    addr: $private_ipv4:4001
+    peer-addr: $private_ipv4:7001
+  fleet:
+    public-ip: $private_ipv4
+    metadata: public_ip=$public_ipv4
+  units:
+    - name: etcd.service
+      command: start
+    - name: fleet.service
+      command: start

ci-env/create_droplets.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# generate discovery URL for the new CoreOS cluster and update the cloud-config file
+DISCOVERY_URL=$(curl -s -X GET "https://discovery.etcd.io/new")
+echo "DISCOVERY_URL: $DISCOVERY_URL"
+DISCOVERY_URL_ESCAPED=$(echo ${DISCOVERY_URL} | sed 's/\//\\\//g')
+sed -i "s/<discovery_url>/${DISCOVERY_URL_ESCAPED}/g" ci-env/cloud-config.yaml
+# cat ci-env/cloud-config.yaml
+
+COREOS_MACHINE_NAMES="\
+ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1 \
+ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 \
+ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3"
+for COREOS_MACHINE in ${COREOS_MACHINE_NAMES}
+do
+  CURL_OUTPUT=$(curl -i -s -X POST https://api.digitalocean.com/v2/droplets \
+                -H 'Content-Type: application/json' \
+                -H "Authorization: Bearer $DO_API_TOKEN" \
+                -d "{
+                  \"name\":\"${COREOS_MACHINE}\",
+                  \"ssh_keys\":[${DO_DROPLET_SSH_PUBLIC_KEY_ID}],"'
+                  "region":"ams3",
+                  "size":"512mb",
+                  "image":"coreos-stable",
+                  "backups":false,
+                  "ipv6":false,
+                  "private_networking":true,
+                  "user_data": "'"$(cat ci-env/cloud-config.yaml | sed 's/"/\\"/g')"'"
+                }')
+  # echo "CURL_OUTPUT: $CURL_OUTPUT"
+
+  STATUS_CODE=$(echo ${CURL_OUTPUT} | awk '{print $2}')
+
+  if [ "$STATUS_CODE" = "202" ]
+  then
+    DROPLET_DETAILS=$(echo "$CURL_OUTPUT" | grep "droplet")
+    #  echo "DROPLET_DETAILS: $DROPLET_DETAILS"
+
+    # split after `:` and `,` characters and extract the droplet ID
+    DROPLET_ID_JUNK_ARRAY=(${DROPLET_DETAILS//:/ })
+    DROPLET_ID_JUNK=${DROPLET_ID_JUNK_ARRAY[2]}
+    DROPLET_ID_ARRAY=(${DROPLET_ID_JUNK//,/ })
+    DROPLET_ID=${DROPLET_ID_ARRAY[0]}
+
+    DROPLET_ID_ACC+="${DROPLET_ID} "
+
+    echo "$COREOS_MACHINE (ID: $DROPLET_ID) droplet creation request accepted"
+
+    # store droplet IDs in a file to be accessible in other script
+    echo ${DROPLET_ID_ACC} > "droplets_${CIRCLE_BUILD_NUM}.txt"
+  else
+    echo "Problem occurred: $COREOS_MACHINE droplet creation request - status code: $STATUS_CODE"
+    exit 1
+  fi
+done

ci-env/wait_for_droplets_and_update_test_inputs.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+
+inc_and_sleep()
+{
+  ((WAITING_TIME+=SLEEP_INTERVAL))
+  sleep ${SLEEP_INTERVAL}
+}
+
+DROPLET_ID_ACC=$(cat "droplets_${CIRCLE_BUILD_NUM}.txt")
+SLEEP_INTERVAL=5 # 5 sec
+TIMEOUT=300 # 5 mins
+
+# retrieve IPv4 addresses of droplets
+for DROPLET_ID in ${DROPLET_ID_ACC}
+do
+  DROPLET_STATUS=''
+  WAITING_TIME=0
+  while [ "$DROPLET_STATUS" != "active" ] && [ "$WAITING_TIME" -lt "$TIMEOUT" ]
+  do
+    CURL_OUTPUT=$(curl -i -s -L -X GET -H 'Content-Type: application/json' -H "Authorization: Bearer ${DO_API_TOKEN}" \
+    "https://api.digitalocean.com/v2/droplets/$DROPLET_ID")
+    # echo "CURL_OUTPUT - GET DROPLET BY ID: $CURL_OUTPUT"
+
+    STATUS_CODE=$(echo "$CURL_OUTPUT" | grep "Status" | awk '{print $2}')
+    # echo "STATUS_CODE: $STATUS_CODE"
+
+    if [ "$STATUS_CODE" = "200" ]
+    then
+      echo "Droplet($DROPLET_ID) information retrieved successfully"
+
+      RESPONSE_BODY_JSON=$(echo "$CURL_OUTPUT" | grep "ip_address")
+      # echo "RESPONSE_BODY_JSON: ${RESPONSE_BODY_JSON}"
+
+      if [ "${RESPONSE_BODY_JSON}" = "" ]
+      then
+        inc_and_sleep
+      else
+        DROPLET_STATUS=$(\
+        echo "$RESPONSE_BODY_JSON" | python -c \
+'
+if True:
+        import json,sys;
+        obj = json.load(sys.stdin);
+        print obj["droplet"]["status"];
+'\
+        )
+        echo "Droplet($DROPLET_ID) status: ${DROPLET_STATUS}"
+
+        if [ "$DROPLET_STATUS" = "active" ]
+        then
+          IP_ADDRESS=$(\
+          echo "$RESPONSE_BODY_JSON" | python -c \
+'
+if True:
+          import json,sys;
+          obj = json.load(sys.stdin);
+          ipv4_container_list = obj["droplet"]["networks"]["v4"];
+          public_ipv4_container_list = filter(lambda x : x["type"] == "public", ipv4_container_list);
+          print public_ipv4_container_list[0]["ip_address"] if len(public_ipv4_container_list) > 0 else "";
+'\
+          )
+          echo "Droplet($DROPLET_ID) IPv4 address: $IP_ADDRESS"
+
+          DROPLET_IP_ADDRESS_ACC+="${IP_ADDRESS} "
+          # echo "DROPLET_IP_ADDRESS_ACC: $DROPLET_IP_ADDRESS_ACC"
+        else
+          inc_and_sleep
+        fi
+      fi
+    else
+      echo "Problem occurred: retrieving droplet($DROPLET_ID) information - status code: $STATUS_CODE"
+      exit 1
+    fi
+  done
+  if [ "$DROPLET_STATUS" != "active" ]
+  then
+    echo "Droplet($DROPLET_ID) is not active after ${WAITING_TIME}"
+    exit 1
+  fi
+done
+
+# update inputs files to use actual IP addresses
+DROPLET_IP_ARRAY=(${DROPLET_IP_ADDRESS_ACC})
+find test -type f -exec sed -i "s/<coreos_host_1>/${DROPLET_IP_ARRAY[0]}/g" {} +
+
+# create ssh private key
+SSH_KEY_PATH=droplets_rsa
+echo -e "${DO_DROPLET_SSH_PRIVATE_KEY}" > ${SSH_KEY_PATH}
+
+# update inputs files to use actual ssh key
+find test -type f -exec sed -i "s/<private_key_file>/${SSH_KEY_PATH}/g" {} +

circle.yml

@@ -12,36 +12,52 @@ general:
     - "builder.log"
 
 machine:
+  environment:
+    # "${CIRCLE_PR_NUMBER}" = "" denotes that the build is not part of a fork PR
+    # ignore test suites on node 1 in fork builds
+    ACTIVE_CI_TEST_SUITES_DROPLETS_NODE: $(if [ "${CIRCLE_PR_NUMBER}" = "" ]; then echo '!default,coreos,cadvisor'; else echo '!default'; fi)
+    RULE_DROPLET_MACHINE_NOT_FORK: '[ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]'
   pre:
     - sudo curl -L -o /usr/bin/docker 'http://s3-external-1.amazonaws.com/circle-downloads/docker-1.6.2-circleci'; sudo chmod 0755 /usr/bin/docker; true
 
   services:
     - docker
 
 dependencies:
-#  cache_directories:
-#    - "~/docker"
   override:
     - docker info
     - docker images
-#    - if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi
-#    - docker pull orius123/dind-ssh
-#    - mkdir -p ~/docker; docker save orius123/dind-ssh > ~/docker/image.tar
 
 test:
   pre:
-#    - docker run --privileged --lxc-conf="lxc.aa_profile=unconfined" -d -p 4444 -p 49153:22 -e PORT=4444 --name docker_host_ssh orius123/dind-ssh
-#    - docker port docker_host_ssh 22
-     - docker run -d -p 49165:8080 jenkins
-     - wget https://github.com/CloudSlang/cloud-slang/releases/download/cloudslang-0.8.RC1/cslang-builder.zip
-     - docker run -d -p 8500:8500 -p 8600:8600/udp fhalim/consul
-     - unzip cslang-builder.zip -d cslang-builder
-     - chmod +x cslang-builder/bin/cslang-builder
-     - mkdir cslang-builder/lib/Lib
-     - pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib
+    - ? > ### machine 0
+        if [ "${CIRCLE_NODE_INDEX}" = "0" ];
+        then docker run -d -p 49165:8080 jenkins
+        && docker run -d -p 8500:8500 -p 8600:8600/udp fhalim/consul;
+        fi
+      : parallel: true
+    - if eval "${RULE_DROPLET_MACHINE_NOT_FORK}"; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi:
+        parallel: true ### machine 1
+    - ? > ### every machine
+        wget https://github.com/CloudSlang/cloud-slang/releases/download/cloudslang-0.8.RC1/cslang-builder.zip
+        && unzip cslang-builder.zip -d cslang-builder
+        && chmod +x cslang-builder/bin/cslang-builder
+        && mkdir cslang-builder/lib/Lib
+        && pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib
+      : parallel: true
+    - ? > ### machine 1
+        if eval "${RULE_DROPLET_MACHINE_NOT_FORK}";
+        then chmod +x ci-env/wait_for_droplets_and_update_test_inputs.sh
+        && ci-env/wait_for_droplets_and_update_test_inputs.sh;
+        fi
+      : parallel: true
   override:
-    - ./cslang-builder/bin/cslang-builder -ts cadvisor,jenkins,consul -cov
+    - ? >
+        case ${CIRCLE_NODE_INDEX} in
+        0) ./cslang-builder/bin/cslang-builder -ts default,jenkins,consul -cov ;;
+        1) ./cslang-builder/bin/cslang-builder -ts ${ACTIVE_CI_TEST_SUITES_DROPLETS_NODE} -cov ;;
+        esac
+      : parallel: true
   post:
-    - docker ps -a
-#    - docker stop $(docker ps -a -q)
-#    - docker rm $(docker ps -a -q)
+    - if eval "${RULE_DROPLET_MACHINE_NOT_FORK}"; then chmod +x ci-env/cleanup_env.sh && ci-env/cleanup_env.sh; fi:
+         parallel: true ### machine 1

content/io/cloudslang/docker/images/clear_unused_and_dangling_images.sl

@@ -88,6 +88,6 @@ flow:
             - amount_of_dangling_images_deleted
   outputs:
     - amount_of_images_deleted: "0 if 'amount_of_images_deleted' not in locals() else amount_of_images_deleted"
-    - amount_of_dangling_images_deleted: "0 if 'amount_of_images_deleted' not in locals() else amount_of_dangling_images_deleted"
+    - amount_of_dangling_images_deleted: "0 if 'amount_of_dangling_images_deleted' not in locals() else amount_of_dangling_images_deleted"
     - dangling_images_list_safe_to_delete
     - images_list_safe_to_delete

content/io/cloudslang/docker/images/get_used_images.sl

@@ -93,7 +93,7 @@ flow:
             - agentForwarding:
                 required: false
         publish:
-          - image_list: returnResult.replace("\n"," ")
+          - image_list: returnResult.replace("\n"," ").replace(":latest", "")
 
   outputs:
     - image_list

content/io/cloudslang/docker/maintenance/images_maintenance.sl

@@ -59,7 +59,7 @@ flow:
           NOT_ENOUGH_DISKSPACE: clear_unused_images
     - clear_unused_images:
         do:
-          docker_images.clear_unused_images:
+          docker_images.clear_unused_and_dangling_images:
             - docker_host
             - docker_username
             - docker_password:

test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml

@@ -7,11 +7,16 @@
 #
 ####################################################
 test_cluster_docker_images_maintenance_success:
-  testSuites: [coreos_local]
-  description: This test needs to be run on local machine by activating the test suite and supplying the inputs
+  testSuites: [coreos]
+  description: >
+    This test cleans the CoreOS hosts in a cluster,
+    prepares a used and an unused Docker image,
+    runs the cluster_docker_images_maintenance flow,
+    checks that only the unused Docker image was deleted,
+    cleans up in the cluster.
   testFlowPath: io.cloudslang.coreos.test_cluster_docker_images_maintenance
   inputs:
-    - coreos_host: 111.111.111.111 # ip address of one host from the cluster
+    - coreos_host: <coreos_host_1> # ip address of one host from the cluster
     - coreos_username: core # username for the machine
-    - private_key_file: c:/Users/bob/ssh_keys/id_rsa # path to private key file
+    - private_key_file: <private_key_file> # path to private key file
   result: SUCCESS

test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml

@@ -7,11 +7,15 @@
 #
 ####################################################
 test_get_container_metrics:
-  testSuites: [cadvisor_local]
-  description: Tests SUCCESS case - this test needs to be run on the Docker machine by activating the test suite and supplying the inputs
+  testSuites: [cadvisor]
+  description: >
+    This test clears the docker containers on the host, starts up a cadvisor container,
+    calls the get_container_metrics flow and validates the result.
   testFlowPath: io.cloudslang.docker.cadvisor.test_get_container_metrics
   inputs:
-    - host: localhost
-    - cadvisor_port: "32952"
+    - host: <coreos_host_1>
+    - username: core
+    - private_key_file: <private_key_file>
+    - cadvisor_port: "32541"
     - cadvisor_container_name: cadvisor
-  result: SUCCESS
+  result: SUCCESS

test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml

@@ -8,10 +8,14 @@
 ####################################################
 test_report_machine_metrics:
   testSuites: [cadvisor]
-  description: Tests SUCCESS case
+  description: >
+    This test clears the docker containers on the host, starts up a cadvisor container,
+    calls the report_machine_metrics flow and validates the result.
   testFlowPath: io.cloudslang.docker.cadvisor.test_report_machine_metrics
   inputs:
-    - host: localhost
+    - host: <coreos_host_1>
+    - username: core
+    - private_key_file: <private_key_file>
+    - cadvisor_port: "32541"
     - cadvisor_container_name: cadvisor
-    - cadvisor_port: "32951"
-  result: SUCCESS
+  result: SUCCESS

test/io/cloudslang/docker/cadvisor/restart_container_base_on_usage.inputs.yaml

@@ -7,16 +7,15 @@
 #
 ########################################################################################################
 test_restart_container_base_on_usage_success:
-  testSuites: [cadvisor_local]
+  testSuites: [cadvisor]
   description: >
-    Tests SUCCESS case - starts a cAdvisor container, waits for the container to start up and runs
-    the restart_container_base_on_usage flow against it. The Docker host is cleared before and after
-    the actual test logic.
+    This test clears the docker containers on the host, starts up a cadvisor container,
+    calls the restart_container_base_on_usage flow and validates the result.
   testFlowPath: io.cloudslang.docker.cadvisor.test_restart_container_base_on_usage
   inputs:
-    - cadvisor_port: "23569"
+    - host: <coreos_host_1>
+    - username: core
+    - private_key_file: <private_key_file>
+    - cadvisor_port: "32541"
     - cadvisor_container_name: cadvisor
-    - host: 111.111.111.111
-    - username: root
-    - private_key_file: "c:/.../id_rsa"
   result: SUCCESS