From 4099dd543fd11165bddd0a6f569ea28de0c8dc5c Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 22 Jul 2015 12:16:54 +0300 Subject: [PATCH 001/104] add prepare_env script - 1 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 15 +++++++++++++++ circle.yml | 2 ++ 2 files changed, 17 insertions(+) create mode 100644 ci-env/prepare_env.sh diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh new file mode 100644 index 0000000000..85c836ef69 --- /dev/null +++ b/ci-env/prepare_env.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# test CircleCI env vars +echo $CIRCLE_PR_NUMBER +echo $CIRCLE_BRANCH +echo $CIRCLE_SHA1 +echo $CIRCLE_BUILD_NUM + +# test CircleCI env vars +echo ${CIRCLE_PR_NUMBER} +echo ${CIRCLE_BRANCH} +echo ${CIRCLE_SHA1} +echo ${CIRCLE_BUILD_NUM} + +curl -X GET -H 'Content-Type: application/json' "http://www.earthtools.org/timezone/40.71417/-74.00639" \ No newline at end of file diff --git a/circle.yml b/circle.yml index 5aac609ce5..c9ab308db0 100644 --- a/circle.yml +++ b/circle.yml @@ -39,6 +39,8 @@ test: - chmod +x cslang-builder/bin/cslang-builder - mkdir cslang-builder/lib/Lib - pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib + - chmod +x ci-env/prepare_env.sh + - ci-env/prepare_env.sh override: - ./cslang-builder/bin/cslang-builder -ts cadvisor,jenkins,consul -cov post: From e9b4833c3b80d01a8f65efe6242e62be4ea82417 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 22 Jul 2015 14:36:06 +0300 Subject: [PATCH 002/104] add prepare_env script - 2 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 85c836ef69..a0dc1ab6ef 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -2,14 +2,22 @@ # test CircleCI env vars echo $CIRCLE_PR_NUMBER -echo $CIRCLE_BRANCH +echo $CIRCLE_BRANCH # first part echo $CIRCLE_SHA1 -echo $CIRCLE_BUILD_NUM +echo $CIRCLE_BUILD_NUM # second part # test CircleCI env vars echo ${CIRCLE_PR_NUMBER} -echo ${CIRCLE_BRANCH} +echo ${CIRCLE_BRANCH} # first part echo ${CIRCLE_SHA1} -echo ${CIRCLE_BUILD_NUM} +echo ${CIRCLE_BUILD_NUM} # second part + +# TODO: make this look nicer +curl -X POST -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ +-d "{\"name\":\"ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1\",\"region\":\"ams3\",\"size\":\"512mb\",\ +\"image\":\"coreos-stable\",\"ssh_keys\":[774367],\"backups\":false,\"ipv6\":false,\ +\"user_data\":\"#cloud-config\n\ncoreos:\n etcd:\n discovery: https://discovery.etcd.io/84b281229d938ba03540624f0252f894\n \ +addr: $private_ipv4:4001\n peer-addr: $private_ipv4:7001\n fleet:\n public-ip: $private_ipv4\n metadata: public_ip=$public_ipv4\n \ +units:\n - name: etcd.service\n command: start\n - name: fleet.service\n command: start\",\"private_networking\":true}" \ +"https://api.digitalocean.com/v2/droplets" -curl -X GET -H 'Content-Type: application/json' "http://www.earthtools.org/timezone/40.71417/-74.00639" \ No newline at end of file From 75d48bdbff5bc8be95b620a15943181080f333c3 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 22 Jul 2015 15:15:20 +0300 Subject: [PATCH 003/104] add prepare_env script - 3 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index a0dc1ab6ef..a5cf490368 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -12,12 +12,21 @@ echo ${CIRCLE_BRANCH} # first part echo ${CIRCLE_SHA1} echo ${CIRCLE_BUILD_NUM} # second part -# TODO: make this look nicer -curl -X POST -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ --d "{\"name\":\"ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1\",\"region\":\"ams3\",\"size\":\"512mb\",\ -\"image\":\"coreos-stable\",\"ssh_keys\":[774367],\"backups\":false,\"ipv6\":false,\ -\"user_data\":\"#cloud-config\n\ncoreos:\n etcd:\n discovery: https://discovery.etcd.io/84b281229d938ba03540624f0252f894\n \ -addr: $private_ipv4:4001\n peer-addr: $private_ipv4:7001\n fleet:\n public-ip: $private_ipv4\n metadata: public_ip=$public_ipv4\n \ -units:\n - name: etcd.service\n command: start\n - name: fleet.service\n command: start\",\"private_networking\":true}" \ -"https://api.digitalocean.com/v2/droplets" +COREOS_MACHINE_NAMES="ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" +for COREOS_MACHINE in $COREOS_MACHINE_NAMES +do + STATUS_CODE=$(curl -X POST -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ + -d "{\"name\":\"$COREOS_MACHINE\",\"region\":\"ams3\",\"size\":\"512mb\",\ + \"image\":\"coreos-stable\",\"ssh_keys\":[774367],\"backups\":false,\"ipv6\":false,\ + \"user_data\":\"#cloud-config\n\ncoreos:\n etcd:\n discovery: https://discovery.etcd.io/84b281229d938ba03540624f0252f894\n \ + addr: $private_ipv4:4001\n peer-addr: $private_ipv4:7001\n fleet:\n public-ip: $private_ipv4\n metadata: public_ip=$public_ipv4\n \ + units:\n - name: etcd.service\n command: start\n - name: fleet.service\n command: start\",\"private_networking\":true}" \ + "https://api.digitalocean.com/v2/droplets" | grep "HTTP/1.1" | awk '{print $2}') + if [ "$STATUS_CODE" = "200" ] + then + echo "$COREOS_MACHINE created" + else + echo "$COREOS_MACHINE NOT created - status code: $STATUS_CODE" + fi +done From 990fcf1cedebf5d6af0dc0eb567d7cd361fc8345 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 22 Jul 2015 15:25:18 +0300 Subject: [PATCH 004/104] add prepare_env script - 3 - fix Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index a5cf490368..cca65c2ad0 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -15,7 +15,7 @@ echo ${CIRCLE_BUILD_NUM} # second part COREOS_MACHINE_NAMES="ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" for COREOS_MACHINE in $COREOS_MACHINE_NAMES do - STATUS_CODE=$(curl -X POST -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ + STATUS_CODE=$(curl -i -s -L -X POST -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ -d "{\"name\":\"$COREOS_MACHINE\",\"region\":\"ams3\",\"size\":\"512mb\",\ \"image\":\"coreos-stable\",\"ssh_keys\":[774367],\"backups\":false,\"ipv6\":false,\ \"user_data\":\"#cloud-config\n\ncoreos:\n etcd:\n discovery: https://discovery.etcd.io/84b281229d938ba03540624f0252f894\n \ From cc43b7f329390dce38ac47ce8a32ffeb5bc135b5 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 22 Jul 2015 15:31:10 +0300 Subject: [PATCH 005/104] add prepare_env script - 4 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 6 +++--- circle.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index cca65c2ad0..2cf57caf70 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -23,10 +23,10 @@ do units:\n - name: etcd.service\n command: start\n - name: fleet.service\n command: start\",\"private_networking\":true}" \ "https://api.digitalocean.com/v2/droplets" | grep "HTTP/1.1" | awk '{print $2}') - if [ "$STATUS_CODE" = "200" ] + if [ "$STATUS_CODE" = "202" ] then - echo "$COREOS_MACHINE created" + echo "$COREOS_MACHINE droplet creation request accepted" else - echo "$COREOS_MACHINE NOT created - status code: $STATUS_CODE" + echo "Problem occurred: $COREOS_MACHINE droplet creation request - status code: $STATUS_CODE" fi done diff --git a/circle.yml b/circle.yml index c9ab308db0..c345546e78 100644 --- a/circle.yml +++ b/circle.yml @@ -30,6 +30,8 @@ dependencies: test: pre: + - chmod +x ci-env/prepare_env.sh + - ci-env/prepare_env.sh # - docker run --privileged --lxc-conf="lxc.aa_profile=unconfined" -d -p 4444 -p 49153:22 -e PORT=4444 --name docker_host_ssh orius123/dind-ssh # - docker port docker_host_ssh 22 - docker run -d -p 49165:8080 jenkins @@ -39,10 +41,8 @@ test: - chmod +x cslang-builder/bin/cslang-builder - mkdir cslang-builder/lib/Lib - pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib - - chmod +x ci-env/prepare_env.sh - - ci-env/prepare_env.sh override: - - ./cslang-builder/bin/cslang-builder -ts cadvisor,jenkins,consul -cov +# - ./cslang-builder/bin/cslang-builder -ts cadvisor,jenkins,consul -cov post: - docker ps -a # - docker stop $(docker ps -a -q) From 1ffb1fa267253164f3b1d1a85fe078c67aa6f845 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 22 Jul 2015 16:08:44 +0300 Subject: [PATCH 006/104] add prepare_env script - 5 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 2cf57caf70..9c87e91370 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -12,16 +12,24 @@ echo ${CIRCLE_BRANCH} # first part echo ${CIRCLE_SHA1} echo ${CIRCLE_BUILD_NUM} # second part -COREOS_MACHINE_NAMES="ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" +COREOS_MACHINE_NAMES="ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1" # ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" for COREOS_MACHINE in $COREOS_MACHINE_NAMES do - STATUS_CODE=$(curl -i -s -L -X POST -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ + CURL_OUTPUT=$(curl -i -s -L -X POST -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ -d "{\"name\":\"$COREOS_MACHINE\",\"region\":\"ams3\",\"size\":\"512mb\",\ \"image\":\"coreos-stable\",\"ssh_keys\":[774367],\"backups\":false,\"ipv6\":false,\ \"user_data\":\"#cloud-config\n\ncoreos:\n etcd:\n discovery: https://discovery.etcd.io/84b281229d938ba03540624f0252f894\n \ addr: $private_ipv4:4001\n peer-addr: $private_ipv4:7001\n fleet:\n public-ip: $private_ipv4\n metadata: public_ip=$public_ipv4\n \ units:\n - name: etcd.service\n command: start\n - name: fleet.service\n command: start\",\"private_networking\":true}" \ - "https://api.digitalocean.com/v2/droplets" | grep "HTTP/1.1" | awk '{print $2}') + "https://api.digitalocean.com/v2/droplets") + + echo $CURL_OUTPUT + + STATUS_CODE=$(echo $CURL_OUTPUT | grep "HTTP/1.1" | awk '{print $2}') + DROPLET_ID=$(echo $CURL_OUTPUT | grep "id" | awk 'NR==1{print $2}') + + echo $STATUS_CODE + echo $DROPLET_ID if [ "$STATUS_CODE" = "202" ] then From f2e1e53b7de8d9bcfaae4946991ad3ce7a66c101 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 22 Jul 2015 16:19:48 +0300 Subject: [PATCH 007/104] add prepare_env script - 6 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 9c87e91370..0cdfb3c28d 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -26,9 +26,12 @@ do echo $CURL_OUTPUT STATUS_CODE=$(echo $CURL_OUTPUT | grep "HTTP/1.1" | awk '{print $2}') - DROPLET_ID=$(echo $CURL_OUTPUT | grep "id" | awk 'NR==1{print $2}') + INTERMEDIATE_RESULT=$(echo $CURL_OUTPUT | grep "id") + echo $INTERMEDIATE_RESULT + DROPLET_ID=$(echo $INTERMEDIATE_RESULT | awk 'NR==1{print $1; print $2; print $3; print $4; print $5; print $16;}') echo $STATUS_CODE + echo $DROPLET_ID if [ "$STATUS_CODE" = "202" ] From da377fd6c6bdf2c036202b4f6739d9dfc5394e9c Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 22 Jul 2015 16:29:45 +0300 Subject: [PATCH 008/104] add prepare_env script - 7 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 0cdfb3c28d..ec4dde2f25 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -23,16 +23,18 @@ do units:\n - name: etcd.service\n command: start\n - name: fleet.service\n command: start\",\"private_networking\":true}" \ "https://api.digitalocean.com/v2/droplets") - echo $CURL_OUTPUT + echo "CURL_OUTPUT: $CURL_OUTPUT" - STATUS_CODE=$(echo $CURL_OUTPUT | grep "HTTP/1.1" | awk '{print $2}') - INTERMEDIATE_RESULT=$(echo $CURL_OUTPUT | grep "id") - echo $INTERMEDIATE_RESULT - DROPLET_ID=$(echo $INTERMEDIATE_RESULT | awk 'NR==1{print $1; print $2; print $3; print $4; print $5; print $16;}') + STATUS_CODE=$(echo $CURL_OUTPUT | awk '{print $2}') - echo $STATUS_CODE +# INTERMEDIATE_RESULT=$(echo $CURL_OUTPUT | grep "id") +# echo $INTERMEDIATE_RESULT +# DROPLET_ID=$(echo $INTERMEDIATE_RESULT | awk 'NR==1{print $1; print $2; print $3; print $4; print $5; print $16;}') - echo $DROPLET_ID + DROPLET_ID=$(echo $CURL_OUTPUT | awk '{print $20;}') + + echo "STATUS_CODE: $STATUS_CODE" + echo "DROPLET_ID: $DROPLET_ID" if [ "$STATUS_CODE" = "202" ] then From c89cf37f5feb9e4b5eeec36fa294c7b24a4bbfc4 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 22 Jul 2015 17:06:24 +0300 Subject: [PATCH 009/104] add prepare_env script - 8 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index ec4dde2f25..70899038f5 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -27,11 +27,9 @@ do STATUS_CODE=$(echo $CURL_OUTPUT | awk '{print $2}') -# INTERMEDIATE_RESULT=$(echo $CURL_OUTPUT | grep "id") -# echo $INTERMEDIATE_RESULT -# DROPLET_ID=$(echo $INTERMEDIATE_RESULT | awk 'NR==1{print $1; print $2; print $3; print $4; print $5; print $16;}') - - DROPLET_ID=$(echo $CURL_OUTPUT | awk '{print $20;}') + DROPLET_DETAILS=$(echo "$CURL_OUTPUT" | grep "droplet") + DROPLET_ID_JUNK=(${DROPLET_DETAILS//:/ })[2] + DROPLET_ID=(${DROPLET_ID_JUNK//,/ })[0] echo "STATUS_CODE: $STATUS_CODE" echo "DROPLET_ID: $DROPLET_ID" From 70d2b34b5d8338c86095773bb4e74f76a5b4717b Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 22 Jul 2015 17:16:29 +0300 Subject: [PATCH 010/104] add prepare_env script - 9 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 70899038f5..8e8dc1a2bf 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -28,8 +28,13 @@ do STATUS_CODE=$(echo $CURL_OUTPUT | awk '{print $2}') DROPLET_DETAILS=$(echo "$CURL_OUTPUT" | grep "droplet") - DROPLET_ID_JUNK=(${DROPLET_DETAILS//:/ })[2] - DROPLET_ID=(${DROPLET_ID_JUNK//,/ })[0] + echo "DROPLET_DETAILS: $DROPLET_DETAILS" + + DROPLET_ID_JUNK_ARRAY=(${DROPLET_DETAILS//:/ }) + DROPLET_ID_JUNK=${DROPLET_ID_JUNK_ARRAY[2]} + + DROPLET_ID_ARRAY=(${DROPLET_ID_JUNK//,/ }) + DROPLET_ID=${DROPLET_ID_ARRAY[0]} echo "STATUS_CODE: $STATUS_CODE" echo "DROPLET_ID: $DROPLET_ID" From 1951186fa1f53f2503b016645cda14045ddb8374 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 22 Jul 2015 17:30:28 +0300 Subject: [PATCH 011/104] add prepare_env script - 10 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 8e8dc1a2bf..32ba921567 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -1,18 +1,6 @@ #!/bin/bash -# test CircleCI env vars -echo $CIRCLE_PR_NUMBER -echo $CIRCLE_BRANCH # first part -echo $CIRCLE_SHA1 -echo $CIRCLE_BUILD_NUM # second part - -# test CircleCI env vars -echo ${CIRCLE_PR_NUMBER} -echo ${CIRCLE_BRANCH} # first part -echo ${CIRCLE_SHA1} -echo ${CIRCLE_BUILD_NUM} # second part - -COREOS_MACHINE_NAMES="ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1" # ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" +COREOS_MACHINE_NAMES="ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" for COREOS_MACHINE in $COREOS_MACHINE_NAMES do CURL_OUTPUT=$(curl -i -s -L -X POST -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ @@ -23,25 +11,22 @@ do units:\n - name: etcd.service\n command: start\n - name: fleet.service\n command: start\",\"private_networking\":true}" \ "https://api.digitalocean.com/v2/droplets") - echo "CURL_OUTPUT: $CURL_OUTPUT" +# echo "CURL_OUTPUT: $CURL_OUTPUT" STATUS_CODE=$(echo $CURL_OUTPUT | awk '{print $2}') - DROPLET_DETAILS=$(echo "$CURL_OUTPUT" | grep "droplet") - echo "DROPLET_DETAILS: $DROPLET_DETAILS" - - DROPLET_ID_JUNK_ARRAY=(${DROPLET_DETAILS//:/ }) - DROPLET_ID_JUNK=${DROPLET_ID_JUNK_ARRAY[2]} - - DROPLET_ID_ARRAY=(${DROPLET_ID_JUNK//,/ }) - DROPLET_ID=${DROPLET_ID_ARRAY[0]} - - echo "STATUS_CODE: $STATUS_CODE" - echo "DROPLET_ID: $DROPLET_ID" - if [ "$STATUS_CODE" = "202" ] then - echo "$COREOS_MACHINE droplet creation request accepted" + DROPLET_DETAILS=$(echo "$CURL_OUTPUT" | grep "droplet") + # echo "DROPLET_DETAILS: $DROPLET_DETAILS" + + # split after `:` and `,` characters and extract the droplet ID + DROPLET_ID_JUNK_ARRAY=(${DROPLET_DETAILS//:/ }) + DROPLET_ID_JUNK=${DROPLET_ID_JUNK_ARRAY[2]} + DROPLET_ID_ARRAY=(${DROPLET_ID_JUNK//,/ }) + DROPLET_ID=${DROPLET_ID_ARRAY[0]} + + echo "$COREOS_MACHINE (ID: $DROPLET_ID) droplet creation request accepted" else echo "Problem occurred: $COREOS_MACHINE droplet creation request - status code: $STATUS_CODE" fi From 0f37318d3cd5ec09e1b063c07511e8df1efa9c9b Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 22 Jul 2015 17:52:51 +0300 Subject: [PATCH 012/104] add prepare_env script - 11 Signed-off-by: Bonczidai Levente --- ci-env/cleanup_env.sh | 4 ++++ ci-env/prepare_env.sh | 11 +++++++++++ circle.yml | 2 ++ 3 files changed, 17 insertions(+) create mode 100644 ci-env/cleanup_env.sh diff --git a/ci-env/cleanup_env.sh b/ci-env/cleanup_env.sh new file mode 100644 index 0000000000..8be2d5605e --- /dev/null +++ b/ci-env/cleanup_env.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +# print droplets +cat < "droplets_${CIRCLE_BUILD_NUM}.txt" diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 32ba921567..4619317fd4 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -1,8 +1,10 @@ #!/bin/bash +DROPLET_ID_ACC="" COREOS_MACHINE_NAMES="ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" for COREOS_MACHINE in $COREOS_MACHINE_NAMES do + # TODO: extract json CURL_OUTPUT=$(curl -i -s -L -X POST -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ -d "{\"name\":\"$COREOS_MACHINE\",\"region\":\"ams3\",\"size\":\"512mb\",\ \"image\":\"coreos-stable\",\"ssh_keys\":[774367],\"backups\":false,\"ipv6\":false,\ @@ -26,8 +28,17 @@ do DROPLET_ID_ARRAY=(${DROPLET_ID_JUNK//,/ }) DROPLET_ID=${DROPLET_ID_ARRAY[0]} + DROPLET_ID_ACC+="${DROPLET_ID} " + echo "$COREOS_MACHINE (ID: $DROPLET_ID) droplet creation request accepted" else echo "Problem occurred: $COREOS_MACHINE droplet creation request - status code: $STATUS_CODE" fi done + +# store droplet IDs in a file to be accessible in cleanup script +echo $DROPLET_ID_ACC +echo $DROPLET_ID_ACC > "droplets_${CIRCLE_BUILD_NUM}.txt" + +# TODO: add waiting loop for droplet startup +# sleep 20 diff --git a/circle.yml b/circle.yml index c345546e78..6acfe7b0d0 100644 --- a/circle.yml +++ b/circle.yml @@ -44,6 +44,8 @@ test: override: # - ./cslang-builder/bin/cslang-builder -ts cadvisor,jenkins,consul -cov post: + - chmod +x ci-env/cleanup_env.sh + - ci-env/cleanup_env.sh - docker ps -a # - docker stop $(docker ps -a -q) # - docker rm $(docker ps -a -q) From 85f8257f34220ae76b5eeaf26660319e0ea948ed Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 23 Jul 2015 11:01:44 +0300 Subject: [PATCH 013/104] add cleanup_env script - 1 Signed-off-by: Bonczidai Levente --- ci-env/cleanup_env.sh | 20 ++++++++++++++++++++ ci-env/prepare_env.sh | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/ci-env/cleanup_env.sh b/ci-env/cleanup_env.sh index 8be2d5605e..f66755b4ed 100644 --- a/ci-env/cleanup_env.sh +++ b/ci-env/cleanup_env.sh @@ -2,3 +2,23 @@ # print droplets cat < "droplets_${CIRCLE_BUILD_NUM}.txt" + +DROPLET_ID_ACC=$(cat < "droplets_${CIRCLE_BUILD_NUM}.txt") +echo "DROPLET_ID_ACC: $DROPLET_ID_ACC" + +for DROPLET_ID in $DROPLET_ID_ACC +do + CURL_OUTPUT=$(curl -i -s -L -X DELETE -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ + "https://api.digitalocean.com/v2/droplets/$DROPLET_ID") + echo "CURL_OUTPUT: $CURL_OUTPUT" + + STATUS_CODE=$(echo "$CURL_OUTPUT" | grep "status" | awk '{print $2}') + echo "STATUS_CODE: $STATUS_CODE" + + if [ "$STATUS_CODE" = "204" ] + then + echo "Droplet($DROPLET_ID) deleted successfully" + else + echo "Problem occurred: destroying droplet($DROPLET_ID) - status code: $STATUS_CODE" + fi +done diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 4619317fd4..ff068db919 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -37,7 +37,7 @@ do done # store droplet IDs in a file to be accessible in cleanup script -echo $DROPLET_ID_ACC +# echo $DROPLET_ID_ACC echo $DROPLET_ID_ACC > "droplets_${CIRCLE_BUILD_NUM}.txt" # TODO: add waiting loop for droplet startup From d068c45ed3f893103d5c40da8714c35227fc6da6 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 23 Jul 2015 11:14:20 +0300 Subject: [PATCH 014/104] add cleanup_env script - 2 Signed-off-by: Bonczidai Levente --- ci-env/cleanup_env.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci-env/cleanup_env.sh b/ci-env/cleanup_env.sh index f66755b4ed..d00f983d36 100644 --- a/ci-env/cleanup_env.sh +++ b/ci-env/cleanup_env.sh @@ -1,18 +1,18 @@ #!/bin/bash # print droplets -cat < "droplets_${CIRCLE_BUILD_NUM}.txt" +# cat < "droplets_${CIRCLE_BUILD_NUM}.txt" DROPLET_ID_ACC=$(cat < "droplets_${CIRCLE_BUILD_NUM}.txt") -echo "DROPLET_ID_ACC: $DROPLET_ID_ACC" +# echo "DROPLET_ID_ACC: $DROPLET_ID_ACC" for DROPLET_ID in $DROPLET_ID_ACC do CURL_OUTPUT=$(curl -i -s -L -X DELETE -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ "https://api.digitalocean.com/v2/droplets/$DROPLET_ID") - echo "CURL_OUTPUT: $CURL_OUTPUT" + # echo "CURL_OUTPUT: $CURL_OUTPUT" - STATUS_CODE=$(echo "$CURL_OUTPUT" | grep "status" | awk '{print $2}') + STATUS_CODE=$(echo "$CURL_OUTPUT" | grep "Status" | awk '{print $2}') echo "STATUS_CODE: $STATUS_CODE" if [ "$STATUS_CODE" = "204" ] From adf7731aa77b28cfd5323e5ff1a53704c4817562 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 27 Jul 2015 10:39:23 +0300 Subject: [PATCH 015/104] retrieve ip addresses Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index ff068db919..5db5c803ff 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -1,6 +1,8 @@ #!/bin/bash DROPLET_ID_ACC="" +DROPLET_IP_ADDRESS_ACC="" + COREOS_MACHINE_NAMES="ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" for COREOS_MACHINE in $COREOS_MACHINE_NAMES do @@ -41,4 +43,30 @@ done echo $DROPLET_ID_ACC > "droplets_${CIRCLE_BUILD_NUM}.txt" # TODO: add waiting loop for droplet startup -# sleep 20 +sleep 20 + +# retrieve IPv4 addresses of droplets +for DROPLET_ID in $DROPLET_ID_ACC +do + CURL_OUTPUT=$(curl -i -s -L -X GET -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ + "https://api.digitalocean.com/v2/droplets/$DROPLET_ID") + echo "CURL_OUTPUT - GET DROPLET BY ID: $CURL_OUTPUT" + + STATUS_CODE=$(echo "$CURL_OUTPUT" | grep "Status" | awk '{print $2}') + echo "STATUS_CODE: $STATUS_CODE" + + if [ "$STATUS_CODE" = "200" ] + then + echo "Droplet($DROPLET_ID) information retrieved successfully" + + IP_ADDRESS_JUNK=$(echo "$CURL_OUTPUT" | grep "ip_address") + IP_ADDRESS_ARRAY=(${IP_ADDRESS_JUNK//\"/ }) + IP_ADDRESS=${IP_ADDRESS_ARRAY[2]} + echo "Droplet($DROPLET_ID) IPv4 address: $IP_ADDRESS" + + DROPLET_IP_ADDRESS_ACC+="${IP_ADDRESS} " + echo "DROPLET_IP_ADDRESS_ACC: $DROPLET_IP_ADDRESS_ACC" + else + echo "Problem occurred: retrieving droplet($DROPLET_ID) information - status code: $STATUS_CODE" + fi +done From 6678a1a5405890a67384254eeb06ab93fdbcc366 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 27 Jul 2015 11:32:14 +0300 Subject: [PATCH 016/104] retrieve ip addresses 2 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 5db5c803ff..fa2800b075 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -50,7 +50,7 @@ for DROPLET_ID in $DROPLET_ID_ACC do CURL_OUTPUT=$(curl -i -s -L -X GET -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ "https://api.digitalocean.com/v2/droplets/$DROPLET_ID") - echo "CURL_OUTPUT - GET DROPLET BY ID: $CURL_OUTPUT" + # echo "CURL_OUTPUT - GET DROPLET BY ID: $CURL_OUTPUT" STATUS_CODE=$(echo "$CURL_OUTPUT" | grep "Status" | awk '{print $2}') echo "STATUS_CODE: $STATUS_CODE" @@ -59,9 +59,14 @@ do then echo "Droplet($DROPLET_ID) information retrieved successfully" - IP_ADDRESS_JUNK=$(echo "$CURL_OUTPUT" | grep "ip_address") - IP_ADDRESS_ARRAY=(${IP_ADDRESS_JUNK//\"/ }) - IP_ADDRESS=${IP_ADDRESS_ARRAY[2]} + RESPONSE_BODY_JSON=$(echo "$CURL_OUTPUT" | grep "ip_address") + echo "IP_ADDRESS_JUNK: $RESPONSE_BODY_JSON" + + IP_ADDRESS=$(echo "$RESPONSE_BODY_JSON" | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["droplet"]["networks"]["v4"][0]["ip_address"]') + + # IP_ADDRESS_ARRAY=(${IP_ADDRESS_JUNK//\"/ }) + # echo "IP_ADDRESS_ARRAY: $IP_ADDRESS_ARRAY" + # IP_ADDRESS=${IP_ADDRESS_ARRAY[2]} echo "Droplet($DROPLET_ID) IPv4 address: $IP_ADDRESS" DROPLET_IP_ADDRESS_ACC+="${IP_ADDRESS} " From a67874b093306b72dc05a137640e95d64e817095 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 27 Jul 2015 12:49:01 +0300 Subject: [PATCH 017/104] remove debug prints Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index fa2800b075..502626ba55 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -53,20 +53,16 @@ do # echo "CURL_OUTPUT - GET DROPLET BY ID: $CURL_OUTPUT" STATUS_CODE=$(echo "$CURL_OUTPUT" | grep "Status" | awk '{print $2}') - echo "STATUS_CODE: $STATUS_CODE" + # echo "STATUS_CODE: $STATUS_CODE" if [ "$STATUS_CODE" = "200" ] then echo "Droplet($DROPLET_ID) information retrieved successfully" RESPONSE_BODY_JSON=$(echo "$CURL_OUTPUT" | grep "ip_address") - echo "IP_ADDRESS_JUNK: $RESPONSE_BODY_JSON" + # echo "IP_ADDRESS_JUNK: $RESPONSE_BODY_JSON" IP_ADDRESS=$(echo "$RESPONSE_BODY_JSON" | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["droplet"]["networks"]["v4"][0]["ip_address"]') - - # IP_ADDRESS_ARRAY=(${IP_ADDRESS_JUNK//\"/ }) - # echo "IP_ADDRESS_ARRAY: $IP_ADDRESS_ARRAY" - # IP_ADDRESS=${IP_ADDRESS_ARRAY[2]} echo "Droplet($DROPLET_ID) IPv4 address: $IP_ADDRESS" DROPLET_IP_ADDRESS_ACC+="${IP_ADDRESS} " From 2826f8ebbcd8e2cc9fbb46e47184758048a646c0 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 27 Jul 2015 14:53:44 +0300 Subject: [PATCH 018/104] test droplet access Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 16 ++++- circle.yml | 6 +- .../test_access_coreos_machine.inputs.yaml | 17 +++++ .../coreos/test_access_coreos_machine.sl | 62 +++++++++++++++++++ 4 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml create mode 100644 test/io/cloudslang/coreos/test_access_coreos_machine.sl diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 502626ba55..c540c8d7fb 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -9,7 +9,7 @@ do # TODO: extract json CURL_OUTPUT=$(curl -i -s -L -X POST -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ -d "{\"name\":\"$COREOS_MACHINE\",\"region\":\"ams3\",\"size\":\"512mb\",\ - \"image\":\"coreos-stable\",\"ssh_keys\":[774367],\"backups\":false,\"ipv6\":false,\ + \"image\":\"coreos-stable\",\"ssh_keys\":[993143],\"backups\":false,\"ipv6\":false,\ \"user_data\":\"#cloud-config\n\ncoreos:\n etcd:\n discovery: https://discovery.etcd.io/84b281229d938ba03540624f0252f894\n \ addr: $private_ipv4:4001\n peer-addr: $private_ipv4:7001\n fleet:\n public-ip: $private_ipv4\n metadata: public_ip=$public_ipv4\n \ units:\n - name: etcd.service\n command: start\n - name: fleet.service\n command: start\",\"private_networking\":true}" \ @@ -71,3 +71,17 @@ do echo "Problem occurred: retrieving droplet($DROPLET_ID) information - status code: $STATUS_CODE" fi done + +# update inputs files to use actual IP addresses +DROPLET_IP_ARRAY=($DROPLET_IP_ADDRESS_ACC) +sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml + +# create ssh private key +SSH_KEY_PATH='/root/.ssh/droplets_rsa' +echo "$SSH_PRIVATE_KEY_CI_ENV_TEMP" > $SSH_KEY_PATH +ls -l /root/.ssh/ + +# update inputs files to use actual ssh key +sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml + +cat test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml diff --git a/circle.yml b/circle.yml index 6acfe7b0d0..72072dc35a 100644 --- a/circle.yml +++ b/circle.yml @@ -30,8 +30,6 @@ dependencies: test: pre: - - chmod +x ci-env/prepare_env.sh - - ci-env/prepare_env.sh # - docker run --privileged --lxc-conf="lxc.aa_profile=unconfined" -d -p 4444 -p 49153:22 -e PORT=4444 --name docker_host_ssh orius123/dind-ssh # - docker port docker_host_ssh 22 - docker run -d -p 49165:8080 jenkins @@ -41,8 +39,10 @@ test: - chmod +x cslang-builder/bin/cslang-builder - mkdir cslang-builder/lib/Lib - pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib + - chmod +x ci-env/prepare_env.sh + - ci-env/prepare_env.sh override: -# - ./cslang-builder/bin/cslang-builder -ts cadvisor,jenkins,consul -cov + - ./cslang-builder/bin/cslang-builder -ts coreos -cov post: - chmod +x ci-env/cleanup_env.sh - ci-env/cleanup_env.sh diff --git a/test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml b/test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml new file mode 100644 index 0000000000..de69abec5b --- /dev/null +++ b/test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml @@ -0,0 +1,17 @@ +# (c) Copyright 2014 Hewlett-Packard Development Company, L.P. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License v2.0 which accompany this distribution. +# +# The Apache License is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# +#################################################### +test_access_coreos_machine: + testSuites: [coreos] + description: test stuff + testFlowPath: io.cloudslang.coreos.test_access_coreos_machine + inputs: + - coreos_host: # ip address of one host from the cluster + - coreos_username: core # username for the machine + - private_key_file: # path to private key file + result: SUCCESS diff --git a/test/io/cloudslang/coreos/test_access_coreos_machine.sl b/test/io/cloudslang/coreos/test_access_coreos_machine.sl new file mode 100644 index 0000000000..09dd99dee4 --- /dev/null +++ b/test/io/cloudslang/coreos/test_access_coreos_machine.sl @@ -0,0 +1,62 @@ +# (c) Copyright 2014 Hewlett-Packard Development Company, L.P. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License v2.0 which accompany this distribution. +# +# The Apache License is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# +#################################################### +# Wrapper flow - logic steps: +# - retrieves the ip addresses of the machines in the cluster +# - cleanup on the machines (so they will not contain any images) +# - prepares a used and an unused Docker image +# - runs the flow +# - verifies only one image remained in the cluster +# - delete the used image +#################################################### + +namespace: io.cloudslang.coreos + +imports: + ssh: io.cloudslang.base.remote_command_execution.ssh + print: io.cloudslang.base.print + +flow: + name: test_access_coreos_machine + inputs: + - coreos_host + - coreos_username + - coreos_password: + required: false + - private_key_file: + required: false + - percentage: + required: false + - timeout: + required: false + - unused_image_name: + default: "'tomcat:7'" + - used_image_name: + default: "'busybox'" + - number_of_images_in_cluster: + default: 0 + overridable: false + + workflow: + - test_ssh_access: + do: + ssh.ssh_flow: + - host: coreos_host + - command: "'pwd'" + - username: coreos_username + - privateKeyFile: + default: private_key_file + required: false + publish: + - returnResult + + - print_result: + do: + print.print_text: + - text: > + 'ssh result -> ' + returnResult From e153c1d13e43889792d6664a9d1bc338e2f8b28e Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 27 Jul 2015 15:06:39 +0300 Subject: [PATCH 019/104] test droplet access 2 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 4 ++-- circle.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index c540c8d7fb..dc47b25b60 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -77,9 +77,9 @@ DROPLET_IP_ARRAY=($DROPLET_IP_ADDRESS_ACC) sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml # create ssh private key -SSH_KEY_PATH='/root/.ssh/droplets_rsa' +SSH_KEY_PATH=droplets_rsa echo "$SSH_PRIVATE_KEY_CI_ENV_TEMP" > $SSH_KEY_PATH -ls -l /root/.ssh/ +ls -l . # update inputs files to use actual ssh key sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml diff --git a/circle.yml b/circle.yml index 72072dc35a..de9aa73d24 100644 --- a/circle.yml +++ b/circle.yml @@ -42,7 +42,7 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - ./cslang-builder/bin/cslang-builder -ts coreos -cov + - ./cslang-builder/bin/cslang-builder -ts coreos,!default -cov post: - chmod +x ci-env/cleanup_env.sh - ci-env/cleanup_env.sh From 658655dc260295f8e4a6234529869c459ee9c86d Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 27 Jul 2015 15:22:44 +0300 Subject: [PATCH 020/104] test droplet access 3 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index dc47b25b60..faf75f905f 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -78,7 +78,7 @@ sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/test # create ssh private key SSH_KEY_PATH=droplets_rsa -echo "$SSH_PRIVATE_KEY_CI_ENV_TEMP" > $SSH_KEY_PATH +echo -e "$SSH_PRIVATE_KEY_CI_ENV_TEMP2" > $SSH_KEY_PATH ls -l . # update inputs files to use actual ssh key From 4170b91abea51c8dbb2f733982887712bd9c77e3 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 27 Jul 2015 15:29:33 +0300 Subject: [PATCH 021/104] test droplet access 4 Signed-off-by: Bonczidai Levente --- ci-env/cleanup_env.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-env/cleanup_env.sh b/ci-env/cleanup_env.sh index d00f983d36..e6f556af03 100644 --- a/ci-env/cleanup_env.sh +++ b/ci-env/cleanup_env.sh @@ -6,6 +6,7 @@ DROPLET_ID_ACC=$(cat < "droplets_${CIRCLE_BUILD_NUM}.txt") # echo "DROPLET_ID_ACC: $DROPLET_ID_ACC" +DROPLET_ID_ACC='' # TODO: remove this - temporary disable droplet cleanup for DROPLET_ID in $DROPLET_ID_ACC do CURL_OUTPUT=$(curl -i -s -L -X DELETE -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ From 81c5273892f0dfb708f732133a22fa5a337e8a13 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 27 Jul 2015 15:40:30 +0300 Subject: [PATCH 022/104] test droplet access 5 Signed-off-by: Bonczidai Levente --- ci-env/cleanup_env.sh | 2 +- ci-env/prepare_env.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-env/cleanup_env.sh b/ci-env/cleanup_env.sh index e6f556af03..39b97ee380 100644 --- a/ci-env/cleanup_env.sh +++ b/ci-env/cleanup_env.sh @@ -6,7 +6,7 @@ DROPLET_ID_ACC=$(cat < "droplets_${CIRCLE_BUILD_NUM}.txt") # echo "DROPLET_ID_ACC: $DROPLET_ID_ACC" -DROPLET_ID_ACC='' # TODO: remove this - temporary disable droplet cleanup +# DROPLET_ID_ACC='' # TODO: remove this - temporary disable droplet cleanup for DROPLET_ID in $DROPLET_ID_ACC do CURL_OUTPUT=$(curl -i -s -L -X DELETE -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index faf75f905f..cb3fb1dde8 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -43,7 +43,7 @@ done echo $DROPLET_ID_ACC > "droplets_${CIRCLE_BUILD_NUM}.txt" # TODO: add waiting loop for droplet startup -sleep 20 +sleep 60 # retrieve IPv4 addresses of droplets for DROPLET_ID in $DROPLET_ID_ACC From d597d2be4c6396d2c589b474579213478f4deacd Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 28 Jul 2015 10:00:45 +0300 Subject: [PATCH 023/104] test droplet access 6 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 1 + .../test_access_coreos_machine.inputs.yaml | 2 +- .../test_access_coreos_machine2.inputs.yaml | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 test/io/cloudslang/coreos/test_access_coreos_machine2.inputs.yaml diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index cb3fb1dde8..fe03546431 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -80,6 +80,7 @@ sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/test SSH_KEY_PATH=droplets_rsa echo -e "$SSH_PRIVATE_KEY_CI_ENV_TEMP2" > $SSH_KEY_PATH ls -l . +cat droplets_rsa # update inputs files to use actual ssh key sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml diff --git a/test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml b/test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml index de69abec5b..162bb58fd0 100644 --- a/test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml +++ b/test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml @@ -11,7 +11,7 @@ test_access_coreos_machine: description: test stuff testFlowPath: io.cloudslang.coreos.test_access_coreos_machine inputs: - - coreos_host: # ip address of one host from the cluster + - coreos_host: 188.166.96.176 # # ip address of one host from the cluster - coreos_username: core # username for the machine - private_key_file: # path to private key file result: SUCCESS diff --git a/test/io/cloudslang/coreos/test_access_coreos_machine2.inputs.yaml b/test/io/cloudslang/coreos/test_access_coreos_machine2.inputs.yaml new file mode 100644 index 0000000000..37bdce16e9 --- /dev/null +++ b/test/io/cloudslang/coreos/test_access_coreos_machine2.inputs.yaml @@ -0,0 +1,17 @@ +# (c) Copyright 2014 Hewlett-Packard Development Company, L.P. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License v2.0 which accompany this distribution. +# +# The Apache License is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# +#################################################### +test_access_coreos_machine: + testSuites: [coreos] + description: test stuff + testFlowPath: io.cloudslang.coreos.test_access_coreos_machine + inputs: + - coreos_host: 188.166.43.167 # # ip address of one host from the cluster + - coreos_username: core # username for the machine + - private_key_file: ./droplets_rsa # path to private key file + result: SUCCESS From ea9234a57cc82ea0992d3d750907f616758aefa6 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 28 Jul 2015 10:07:24 +0300 Subject: [PATCH 024/104] test droplet access 7 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 3 +-- .../test_access_coreos_machine2.inputs.yaml | 17 ----------------- 2 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 test/io/cloudslang/coreos/test_access_coreos_machine2.inputs.yaml diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index fe03546431..09213bb6af 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -79,8 +79,7 @@ sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/test # create ssh private key SSH_KEY_PATH=droplets_rsa echo -e "$SSH_PRIVATE_KEY_CI_ENV_TEMP2" > $SSH_KEY_PATH -ls -l . -cat droplets_rsa +# ls -l . # update inputs files to use actual ssh key sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml diff --git a/test/io/cloudslang/coreos/test_access_coreos_machine2.inputs.yaml b/test/io/cloudslang/coreos/test_access_coreos_machine2.inputs.yaml deleted file mode 100644 index 37bdce16e9..0000000000 --- a/test/io/cloudslang/coreos/test_access_coreos_machine2.inputs.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License v2.0 which accompany this distribution. -# -# The Apache License is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# -#################################################### -test_access_coreos_machine: - testSuites: [coreos] - description: test stuff - testFlowPath: io.cloudslang.coreos.test_access_coreos_machine - inputs: - - coreos_host: 188.166.43.167 # # ip address of one host from the cluster - - coreos_username: core # username for the machine - - private_key_file: ./droplets_rsa # path to private key file - result: SUCCESS From 39352fa1bf6637c68a30a6670894209aeb2c8c2d Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 28 Jul 2015 15:41:39 +0300 Subject: [PATCH 025/104] test droplet access 8 Signed-off-by: Bonczidai Levente --- ci-env/cloud-config.yaml | 15 +++++++++ ci-env/prepare_env.sh | 33 ++++++++++++------- .../coreos/test_access_coreos_machine.sl | 2 +- 3 files changed, 38 insertions(+), 12 deletions(-) create mode 100644 ci-env/cloud-config.yaml diff --git a/ci-env/cloud-config.yaml b/ci-env/cloud-config.yaml new file mode 100644 index 0000000000..e286827334 --- /dev/null +++ b/ci-env/cloud-config.yaml @@ -0,0 +1,15 @@ +#cloud-config + +coreos: + etcd: + discovery: + addr: $private_ipv4:4001 + peer-addr: $private_ipv4:7001 + fleet: + public-ip: $private_ipv4 + metadata: public_ip=$public_ipv4 + units: + - name: etcd.service + command: start + - name: fleet.service + command: start diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 09213bb6af..c7a7036107 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -3,19 +3,30 @@ DROPLET_ID_ACC="" DROPLET_IP_ADDRESS_ACC="" -COREOS_MACHINE_NAMES="ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" +DISCOVERY_URL=$(curl -X GET "https://discovery.etcd.io/new") +echo "DISCOVERY_URL: $DISCOVERY_URL" +sed -i "s//${DISCOVERY_URL}/g" ci-env/cloud-config.yaml +cat ci-env/cloud-config.yaml + +COREOS_MACHINE_NAMES="ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1" # ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" for COREOS_MACHINE in $COREOS_MACHINE_NAMES do - # TODO: extract json - CURL_OUTPUT=$(curl -i -s -L -X POST -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ - -d "{\"name\":\"$COREOS_MACHINE\",\"region\":\"ams3\",\"size\":\"512mb\",\ - \"image\":\"coreos-stable\",\"ssh_keys\":[993143],\"backups\":false,\"ipv6\":false,\ - \"user_data\":\"#cloud-config\n\ncoreos:\n etcd:\n discovery: https://discovery.etcd.io/84b281229d938ba03540624f0252f894\n \ - addr: $private_ipv4:4001\n peer-addr: $private_ipv4:7001\n fleet:\n public-ip: $private_ipv4\n metadata: public_ip=$public_ipv4\n \ - units:\n - name: etcd.service\n command: start\n - name: fleet.service\n command: start\",\"private_networking\":true}" \ - "https://api.digitalocean.com/v2/droplets") - -# echo "CURL_OUTPUT: $CURL_OUTPUT" + CURL_OUTPUT=$(curl -i -s -X POST https://api.digitalocean.com/v2/droplets \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $DO_API_TOKEN" \ + -d '{ + "name":"${COREOS_MACHINE}", + "region":"ams3", + "size":"512mb", + "image":"coreos-stable", + "ssh_keys":[993143], + "backups":false, + "ipv6":false, + "private_networking":true, + "user_data": "'"$(cat ci-env/cloud-config.yaml | sed 's/"/\\"/g')"'" + }') + + echo "CURL_OUTPUT: $CURL_OUTPUT" STATUS_CODE=$(echo $CURL_OUTPUT | awk '{print $2}') diff --git a/test/io/cloudslang/coreos/test_access_coreos_machine.sl b/test/io/cloudslang/coreos/test_access_coreos_machine.sl index 09dd99dee4..afdde01539 100644 --- a/test/io/cloudslang/coreos/test_access_coreos_machine.sl +++ b/test/io/cloudslang/coreos/test_access_coreos_machine.sl @@ -47,7 +47,7 @@ flow: do: ssh.ssh_flow: - host: coreos_host - - command: "'pwd'" + - command: "'fleetctl list-machines'" - username: coreos_username - privateKeyFile: default: private_key_file From 06b6701b77f6e22e569e986a1d214206f1b2b0f4 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 28 Jul 2015 15:59:36 +0300 Subject: [PATCH 026/104] test droplet access 9 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index c7a7036107..23ca8d0ba4 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -5,7 +5,8 @@ DROPLET_IP_ADDRESS_ACC="" DISCOVERY_URL=$(curl -X GET "https://discovery.etcd.io/new") echo "DISCOVERY_URL: $DISCOVERY_URL" -sed -i "s//${DISCOVERY_URL}/g" ci-env/cloud-config.yaml +DISCOVERY_URL_ESCAPED=$(echo $DISCOVERY_URL | sed 's/\//\\\//g') +sed -i "s//${DISCOVERY_URL_ESCAPED}/g" ci-env/cloud-config.yaml cat ci-env/cloud-config.yaml COREOS_MACHINE_NAMES="ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1" # ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" From a7b9611fb41eb9f6c9c8c63d1d2bb0787d7aea49 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 28 Jul 2015 16:22:44 +0300 Subject: [PATCH 027/104] test droplet access 10 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 23ca8d0ba4..e878397c21 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -15,8 +15,8 @@ do CURL_OUTPUT=$(curl -i -s -X POST https://api.digitalocean.com/v2/droplets \ -H 'Content-Type: application/json' \ -H "Authorization: Bearer $DO_API_TOKEN" \ - -d '{ - "name":"${COREOS_MACHINE}", + -d "{ + \"name\":\"${COREOS_MACHINE}\","' "region":"ams3", "size":"512mb", "image":"coreos-stable", From 837d5a93c26eeb30b090a801785b93ac6416e18b Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 29 Jul 2015 09:46:11 +0300 Subject: [PATCH 028/104] test droplet access 11 Signed-off-by: Bonczidai Levente --- .../io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml b/test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml index 162bb58fd0..de69abec5b 100644 --- a/test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml +++ b/test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml @@ -11,7 +11,7 @@ test_access_coreos_machine: description: test stuff testFlowPath: io.cloudslang.coreos.test_access_coreos_machine inputs: - - coreos_host: 188.166.96.176 # # ip address of one host from the cluster + - coreos_host: # ip address of one host from the cluster - coreos_username: core # username for the machine - private_key_file: # path to private key file result: SUCCESS From 2efd6e115f0e14b1b924a61e7adca6352ec2c4c5 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 29 Jul 2015 09:54:30 +0300 Subject: [PATCH 029/104] test droplet access 12 Signed-off-by: Bonczidai Levente --- ci-env/cleanup_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-env/cleanup_env.sh b/ci-env/cleanup_env.sh index 39b97ee380..e6f556af03 100644 --- a/ci-env/cleanup_env.sh +++ b/ci-env/cleanup_env.sh @@ -6,7 +6,7 @@ DROPLET_ID_ACC=$(cat < "droplets_${CIRCLE_BUILD_NUM}.txt") # echo "DROPLET_ID_ACC: $DROPLET_ID_ACC" -# DROPLET_ID_ACC='' # TODO: remove this - temporary disable droplet cleanup +DROPLET_ID_ACC='' # TODO: remove this - temporary disable droplet cleanup for DROPLET_ID in $DROPLET_ID_ACC do CURL_OUTPUT=$(curl -i -s -L -X DELETE -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ From be6762b98f2407f6f5d257a7ef6079a834f50862 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 29 Jul 2015 10:20:40 +0300 Subject: [PATCH 030/104] check private / public ip types Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index e878397c21..7be326815f 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -74,7 +74,19 @@ do RESPONSE_BODY_JSON=$(echo "$CURL_OUTPUT" | grep "ip_address") # echo "IP_ADDRESS_JUNK: $RESPONSE_BODY_JSON" - IP_ADDRESS=$(echo "$RESPONSE_BODY_JSON" | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["droplet"]["networks"]["v4"][0]["ip_address"]') + IP_ADDRESS=$(\ + echo "$RESPONSE_BODY_JSON" | python -c \ + ' + import json,sys; + obj = json.load(sys.stdin); + ipv4_list = obj["droplet"]["networks"]["v4"]; + ip = "" + for ip_obj in ipv4_list: + if ip_obj["type"] == "public": + ip = ip_obj["ip_address"]; + break; + print ip; + ') echo "Droplet($DROPLET_ID) IPv4 address: $IP_ADDRESS" DROPLET_IP_ADDRESS_ACC+="${IP_ADDRESS} " From b1f00430f85c08790b6df4faaeaaea6e82bcb4e3 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 29 Jul 2015 10:29:53 +0300 Subject: [PATCH 031/104] check private / public ip types - resolve python indentation issue Signed-off-by: Bonczidai Levente --- ci-env/cleanup_env.sh | 2 +- ci-env/prepare_env.sh | 23 ++++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ci-env/cleanup_env.sh b/ci-env/cleanup_env.sh index e6f556af03..39b97ee380 100644 --- a/ci-env/cleanup_env.sh +++ b/ci-env/cleanup_env.sh @@ -6,7 +6,7 @@ DROPLET_ID_ACC=$(cat < "droplets_${CIRCLE_BUILD_NUM}.txt") # echo "DROPLET_ID_ACC: $DROPLET_ID_ACC" -DROPLET_ID_ACC='' # TODO: remove this - temporary disable droplet cleanup +# DROPLET_ID_ACC='' # TODO: remove this - temporary disable droplet cleanup for DROPLET_ID in $DROPLET_ID_ACC do CURL_OUTPUT=$(curl -i -s -L -X DELETE -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 7be326815f..1a4e4073ce 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -76,17 +76,18 @@ do IP_ADDRESS=$(\ echo "$RESPONSE_BODY_JSON" | python -c \ - ' - import json,sys; - obj = json.load(sys.stdin); - ipv4_list = obj["droplet"]["networks"]["v4"]; - ip = "" - for ip_obj in ipv4_list: - if ip_obj["type"] == "public": - ip = ip_obj["ip_address"]; - break; - print ip; - ') +' +import json,sys; +obj = json.load(sys.stdin); +ipv4_list = obj["droplet"]["networks"]["v4"]; +ip = "" +for ip_obj in ipv4_list: + if ip_obj["type"] == "public": + ip = ip_obj["ip_address"]; + break; +print ip; +'\ + ) echo "Droplet($DROPLET_ID) IPv4 address: $IP_ADDRESS" DROPLET_IP_ADDRESS_ACC+="${IP_ADDRESS} " From 53734527ed490df50978c0b3044e6ec736b047ae Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 29 Jul 2015 10:40:09 +0300 Subject: [PATCH 032/104] add cluster_docker_images_maintenance to ci Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 3 +++ .../cluster_docker_images_maintenance.inputs.yaml | 13 +++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 1a4e4073ce..5eb7ff9e6f 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -100,6 +100,7 @@ done # update inputs files to use actual IP addresses DROPLET_IP_ARRAY=($DROPLET_IP_ADDRESS_ACC) sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml +sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml # create ssh private key SSH_KEY_PATH=droplets_rsa @@ -108,5 +109,7 @@ echo -e "$SSH_PRIVATE_KEY_CI_ENV_TEMP2" > $SSH_KEY_PATH # update inputs files to use actual ssh key sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml +sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml cat test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml +cat test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml diff --git a/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml b/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml index ca360acfd0..ccadfc91c7 100644 --- a/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml +++ b/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml @@ -7,11 +7,16 @@ # #################################################### test_cluster_docker_images_maintenance_success: - testSuites: [coreos_local] - description: This test needs to be run on local machine by activating the test suite and supplying the inputs + testSuites: [coreos] + description: > + This test cleans the CoreOS hosts in a cluster, + prepares a used and an unused Docker image, + runs the cluster_docker_images_maintenance flow, + checks that only the unused Docker image was deleted, + cleans up in the cluster. testFlowPath: io.cloudslang.coreos.test_cluster_docker_images_maintenance inputs: - - coreos_host: 111.111.111.111 # ip address of one host from the cluster + - coreos_host: # ip address of one host from the cluster - coreos_username: core # username for the machine - - private_key_file: c:/Users/bob/ssh_keys/id_rsa # path to private key file + - private_key_file: # path to private key file result: SUCCESS From 47696a20e9c952ad7a95338d09abd9852219dab1 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 29 Jul 2015 10:50:57 +0300 Subject: [PATCH 033/104] temporarily disable droplet removal Signed-off-by: Bonczidai Levente --- ci-env/cleanup_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-env/cleanup_env.sh b/ci-env/cleanup_env.sh index 39b97ee380..e6f556af03 100644 --- a/ci-env/cleanup_env.sh +++ b/ci-env/cleanup_env.sh @@ -6,7 +6,7 @@ DROPLET_ID_ACC=$(cat < "droplets_${CIRCLE_BUILD_NUM}.txt") # echo "DROPLET_ID_ACC: $DROPLET_ID_ACC" -# DROPLET_ID_ACC='' # TODO: remove this - temporary disable droplet cleanup +DROPLET_ID_ACC='' # TODO: remove this - temporary disable droplet cleanup for DROPLET_ID in $DROPLET_ID_ACC do CURL_OUTPUT=$(curl -i -s -L -X DELETE -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ From 720bb049d0b36ef2e3229bd6b49fb232f1e4fadc Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 29 Jul 2015 10:51:39 +0300 Subject: [PATCH 034/104] enable multiple machines in the cluster Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 5eb7ff9e6f..a0966010d4 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -9,7 +9,7 @@ DISCOVERY_URL_ESCAPED=$(echo $DISCOVERY_URL | sed 's/\//\\\//g') sed -i "s//${DISCOVERY_URL_ESCAPED}/g" ci-env/cloud-config.yaml cat ci-env/cloud-config.yaml -COREOS_MACHINE_NAMES="ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1" # ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" +COREOS_MACHINE_NAMES="ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" for COREOS_MACHINE in $COREOS_MACHINE_NAMES do CURL_OUTPUT=$(curl -i -s -X POST https://api.digitalocean.com/v2/droplets \ From 157c6456ad6eeb393012a021322b30ab92ddd6cf Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 29 Jul 2015 11:31:37 +0300 Subject: [PATCH 035/104] verify machines in cluster Signed-off-by: Bonczidai Levente --- ci-env/cleanup_env.sh | 2 +- .../coreos/cluster_docker_images_maintenance.inputs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-env/cleanup_env.sh b/ci-env/cleanup_env.sh index e6f556af03..39b97ee380 100644 --- a/ci-env/cleanup_env.sh +++ b/ci-env/cleanup_env.sh @@ -6,7 +6,7 @@ DROPLET_ID_ACC=$(cat < "droplets_${CIRCLE_BUILD_NUM}.txt") # echo "DROPLET_ID_ACC: $DROPLET_ID_ACC" -DROPLET_ID_ACC='' # TODO: remove this - temporary disable droplet cleanup +# DROPLET_ID_ACC='' # TODO: remove this - temporary disable droplet cleanup for DROPLET_ID in $DROPLET_ID_ACC do CURL_OUTPUT=$(curl -i -s -L -X DELETE -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ diff --git a/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml b/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml index ccadfc91c7..74d603e36c 100644 --- a/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml +++ b/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml @@ -7,7 +7,7 @@ # #################################################### test_cluster_docker_images_maintenance_success: - testSuites: [coreos] + testSuites: [coreos_local] description: > This test cleans the CoreOS hosts in a cluster, prepares a used and an unused Docker image, From f398e26a5395656f3fde085340497935b2b68e4e Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 29 Jul 2015 14:08:41 +0300 Subject: [PATCH 036/104] fix cluster_docker_images_maintenance use case Signed-off-by: Bonczidai Levente --- .../docker/images/clear_unused_and_dangling_images.sl | 2 +- content/io/cloudslang/docker/images/get_used_images.sl | 2 +- content/io/cloudslang/docker/maintenance/images_maintenance.sl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/content/io/cloudslang/docker/images/clear_unused_and_dangling_images.sl b/content/io/cloudslang/docker/images/clear_unused_and_dangling_images.sl index 1410d488c1..f31fd8f759 100644 --- a/content/io/cloudslang/docker/images/clear_unused_and_dangling_images.sl +++ b/content/io/cloudslang/docker/images/clear_unused_and_dangling_images.sl @@ -88,6 +88,6 @@ flow: - amount_of_dangling_images_deleted outputs: - amount_of_images_deleted: "0 if 'amount_of_images_deleted' not in locals() else amount_of_images_deleted" - - amount_of_dangling_images_deleted: "0 if 'amount_of_images_deleted' not in locals() else amount_of_dangling_images_deleted" + - amount_of_dangling_images_deleted: "0 if 'amount_of_dangling_images_deleted' not in locals() else amount_of_dangling_images_deleted" - dangling_images_list_safe_to_delete - images_list_safe_to_delete diff --git a/content/io/cloudslang/docker/images/get_used_images.sl b/content/io/cloudslang/docker/images/get_used_images.sl index 9022236977..5f5cbb0d7e 100644 --- a/content/io/cloudslang/docker/images/get_used_images.sl +++ b/content/io/cloudslang/docker/images/get_used_images.sl @@ -93,7 +93,7 @@ flow: - agentForwarding: required: false publish: - - image_list: returnResult.replace("\n"," ") + - image_list: returnResult.replace("\n"," ").replace(":latest", "") outputs: - image_list diff --git a/content/io/cloudslang/docker/maintenance/images_maintenance.sl b/content/io/cloudslang/docker/maintenance/images_maintenance.sl index b727216c14..f0a8985358 100644 --- a/content/io/cloudslang/docker/maintenance/images_maintenance.sl +++ b/content/io/cloudslang/docker/maintenance/images_maintenance.sl @@ -59,7 +59,7 @@ flow: NOT_ENOUGH_DISKSPACE: clear_unused_images - clear_unused_images: do: - docker_images.clear_unused_images: + docker_images.clear_unused_and_dangling_images: - docker_host - docker_username - docker_password: From d36bd311797af54ba843d520140a022a9e632f30 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 29 Jul 2015 14:10:39 +0300 Subject: [PATCH 037/104] activate test suite Signed-off-by: Bonczidai Levente --- .../coreos/cluster_docker_images_maintenance.inputs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml b/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml index 74d603e36c..ccadfc91c7 100644 --- a/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml +++ b/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml @@ -7,7 +7,7 @@ # #################################################### test_cluster_docker_images_maintenance_success: - testSuites: [coreos_local] + testSuites: [coreos] description: > This test cleans the CoreOS hosts in a cluster, prepares a used and an unused Docker image, From 593462f0fe4085bdf4afed8c76f5b4048acef804 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 29 Jul 2015 16:06:18 +0300 Subject: [PATCH 038/104] add waiting logic for droplet startup Signed-off-by: Bonczidai Levente --- .../coreos/cluster_docker_images_maintenance.inputs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml b/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml index ccadfc91c7..74d603e36c 100644 --- a/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml +++ b/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml @@ -7,7 +7,7 @@ # #################################################### test_cluster_docker_images_maintenance_success: - testSuites: [coreos] + testSuites: [coreos_local] description: > This test cleans the CoreOS hosts in a cluster, prepares a used and an unused Docker image, From c16dee54b93fb811aa91f1fe6e65b2c3f750e7c4 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 29 Jul 2015 16:06:29 +0300 Subject: [PATCH 039/104] add waiting logic for droplet startup Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 74 ++++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index a0966010d4..eb9a1af943 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -54,28 +54,42 @@ done # echo $DROPLET_ID_ACC echo $DROPLET_ID_ACC > "droplets_${CIRCLE_BUILD_NUM}.txt" -# TODO: add waiting loop for droplet startup -sleep 60 +SLEEP_INTERVAL=5 # 5 sec +TIMEOUT=300 # 5 mins # retrieve IPv4 addresses of droplets -for DROPLET_ID in $DROPLET_ID_ACC +for DROPLET_ID in ${DROPLET_ID_ACC} do - CURL_OUTPUT=$(curl -i -s -L -X GET -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ - "https://api.digitalocean.com/v2/droplets/$DROPLET_ID") - # echo "CURL_OUTPUT - GET DROPLET BY ID: $CURL_OUTPUT" - - STATUS_CODE=$(echo "$CURL_OUTPUT" | grep "Status" | awk '{print $2}') - # echo "STATUS_CODE: $STATUS_CODE" - - if [ "$STATUS_CODE" = "200" ] - then - echo "Droplet($DROPLET_ID) information retrieved successfully" - - RESPONSE_BODY_JSON=$(echo "$CURL_OUTPUT" | grep "ip_address") - # echo "IP_ADDRESS_JUNK: $RESPONSE_BODY_JSON" + DROPLET_STATUS='' + WAITING_TIME=0 + while [ "$DROPLET_STATUS" != "active" ] && [ "$WAITING_TIME" -lt "$TIMEOUT" ] + do + CURL_OUTPUT=$(curl -i -s -L -X GET -H 'Content-Type: application/json' -H "Authorization: Bearer ${DO_API_TOKEN}" \ + "https://api.digitalocean.com/v2/droplets/$DROPLET_ID") + # echo "CURL_OUTPUT - GET DROPLET BY ID: $CURL_OUTPUT" + + STATUS_CODE=$(echo "$CURL_OUTPUT" | grep "Status" | awk '{print $2}') + # echo "STATUS_CODE: $STATUS_CODE" + + if [ "$STATUS_CODE" = "200" ] + then + echo "Droplet($DROPLET_ID) information retrieved successfully" + + RESPONSE_BODY_JSON=$(echo "$CURL_OUTPUT" | grep "ip_address") + DROPLET_STATUS=$(\ + echo "$RESPONSE_BODY_JSON" | python -c \ +' +import json,sys; +obj = json.load(sys.stdin); +print obj["droplet"]["status"]; +'\ + ) + echo "Droplet($DROPLET_ID) status: ${DROPLET_STATUS}" - IP_ADDRESS=$(\ - echo "$RESPONSE_BODY_JSON" | python -c \ + if [ "$DROPLET_STATUS" = "active" ] + then + IP_ADDRESS=$(\ + echo "$RESPONSE_BODY_JSON" | python -c \ ' import json,sys; obj = json.load(sys.stdin); @@ -87,13 +101,23 @@ for ip_obj in ipv4_list: break; print ip; '\ - ) - echo "Droplet($DROPLET_ID) IPv4 address: $IP_ADDRESS" - - DROPLET_IP_ADDRESS_ACC+="${IP_ADDRESS} " - echo "DROPLET_IP_ADDRESS_ACC: $DROPLET_IP_ADDRESS_ACC" - else - echo "Problem occurred: retrieving droplet($DROPLET_ID) information - status code: $STATUS_CODE" + ) + echo "Droplet($DROPLET_ID) IPv4 address: $IP_ADDRESS" + + DROPLET_IP_ADDRESS_ACC+="${IP_ADDRESS} " + # echo "DROPLET_IP_ADDRESS_ACC: $DROPLET_IP_ADDRESS_ACC" + else + ((WAITING_TIME+=SLEEP_INTERVAL)) + sleep ${SLEEP_INTERVAL} + fi + else + echo "Problem occurred: retrieving droplet($DROPLET_ID) information - status code: $STATUS_CODE" + fi + done + if [ "$DROPLET_STATUS" != "active" ] + then + echo "Droplet($DROPLET_ID) is not active after ${WAITING_TIME}" + exit 1 fi done From d571c3676be52f04b422be4e0c6190e3ffce3d89 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 29 Jul 2015 16:15:45 +0300 Subject: [PATCH 040/104] enhancements Signed-off-by: Bonczidai Levente --- ci-env/cleanup_env.sh | 2 +- ci-env/prepare_env.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci-env/cleanup_env.sh b/ci-env/cleanup_env.sh index 39b97ee380..df77eba17a 100644 --- a/ci-env/cleanup_env.sh +++ b/ci-env/cleanup_env.sh @@ -7,7 +7,7 @@ DROPLET_ID_ACC=$(cat < "droplets_${CIRCLE_BUILD_NUM}.txt") # echo "DROPLET_ID_ACC: $DROPLET_ID_ACC" # DROPLET_ID_ACC='' # TODO: remove this - temporary disable droplet cleanup -for DROPLET_ID in $DROPLET_ID_ACC +for DROPLET_ID in ${DROPLET_ID_ACC} do CURL_OUTPUT=$(curl -i -s -L -X DELETE -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ "https://api.digitalocean.com/v2/droplets/$DROPLET_ID") diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index eb9a1af943..4f21642b2d 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -5,12 +5,12 @@ DROPLET_IP_ADDRESS_ACC="" DISCOVERY_URL=$(curl -X GET "https://discovery.etcd.io/new") echo "DISCOVERY_URL: $DISCOVERY_URL" -DISCOVERY_URL_ESCAPED=$(echo $DISCOVERY_URL | sed 's/\//\\\//g') +DISCOVERY_URL_ESCAPED=$(echo ${DISCOVERY_URL} | sed 's/\//\\\//g') sed -i "s//${DISCOVERY_URL_ESCAPED}/g" ci-env/cloud-config.yaml cat ci-env/cloud-config.yaml COREOS_MACHINE_NAMES="ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" -for COREOS_MACHINE in $COREOS_MACHINE_NAMES +for COREOS_MACHINE in ${COREOS_MACHINE_NAMES} do CURL_OUTPUT=$(curl -i -s -X POST https://api.digitalocean.com/v2/droplets \ -H 'Content-Type: application/json' \ @@ -29,7 +29,7 @@ do echo "CURL_OUTPUT: $CURL_OUTPUT" - STATUS_CODE=$(echo $CURL_OUTPUT | awk '{print $2}') + STATUS_CODE=$(echo ${CURL_OUTPUT} | awk '{print $2}') if [ "$STATUS_CODE" = "202" ] then From 846c5725072f5161b5ddc7fc7abb56dbd452b42e Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 09:55:28 +0300 Subject: [PATCH 041/104] enhancements 2 Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 4f21642b2d..4b0f81576b 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -1,15 +1,16 @@ #!/bin/bash -DROPLET_ID_ACC="" -DROPLET_IP_ADDRESS_ACC="" - -DISCOVERY_URL=$(curl -X GET "https://discovery.etcd.io/new") +# generate discovery URL for the new CoreOS cluster and update the cloud-config file +DISCOVERY_URL=$(curl -s -X GET "https://discovery.etcd.io/new") echo "DISCOVERY_URL: $DISCOVERY_URL" DISCOVERY_URL_ESCAPED=$(echo ${DISCOVERY_URL} | sed 's/\//\\\//g') sed -i "s//${DISCOVERY_URL_ESCAPED}/g" ci-env/cloud-config.yaml -cat ci-env/cloud-config.yaml +# cat ci-env/cloud-config.yaml -COREOS_MACHINE_NAMES="ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" +COREOS_MACHINE_NAMES="\ +ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1 \ +ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 \ +ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" for COREOS_MACHINE in ${COREOS_MACHINE_NAMES} do CURL_OUTPUT=$(curl -i -s -X POST https://api.digitalocean.com/v2/droplets \ @@ -52,7 +53,7 @@ done # store droplet IDs in a file to be accessible in cleanup script # echo $DROPLET_ID_ACC -echo $DROPLET_ID_ACC > "droplets_${CIRCLE_BUILD_NUM}.txt" +echo ${DROPLET_ID_ACC} > "droplets_${CIRCLE_BUILD_NUM}.txt" SLEEP_INTERVAL=5 # 5 sec TIMEOUT=300 # 5 mins @@ -66,7 +67,7 @@ do do CURL_OUTPUT=$(curl -i -s -L -X GET -H 'Content-Type: application/json' -H "Authorization: Bearer ${DO_API_TOKEN}" \ "https://api.digitalocean.com/v2/droplets/$DROPLET_ID") - # echo "CURL_OUTPUT - GET DROPLET BY ID: $CURL_OUTPUT" + echo "CURL_OUTPUT - GET DROPLET BY ID: $CURL_OUTPUT" STATUS_CODE=$(echo "$CURL_OUTPUT" | grep "Status" | awk '{print $2}') # echo "STATUS_CODE: $STATUS_CODE" @@ -76,6 +77,8 @@ do echo "Droplet($DROPLET_ID) information retrieved successfully" RESPONSE_BODY_JSON=$(echo "$CURL_OUTPUT" | grep "ip_address") + echo "RESPONSE_BODY_JSON: ${RESPONSE_BODY_JSON}" + DROPLET_STATUS=$(\ echo "$RESPONSE_BODY_JSON" | python -c \ ' @@ -117,7 +120,6 @@ print ip; if [ "$DROPLET_STATUS" != "active" ] then echo "Droplet($DROPLET_ID) is not active after ${WAITING_TIME}" - exit 1 fi done From dd35c96fd905c6297360e87c34a29f700f7005b2 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 10:19:40 +0300 Subject: [PATCH 042/104] check empty json info about droplet Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 4b0f81576b..24fb745f12 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -1,5 +1,11 @@ #!/bin/bash +inc_and_sleep() +{ + ((WAITING_TIME+=SLEEP_INTERVAL)) + sleep ${SLEEP_INTERVAL} +} + # generate discovery URL for the new CoreOS cluster and update the cloud-config file DISCOVERY_URL=$(curl -s -X GET "https://discovery.etcd.io/new") echo "DISCOVERY_URL: $DISCOVERY_URL" @@ -77,22 +83,26 @@ do echo "Droplet($DROPLET_ID) information retrieved successfully" RESPONSE_BODY_JSON=$(echo "$CURL_OUTPUT" | grep "ip_address") - echo "RESPONSE_BODY_JSON: ${RESPONSE_BODY_JSON}" + # echo "RESPONSE_BODY_JSON: ${RESPONSE_BODY_JSON}" - DROPLET_STATUS=$(\ - echo "$RESPONSE_BODY_JSON" | python -c \ + if [ "${RESPONSE_BODY_JSON}" = "" ] + then + inc_and_sleep + else + DROPLET_STATUS=$(\ + echo "$RESPONSE_BODY_JSON" | python -c \ ' import json,sys; obj = json.load(sys.stdin); print obj["droplet"]["status"]; '\ ) - echo "Droplet($DROPLET_ID) status: ${DROPLET_STATUS}" + echo "Droplet($DROPLET_ID) status: ${DROPLET_STATUS}" - if [ "$DROPLET_STATUS" = "active" ] - then - IP_ADDRESS=$(\ - echo "$RESPONSE_BODY_JSON" | python -c \ + if [ "$DROPLET_STATUS" = "active" ] + then + IP_ADDRESS=$(\ + echo "$RESPONSE_BODY_JSON" | python -c \ ' import json,sys; obj = json.load(sys.stdin); @@ -105,13 +115,13 @@ for ip_obj in ipv4_list: print ip; '\ ) - echo "Droplet($DROPLET_ID) IPv4 address: $IP_ADDRESS" + echo "Droplet($DROPLET_ID) IPv4 address: $IP_ADDRESS" - DROPLET_IP_ADDRESS_ACC+="${IP_ADDRESS} " - # echo "DROPLET_IP_ADDRESS_ACC: $DROPLET_IP_ADDRESS_ACC" - else - ((WAITING_TIME+=SLEEP_INTERVAL)) - sleep ${SLEEP_INTERVAL} + DROPLET_IP_ADDRESS_ACC+="${IP_ADDRESS} " + # echo "DROPLET_IP_ADDRESS_ACC: $DROPLET_IP_ADDRESS_ACC" + else + inc_and_sleep + fi fi else echo "Problem occurred: retrieving droplet($DROPLET_ID) information - status code: $STATUS_CODE" From f458b981856b5562c823d334c32b93741ee203d6 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 10:35:45 +0300 Subject: [PATCH 043/104] extract env variables Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 24fb745f12..cba9691b42 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -23,18 +23,18 @@ do -H 'Content-Type: application/json' \ -H "Authorization: Bearer $DO_API_TOKEN" \ -d "{ - \"name\":\"${COREOS_MACHINE}\","' + \"name\":\"${COREOS_MACHINE}\", + \"ssh_keys\":[${DO_DROPLET_SSH_PUBLIC_KEY_ID}],"' "region":"ams3", "size":"512mb", "image":"coreos-stable", - "ssh_keys":[993143], "backups":false, "ipv6":false, "private_networking":true, "user_data": "'"$(cat ci-env/cloud-config.yaml | sed 's/"/\\"/g')"'" }') - echo "CURL_OUTPUT: $CURL_OUTPUT" + # echo "CURL_OUTPUT: $CURL_OUTPUT" STATUS_CODE=$(echo ${CURL_OUTPUT} | awk '{print $2}') @@ -73,7 +73,7 @@ do do CURL_OUTPUT=$(curl -i -s -L -X GET -H 'Content-Type: application/json' -H "Authorization: Bearer ${DO_API_TOKEN}" \ "https://api.digitalocean.com/v2/droplets/$DROPLET_ID") - echo "CURL_OUTPUT - GET DROPLET BY ID: $CURL_OUTPUT" + # echo "CURL_OUTPUT - GET DROPLET BY ID: $CURL_OUTPUT" STATUS_CODE=$(echo "$CURL_OUTPUT" | grep "Status" | awk '{print $2}') # echo "STATUS_CODE: $STATUS_CODE" @@ -134,13 +134,14 @@ print ip; done # update inputs files to use actual IP addresses -DROPLET_IP_ARRAY=($DROPLET_IP_ADDRESS_ACC) +DROPLET_IP_ARRAY=(${DROPLET_IP_ADDRESS_ACC}) sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml # create ssh private key SSH_KEY_PATH=droplets_rsa -echo -e "$SSH_PRIVATE_KEY_CI_ENV_TEMP2" > $SSH_KEY_PATH +echo -e "${DO_DROPLET_SSH_PRIVATE_KEY}" > ${SSH_KEY_PATH} +cat "${SSH_KEY_PATH}" # ls -l . # update inputs files to use actual ssh key From 2f91754cc202a41ce33211832dae1b2787c64a45 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 10:36:00 +0300 Subject: [PATCH 044/104] activate test suite Signed-off-by: Bonczidai Levente --- .../coreos/cluster_docker_images_maintenance.inputs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml b/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml index 74d603e36c..ccadfc91c7 100644 --- a/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml +++ b/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml @@ -7,7 +7,7 @@ # #################################################### test_cluster_docker_images_maintenance_success: - testSuites: [coreos_local] + testSuites: [coreos] description: > This test cleans the CoreOS hosts in a cluster, prepares a used and an unused Docker image, From a5ee9e54b12b508cad22c43efdabeff81265526d Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 10:48:51 +0300 Subject: [PATCH 045/104] extract create_droplets script Signed-off-by: Bonczidai Levente --- ci-env/create_droplets.sh | 54 ++++++++++++++++++++++++++++++++++ ci-env/prepare_env.sh | 61 +-------------------------------------- circle.yml | 2 ++ 3 files changed, 57 insertions(+), 60 deletions(-) create mode 100644 ci-env/create_droplets.sh diff --git a/ci-env/create_droplets.sh b/ci-env/create_droplets.sh new file mode 100644 index 0000000000..bde426053b --- /dev/null +++ b/ci-env/create_droplets.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# generate discovery URL for the new CoreOS cluster and update the cloud-config file +DISCOVERY_URL=$(curl -s -X GET "https://discovery.etcd.io/new") +echo "DISCOVERY_URL: $DISCOVERY_URL" +DISCOVERY_URL_ESCAPED=$(echo ${DISCOVERY_URL} | sed 's/\//\\\//g') +sed -i "s//${DISCOVERY_URL_ESCAPED}/g" ci-env/cloud-config.yaml +# cat ci-env/cloud-config.yaml + +COREOS_MACHINE_NAMES="\ +ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1 \ +ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 \ +ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" +for COREOS_MACHINE in ${COREOS_MACHINE_NAMES} +do + CURL_OUTPUT=$(curl -i -s -X POST https://api.digitalocean.com/v2/droplets \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $DO_API_TOKEN" \ + -d "{ + \"name\":\"${COREOS_MACHINE}\", + \"ssh_keys\":[${DO_DROPLET_SSH_PUBLIC_KEY_ID}],"' + "region":"ams3", + "size":"512mb", + "image":"coreos-stable", + "backups":false, + "ipv6":false, + "private_networking":true, + "user_data": "'"$(cat ci-env/cloud-config.yaml | sed 's/"/\\"/g')"'" + }') + # echo "CURL_OUTPUT: $CURL_OUTPUT" + + STATUS_CODE=$(echo ${CURL_OUTPUT} | awk '{print $2}') + + if [ "$STATUS_CODE" = "202" ] + then + DROPLET_DETAILS=$(echo "$CURL_OUTPUT" | grep "droplet") + # echo "DROPLET_DETAILS: $DROPLET_DETAILS" + + # split after `:` and `,` characters and extract the droplet ID + DROPLET_ID_JUNK_ARRAY=(${DROPLET_DETAILS//:/ }) + DROPLET_ID_JUNK=${DROPLET_ID_JUNK_ARRAY[2]} + DROPLET_ID_ARRAY=(${DROPLET_ID_JUNK//,/ }) + DROPLET_ID=${DROPLET_ID_ARRAY[0]} + + DROPLET_ID_ACC+="${DROPLET_ID} " + + echo "$COREOS_MACHINE (ID: $DROPLET_ID) droplet creation request accepted" + else + echo "Problem occurred: $COREOS_MACHINE droplet creation request - status code: $STATUS_CODE" + fi +done + +# store droplet IDs in a file to be accessible in other script +echo ${DROPLET_ID_ACC} > "droplets_${CIRCLE_BUILD_NUM}.txt" diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index cba9691b42..0d1937857f 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -6,61 +6,7 @@ inc_and_sleep() sleep ${SLEEP_INTERVAL} } -# generate discovery URL for the new CoreOS cluster and update the cloud-config file -DISCOVERY_URL=$(curl -s -X GET "https://discovery.etcd.io/new") -echo "DISCOVERY_URL: $DISCOVERY_URL" -DISCOVERY_URL_ESCAPED=$(echo ${DISCOVERY_URL} | sed 's/\//\\\//g') -sed -i "s//${DISCOVERY_URL_ESCAPED}/g" ci-env/cloud-config.yaml -# cat ci-env/cloud-config.yaml - -COREOS_MACHINE_NAMES="\ -ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-1 \ -ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-2 \ -ci-${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}-coreos-3" -for COREOS_MACHINE in ${COREOS_MACHINE_NAMES} -do - CURL_OUTPUT=$(curl -i -s -X POST https://api.digitalocean.com/v2/droplets \ - -H 'Content-Type: application/json' \ - -H "Authorization: Bearer $DO_API_TOKEN" \ - -d "{ - \"name\":\"${COREOS_MACHINE}\", - \"ssh_keys\":[${DO_DROPLET_SSH_PUBLIC_KEY_ID}],"' - "region":"ams3", - "size":"512mb", - "image":"coreos-stable", - "backups":false, - "ipv6":false, - "private_networking":true, - "user_data": "'"$(cat ci-env/cloud-config.yaml | sed 's/"/\\"/g')"'" - }') - - # echo "CURL_OUTPUT: $CURL_OUTPUT" - - STATUS_CODE=$(echo ${CURL_OUTPUT} | awk '{print $2}') - - if [ "$STATUS_CODE" = "202" ] - then - DROPLET_DETAILS=$(echo "$CURL_OUTPUT" | grep "droplet") - # echo "DROPLET_DETAILS: $DROPLET_DETAILS" - - # split after `:` and `,` characters and extract the droplet ID - DROPLET_ID_JUNK_ARRAY=(${DROPLET_DETAILS//:/ }) - DROPLET_ID_JUNK=${DROPLET_ID_JUNK_ARRAY[2]} - DROPLET_ID_ARRAY=(${DROPLET_ID_JUNK//,/ }) - DROPLET_ID=${DROPLET_ID_ARRAY[0]} - - DROPLET_ID_ACC+="${DROPLET_ID} " - - echo "$COREOS_MACHINE (ID: $DROPLET_ID) droplet creation request accepted" - else - echo "Problem occurred: $COREOS_MACHINE droplet creation request - status code: $STATUS_CODE" - fi -done - -# store droplet IDs in a file to be accessible in cleanup script -# echo $DROPLET_ID_ACC -echo ${DROPLET_ID_ACC} > "droplets_${CIRCLE_BUILD_NUM}.txt" - +DROPLET_ID_ACC=$(cat "droplets_${CIRCLE_BUILD_NUM}.txt") SLEEP_INTERVAL=5 # 5 sec TIMEOUT=300 # 5 mins @@ -141,12 +87,7 @@ sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/clus # create ssh private key SSH_KEY_PATH=droplets_rsa echo -e "${DO_DROPLET_SSH_PRIVATE_KEY}" > ${SSH_KEY_PATH} -cat "${SSH_KEY_PATH}" -# ls -l . # update inputs files to use actual ssh key sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml - -cat test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml -cat test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml diff --git a/circle.yml b/circle.yml index 633d18c435..c2821f4a22 100644 --- a/circle.yml +++ b/circle.yml @@ -30,6 +30,8 @@ dependencies: test: pre: + - chmod +x ci-env/create_droplets.sh + - ci-env/create_droplets.sh # - docker run --privileged --lxc-conf="lxc.aa_profile=unconfined" -d -p 4444 -p 49153:22 -e PORT=4444 --name docker_host_ssh orius123/dind-ssh # - docker port docker_host_ssh 22 - docker run -d -p 49165:8080 jenkins From 4f984f694c414ac4d234297271a1ca3c4c75bdcd Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 11:01:46 +0300 Subject: [PATCH 046/104] remove auxiliary files Signed-off-by: Bonczidai Levente --- .../test_access_coreos_machine.inputs.yaml | 17 ----- .../coreos/test_access_coreos_machine.sl | 62 ------------------- 2 files changed, 79 deletions(-) delete mode 100644 test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml delete mode 100644 test/io/cloudslang/coreos/test_access_coreos_machine.sl diff --git a/test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml b/test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml deleted file mode 100644 index de69abec5b..0000000000 --- a/test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License v2.0 which accompany this distribution. -# -# The Apache License is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# -#################################################### -test_access_coreos_machine: - testSuites: [coreos] - description: test stuff - testFlowPath: io.cloudslang.coreos.test_access_coreos_machine - inputs: - - coreos_host: # ip address of one host from the cluster - - coreos_username: core # username for the machine - - private_key_file: # path to private key file - result: SUCCESS diff --git a/test/io/cloudslang/coreos/test_access_coreos_machine.sl b/test/io/cloudslang/coreos/test_access_coreos_machine.sl deleted file mode 100644 index afdde01539..0000000000 --- a/test/io/cloudslang/coreos/test_access_coreos_machine.sl +++ /dev/null @@ -1,62 +0,0 @@ -# (c) Copyright 2014 Hewlett-Packard Development Company, L.P. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License v2.0 which accompany this distribution. -# -# The Apache License is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# -#################################################### -# Wrapper flow - logic steps: -# - retrieves the ip addresses of the machines in the cluster -# - cleanup on the machines (so they will not contain any images) -# - prepares a used and an unused Docker image -# - runs the flow -# - verifies only one image remained in the cluster -# - delete the used image -#################################################### - -namespace: io.cloudslang.coreos - -imports: - ssh: io.cloudslang.base.remote_command_execution.ssh - print: io.cloudslang.base.print - -flow: - name: test_access_coreos_machine - inputs: - - coreos_host - - coreos_username - - coreos_password: - required: false - - private_key_file: - required: false - - percentage: - required: false - - timeout: - required: false - - unused_image_name: - default: "'tomcat:7'" - - used_image_name: - default: "'busybox'" - - number_of_images_in_cluster: - default: 0 - overridable: false - - workflow: - - test_ssh_access: - do: - ssh.ssh_flow: - - host: coreos_host - - command: "'fleetctl list-machines'" - - username: coreos_username - - privateKeyFile: - default: private_key_file - required: false - publish: - - returnResult - - - print_result: - do: - print.print_text: - - text: > - 'ssh result -> ' + returnResult From f48a9f80426955fb14bad9a752c770dab91a2eee Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 11:02:13 +0300 Subject: [PATCH 047/104] use sed input replace for all files Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index 0d1937857f..b09d382b31 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -81,13 +81,11 @@ done # update inputs files to use actual IP addresses DROPLET_IP_ARRAY=(${DROPLET_IP_ADDRESS_ACC}) -sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml -sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml +sed -i "s//${DROPLET_IP_ARRAY[0]}/g" * # create ssh private key SSH_KEY_PATH=droplets_rsa echo -e "${DO_DROPLET_SSH_PRIVATE_KEY}" > ${SSH_KEY_PATH} # update inputs files to use actual ssh key -sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/coreos/test_access_coreos_machine.inputs.yaml -sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml +sed -i "s//${SSH_KEY_PATH}/g" * From 24fe8ee3d2c6439a48f75f0ecb9b439437b75d8a Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 11:33:55 +0300 Subject: [PATCH 048/104] modify test suites Signed-off-by: Bonczidai Levente --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index c2821f4a22..04e76dc383 100644 --- a/circle.yml +++ b/circle.yml @@ -44,7 +44,7 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - ./cslang-builder/bin/cslang-builder -ts coreos,!default -cov + - ./cslang-builder/bin/cslang-builder -ts cadvisor,jenkins,consul,coreos -cov post: - chmod +x ci-env/cleanup_env.sh - ci-env/cleanup_env.sh From e9ae01b4447ae0be3fb3875c0e1e4524890ce912 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 11:38:07 +0300 Subject: [PATCH 049/104] make sed use explicit file path Signed-off-by: Bonczidai Levente --- ci-env/prepare_env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-env/prepare_env.sh b/ci-env/prepare_env.sh index b09d382b31..00c4ec3a03 100644 --- a/ci-env/prepare_env.sh +++ b/ci-env/prepare_env.sh @@ -81,11 +81,11 @@ done # update inputs files to use actual IP addresses DROPLET_IP_ARRAY=(${DROPLET_IP_ADDRESS_ACC}) -sed -i "s//${DROPLET_IP_ARRAY[0]}/g" * +sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml # create ssh private key SSH_KEY_PATH=droplets_rsa echo -e "${DO_DROPLET_SSH_PRIVATE_KEY}" > ${SSH_KEY_PATH} # update inputs files to use actual ssh key -sed -i "s//${SSH_KEY_PATH}/g" * +sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml From eace28efff2ce2ea4d029b87e1b1f87a1623a703 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 13:32:15 +0300 Subject: [PATCH 050/104] ignore test suites Signed-off-by: Bonczidai Levente --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 04e76dc383..c2821f4a22 100644 --- a/circle.yml +++ b/circle.yml @@ -44,7 +44,7 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - ./cslang-builder/bin/cslang-builder -ts cadvisor,jenkins,consul,coreos -cov + - ./cslang-builder/bin/cslang-builder -ts coreos,!default -cov post: - chmod +x ci-env/cleanup_env.sh - ci-env/cleanup_env.sh From e44afc2f557032c979d37b2ac6e291c5c5f1153f Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 13:40:57 +0300 Subject: [PATCH 051/104] add cadvisor test suite Signed-off-by: Bonczidai Levente --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index c2821f4a22..8fc8a91c22 100644 --- a/circle.yml +++ b/circle.yml @@ -44,7 +44,7 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - ./cslang-builder/bin/cslang-builder -ts coreos,!default -cov + - ./cslang-builder/bin/cslang-builder -ts cadvisor,coreos,!default -cov post: - chmod +x ci-env/cleanup_env.sh - ci-env/cleanup_env.sh From ffbd8c3661c11273e46f702c9b56141155dcc4cb Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 13:48:55 +0300 Subject: [PATCH 052/104] add jenkins test suite Signed-off-by: Bonczidai Levente --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 8fc8a91c22..e2d6b65c48 100644 --- a/circle.yml +++ b/circle.yml @@ -44,7 +44,7 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - ./cslang-builder/bin/cslang-builder -ts cadvisor,coreos,!default -cov + - ./cslang-builder/bin/cslang-builder -ts cadvisor,jenkins,coreos,!default -cov post: - chmod +x ci-env/cleanup_env.sh - ci-env/cleanup_env.sh From 69090192d97ebbb52d098114b06a12c18336f142 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 14:02:52 +0300 Subject: [PATCH 053/104] ignore cadvisor test suite Signed-off-by: Bonczidai Levente --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index e2d6b65c48..a9ca48d5cb 100644 --- a/circle.yml +++ b/circle.yml @@ -44,7 +44,7 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - ./cslang-builder/bin/cslang-builder -ts cadvisor,jenkins,coreos,!default -cov + - ./cslang-builder/bin/cslang-builder -ts jenkins,coreos,!default -cov post: - chmod +x ci-env/cleanup_env.sh - ci-env/cleanup_env.sh From dffe51f8e08564a7d896af982c5325c9172497ac Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 14:11:09 +0300 Subject: [PATCH 054/104] add consul test suite Signed-off-by: Bonczidai Levente --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index a9ca48d5cb..5fd0556bbc 100644 --- a/circle.yml +++ b/circle.yml @@ -44,7 +44,7 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - ./cslang-builder/bin/cslang-builder -ts jenkins,coreos,!default -cov + - ./cslang-builder/bin/cslang-builder -ts jenkins,consul,coreos,!default -cov post: - chmod +x ci-env/cleanup_env.sh - ci-env/cleanup_env.sh From cafea71c3b68c8e1bacf7f90848ac3da54da3974 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 14:23:51 +0300 Subject: [PATCH 055/104] add default test suite Signed-off-by: Bonczidai Levente --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 5fd0556bbc..9638a8b0c3 100644 --- a/circle.yml +++ b/circle.yml @@ -44,7 +44,7 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - ./cslang-builder/bin/cslang-builder -ts jenkins,consul,coreos,!default -cov + - ./cslang-builder/bin/cslang-builder -ts jenkins,consul,coreos -cov post: - chmod +x ci-env/cleanup_env.sh - ci-env/cleanup_env.sh From d644677b4f368c76b10773f416249ce19a5e2f17 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 14:31:11 +0300 Subject: [PATCH 056/104] ignore jenkins test suite Signed-off-by: Bonczidai Levente --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 9638a8b0c3..de7745d08f 100644 --- a/circle.yml +++ b/circle.yml @@ -44,7 +44,7 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - ./cslang-builder/bin/cslang-builder -ts jenkins,consul,coreos -cov + - ./cslang-builder/bin/cslang-builder -ts consul,coreos -cov post: - chmod +x ci-env/cleanup_env.sh - ci-env/cleanup_env.sh From 0cbf8728c82b6b8c26fe843ef42abf0d32aa8626 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 14:46:59 +0300 Subject: [PATCH 057/104] remove TODO item Signed-off-by: Bonczidai Levente --- ci-env/cleanup_env.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci-env/cleanup_env.sh b/ci-env/cleanup_env.sh index df77eba17a..58839969eb 100644 --- a/ci-env/cleanup_env.sh +++ b/ci-env/cleanup_env.sh @@ -6,7 +6,6 @@ DROPLET_ID_ACC=$(cat < "droplets_${CIRCLE_BUILD_NUM}.txt") # echo "DROPLET_ID_ACC: $DROPLET_ID_ACC" -# DROPLET_ID_ACC='' # TODO: remove this - temporary disable droplet cleanup for DROPLET_ID in ${DROPLET_ID_ACC} do CURL_OUTPUT=$(curl -i -s -L -X DELETE -H 'Content-Type: application/json' -H "Authorization: Bearer $DO_API_TOKEN" \ From 3111529496f2169b73a6dadc86d4333ebed61c54 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 14:55:11 +0300 Subject: [PATCH 058/104] ignore consul test suite Signed-off-by: Bonczidai Levente --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index de7745d08f..05e0ade746 100644 --- a/circle.yml +++ b/circle.yml @@ -44,7 +44,7 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - ./cslang-builder/bin/cslang-builder -ts consul,coreos -cov + - ./cslang-builder/bin/cslang-builder -ts coreos -cov post: - chmod +x ci-env/cleanup_env.sh - ci-env/cleanup_env.sh From 06e8bb64a316988acdf6be408bba6acd63cfbe87 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 30 Jul 2015 15:03:36 +0300 Subject: [PATCH 059/104] ignore default test suite Signed-off-by: Bonczidai Levente --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 05e0ade746..c2821f4a22 100644 --- a/circle.yml +++ b/circle.yml @@ -44,7 +44,7 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - ./cslang-builder/bin/cslang-builder -ts coreos -cov + - ./cslang-builder/bin/cslang-builder -ts coreos,!default -cov post: - chmod +x ci-env/cleanup_env.sh - ci-env/cleanup_env.sh From 6de2f1f9d60d9fd0d89d6213b810b4bd31c216a1 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 3 Aug 2015 11:05:40 +0300 Subject: [PATCH 060/104] activate default test suite Signed-off-by: Bonczidai Levente --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index c2821f4a22..05e0ade746 100644 --- a/circle.yml +++ b/circle.yml @@ -44,7 +44,7 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - ./cslang-builder/bin/cslang-builder -ts coreos,!default -cov + - ./cslang-builder/bin/cslang-builder -ts coreos -cov post: - chmod +x ci-env/cleanup_env.sh - ci-env/cleanup_env.sh From 7294ad4d25d91cc6da2d5807af3c28c126732c7b Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 3 Aug 2015 15:30:53 +0300 Subject: [PATCH 061/104] temporary ignore machine removal Signed-off-by: Bonczidai Levente --- ci-env/cleanup_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-env/cleanup_env.sh b/ci-env/cleanup_env.sh index 58839969eb..9b4abc075f 100644 --- a/ci-env/cleanup_env.sh +++ b/ci-env/cleanup_env.sh @@ -3,7 +3,7 @@ # print droplets # cat < "droplets_${CIRCLE_BUILD_NUM}.txt" -DROPLET_ID_ACC=$(cat < "droplets_${CIRCLE_BUILD_NUM}.txt") +# DROPLET_ID_ACC=$(cat < "droplets_${CIRCLE_BUILD_NUM}.txt") # TODO: unignore # echo "DROPLET_ID_ACC: $DROPLET_ID_ACC" for DROPLET_ID in ${DROPLET_ID_ACC} From 6fecfd97ea4e7978f85c11594751ee06afb2cea0 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 3 Aug 2015 17:17:06 +0300 Subject: [PATCH 062/104] unignore machine removal Signed-off-by: Bonczidai Levente --- ci-env/cleanup_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-env/cleanup_env.sh b/ci-env/cleanup_env.sh index 9b4abc075f..58839969eb 100644 --- a/ci-env/cleanup_env.sh +++ b/ci-env/cleanup_env.sh @@ -3,7 +3,7 @@ # print droplets # cat < "droplets_${CIRCLE_BUILD_NUM}.txt" -# DROPLET_ID_ACC=$(cat < "droplets_${CIRCLE_BUILD_NUM}.txt") # TODO: unignore +DROPLET_ID_ACC=$(cat < "droplets_${CIRCLE_BUILD_NUM}.txt") # echo "DROPLET_ID_ACC: $DROPLET_ID_ACC" for DROPLET_ID in ${DROPLET_ID_ACC} From 73171dc56774f47b8999315c0fa79edf82ba2eba Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 3 Aug 2015 17:24:25 +0300 Subject: [PATCH 063/104] make test suites run in parallel Signed-off-by: Bonczidai Levente --- circle.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 05e0ade746..00348a18cf 100644 --- a/circle.yml +++ b/circle.yml @@ -44,7 +44,11 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - ./cslang-builder/bin/cslang-builder -ts coreos -cov + - ./cslang-builder/bin/cslang-builder -cov -ts : + parallel: true + files: + - default + - coreos post: - chmod +x ci-env/cleanup_env.sh - ci-env/cleanup_env.sh From 8992215c7d86178304c8da19613a430e80b7365c Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 3 Aug 2015 17:27:13 +0300 Subject: [PATCH 064/104] make test suites run in parallel 2 Signed-off-by: Bonczidai Levente --- circle.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/circle.yml b/circle.yml index 00348a18cf..ff7a3124a6 100644 --- a/circle.yml +++ b/circle.yml @@ -44,11 +44,8 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - ./cslang-builder/bin/cslang-builder -cov -ts : + - case $CIRCLE_NODE_INDEX in 0) ./cslang-builder/bin/cslang-builder -ts coreos,!default -cov ;; 1) ./cslang-builder/bin/cslang-builder -ts default -cov ;; esac: parallel: true - files: - - default - - coreos post: - chmod +x ci-env/cleanup_env.sh - ci-env/cleanup_env.sh From ce3ae6930e999d40bf09c64b35c567a6c4da9b5f Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 3 Aug 2015 17:38:32 +0300 Subject: [PATCH 065/104] make test suites run in parallel 3 Signed-off-by: Bonczidai Levente --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index ff7a3124a6..258ee1552a 100644 --- a/circle.yml +++ b/circle.yml @@ -44,7 +44,7 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - case $CIRCLE_NODE_INDEX in 0) ./cslang-builder/bin/cslang-builder -ts coreos,!default -cov ;; 1) ./cslang-builder/bin/cslang-builder -ts default -cov ;; esac: + - case $CIRCLE_NODE_INDEX in 0) ./cslang-builder/bin/cslang-builder -ts coreos,!default -cov ;; 1) ls -l && ./cslang-builder/bin/cslang-builder -cov ;; esac: parallel: true post: - chmod +x ci-env/cleanup_env.sh From 49c2e123e86be99bda65a487084b75d7dff1c844 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 3 Aug 2015 17:46:01 +0300 Subject: [PATCH 066/104] add parallel modifiers for commands Signed-off-by: Bonczidai Levente --- circle.yml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/circle.yml b/circle.yml index 258ee1552a..b01f3f89b8 100644 --- a/circle.yml +++ b/circle.yml @@ -32,23 +32,23 @@ test: pre: - chmod +x ci-env/create_droplets.sh - ci-env/create_droplets.sh -# - docker run --privileged --lxc-conf="lxc.aa_profile=unconfined" -d -p 4444 -p 49153:22 -e PORT=4444 --name docker_host_ssh orius123/dind-ssh -# - docker port docker_host_ssh 22 - docker run -d -p 49165:8080 jenkins - - wget https://github.com/CloudSlang/cloud-slang/releases/download/cloudslang-0.8.RC1/cslang-builder.zip + - wget https://github.com/CloudSlang/cloud-slang/releases/download/cloudslang-0.8.RC1/cslang-builder.zip: + parallel: true - docker run -d -p 8500:8500 -p 8600:8600/udp fhalim/consul - - unzip cslang-builder.zip -d cslang-builder - - chmod +x cslang-builder/bin/cslang-builder - - mkdir cslang-builder/lib/Lib - - pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib + - unzip cslang-builder.zip -d cslang-builder: + parallel: true + - chmod +x cslang-builder/bin/cslang-builder: + parallel: true + - mkdir cslang-builder/lib/Lib: + parallel: true + - pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib: + parallel: true - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - case $CIRCLE_NODE_INDEX in 0) ./cslang-builder/bin/cslang-builder -ts coreos,!default -cov ;; 1) ls -l && ./cslang-builder/bin/cslang-builder -cov ;; esac: + - case $CIRCLE_NODE_INDEX in 0) ls -l && ./cslang-builder/bin/cslang-builder -ts coreos,!default -cov ;; 1) ls -l && ./cslang-builder/bin/cslang-builder -cov ;; esac: parallel: true post: - chmod +x ci-env/cleanup_env.sh - ci-env/cleanup_env.sh - - docker ps -a -# - docker stop $(docker ps -a -q) -# - docker rm $(docker ps -a -q) From 48191c9f92ac875cc5f1711abbaba8817fa9dfb0 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 3 Aug 2015 18:02:08 +0300 Subject: [PATCH 067/104] add jenkins test suite Signed-off-by: Bonczidai Levente --- circle.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/circle.yml b/circle.yml index b01f3f89b8..4f11921812 100644 --- a/circle.yml +++ b/circle.yml @@ -25,14 +25,12 @@ dependencies: - docker info - docker images # - if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi -# - docker pull orius123/dind-ssh -# - mkdir -p ~/docker; docker save orius123/dind-ssh > ~/docker/image.tar test: pre: - chmod +x ci-env/create_droplets.sh - ci-env/create_droplets.sh - - docker run -d -p 49165:8080 jenkins + - if [ "${CIRCLE_NODE_INDEX}" = "1" ]; then docker run -d -p 49165:8080 jenkins; fi - wget https://github.com/CloudSlang/cloud-slang/releases/download/cloudslang-0.8.RC1/cslang-builder.zip: parallel: true - docker run -d -p 8500:8500 -p 8600:8600/udp fhalim/consul @@ -47,7 +45,7 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - case $CIRCLE_NODE_INDEX in 0) ls -l && ./cslang-builder/bin/cslang-builder -ts coreos,!default -cov ;; 1) ls -l && ./cslang-builder/bin/cslang-builder -cov ;; esac: + - case $CIRCLE_NODE_INDEX in 0) ls -l && ./cslang-builder/bin/cslang-builder -ts coreos,!default -cov ;; 1) ls -l && ./cslang-builder/bin/cslang-builder -ts default,jenkins -cov ;; esac: # TODO: remove ls parallel: true post: - chmod +x ci-env/cleanup_env.sh From 7abec44a048f5dd8f70d1ac5aed6da693b6afed7 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 3 Aug 2015 18:04:49 +0300 Subject: [PATCH 068/104] add jenkins test suite - add parallel modifier Signed-off-by: Bonczidai Levente --- circle.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 4f11921812..f5ec9a5b7a 100644 --- a/circle.yml +++ b/circle.yml @@ -30,7 +30,8 @@ test: pre: - chmod +x ci-env/create_droplets.sh - ci-env/create_droplets.sh - - if [ "${CIRCLE_NODE_INDEX}" = "1" ]; then docker run -d -p 49165:8080 jenkins; fi + - if [ "${CIRCLE_NODE_INDEX}" = "1" ]; then docker run -d -p 49165:8080 jenkins; fi: + parallel: true - wget https://github.com/CloudSlang/cloud-slang/releases/download/cloudslang-0.8.RC1/cslang-builder.zip: parallel: true - docker run -d -p 8500:8500 -p 8600:8600/udp fhalim/consul From 120ef76eb5abb91c67512404c3754cced3735e6c Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Mon, 3 Aug 2015 18:13:55 +0300 Subject: [PATCH 069/104] add consul test suite Signed-off-by: Bonczidai Levente --- circle.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index f5ec9a5b7a..eab4862b06 100644 --- a/circle.yml +++ b/circle.yml @@ -32,9 +32,10 @@ test: - ci-env/create_droplets.sh - if [ "${CIRCLE_NODE_INDEX}" = "1" ]; then docker run -d -p 49165:8080 jenkins; fi: parallel: true + - if [ "${CIRCLE_NODE_INDEX}" = "1" ]; then docker run -d -p 8500:8500 -p 8600:8600/udp fhalim/consul; fi: + parallel: true - wget https://github.com/CloudSlang/cloud-slang/releases/download/cloudslang-0.8.RC1/cslang-builder.zip: parallel: true - - docker run -d -p 8500:8500 -p 8600:8600/udp fhalim/consul - unzip cslang-builder.zip -d cslang-builder: parallel: true - chmod +x cslang-builder/bin/cslang-builder: @@ -46,7 +47,7 @@ test: - chmod +x ci-env/prepare_env.sh - ci-env/prepare_env.sh override: - - case $CIRCLE_NODE_INDEX in 0) ls -l && ./cslang-builder/bin/cslang-builder -ts coreos,!default -cov ;; 1) ls -l && ./cslang-builder/bin/cslang-builder -ts default,jenkins -cov ;; esac: # TODO: remove ls + - case $CIRCLE_NODE_INDEX in 0) ls -l && ./cslang-builder/bin/cslang-builder -ts coreos,!default -cov ;; 1) ls -l && ./cslang-builder/bin/cslang-builder -ts default,jenkins,consul -cov ;; esac: # TODO: remove ls parallel: true post: - chmod +x ci-env/cleanup_env.sh From 0bd1f02d859ddb786bef8c3fbdce39c9d8ddabf4 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 4 Aug 2015 10:12:16 +0300 Subject: [PATCH 070/104] switch containers for commands Signed-off-by: Bonczidai Levente --- circle.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/circle.yml b/circle.yml index eab4862b06..89ee3859c6 100644 --- a/circle.yml +++ b/circle.yml @@ -19,20 +19,16 @@ machine: - docker dependencies: -# cache_directories: -# - "~/docker" override: - docker info - docker images -# - if [[ -e ~/docker/image.tar ]]; then docker load -i ~/docker/image.tar; fi test: pre: - - chmod +x ci-env/create_droplets.sh - - ci-env/create_droplets.sh - - if [ "${CIRCLE_NODE_INDEX}" = "1" ]; then docker run -d -p 49165:8080 jenkins; fi: + - chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh + - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; then docker run -d -p 49165:8080 jenkins; fi: parallel: true - - if [ "${CIRCLE_NODE_INDEX}" = "1" ]; then docker run -d -p 8500:8500 -p 8600:8600/udp fhalim/consul; fi: + - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; then docker run -d -p 8500:8500 -p 8600:8600/udp fhalim/consul; fi: parallel: true - wget https://github.com/CloudSlang/cloud-slang/releases/download/cloudslang-0.8.RC1/cslang-builder.zip: parallel: true @@ -44,11 +40,9 @@ test: parallel: true - pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib: parallel: true - - chmod +x ci-env/prepare_env.sh - - ci-env/prepare_env.sh + - chmod +x ci-env/prepare_env.sh && ci-env/prepare_env.sh override: - - case $CIRCLE_NODE_INDEX in 0) ls -l && ./cslang-builder/bin/cslang-builder -ts coreos,!default -cov ;; 1) ls -l && ./cslang-builder/bin/cslang-builder -ts default,jenkins,consul -cov ;; esac: # TODO: remove ls + - case $CIRCLE_NODE_INDEX in 0) ./cslang-builder/bin/cslang-builder -ts default,jenkins,consul -cov ;; 1) ./cslang-builder/bin/cslang-builder -ts !default,coreos -cov ;; esac: parallel: true post: - - chmod +x ci-env/cleanup_env.sh - - ci-env/cleanup_env.sh + - chmod +x ci-env/cleanup_env.sh && ci-env/cleanup_env.sh From 4b28033fe17f4b711efbd916b9635bed2d8286ee Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 4 Aug 2015 10:34:15 +0300 Subject: [PATCH 071/104] execute droplet scripts in second container Signed-off-by: Bonczidai Levente --- ...nv.sh => wait_for_droplets_and_update_test_inputs.sh} | 0 circle.yml | 9 ++++++--- 2 files changed, 6 insertions(+), 3 deletions(-) rename ci-env/{prepare_env.sh => wait_for_droplets_and_update_test_inputs.sh} (100%) diff --git a/ci-env/prepare_env.sh b/ci-env/wait_for_droplets_and_update_test_inputs.sh similarity index 100% rename from ci-env/prepare_env.sh rename to ci-env/wait_for_droplets_and_update_test_inputs.sh diff --git a/circle.yml b/circle.yml index 89ee3859c6..2b1baf3b0f 100644 --- a/circle.yml +++ b/circle.yml @@ -25,7 +25,8 @@ dependencies: test: pre: - - chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh + - if [ "${CIRCLE_NODE_INDEX}" = "1" ]; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: + parallel: true - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; then docker run -d -p 49165:8080 jenkins; fi: parallel: true - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; then docker run -d -p 8500:8500 -p 8600:8600/udp fhalim/consul; fi: @@ -40,9 +41,11 @@ test: parallel: true - pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib: parallel: true - - chmod +x ci-env/prepare_env.sh && ci-env/prepare_env.sh + - if [ "${CIRCLE_NODE_INDEX}" = "1" ]; then chmod +x ci-env/wait_for_droplets_and_update_test_inputs.sh && ci-env/wait_for_droplets_and_update_test_inputs.sh; fi: + parallel: true override: - case $CIRCLE_NODE_INDEX in 0) ./cslang-builder/bin/cslang-builder -ts default,jenkins,consul -cov ;; 1) ./cslang-builder/bin/cslang-builder -ts !default,coreos -cov ;; esac: parallel: true post: - - chmod +x ci-env/cleanup_env.sh && ci-env/cleanup_env.sh + - if [ "${CIRCLE_NODE_INDEX}" = "1" ]; then chmod +x ci-env/cleanup_env.sh && ci-env/cleanup_env.sh; fi: + parallel: true From dc2c4c4dd228cde10b893a8bdcb3689151ddfdc2 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 4 Aug 2015 12:01:59 +0300 Subject: [PATCH 072/104] add test_get_container_metrics test case to ci Signed-off-by: Bonczidai Levente --- ...ait_for_droplets_and_update_test_inputs.sh | 2 + circle.yml | 2 +- .../get_container_metrics.inputs.yaml | 14 +++-- .../report_machine_metrics.inputs.yaml | 2 +- .../cadvisor/test_get_container_metrics.sl | 52 +++++++++++-------- 5 files changed, 44 insertions(+), 28 deletions(-) diff --git a/ci-env/wait_for_droplets_and_update_test_inputs.sh b/ci-env/wait_for_droplets_and_update_test_inputs.sh index 00c4ec3a03..64b2ec285b 100644 --- a/ci-env/wait_for_droplets_and_update_test_inputs.sh +++ b/ci-env/wait_for_droplets_and_update_test_inputs.sh @@ -82,6 +82,7 @@ done # update inputs files to use actual IP addresses DROPLET_IP_ARRAY=(${DROPLET_IP_ADDRESS_ACC}) sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml +sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml # create ssh private key SSH_KEY_PATH=droplets_rsa @@ -89,3 +90,4 @@ echo -e "${DO_DROPLET_SSH_PRIVATE_KEY}" > ${SSH_KEY_PATH} # update inputs files to use actual ssh key sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml +sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml diff --git a/circle.yml b/circle.yml index 2b1baf3b0f..c909d28db6 100644 --- a/circle.yml +++ b/circle.yml @@ -44,7 +44,7 @@ test: - if [ "${CIRCLE_NODE_INDEX}" = "1" ]; then chmod +x ci-env/wait_for_droplets_and_update_test_inputs.sh && ci-env/wait_for_droplets_and_update_test_inputs.sh; fi: parallel: true override: - - case $CIRCLE_NODE_INDEX in 0) ./cslang-builder/bin/cslang-builder -ts default,jenkins,consul -cov ;; 1) ./cslang-builder/bin/cslang-builder -ts !default,coreos -cov ;; esac: + - case $CIRCLE_NODE_INDEX in 0) ./cslang-builder/bin/cslang-builder -ts default,jenkins,consul -cov ;; 1) ./cslang-builder/bin/cslang-builder -ts !default,coreos,cadvisor -cov ;; esac: parallel: true post: - if [ "${CIRCLE_NODE_INDEX}" = "1" ]; then chmod +x ci-env/cleanup_env.sh && ci-env/cleanup_env.sh; fi: diff --git a/test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml b/test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml index 7a004210a5..30faa28be9 100644 --- a/test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml +++ b/test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml @@ -7,11 +7,15 @@ # #################################################### test_get_container_metrics: - testSuites: [cadvisor_local] - description: Tests SUCCESS case - this test needs to be run on the Docker machine by activating the test suite and supplying the inputs + testSuites: [cadvisor] + description: > + This test clears the docker containers on the host, starts up a cadvisor container, + calls the get_container_metrics flow and validates the result. testFlowPath: io.cloudslang.docker.cadvisor.test_get_container_metrics inputs: - - host: localhost - - cadvisor_port: "32952" + - host: + - username: core + - private_key_file: + - cadvisor_port: "32541" - cadvisor_container_name: cadvisor - result: SUCCESS \ No newline at end of file + result: SUCCESS diff --git a/test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml b/test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml index 67d434f985..e90e61fc26 100644 --- a/test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml +++ b/test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml @@ -7,7 +7,7 @@ # #################################################### test_report_machine_metrics: - testSuites: [cadvisor] + testSuites: [cadvisor_local] description: Tests SUCCESS case testFlowPath: io.cloudslang.docker.cadvisor.test_report_machine_metrics inputs: diff --git a/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl b/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl index 9424db2e9f..8800cb3e77 100644 --- a/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl +++ b/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl @@ -11,8 +11,8 @@ namespace: io.cloudslang.docker.cadvisor imports: cadvisor: io.cloudslang.docker.cadvisor - cmd: io.cloudslang.base.cmd strings: io.cloudslang.base.strings + containers: io.cloudslang.docker.containers flow: @@ -20,16 +20,28 @@ flow: inputs: - host + - username + - private_key_file - cadvisor_port - cadvisor_container_name workflow: + - clear_docker_containers: + do: + containers.clear_containers: + - docker_host: host + - docker_username: username + - private_key_file + navigate: + SUCCESS: create_cAdvisor_container + FAILURE: CLEAR_DOCKER_CONTAINERS_PROBLEM - create_cAdvisor_container: do: - cmd.run_command: - - command: > - 'docker run --privileged -d --name ' + cadvisor_container_name + ' ' + + containers.run_container: + - container_name: cadvisor_container_name + - container_params: > + '--privileged --publish=' + cadvisor_port + ':8080 ' + '--volume=/:/rootfs:ro ' + '--volume=/var/run:/var/run:rw ' + '--volume=/sys:/sys:ro ' + @@ -38,15 +50,16 @@ flow: '--volume=/sys/fs/cgroup/cpuacct:/cgroup/cpuacct ' + '--volume=/sys/fs/cgroup/cpuset:/cgroup/cpuset ' + '--volume=/sys/fs/cgroup/memory:/cgroup/memory ' + - '--volume=/sys/fs/cgroup/blkio:/cgroup/blkio ' + - '--publish=' + cadvisor_port + ':8080 ' + - 'google/cadvisor:latest --logtostderr' - - overridable: false + '--volume=/sys/fs/cgroup/blkio:/cgroup/blkio' + - image_name: "'google/cadvisor:latest'" + - host + - username + - private_key_file navigate: - SUCCESS: validate_success_get_container_metrics + SUCCESS: call_get_container_metrics FAILURE: C_ADVISOR_CONTAINER_STARTUP_PROBLEM - - validate_success_get_container_metrics: + - call_get_container_metrics: do: cadvisor.get_container_metrics: - host @@ -54,6 +67,9 @@ flow: - container: cadvisor_container_name publish: - returnResult + navigate: + SUCCESS: validate_response_is_not_empty + FAILURE: CALL_GET_CONTAINER_METRICS_PROBLEM - validate_response_is_not_empty: do: @@ -61,18 +77,12 @@ flow: - string_in_which_to_search: returnResult - string_to_find: "'cpu'" navigate: - SUCCESS: clear_docker_host_after - FAILURE: VERIFY_FAILURE - - - clear_docker_host_after: - do: - cmd.run_command: - - command: > - 'docker rm -f ' + cadvisor_container_name - - overridable: false + SUCCESS: SUCCESS + FAILURE: VALIDATE_RESPONSE_IS_NOT_EMPTY_PROBLEM results: - SUCCESS - - FAILURE + - CLEAR_DOCKER_CONTAINERS_PROBLEM - C_ADVISOR_CONTAINER_STARTUP_PROBLEM - - VERIFY_FAILURE + - CALL_GET_CONTAINER_METRICS_PROBLEM + - VALIDATE_RESPONSE_IS_NOT_EMPTY_PROBLEM From 1e07070879d9d146a507c33e7ce653fad00bc459 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 4 Aug 2015 13:30:37 +0300 Subject: [PATCH 073/104] ssh test Signed-off-by: Bonczidai Levente --- .../base/os/linux/validate_linux_machine_ssh_access.sl | 2 +- .../cloudslang/base/remote_command_execution/ssh/ssh_command.sl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/content/io/cloudslang/base/os/linux/validate_linux_machine_ssh_access.sl b/content/io/cloudslang/base/os/linux/validate_linux_machine_ssh_access.sl index d1b054baa7..e0c836dda2 100644 --- a/content/io/cloudslang/base/os/linux/validate_linux_machine_ssh_access.sl +++ b/content/io/cloudslang/base/os/linux/validate_linux_machine_ssh_access.sl @@ -41,7 +41,7 @@ operation: - privateKeyFile: required: false - command: - default: "' '" + default: "'ls'" overridable: false - arguments: required: false diff --git a/content/io/cloudslang/base/remote_command_execution/ssh/ssh_command.sl b/content/io/cloudslang/base/remote_command_execution/ssh/ssh_command.sl index dc1e5e45b2..d8520b49f1 100644 --- a/content/io/cloudslang/base/remote_command_execution/ssh/ssh_command.sl +++ b/content/io/cloudslang/base/remote_command_execution/ssh/ssh_command.sl @@ -52,7 +52,7 @@ operation: required: false - timeout: "'90000'" - characterSet: "'UTF-8'" - - closeSession: "'false'" + - closeSession: "'true'" - agentForwarding: required: false action: From 4ce0c03cf66bff68cbf8d85c37cde39fc0576c85 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 4 Aug 2015 13:43:13 +0300 Subject: [PATCH 074/104] add sleep to test_get_container_metrics Signed-off-by: Bonczidai Levente --- .../docker/cadvisor/test_get_container_metrics.sl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl b/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl index 8800cb3e77..09c50af471 100644 --- a/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl +++ b/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl @@ -13,6 +13,7 @@ imports: cadvisor: io.cloudslang.docker.cadvisor strings: io.cloudslang.base.strings containers: io.cloudslang.docker.containers + utils: io.cloudslang.base.utils flow: @@ -56,9 +57,14 @@ flow: - username - private_key_file navigate: - SUCCESS: call_get_container_metrics + SUCCESS: sleep FAILURE: C_ADVISOR_CONTAINER_STARTUP_PROBLEM + - sleep: + do: + utils.sleep: + - seconds: 5 + - call_get_container_metrics: do: cadvisor.get_container_metrics: From cb9835183fef8b9cef9d12fc09790cbb58aa97f0 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 4 Aug 2015 13:47:47 +0300 Subject: [PATCH 075/104] add FAILURE result Signed-off-by: Bonczidai Levente --- test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl b/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl index 09c50af471..20010d8195 100644 --- a/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl +++ b/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl @@ -88,6 +88,7 @@ flow: results: - SUCCESS + - FAILURE - CLEAR_DOCKER_CONTAINERS_PROBLEM - C_ADVISOR_CONTAINER_STARTUP_PROBLEM - CALL_GET_CONTAINER_METRICS_PROBLEM From c1a1c06b3bcfcdc51c5170696a61c76720df62af Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 4 Aug 2015 13:56:35 +0300 Subject: [PATCH 076/104] revert validate machine ssh access changes Signed-off-by: Bonczidai Levente --- .../base/os/linux/validate_linux_machine_ssh_access.sl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/io/cloudslang/base/os/linux/validate_linux_machine_ssh_access.sl b/content/io/cloudslang/base/os/linux/validate_linux_machine_ssh_access.sl index e0c836dda2..d1b054baa7 100644 --- a/content/io/cloudslang/base/os/linux/validate_linux_machine_ssh_access.sl +++ b/content/io/cloudslang/base/os/linux/validate_linux_machine_ssh_access.sl @@ -41,7 +41,7 @@ operation: - privateKeyFile: required: false - command: - default: "'ls'" + default: "' '" overridable: false - arguments: required: false From a2f06cd25e37bef7c034778947006cbaf4435d82 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 4 Aug 2015 13:56:47 +0300 Subject: [PATCH 077/104] use default navigation Signed-off-by: Bonczidai Levente --- .../cloudslang/docker/cadvisor/test_get_container_metrics.sl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl b/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl index 20010d8195..e93a0505e6 100644 --- a/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl +++ b/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl @@ -64,6 +64,8 @@ flow: do: utils.sleep: - seconds: 5 + navigate: + SUCCESS: call_get_container_metrics - call_get_container_metrics: do: @@ -88,7 +90,6 @@ flow: results: - SUCCESS - - FAILURE - CLEAR_DOCKER_CONTAINERS_PROBLEM - C_ADVISOR_CONTAINER_STARTUP_PROBLEM - CALL_GET_CONTAINER_METRICS_PROBLEM From b89d51d3ca324ae36590ecf7fc07c1281f6afba8 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 4 Aug 2015 14:03:14 +0300 Subject: [PATCH 078/104] revert close session changes Signed-off-by: Bonczidai Levente --- .../cloudslang/base/remote_command_execution/ssh/ssh_command.sl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/io/cloudslang/base/remote_command_execution/ssh/ssh_command.sl b/content/io/cloudslang/base/remote_command_execution/ssh/ssh_command.sl index d8520b49f1..dc1e5e45b2 100644 --- a/content/io/cloudslang/base/remote_command_execution/ssh/ssh_command.sl +++ b/content/io/cloudslang/base/remote_command_execution/ssh/ssh_command.sl @@ -52,7 +52,7 @@ operation: required: false - timeout: "'90000'" - characterSet: "'UTF-8'" - - closeSession: "'true'" + - closeSession: "'false'" - agentForwarding: required: false action: From 54bdbdc2e3ede2f5f0690fdbaeea176a94c42529 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 4 Aug 2015 14:25:14 +0300 Subject: [PATCH 079/104] add get_container_metrics test case to ci Signed-off-by: Bonczidai Levente --- ...ait_for_droplets_and_update_test_inputs.sh | 2 + .../report_machine_metrics.inputs.yaml | 14 ++-- .../cadvisor/test_report_machine_metrics.sl | 66 ++++++++++++++----- 3 files changed, 61 insertions(+), 21 deletions(-) diff --git a/ci-env/wait_for_droplets_and_update_test_inputs.sh b/ci-env/wait_for_droplets_and_update_test_inputs.sh index 64b2ec285b..ca8c80d2c6 100644 --- a/ci-env/wait_for_droplets_and_update_test_inputs.sh +++ b/ci-env/wait_for_droplets_and_update_test_inputs.sh @@ -83,6 +83,7 @@ done DROPLET_IP_ARRAY=(${DROPLET_IP_ADDRESS_ACC}) sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml +sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/docker/cadvisor/test_report_machine_metrics.sl # create ssh private key SSH_KEY_PATH=droplets_rsa @@ -91,3 +92,4 @@ echo -e "${DO_DROPLET_SSH_PRIVATE_KEY}" > ${SSH_KEY_PATH} # update inputs files to use actual ssh key sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml +sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/docker/cadvisor/test_report_machine_metrics.sl diff --git a/test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml b/test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml index e90e61fc26..1a6e70c107 100644 --- a/test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml +++ b/test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml @@ -7,11 +7,15 @@ # #################################################### test_report_machine_metrics: - testSuites: [cadvisor_local] - description: Tests SUCCESS case + testSuites: [cadvisor] + description: > + This test clears the docker containers on the host, starts up a cadvisor container, + calls the report_machine_metrics flow and validates the result. testFlowPath: io.cloudslang.docker.cadvisor.test_report_machine_metrics inputs: - - host: localhost + - host: + - username: core + - private_key_file: + - cadvisor_port: "32541" - cadvisor_container_name: cadvisor - - cadvisor_port: "32951" - result: SUCCESS \ No newline at end of file + result: SUCCESS diff --git a/test/io/cloudslang/docker/cadvisor/test_report_machine_metrics.sl b/test/io/cloudslang/docker/cadvisor/test_report_machine_metrics.sl index 94f0dec63d..41d7cbfe7a 100644 --- a/test/io/cloudslang/docker/cadvisor/test_report_machine_metrics.sl +++ b/test/io/cloudslang/docker/cadvisor/test_report_machine_metrics.sl @@ -10,35 +10,51 @@ namespace: io.cloudslang.docker.cadvisor imports: - cadvisor: io.cloudslang.docker.cadvisor - cmd: io.cloudslang.base.cmd + strings: io.cloudslang.base.strings + containers: io.cloudslang.docker.containers utils: io.cloudslang.base.utils + flow: name: test_report_machine_metrics inputs: - host - - cadvisor_container_name: - default: "'cadvisor'" - - cadvisor_port: - default: "'8080'" + - username + - private_key_file + - cadvisor_port + - cadvisor_container_name workflow: + - clear_docker_containers: + do: + containers.clear_containers: + - docker_host: host + - docker_username: username + - private_key_file + navigate: + SUCCESS: create_cAdvisor_container + FAILURE: CLEAR_DOCKER_CONTAINERS_PROBLEM - create_cAdvisor_container: do: - cmd.run_command: - - command: > - 'docker run --privileged -d --name ' + cadvisor_container_name + ' ' + + containers.run_container: + - container_name: cadvisor_container_name + - container_params: > + '--privileged --publish=' + cadvisor_port + ':8080 ' + '--volume=/:/rootfs:ro ' + '--volume=/var/run:/var/run:rw ' + '--volume=/sys:/sys:ro ' + '--volume=/var/lib/docker/:/var/lib/docker:ro ' + - '--volume=/cgroup:/cgroup ' + - '--publish=' + cadvisor_port + ':8080 ' + - 'google/cadvisor:latest' - - overridable: false + '--volume=/sys/fs/cgroup/cpu:/cgroup/cpu ' + + '--volume=/sys/fs/cgroup/cpuacct:/cgroup/cpuacct ' + + '--volume=/sys/fs/cgroup/cpuset:/cgroup/cpuset ' + + '--volume=/sys/fs/cgroup/memory:/cgroup/memory ' + + '--volume=/sys/fs/cgroup/blkio:/cgroup/blkio' + - image_name: "'google/cadvisor:latest'" + - host + - username + - private_key_file navigate: SUCCESS: sleep FAILURE: C_ADVISOR_CONTAINER_STARTUP_PROBLEM @@ -47,14 +63,32 @@ flow: do: utils.sleep: - seconds: 5 + navigate: + SUCCESS: call_get_container_metrics - - validate_success_report_machine_metrics: + - call_report_machine_metrics: do: - cadvisor.report_machine_metrics: + report_machine_metrics: - host - cadvisor_port + publish: + - memory_capacity + navigate: + SUCCESS: validate_response_is_not_empty + FAILURE: CALL_GET_CONTAINER_METRICS_PROBLEM + + - validate_response_is_not_empty: + do: + strings.string_equals: + - first_string: str(len(memory_capacity.strip())) + - second_string: "'0'" + navigate: + SUCCESS: VALIDATE_RESPONSE_IS_NOT_EMPTY_PROBLEM + FAILURE: SUCCESS results: - SUCCESS - - FAILURE + - CLEAR_DOCKER_CONTAINERS_PROBLEM - C_ADVISOR_CONTAINER_STARTUP_PROBLEM + - CALL_GET_CONTAINER_METRICS_PROBLEM + - VALIDATE_RESPONSE_IS_NOT_EMPTY_PROBLEM From 1dea7d609191565780a13ef3ffc7b26ddf0cd617 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 4 Aug 2015 14:26:09 +0300 Subject: [PATCH 080/104] use default namespace Signed-off-by: Bonczidai Levente --- .../cloudslang/docker/cadvisor/test_get_container_metrics.sl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl b/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl index e93a0505e6..3b5f7942b0 100644 --- a/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl +++ b/test/io/cloudslang/docker/cadvisor/test_get_container_metrics.sl @@ -10,7 +10,6 @@ namespace: io.cloudslang.docker.cadvisor imports: - cadvisor: io.cloudslang.docker.cadvisor strings: io.cloudslang.base.strings containers: io.cloudslang.docker.containers utils: io.cloudslang.base.utils @@ -69,7 +68,7 @@ flow: - call_get_container_metrics: do: - cadvisor.get_container_metrics: + get_container_metrics: - host - cadvisor_port - container: cadvisor_container_name From 31d8f0483c813085e01b1e8ffe4dc67677a4ef56 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 4 Aug 2015 14:33:41 +0300 Subject: [PATCH 081/104] update navigation Signed-off-by: Bonczidai Levente --- .../cloudslang/docker/cadvisor/test_report_machine_metrics.sl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/io/cloudslang/docker/cadvisor/test_report_machine_metrics.sl b/test/io/cloudslang/docker/cadvisor/test_report_machine_metrics.sl index 41d7cbfe7a..549394e340 100644 --- a/test/io/cloudslang/docker/cadvisor/test_report_machine_metrics.sl +++ b/test/io/cloudslang/docker/cadvisor/test_report_machine_metrics.sl @@ -64,7 +64,7 @@ flow: utils.sleep: - seconds: 5 navigate: - SUCCESS: call_get_container_metrics + SUCCESS: call_report_machine_metrics - call_report_machine_metrics: do: From 9b1a6c4ecd8e2a19a41c5b597471550b0bc2d9ca Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 4 Aug 2015 14:42:10 +0300 Subject: [PATCH 082/104] fix test names Signed-off-by: Bonczidai Levente --- ci-env/wait_for_droplets_and_update_test_inputs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-env/wait_for_droplets_and_update_test_inputs.sh b/ci-env/wait_for_droplets_and_update_test_inputs.sh index ca8c80d2c6..6e95d2febf 100644 --- a/ci-env/wait_for_droplets_and_update_test_inputs.sh +++ b/ci-env/wait_for_droplets_and_update_test_inputs.sh @@ -83,7 +83,7 @@ done DROPLET_IP_ARRAY=(${DROPLET_IP_ADDRESS_ACC}) sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml -sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/docker/cadvisor/test_report_machine_metrics.sl +sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml # create ssh private key SSH_KEY_PATH=droplets_rsa @@ -92,4 +92,4 @@ echo -e "${DO_DROPLET_SSH_PRIVATE_KEY}" > ${SSH_KEY_PATH} # update inputs files to use actual ssh key sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml -sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/docker/cadvisor/test_report_machine_metrics.sl +sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml From c6616abcdad3ae8301bee96a2a70936fe6ca12eb Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 4 Aug 2015 14:42:32 +0300 Subject: [PATCH 083/104] use default close session Signed-off-by: Bonczidai Levente --- .../cloudslang/base/remote_command_execution/ssh/ssh_command.sl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/io/cloudslang/base/remote_command_execution/ssh/ssh_command.sl b/content/io/cloudslang/base/remote_command_execution/ssh/ssh_command.sl index dc1e5e45b2..d8520b49f1 100644 --- a/content/io/cloudslang/base/remote_command_execution/ssh/ssh_command.sl +++ b/content/io/cloudslang/base/remote_command_execution/ssh/ssh_command.sl @@ -52,7 +52,7 @@ operation: required: false - timeout: "'90000'" - characterSet: "'UTF-8'" - - closeSession: "'false'" + - closeSession: "'true'" - agentForwarding: required: false action: From 5286da9eaf17322352fc2c30547a4ef679f6a8bf Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 4 Aug 2015 14:53:08 +0300 Subject: [PATCH 084/104] fix python expression Signed-off-by: Bonczidai Levente --- .../cloudslang/docker/cadvisor/test_report_machine_metrics.sl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/io/cloudslang/docker/cadvisor/test_report_machine_metrics.sl b/test/io/cloudslang/docker/cadvisor/test_report_machine_metrics.sl index 549394e340..2c9dca6d75 100644 --- a/test/io/cloudslang/docker/cadvisor/test_report_machine_metrics.sl +++ b/test/io/cloudslang/docker/cadvisor/test_report_machine_metrics.sl @@ -80,7 +80,7 @@ flow: - validate_response_is_not_empty: do: strings.string_equals: - - first_string: str(len(memory_capacity.strip())) + - first_string: str(memory_capacity) - second_string: "'0'" navigate: SUCCESS: VALIDATE_RESPONSE_IS_NOT_EMPTY_PROBLEM From 9d22b9ebc4e8bca1ddb180df4e80afdcd3a4eeb1 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Tue, 4 Aug 2015 16:56:25 +0300 Subject: [PATCH 085/104] add restart_container_base_on_usage test case to ci Signed-off-by: Bonczidai Levente --- ...ait_for_droplets_and_update_test_inputs.sh | 6 +- ...estart_container_base_on_usage.inputs.yaml | 15 ++- .../test_restart_container_base_on_usage.sl | 91 ++++--------------- 3 files changed, 28 insertions(+), 84 deletions(-) diff --git a/ci-env/wait_for_droplets_and_update_test_inputs.sh b/ci-env/wait_for_droplets_and_update_test_inputs.sh index 6e95d2febf..a7077d8460 100644 --- a/ci-env/wait_for_droplets_and_update_test_inputs.sh +++ b/ci-env/wait_for_droplets_and_update_test_inputs.sh @@ -82,8 +82,7 @@ done # update inputs files to use actual IP addresses DROPLET_IP_ARRAY=(${DROPLET_IP_ADDRESS_ACC}) sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml -sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml -sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml +sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/docker/cadvisor/*.inputs.yaml # create ssh private key SSH_KEY_PATH=droplets_rsa @@ -91,5 +90,4 @@ echo -e "${DO_DROPLET_SSH_PRIVATE_KEY}" > ${SSH_KEY_PATH} # update inputs files to use actual ssh key sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml -sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml -sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml +sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/docker/cadvisor/*.inputs.yaml diff --git a/test/io/cloudslang/docker/cadvisor/restart_container_base_on_usage.inputs.yaml b/test/io/cloudslang/docker/cadvisor/restart_container_base_on_usage.inputs.yaml index 491ccefdd4..a9201aeb1b 100644 --- a/test/io/cloudslang/docker/cadvisor/restart_container_base_on_usage.inputs.yaml +++ b/test/io/cloudslang/docker/cadvisor/restart_container_base_on_usage.inputs.yaml @@ -7,16 +7,15 @@ # ######################################################################################################## test_restart_container_base_on_usage_success: - testSuites: [cadvisor_local] + testSuites: [cadvisor] description: > - Tests SUCCESS case - starts a cAdvisor container, waits for the container to start up and runs - the restart_container_base_on_usage flow against it. The Docker host is cleared before and after - the actual test logic. + This test clears the docker containers on the host, starts up a cadvisor container, + calls the restart_container_base_on_usage flow and validates the result. testFlowPath: io.cloudslang.docker.cadvisor.test_restart_container_base_on_usage inputs: - - cadvisor_port: "23569" + - host: + - username: core + - private_key_file: + - cadvisor_port: "32541" - cadvisor_container_name: cadvisor - - host: 111.111.111.111 - - username: root - - private_key_file: "c:/.../id_rsa" result: SUCCESS diff --git a/test/io/cloudslang/docker/cadvisor/test_restart_container_base_on_usage.sl b/test/io/cloudslang/docker/cadvisor/test_restart_container_base_on_usage.sl index e6fbe439e9..5cd55aa07d 100644 --- a/test/io/cloudslang/docker/cadvisor/test_restart_container_base_on_usage.sl +++ b/test/io/cloudslang/docker/cadvisor/test_restart_container_base_on_usage.sl @@ -5,13 +5,11 @@ # The Apache License is available at # http://www.apache.org/licenses/LICENSE-2.0 # -######################################################################################################## +#################################################### namespace: io.cloudslang.docker.cadvisor imports: - cadvisor: io.cloudslang.docker.cadvisor - maintenance: io.cloudslang.docker.maintenance containers: io.cloudslang.docker.containers utils: io.cloudslang.base.utils @@ -20,37 +18,22 @@ flow: name: test_restart_container_base_on_usage inputs: - - cadvisor_port - - cadvisor_container_name - host - - port: - required: false - username - - password: - required: false - - private_key_file: - required: false - - timeout: - required: false + - private_key_file + - cadvisor_port + - cadvisor_container_name workflow: - - pre_clear_machine: - do: - maintenance.clear_host: - - docker_host: host - - docker_username: username - - docker_password: - default: password - required: false - - private_key_file: - required: false - - timeout: - required: false - - port: - required: false - navigate: - SUCCESS: create_cAdvisor_container - FAILURE: PRE_CLEAR_MACHINE_PROBLEM + - clear_docker_containers: + do: + containers.clear_containers: + - docker_host: host + - docker_username: username + - private_key_file + navigate: + SUCCESS: create_cAdvisor_container + FAILURE: CLEAR_DOCKER_CONTAINERS_PROBLEM - create_cAdvisor_container: do: @@ -66,18 +49,11 @@ flow: '--volume=/sys/fs/cgroup/cpuacct:/cgroup/cpuacct ' + '--volume=/sys/fs/cgroup/cpuset:/cgroup/cpuset ' + '--volume=/sys/fs/cgroup/memory:/cgroup/memory ' + - '--volume=/sys/fs/cgroup/blkio:/cgroup/blkio ' + '--volume=/sys/fs/cgroup/blkio:/cgroup/blkio' - image_name: "'google/cadvisor:latest'" - host - - port: - required: false - username - - password: - required: false - - private_key_file: - required: false - - timeout: - required: false + - private_key_file navigate: SUCCESS: sleep FAILURE: C_ADVISOR_CONTAINER_STARTUP_PROBLEM @@ -88,49 +64,20 @@ flow: - seconds: 5 navigate: SUCCESS: call_restart_container_base_on_usage - FAILURE: FAILED_TO_SLEEP - call_restart_container_base_on_usage: do: - cadvisor.restart_container_base_on_usage: + restart_container_base_on_usage: - container: cadvisor_container_name - host - cadvisor_port - - machine_connect_port: - default: port - required: false - username - - password: - required: false - - privateKeyFile: - default: private_key_file - required: false - navigate: - SUCCESS: post_clear_machine - FAILURE: CALL_RESTART_CONTAINER_BASE_ON_USAGE_PROBLEM - - - post_clear_machine: - do: - maintenance.clear_host: - - docker_host: host - - docker_username: username - - docker_password: - default: password - required: false - - private_key_file: - required: false - - timeout: - required: false - - port: - required: false + - privateKeyFile: private_key_file navigate: SUCCESS: SUCCESS - FAILURE: POST_CLEAR_MACHINE_PROBLEM + FAILURE: CALL_RESTART_CONTAINER_BASE_ON_USAGE_PROBLEM results: - SUCCESS - - FAILURE - - PRE_CLEAR_MACHINE_PROBLEM + - CLEAR_DOCKER_CONTAINERS_PROBLEM - C_ADVISOR_CONTAINER_STARTUP_PROBLEM - CALL_RESTART_CONTAINER_BASE_ON_USAGE_PROBLEM - - POST_CLEAR_MACHINE_PROBLEM - - FAILED_TO_SLEEP From a26cb5e7d08254ed2f456bda93e89d1d95f46d71 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 5 Aug 2015 10:40:41 +0300 Subject: [PATCH 086/104] revert close session to `false` default value Signed-off-by: Bonczidai Levente --- .../cloudslang/base/remote_command_execution/ssh/ssh_command.sl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/io/cloudslang/base/remote_command_execution/ssh/ssh_command.sl b/content/io/cloudslang/base/remote_command_execution/ssh/ssh_command.sl index d8520b49f1..dc1e5e45b2 100644 --- a/content/io/cloudslang/base/remote_command_execution/ssh/ssh_command.sl +++ b/content/io/cloudslang/base/remote_command_execution/ssh/ssh_command.sl @@ -52,7 +52,7 @@ operation: required: false - timeout: "'90000'" - characterSet: "'UTF-8'" - - closeSession: "'true'" + - closeSession: "'false'" - agentForwarding: required: false action: From de556506420c28a46c872641af69b780eee4c6bc Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 5 Aug 2015 11:30:49 +0300 Subject: [PATCH 087/104] check fork env var Signed-off-by: Bonczidai Levente --- ci-env/create_droplets.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci-env/create_droplets.sh b/ci-env/create_droplets.sh index bde426053b..2c8db759e2 100644 --- a/ci-env/create_droplets.sh +++ b/ci-env/create_droplets.sh @@ -1,5 +1,12 @@ #!/bin/bash +# don't run script in PRs from forks +echo "CIRCLE_PR_NUMBER: ${CIRCLE_PR_NUMBER}" +if [ "${CIRCLE_PR_NUMBER}" != "" ] +then + exit 0 +fi + # generate discovery URL for the new CoreOS cluster and update the cloud-config file DISCOVERY_URL=$(curl -s -X GET "https://discovery.etcd.io/new") echo "DISCOVERY_URL: $DISCOVERY_URL" From 8d703ba980ff2779583fd8698c87ca350937e874 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Wed, 5 Aug 2015 11:56:35 +0300 Subject: [PATCH 088/104] ignore builds in fork PRs Signed-off-by: Bonczidai Levente --- ci-env/create_droplets.sh | 7 ------- circle.yml | 12 ++++++++---- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/ci-env/create_droplets.sh b/ci-env/create_droplets.sh index 2c8db759e2..bde426053b 100644 --- a/ci-env/create_droplets.sh +++ b/ci-env/create_droplets.sh @@ -1,12 +1,5 @@ #!/bin/bash -# don't run script in PRs from forks -echo "CIRCLE_PR_NUMBER: ${CIRCLE_PR_NUMBER}" -if [ "${CIRCLE_PR_NUMBER}" != "" ] -then - exit 0 -fi - # generate discovery URL for the new CoreOS cluster and update the cloud-config file DISCOVERY_URL=$(curl -s -X GET "https://discovery.etcd.io/new") echo "DISCOVERY_URL: $DISCOVERY_URL" diff --git a/circle.yml b/circle.yml index c909d28db6..8998e5ae58 100644 --- a/circle.yml +++ b/circle.yml @@ -12,6 +12,10 @@ general: - "builder.log" machine: + environment: + # "${CIRCLE_PR_NUMBER}" = "" denotes that the build is not part of a fork PR + # ignore test suites on node 1 in fork builds + ACTIVE_CI_TEST_SUITES_NODE_1: $(if [ "${CIRCLE_PR_NUMBER}" = "" ]; then echo '!default,coreos,cadvisor'; else echo '!default'; fi) pre: - sudo curl -L -o /usr/bin/docker 'http://s3-external-1.amazonaws.com/circle-downloads/docker-1.6.2-circleci'; sudo chmod 0755 /usr/bin/docker; true @@ -25,7 +29,7 @@ dependencies: test: pre: - - if [ "${CIRCLE_NODE_INDEX}" = "1" ]; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: + - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: parallel: true - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; then docker run -d -p 49165:8080 jenkins; fi: parallel: true @@ -41,11 +45,11 @@ test: parallel: true - pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib: parallel: true - - if [ "${CIRCLE_NODE_INDEX}" = "1" ]; then chmod +x ci-env/wait_for_droplets_and_update_test_inputs.sh && ci-env/wait_for_droplets_and_update_test_inputs.sh; fi: + - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/wait_for_droplets_and_update_test_inputs.sh && ci-env/wait_for_droplets_and_update_test_inputs.sh; fi: parallel: true override: - - case $CIRCLE_NODE_INDEX in 0) ./cslang-builder/bin/cslang-builder -ts default,jenkins,consul -cov ;; 1) ./cslang-builder/bin/cslang-builder -ts !default,coreos,cadvisor -cov ;; esac: + - case $CIRCLE_NODE_INDEX in 0) ./cslang-builder/bin/cslang-builder -ts default,jenkins,consul -cov ;; 1) ./cslang-builder/bin/cslang-builder -ts ${ACTIVE_CI_TEST_SUITES_NODE_1} -cov ;; esac: parallel: true post: - - if [ "${CIRCLE_NODE_INDEX}" = "1" ]; then chmod +x ci-env/cleanup_env.sh && ci-env/cleanup_env.sh; fi: + - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/cleanup_env.sh && ci-env/cleanup_env.sh; fi: parallel: true From deea1b66eff54fff64fc19458fcac3b138485819 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 6 Aug 2015 14:06:37 +0300 Subject: [PATCH 089/104] use -z and test yaml syntax Signed-off-by: Bonczidai Levente --- circle.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 8998e5ae58..fceb22a9dd 100644 --- a/circle.yml +++ b/circle.yml @@ -15,7 +15,7 @@ machine: environment: # "${CIRCLE_PR_NUMBER}" = "" denotes that the build is not part of a fork PR # ignore test suites on node 1 in fork builds - ACTIVE_CI_TEST_SUITES_NODE_1: $(if [ "${CIRCLE_PR_NUMBER}" = "" ]; then echo '!default,coreos,cadvisor'; else echo '!default'; fi) + ACTIVE_CI_TEST_SUITES_NODE_1: $(if [ -z "${CIRCLE_PR_NUMBER}" ]; then echo '!default,coreos,cadvisor'; else echo '!default'; fi) pre: - sudo curl -L -o /usr/bin/docker 'http://s3-external-1.amazonaws.com/circle-downloads/docker-1.6.2-circleci'; sudo chmod 0755 /usr/bin/docker; true @@ -29,6 +29,9 @@ dependencies: test: pre: + - > + echo "1" + && echo "2" - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: parallel: true - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; then docker run -d -p 49165:8080 jenkins; fi: From 44b36e1cc3e6c23ca080a0f81e000ef652bb0d0a Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 6 Aug 2015 14:26:55 +0300 Subject: [PATCH 090/104] don't use -z and test yaml syntax Signed-off-by: Bonczidai Levente --- circle.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/circle.yml b/circle.yml index fceb22a9dd..3e657fb721 100644 --- a/circle.yml +++ b/circle.yml @@ -15,7 +15,7 @@ machine: environment: # "${CIRCLE_PR_NUMBER}" = "" denotes that the build is not part of a fork PR # ignore test suites on node 1 in fork builds - ACTIVE_CI_TEST_SUITES_NODE_1: $(if [ -z "${CIRCLE_PR_NUMBER}" ]; then echo '!default,coreos,cadvisor'; else echo '!default'; fi) + ACTIVE_CI_TEST_SUITES_NODE_1: $(if [ "${CIRCLE_PR_NUMBER}" = "" ]; then echo '!default,coreos,cadvisor'; else echo '!default'; fi) pre: - sudo curl -L -o /usr/bin/docker 'http://s3-external-1.amazonaws.com/circle-downloads/docker-1.6.2-circleci'; sudo chmod 0755 /usr/bin/docker; true @@ -29,9 +29,9 @@ dependencies: test: pre: - - > + - | echo "1" - && echo "2" + echo "2" - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: parallel: true - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; then docker run -d -p 49165:8080 jenkins; fi: From 3db417a3225d5e74ddd666bb95071a66ceede029 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 6 Aug 2015 14:57:14 +0300 Subject: [PATCH 091/104] test yaml syntax 3 Signed-off-by: Bonczidai Levente --- circle.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/circle.yml b/circle.yml index 3e657fb721..27d070cb05 100644 --- a/circle.yml +++ b/circle.yml @@ -30,11 +30,12 @@ dependencies: test: pre: - | - echo "1" - echo "2" - - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: + if [ "${CIRCLE_NODE_INDEX}" = "0" ] + then + docker run -d -p 49165:8080 jenkins + fi: parallel: true - - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; then docker run -d -p 49165:8080 jenkins; fi: + - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: parallel: true - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; then docker run -d -p 8500:8500 -p 8600:8600/udp fhalim/consul; fi: parallel: true From 56639b045a51d20910d8aaf9e7b625d2d4bbe54f Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 6 Aug 2015 15:29:40 +0300 Subject: [PATCH 092/104] test yaml syntax 4 Signed-off-by: Bonczidai Levente --- circle.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/circle.yml b/circle.yml index 27d070cb05..0daf005a72 100644 --- a/circle.yml +++ b/circle.yml @@ -29,11 +29,9 @@ dependencies: test: pre: - - | - if [ "${CIRCLE_NODE_INDEX}" = "0" ] - then - docker run -d -p 49165:8080 jenkins - fi: + - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; \ + then docker run -d -p 49165:8080 jenkins; \ + fi: parallel: true - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: parallel: true From 9c20cfa84d73472faa29f6307f2ffc352dc16066 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 6 Aug 2015 17:02:05 +0300 Subject: [PATCH 093/104] enhancements Signed-off-by: Bonczidai Levente --- ci-env/create_droplets.sh | 7 ++++--- .../wait_for_droplets_and_update_test_inputs.sh | 15 +++++---------- circle.yml | 13 ++++++------- .../cluster_docker_images_maintenance.inputs.yaml | 2 +- .../cadvisor/get_container_metrics.inputs.yaml | 2 +- .../cadvisor/report_machine_metrics.inputs.yaml | 2 +- .../restart_container_base_on_usage.inputs.yaml | 2 +- 7 files changed, 19 insertions(+), 24 deletions(-) diff --git a/ci-env/create_droplets.sh b/ci-env/create_droplets.sh index bde426053b..d3e7e232e9 100644 --- a/ci-env/create_droplets.sh +++ b/ci-env/create_droplets.sh @@ -45,10 +45,11 @@ do DROPLET_ID_ACC+="${DROPLET_ID} " echo "$COREOS_MACHINE (ID: $DROPLET_ID) droplet creation request accepted" + + # store droplet IDs in a file to be accessible in other script + echo ${DROPLET_ID_ACC} > "droplets_${CIRCLE_BUILD_NUM}.txt" else echo "Problem occurred: $COREOS_MACHINE droplet creation request - status code: $STATUS_CODE" + exit 1 fi done - -# store droplet IDs in a file to be accessible in other script -echo ${DROPLET_ID_ACC} > "droplets_${CIRCLE_BUILD_NUM}.txt" diff --git a/ci-env/wait_for_droplets_and_update_test_inputs.sh b/ci-env/wait_for_droplets_and_update_test_inputs.sh index a7077d8460..52d51462f8 100644 --- a/ci-env/wait_for_droplets_and_update_test_inputs.sh +++ b/ci-env/wait_for_droplets_and_update_test_inputs.sh @@ -53,12 +53,8 @@ print obj["droplet"]["status"]; import json,sys; obj = json.load(sys.stdin); ipv4_list = obj["droplet"]["networks"]["v4"]; -ip = "" -for ip_obj in ipv4_list: - if ip_obj["type"] == "public": - ip = ip_obj["ip_address"]; - break; -print ip; +public_ipv4_list = filter(lambda x : x["type"] == "public", ipv4_list); +print public_ipv4_list[0] if len(public_ipv4_list) > 0 else ''; '\ ) echo "Droplet($DROPLET_ID) IPv4 address: $IP_ADDRESS" @@ -76,18 +72,17 @@ print ip; if [ "$DROPLET_STATUS" != "active" ] then echo "Droplet($DROPLET_ID) is not active after ${WAITING_TIME}" + exit 1 fi done # update inputs files to use actual IP addresses DROPLET_IP_ARRAY=(${DROPLET_IP_ADDRESS_ACC}) -sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml -sed -i "s//${DROPLET_IP_ARRAY[0]}/g" test/io/cloudslang/docker/cadvisor/*.inputs.yaml +find test -type f -exec sed -i "s//${DROPLET_IP_ARRAY[0]}/g" {} + # create ssh private key SSH_KEY_PATH=droplets_rsa echo -e "${DO_DROPLET_SSH_PRIVATE_KEY}" > ${SSH_KEY_PATH} # update inputs files to use actual ssh key -sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml -sed -i "s//${SSH_KEY_PATH}/g" test/io/cloudslang/docker/cadvisor/*.inputs.yaml +find test -type f -exec sed -i "s//${SSH_KEY_PATH}/g" {} + diff --git a/circle.yml b/circle.yml index 0daf005a72..aa98d1eca5 100644 --- a/circle.yml +++ b/circle.yml @@ -16,6 +16,7 @@ machine: # "${CIRCLE_PR_NUMBER}" = "" denotes that the build is not part of a fork PR # ignore test suites on node 1 in fork builds ACTIVE_CI_TEST_SUITES_NODE_1: $(if [ "${CIRCLE_PR_NUMBER}" = "" ]; then echo '!default,coreos,cadvisor'; else echo '!default'; fi) + RULE_NODE_1_NOT_IN_FORK: '[ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]' pre: - sudo curl -L -o /usr/bin/docker 'http://s3-external-1.amazonaws.com/circle-downloads/docker-1.6.2-circleci'; sudo chmod 0755 /usr/bin/docker; true @@ -29,11 +30,9 @@ dependencies: test: pre: - - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; \ - then docker run -d -p 49165:8080 jenkins; \ - fi: + - if ${RULE_NODE_1_NOT_IN_FORK}; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: parallel: true - - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: + - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; then docker run -d -p 49165:8080 jenkins; fi: parallel: true - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; then docker run -d -p 8500:8500 -p 8600:8600/udp fhalim/consul; fi: parallel: true @@ -47,11 +46,11 @@ test: parallel: true - pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib: parallel: true - - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/wait_for_droplets_and_update_test_inputs.sh && ci-env/wait_for_droplets_and_update_test_inputs.sh; fi: + - if ${RULE_NODE_1_NOT_IN_FORK}; then chmod +x ci-env/wait_for_droplets_and_update_test_inputs.sh && ci-env/wait_for_droplets_and_update_test_inputs.sh; fi: parallel: true override: - - case $CIRCLE_NODE_INDEX in 0) ./cslang-builder/bin/cslang-builder -ts default,jenkins,consul -cov ;; 1) ./cslang-builder/bin/cslang-builder -ts ${ACTIVE_CI_TEST_SUITES_NODE_1} -cov ;; esac: + - case ${CIRCLE_NODE_INDEX} in 0) ./cslang-builder/bin/cslang-builder -ts default,jenkins,consul -cov ;; 1) ./cslang-builder/bin/cslang-builder -ts ${ACTIVE_CI_TEST_SUITES_NODE_1} -cov ;; esac: parallel: true post: - - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/cleanup_env.sh && ci-env/cleanup_env.sh; fi: + - if ${RULE_NODE_1_NOT_IN_FORK}; then chmod +x ci-env/cleanup_env.sh && ci-env/cleanup_env.sh; fi: parallel: true diff --git a/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml b/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml index ccadfc91c7..06f5588093 100644 --- a/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml +++ b/test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml @@ -16,7 +16,7 @@ test_cluster_docker_images_maintenance_success: cleans up in the cluster. testFlowPath: io.cloudslang.coreos.test_cluster_docker_images_maintenance inputs: - - coreos_host: # ip address of one host from the cluster + - coreos_host: # ip address of one host from the cluster - coreos_username: core # username for the machine - private_key_file: # path to private key file result: SUCCESS diff --git a/test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml b/test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml index 30faa28be9..06595dbbd2 100644 --- a/test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml +++ b/test/io/cloudslang/docker/cadvisor/get_container_metrics.inputs.yaml @@ -13,7 +13,7 @@ test_get_container_metrics: calls the get_container_metrics flow and validates the result. testFlowPath: io.cloudslang.docker.cadvisor.test_get_container_metrics inputs: - - host: + - host: - username: core - private_key_file: - cadvisor_port: "32541" diff --git a/test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml b/test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml index 1a6e70c107..c39f7981ff 100644 --- a/test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml +++ b/test/io/cloudslang/docker/cadvisor/report_machine_metrics.inputs.yaml @@ -13,7 +13,7 @@ test_report_machine_metrics: calls the report_machine_metrics flow and validates the result. testFlowPath: io.cloudslang.docker.cadvisor.test_report_machine_metrics inputs: - - host: + - host: - username: core - private_key_file: - cadvisor_port: "32541" diff --git a/test/io/cloudslang/docker/cadvisor/restart_container_base_on_usage.inputs.yaml b/test/io/cloudslang/docker/cadvisor/restart_container_base_on_usage.inputs.yaml index a9201aeb1b..a20cb98404 100644 --- a/test/io/cloudslang/docker/cadvisor/restart_container_base_on_usage.inputs.yaml +++ b/test/io/cloudslang/docker/cadvisor/restart_container_base_on_usage.inputs.yaml @@ -13,7 +13,7 @@ test_restart_container_base_on_usage_success: calls the restart_container_base_on_usage flow and validates the result. testFlowPath: io.cloudslang.docker.cadvisor.test_restart_container_base_on_usage inputs: - - host: + - host: - username: core - private_key_file: - cadvisor_port: "32541" From 05683f4dcd83def64cec6601fbb11d5873c25c0d Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 6 Aug 2015 17:21:34 +0300 Subject: [PATCH 094/104] use explicit rule Signed-off-by: Bonczidai Levente --- circle.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/circle.yml b/circle.yml index aa98d1eca5..4f88a6b361 100644 --- a/circle.yml +++ b/circle.yml @@ -16,7 +16,6 @@ machine: # "${CIRCLE_PR_NUMBER}" = "" denotes that the build is not part of a fork PR # ignore test suites on node 1 in fork builds ACTIVE_CI_TEST_SUITES_NODE_1: $(if [ "${CIRCLE_PR_NUMBER}" = "" ]; then echo '!default,coreos,cadvisor'; else echo '!default'; fi) - RULE_NODE_1_NOT_IN_FORK: '[ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]' pre: - sudo curl -L -o /usr/bin/docker 'http://s3-external-1.amazonaws.com/circle-downloads/docker-1.6.2-circleci'; sudo chmod 0755 /usr/bin/docker; true @@ -30,7 +29,7 @@ dependencies: test: pre: - - if ${RULE_NODE_1_NOT_IN_FORK}; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: + - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: parallel: true - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; then docker run -d -p 49165:8080 jenkins; fi: parallel: true @@ -46,11 +45,11 @@ test: parallel: true - pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib: parallel: true - - if ${RULE_NODE_1_NOT_IN_FORK}; then chmod +x ci-env/wait_for_droplets_and_update_test_inputs.sh && ci-env/wait_for_droplets_and_update_test_inputs.sh; fi: + - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/wait_for_droplets_and_update_test_inputs.sh && ci-env/wait_for_droplets_and_update_test_inputs.sh; fi: parallel: true override: - case ${CIRCLE_NODE_INDEX} in 0) ./cslang-builder/bin/cslang-builder -ts default,jenkins,consul -cov ;; 1) ./cslang-builder/bin/cslang-builder -ts ${ACTIVE_CI_TEST_SUITES_NODE_1} -cov ;; esac: parallel: true post: - - if ${RULE_NODE_1_NOT_IN_FORK}; then chmod +x ci-env/cleanup_env.sh && ci-env/cleanup_env.sh; fi: + - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/cleanup_env.sh && ci-env/cleanup_env.sh; fi: parallel: true From 42b910608347ed0aa1919c315cd63129b46e2c2c Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 6 Aug 2015 17:34:19 +0300 Subject: [PATCH 095/104] add print Signed-off-by: Bonczidai Levente --- ci-env/wait_for_droplets_and_update_test_inputs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci-env/wait_for_droplets_and_update_test_inputs.sh b/ci-env/wait_for_droplets_and_update_test_inputs.sh index 52d51462f8..b762ae0f72 100644 --- a/ci-env/wait_for_droplets_and_update_test_inputs.sh +++ b/ci-env/wait_for_droplets_and_update_test_inputs.sh @@ -86,3 +86,5 @@ echo -e "${DO_DROPLET_SSH_PRIVATE_KEY}" > ${SSH_KEY_PATH} # update inputs files to use actual ssh key find test -type f -exec sed -i "s//${SSH_KEY_PATH}/g" {} + + +cat test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml # TODO remove From 6c296b4ded5dd6dffa067f9e41dbd182a9809471 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Thu, 6 Aug 2015 17:40:17 +0300 Subject: [PATCH 096/104] fix quotes Signed-off-by: Bonczidai Levente --- ci-env/wait_for_droplets_and_update_test_inputs.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci-env/wait_for_droplets_and_update_test_inputs.sh b/ci-env/wait_for_droplets_and_update_test_inputs.sh index b762ae0f72..bea05f0cee 100644 --- a/ci-env/wait_for_droplets_and_update_test_inputs.sh +++ b/ci-env/wait_for_droplets_and_update_test_inputs.sh @@ -54,7 +54,7 @@ import json,sys; obj = json.load(sys.stdin); ipv4_list = obj["droplet"]["networks"]["v4"]; public_ipv4_list = filter(lambda x : x["type"] == "public", ipv4_list); -print public_ipv4_list[0] if len(public_ipv4_list) > 0 else ''; +print public_ipv4_list[0] if len(public_ipv4_list) > 0 else ""; '\ ) echo "Droplet($DROPLET_ID) IPv4 address: $IP_ADDRESS" @@ -86,5 +86,3 @@ echo -e "${DO_DROPLET_SSH_PRIVATE_KEY}" > ${SSH_KEY_PATH} # update inputs files to use actual ssh key find test -type f -exec sed -i "s//${SSH_KEY_PATH}/g" {} + - -cat test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml # TODO remove From 785843c68a47d7a5459190cf75eba8d16bb1006f Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Fri, 7 Aug 2015 09:37:59 +0300 Subject: [PATCH 097/104] print again Signed-off-by: Bonczidai Levente --- ci-env/wait_for_droplets_and_update_test_inputs.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci-env/wait_for_droplets_and_update_test_inputs.sh b/ci-env/wait_for_droplets_and_update_test_inputs.sh index bea05f0cee..29860058a8 100644 --- a/ci-env/wait_for_droplets_and_update_test_inputs.sh +++ b/ci-env/wait_for_droplets_and_update_test_inputs.sh @@ -86,3 +86,5 @@ echo -e "${DO_DROPLET_SSH_PRIVATE_KEY}" > ${SSH_KEY_PATH} # update inputs files to use actual ssh key find test -type f -exec sed -i "s//${SSH_KEY_PATH}/g" {} + + +cat test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml # TODO remove \ No newline at end of file From 5a7095cea8e9eb41fe86531b23098e5834d51928 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Fri, 7 Aug 2015 09:45:14 +0300 Subject: [PATCH 098/104] python script: get ip_address from obj Signed-off-by: Bonczidai Levente --- ci-env/wait_for_droplets_and_update_test_inputs.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci-env/wait_for_droplets_and_update_test_inputs.sh b/ci-env/wait_for_droplets_and_update_test_inputs.sh index 29860058a8..39aab5fb90 100644 --- a/ci-env/wait_for_droplets_and_update_test_inputs.sh +++ b/ci-env/wait_for_droplets_and_update_test_inputs.sh @@ -52,9 +52,9 @@ print obj["droplet"]["status"]; ' import json,sys; obj = json.load(sys.stdin); -ipv4_list = obj["droplet"]["networks"]["v4"]; -public_ipv4_list = filter(lambda x : x["type"] == "public", ipv4_list); -print public_ipv4_list[0] if len(public_ipv4_list) > 0 else ""; +ipv4_container_list = obj["droplet"]["networks"]["v4"]; +public_ipv4_container_list = filter(lambda x : x["type"] == "public", ipv4_container_list); +print public_ipv4_container_list[0]["ip_address"] if len(public_ipv4_container_list) > 0 else ""; '\ ) echo "Droplet($DROPLET_ID) IPv4 address: $IP_ADDRESS" From b541125743725d1a4d5283db85f35e1f7839fbb9 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Fri, 7 Aug 2015 09:59:24 +0300 Subject: [PATCH 099/104] remove debug print Signed-off-by: Bonczidai Levente --- ci-env/wait_for_droplets_and_update_test_inputs.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci-env/wait_for_droplets_and_update_test_inputs.sh b/ci-env/wait_for_droplets_and_update_test_inputs.sh index 39aab5fb90..11019f7231 100644 --- a/ci-env/wait_for_droplets_and_update_test_inputs.sh +++ b/ci-env/wait_for_droplets_and_update_test_inputs.sh @@ -86,5 +86,3 @@ echo -e "${DO_DROPLET_SSH_PRIVATE_KEY}" > ${SSH_KEY_PATH} # update inputs files to use actual ssh key find test -type f -exec sed -i "s//${SSH_KEY_PATH}/g" {} + - -cat test/io/cloudslang/coreos/cluster_docker_images_maintenance.inputs.yaml # TODO remove \ No newline at end of file From 264f815f91b5316042b9388d0e6f7c5b60053ab9 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Fri, 7 Aug 2015 10:02:49 +0300 Subject: [PATCH 100/104] try to indent python scripts Signed-off-by: Bonczidai Levente --- ...ait_for_droplets_and_update_test_inputs.sh | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/ci-env/wait_for_droplets_and_update_test_inputs.sh b/ci-env/wait_for_droplets_and_update_test_inputs.sh index 11019f7231..62c2784b66 100644 --- a/ci-env/wait_for_droplets_and_update_test_inputs.sh +++ b/ci-env/wait_for_droplets_and_update_test_inputs.sh @@ -38,11 +38,12 @@ do DROPLET_STATUS=$(\ echo "$RESPONSE_BODY_JSON" | python -c \ ' -import json,sys; -obj = json.load(sys.stdin); -print obj["droplet"]["status"]; +if True: + import json,sys; + obj = json.load(sys.stdin); + print obj["droplet"]["status"]; '\ - ) + ) echo "Droplet($DROPLET_ID) status: ${DROPLET_STATUS}" if [ "$DROPLET_STATUS" = "active" ] @@ -50,13 +51,14 @@ print obj["droplet"]["status"]; IP_ADDRESS=$(\ echo "$RESPONSE_BODY_JSON" | python -c \ ' -import json,sys; -obj = json.load(sys.stdin); -ipv4_container_list = obj["droplet"]["networks"]["v4"]; -public_ipv4_container_list = filter(lambda x : x["type"] == "public", ipv4_container_list); -print public_ipv4_container_list[0]["ip_address"] if len(public_ipv4_container_list) > 0 else ""; +if True: + import json,sys; + obj = json.load(sys.stdin); + ipv4_container_list = obj["droplet"]["networks"]["v4"]; + public_ipv4_container_list = filter(lambda x : x["type"] == "public", ipv4_container_list); + print public_ipv4_container_list[0]["ip_address"] if len(public_ipv4_container_list) > 0 else ""; '\ - ) + ) echo "Droplet($DROPLET_ID) IPv4 address: $IP_ADDRESS" DROPLET_IP_ADDRESS_ACC+="${IP_ADDRESS} " From b3e5cccb4be1c9072d376f28b3eb410ad49cd2d5 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Fri, 7 Aug 2015 10:25:45 +0300 Subject: [PATCH 101/104] extract rule Signed-off-by: Bonczidai Levente --- circle.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/circle.yml b/circle.yml index 4f88a6b361..942e40eb72 100644 --- a/circle.yml +++ b/circle.yml @@ -16,6 +16,7 @@ machine: # "${CIRCLE_PR_NUMBER}" = "" denotes that the build is not part of a fork PR # ignore test suites on node 1 in fork builds ACTIVE_CI_TEST_SUITES_NODE_1: $(if [ "${CIRCLE_PR_NUMBER}" = "" ]; then echo '!default,coreos,cadvisor'; else echo '!default'; fi) + RULE_DROPLET_MACHINE_NOT_FORK: '[ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]' pre: - sudo curl -L -o /usr/bin/docker 'http://s3-external-1.amazonaws.com/circle-downloads/docker-1.6.2-circleci'; sudo chmod 0755 /usr/bin/docker; true @@ -29,7 +30,7 @@ dependencies: test: pre: - - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: + - if eval "${RULE_DROPLET_MACHINE_NOT_FORK}"; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: parallel: true - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; then docker run -d -p 49165:8080 jenkins; fi: parallel: true @@ -45,11 +46,11 @@ test: parallel: true - pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib: parallel: true - - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/wait_for_droplets_and_update_test_inputs.sh && ci-env/wait_for_droplets_and_update_test_inputs.sh; fi: + - if eval "${RULE_DROPLET_MACHINE_NOT_FORK}"; then chmod +x ci-env/wait_for_droplets_and_update_test_inputs.sh && ci-env/wait_for_droplets_and_update_test_inputs.sh; fi: parallel: true override: - case ${CIRCLE_NODE_INDEX} in 0) ./cslang-builder/bin/cslang-builder -ts default,jenkins,consul -cov ;; 1) ./cslang-builder/bin/cslang-builder -ts ${ACTIVE_CI_TEST_SUITES_NODE_1} -cov ;; esac: parallel: true post: - - if [ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]; then chmod +x ci-env/cleanup_env.sh && ci-env/cleanup_env.sh; fi: + - if eval "${RULE_DROPLET_MACHINE_NOT_FORK}"; then chmod +x ci-env/cleanup_env.sh && ci-env/cleanup_env.sh; fi: parallel: true From c52b31aa2edc510e89aa12a5d56e54043594e68b Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Fri, 7 Aug 2015 11:16:36 +0300 Subject: [PATCH 102/104] group commands after machines - use multi line keys 1 Signed-off-by: Bonczidai Levente --- circle.yml | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/circle.yml b/circle.yml index 942e40eb72..c4e1530a85 100644 --- a/circle.yml +++ b/circle.yml @@ -30,24 +30,26 @@ dependencies: test: pre: - - if eval "${RULE_DROPLET_MACHINE_NOT_FORK}"; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: - parallel: true - - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; then docker run -d -p 49165:8080 jenkins; fi: - parallel: true - - if [ "${CIRCLE_NODE_INDEX}" = "0" ]; then docker run -d -p 8500:8500 -p 8600:8600/udp fhalim/consul; fi: - parallel: true - - wget https://github.com/CloudSlang/cloud-slang/releases/download/cloudslang-0.8.RC1/cslang-builder.zip: - parallel: true - - unzip cslang-builder.zip -d cslang-builder: - parallel: true - - chmod +x cslang-builder/bin/cslang-builder: - parallel: true - - mkdir cslang-builder/lib/Lib: - parallel: true - - pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib: - parallel: true - - if eval "${RULE_DROPLET_MACHINE_NOT_FORK}"; then chmod +x ci-env/wait_for_droplets_and_update_test_inputs.sh && ci-env/wait_for_droplets_and_update_test_inputs.sh; fi: - parallel: true + - ? > + if [ "${CIRCLE_NODE_INDEX}" = "0" ]; + then docker run -d -p 49165:8080 jenkins + && docker run -d -p 8500:8500 -p 8600:8600/udp fhalim/consul; + fi + : parallel: true + - if eval "${RULE_DROPLET_MACHINE_NOT_FORK}"; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: + parallel: true + - wget https://github.com/CloudSlang/cloud-slang/releases/download/cloudslang-0.8.RC1/cslang-builder.zip: + parallel: true + - unzip cslang-builder.zip -d cslang-builder: + parallel: true + - chmod +x cslang-builder/bin/cslang-builder: + parallel: true + - mkdir cslang-builder/lib/Lib: + parallel: true + - pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib: + parallel: true + - if eval "${RULE_DROPLET_MACHINE_NOT_FORK}"; then chmod +x ci-env/wait_for_droplets_and_update_test_inputs.sh && ci-env/wait_for_droplets_and_update_test_inputs.sh; fi: + parallel: true override: - case ${CIRCLE_NODE_INDEX} in 0) ./cslang-builder/bin/cslang-builder -ts default,jenkins,consul -cov ;; 1) ./cslang-builder/bin/cslang-builder -ts ${ACTIVE_CI_TEST_SUITES_NODE_1} -cov ;; esac: parallel: true From 4df6168920215da07758a7b0a21a48df7e5871f8 Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Fri, 7 Aug 2015 11:33:30 +0300 Subject: [PATCH 103/104] group commands after machines - use multi line keys 2 Signed-off-by: Bonczidai Levente --- circle.yml | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/circle.yml b/circle.yml index c4e1530a85..3de525400a 100644 --- a/circle.yml +++ b/circle.yml @@ -15,7 +15,7 @@ machine: environment: # "${CIRCLE_PR_NUMBER}" = "" denotes that the build is not part of a fork PR # ignore test suites on node 1 in fork builds - ACTIVE_CI_TEST_SUITES_NODE_1: $(if [ "${CIRCLE_PR_NUMBER}" = "" ]; then echo '!default,coreos,cadvisor'; else echo '!default'; fi) + ACTIVE_CI_TEST_SUITES_DROPLETS_NODE: $(if [ "${CIRCLE_PR_NUMBER}" = "" ]; then echo '!default,coreos,cadvisor'; else echo '!default'; fi) RULE_DROPLET_MACHINE_NOT_FORK: '[ "${CIRCLE_NODE_INDEX}" = "1" ] && [ "${CIRCLE_PR_NUMBER}" = "" ]' pre: - sudo curl -L -o /usr/bin/docker 'http://s3-external-1.amazonaws.com/circle-downloads/docker-1.6.2-circleci'; sudo chmod 0755 /usr/bin/docker; true @@ -30,29 +30,34 @@ dependencies: test: pre: - - ? > + - ? > ### machine 0 if [ "${CIRCLE_NODE_INDEX}" = "0" ]; then docker run -d -p 49165:8080 jenkins && docker run -d -p 8500:8500 -p 8600:8600/udp fhalim/consul; fi : parallel: true - if eval "${RULE_DROPLET_MACHINE_NOT_FORK}"; then chmod +x ci-env/create_droplets.sh && ci-env/create_droplets.sh; fi: - parallel: true - - wget https://github.com/CloudSlang/cloud-slang/releases/download/cloudslang-0.8.RC1/cslang-builder.zip: - parallel: true - - unzip cslang-builder.zip -d cslang-builder: - parallel: true - - chmod +x cslang-builder/bin/cslang-builder: - parallel: true - - mkdir cslang-builder/lib/Lib: - parallel: true - - pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib: - parallel: true - - if eval "${RULE_DROPLET_MACHINE_NOT_FORK}"; then chmod +x ci-env/wait_for_droplets_and_update_test_inputs.sh && ci-env/wait_for_droplets_and_update_test_inputs.sh; fi: - parallel: true + parallel: true ### machine 1 + - ? > ### every machine + wget https://github.com/CloudSlang/cloud-slang/releases/download/cloudslang-0.8.RC1/cslang-builder.zip + && unzip cslang-builder.zip -d cslang-builder + && chmod +x cslang-builder/bin/cslang-builder + && mkdir cslang-builder/lib/Lib + && pip install -r python-lib/requirements.txt -t cslang-builder/lib/Lib + : parallel: true + - ? > ### machine 1 + if eval "${RULE_DROPLET_MACHINE_NOT_FORK}"; + then chmod +x ci-env/wait_for_droplets_and_update_test_inputs.sh + && ci-env/wait_for_droplets_and_update_test_inputs.sh; + fi + : parallel: true override: - - case ${CIRCLE_NODE_INDEX} in 0) ./cslang-builder/bin/cslang-builder -ts default,jenkins,consul -cov ;; 1) ./cslang-builder/bin/cslang-builder -ts ${ACTIVE_CI_TEST_SUITES_NODE_1} -cov ;; esac: - parallel: true + - ? > + case ${CIRCLE_NODE_INDEX} in + 0) ./cslang-builder/bin/cslang-builder -ts default,jenkins,consul -cov ;; + 1) ./cslang-builder/bin/cslang-builder -ts ${ACTIVE_CI_TEST_SUITES_DROPLETS_NODE} -cov ;; + esac + : parallel: true post: - if eval "${RULE_DROPLET_MACHINE_NOT_FORK}"; then chmod +x ci-env/cleanup_env.sh && ci-env/cleanup_env.sh; fi: - parallel: true + parallel: true ### machine 1 From 2f598b3fcd1954df1f5be2d8458942793527f24a Mon Sep 17 00:00:00 2001 From: Bonczidai Levente Date: Fri, 7 Aug 2015 16:26:41 +0300 Subject: [PATCH 104/104] add exit Signed-off-by: Bonczidai Levente --- ci-env/wait_for_droplets_and_update_test_inputs.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-env/wait_for_droplets_and_update_test_inputs.sh b/ci-env/wait_for_droplets_and_update_test_inputs.sh index 62c2784b66..ae7aebc1a5 100644 --- a/ci-env/wait_for_droplets_and_update_test_inputs.sh +++ b/ci-env/wait_for_droplets_and_update_test_inputs.sh @@ -69,6 +69,7 @@ if True: fi else echo "Problem occurred: retrieving droplet($DROPLET_ID) information - status code: $STATUS_CODE" + exit 1 fi done if [ "$DROPLET_STATUS" != "active" ]