Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

AO CI: Updating the CI system to install and run minikube #42

Merged
merged 2 commits into from
Dec 11, 2023
Merged

AO CI: Updating the CI system to install and run minikube #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f7a096e
An attempt to configure minikube
Dec 4, 2023
d2af4f2
Update to aws test: replacing pull_request with pull_request_target
Dec 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 18 additions & 4 deletions .github/workflows/awstest.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: aws_test
on:
push:
branches: [ "main" ]
pull_request:
pull_request_target:
branches: [ "main" ]

workflow_dispatch:
Expand All @@ -25,11 +25,12 @@ jobs:
# ---------------------------
# install and configure AWSCLI
# ---------------------------
- name: install awscli
- name: install awscli and helm
run: |
. ./util/awscli_util.sh
#awscli_install
awscli_config
helm_install
env:
AWS_KEYPAIR: ${{secrets.GEORGES_AWS_KEYPAIR}}
AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
Expand All @@ -41,7 +42,7 @@ jobs:
- name: create EC2 VM
run: |
. ./util/awscli_util.sh
instanceid=$(awscli_launch)
instanceid=$(awscli_launch ao)
echo "instanceid=${instanceid}" >> ${GITHUB_ENV}
awscli_wait_run ${instanceid} || exit -1
echo "ipaddr=$(awscli_get_ipaddr ${instanceid})" >> ${GITHUB_ENV}
Expand All @@ -54,11 +55,24 @@ jobs:
. ./util/awscli_util.sh
awscli_install_minikube ${ipaddr}

# ---------------------------
# download minikube credentials and open port access
# ---------------------------
- name: download minikube creds
run: |
. ./util/awscli_util.sh
awscli_access_minikube_start ${ipaddr}



# ---------------------------
# terminate the VM
# ---------------------------
- name: finish
if: success() || failure()
run: |
. ./util/awscli_util.sh
if [[ ${instanceid} != "" ]] ; then awscli_terminate ${instanceid} ; fi
if [[ ${instanceid} != "" ]]
then
awscli_terminate ${instanceid}
fi
171 changes: 134 additions & 37 deletions util/awscli_util.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,27 @@


# IMAGEID is defined to have UEFI and TPM support
# SGNAME is defined to have ssh access. Helm/kube access TBD.
# SGNAME is defined to have ssh access.

export IMAGEID=${IMAGEID:-ami-025d6a3788eadba52}
export KEYNAME=${KEYNAME:-george_aws_keypair}
export SGNAME=${SGNAME:-sg-05863e2cac3b4e3ea}
export INSTANCETYPE=${INSTANCETYPE:-t3.medium}

# #############################################################
# install awscli
# utility: install helm locally
# (not in use because the github action docker container has helm installed)
# #############################################################

function helm_install() {
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
chmod 700 get_helm.sh
./get_helm.sh
}

# #############################################################
# utility: install awscli
# (not in use because the github action docker container has awscli installed)
# #############################################################

function awscli_install() {
Expand All @@ -27,58 +39,67 @@ function awscli_install() {
# #############################################################
# configure AWS CLI for operation:
# copy github action secrets into local environment
# requires AWS_KEYPAIR (the private key or keypair corresponding to AWS key named ${KEYNAME})
# requires AWS_ACCESS_KEY_ID and AWS_ACCESS_KEY_SECRET for authenticating awscli
# #############################################################

function awscli_config() {
echo "awscli_config: creating AWS/SSH configuration and credentials"
# check whether secrets exist as env vars
if [[ "${AWS_KEYPAIR}" == "" ]]
then
echo "AWS keypair secret undefined. Exiting."
echo "ERROR: AWS keypair secret undefined. Exiting."
exit -1
fi

if [[ "${AWS_ACCESS_KEY_ID}" == "" ]]
then
echo "AWS access key ID undefined. Exiting."
echo "ERROR: AWS access key ID undefined. Exiting."
exit -1
fi

if [[ "${AWS_ACCESS_KEY_SECRET}" == "" ]]
then
echo "AWS secret undefined. Exiting."
echo "ERROR: AWS secret undefined. Exiting."
exit -1
fi

# create ssh configuration and credentials
echo "==> Creating AWS/SSH configuration and credentials"
mkdir ~/.ssh
cat > ~/.ssh/config <<EOF
mkdir ${HOME}/.ssh
cat > ${HOME}/.ssh/config <<EOF
StrictHostKeyChecking=no
UserKnownHostsFile=/dev/null
LogLevel=ERROR
EOF
echo "${AWS_KEYPAIR}" > ~/.ssh/aws.pem
chmod 600 ~/.ssh/aws.pem
echo "${AWS_KEYPAIR}" > ${HOME}/.ssh/aws.pem
chmod 600 ${HOME}/.ssh/aws.pem

# create AWS CLI configuration and credentials
echo "==> Creating AWSCLI configuration and credentials"
mkdir ~/.aws
cat > ~/.aws/config <<EOF
mkdir ${HOME}/.aws
cat > ${HOME}/.aws/config <<EOF
[default]
region = us-east-1
EOF
chmod 0600 ~/.aws/config
cat > ~/.aws/credentials <<EOF
chmod 0600 ${HOME}/.aws/config
cat > ${HOME}/.aws/credentials <<EOF
[default]
aws_access_key_id = ${AWS_ACCESS_KEY_ID}
aws_secret_access_key = ${AWS_ACCESS_KEY_SECRET}
EOF
chmod 0600 ~/.aws/credentials
chmod 0600 ${HOME}/.aws/credentials
return 0
}

# #############################################################
# Launch an AWS instance with TPM support.
# * IMAGEID is a pre-created AWS image with UEFI and TPM support
# * KEYNAME is a pre-created AWS keypair for accessing the VM
# * SGNAME is a pre-creates AWS security group with port 22 opened
# * INSTANCETYPE describes the AWS EC2 instance type, currently t3.medium
# * TODO add configurable disk size
# #############################################################
# \param vmname -- the name of the virtual machine to create.
# \returns instance ID in AWS EC2 format, or nonzero exit code.
# #############################################################

function awscli_launch() {
Expand All @@ -92,8 +113,8 @@ function awscli_launch() {
--block-device-mappings '[{"DeviceName": "/dev/xvda", "Ebs": {"VolumeSize": 25}}]' )
if [[ $? != 0 ]]
then
echo "Launch failed"
return 1
echo "ERROR: EC2 launch failed"
exit -1
fi
local instanceid=$(echo "${output}" | jq -r .Instances[0].InstanceId -)
aws ec2 create-tags --resources ${instanceid} --tags="Key=Name,Value=${vmname}-$$" >/dev/null 2>&1
Expand Down Expand Up @@ -130,7 +151,7 @@ function awscli_wait_run() {
local tend=$((t0+timeout))

# step 1: wait for instance to reach "running" state
echo -n "Waiting for ${instanceid} to reach run state: "
echo -n "awscli_wait_run: waiting for ${instanceid} to run: "
local running=0
while [[ $(date +%s) < $tend ]]
do
Expand All @@ -145,16 +166,16 @@ function awscli_wait_run() {
done
if [[ ${running} == 0 ]]
then
echo "Timed out"
echo "ERROR: Timed out"
exit -1
else
local t1=$(date +%s)
echo "done, $((t1-t0)) seconds"
fi

# step 2: wait for instsance to have a public IP
# step 2: wait for instance to have a public IP
local ipcmd="aws ec2 describe-instances | jq -r '.Reservations[].Instances[] | select(.InstanceId==\"${instanceid}\") | .PublicIpAddress'"
echo -n "Waiting for ${instanceid} IP address: "
echo -n "awscli_wait_run: waiting for ${instanceid} IP address: "
while [[ $(date +%s) < $tend ]]
do
local ipaddr=$(eval ${ipcmd})
Expand All @@ -164,27 +185,28 @@ function awscli_wait_run() {
done
if [[ ${ipaddr} == "" ]]
then
echo "Timed out"
echo "ERROR: Timed out"
exit -1
else
local t1=$(date +%s)
echo "${ipaddr}, took $((t1-t0)) seconds"
fi

# step 3: test public IP
echo -n "Performing uptime test: "
echo -n "awscli_wait_run: performing uptime test"
while [[ $(date +%s) < $tend ]]
do
if ssh -i ~/.ssh/aws.pem ubuntu@${ipaddr} uptime > /dev/null 2>&1
if ssh -i ${HOME}/.ssh/aws.pem ubuntu@${ipaddr} uptime > /dev/null 2>&1
then
local t1=$(date +%s)
echo "done, $((t1-t0)) total seconds to launch"
echo "done."
echo "awscli_wait_run: SUCCESS after $((t1-t0)) seconds."
return 0
fi
echo -n "."
sleep 10
done
echo "Timed out"
echo "ERROR: Timed out"
return -1
}

Expand All @@ -193,33 +215,108 @@ function awscli_wait_run() {
# #############################################################

function awscli_terminate() {
aws ec2 terminate-instances --instance-ids "${1}"
echo "awscli_terminate: destroying EC2 VM ID ${1}"
aws ec2 terminate-instances --instance-ids "${1}" > /dev/null 2>&1
}

# #############################################################
# install minikube on the AWS instance
# #############################################################

function awscli_install_minikube() {
function awscli_start_minikube() {
local ipaddr=${1}
local t0=$(date +%s)
# install docker
ssh -i ~/.ssh/aws.pem ubuntu@${ipaddr} <<EOF
echo "awscli_start_minikube on ${ipaddr}: installing docker"
ssh -i ${HOME}/.ssh/aws.pem ubuntu@${ipaddr} > /tmp/docker-install.log 2>&1 <<EOF
sudo apt-get update
sudo apt-get install -y docker.io
sudo usermod -aG docker ubuntu
EOF
if [[ $? != 0 ]]
then
echo "ERROR: docker installation failed. Attaching log."
cat /tmp/docker-install.log
exit -1
fi
# install and start minikube
ssh -i ~/.ssh/aws.pem ubuntu@${ipaddr} <<EOF
echo "awscli_start_minikube on ${ipaddr}: installing minikube"
ssh -i ${HOME}/.ssh/aws.pem ubuntu@${ipaddr} > /tmp/minikube-install.log 2>&1 <<EOF
curl https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 -o /tmp/minikube-linux-amd64
sudo mv /tmp/minikube-linux-amd64 /usr/local/bin/minikube
sudo chmod 755 /usr/local/bin/minikube
/usr/local/bin/minikube start
/usr/local/bin/minikube kubectl get nodes
EOF
# install helm (?)
# ssh -i ~/.ssh/aws.pem ubuntu@${ipaddr} <<EOF
#curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
#chmod 700 get_helm.sh
#./get_helm.sh || exit -1
#EOF
if [[ $? != 0 ]]
then
echo "ERROR: minikube installation failed. Attaching log."
cat /tmp/minikube-install.log
exit -1
fi
local t1=$(date +%s)
echo "awscli_start_minikube: SUCCESS, total time=$((t1-t0))"
return 0
}


# #############################################################
# access minikube from the github action container
# * copy credentials from EC2 VM with scp
# * fix up kube configuration
# * create a ssh tunnel on local port 8443
# * use tunnel to check minikube function
# #############################################################

function awscli_access_minikube() {
local ipaddr=${1}
local t0=$(date +%s)
echo "awscli_access_minikube: copying credentials from ${ipaddr}"
mkdir -p ${HOME}/.kube
scp -i ${HOME}/.ssh/aws.pem ubuntu@${ipaddr}:.kube/config ${HOME}/.kube/config && \
scp -i ${HOME}/.ssh/aws.pem ubuntu@${ipaddr}:.minikube/ca.crt ${HOME}/.kube/ca.crt && \
scp -i ${HOME}/.ssh/aws.pem ubuntu@${ipaddr}:.minikube/profiles/minikube/client.crt ${HOME}/.kube/client.crt && \
scp -i ${HOME}/.ssh/aws.pem ubuntu@${ipaddr}:.minikube/profiles/minikube/client.key ${HOME}/.kube/client.key
if [[ $? != 0 ]]
then
echo "ERROR: failed to copy credentials from EC2 VM"
exit -1
fi

local serverip=$(yq -r .clusters[0].cluster.server ${HOME}/.kube/config | sed "s%https://%%" | sed "s/:.*//")
echo "awscli_access_minikube: server-local minikube address is ${serverip}"

# change the kube configuration
echo "awscli_access_minikube: patching .kube/config"
sed -i "s%certificate-authority:.*%certificate-authority: ${HOME}/.kube/ca.crt%" ${HOME}/.kube/config && \
sed -i "s%client-certificate:.*%client-certificate: ${HOME}/.kube/client.crt%" ${HOME}/.kube/config && \
sed -i "s%client-key:.*%client-key: ${HOME}/.kube/client.key%" ${HOME}/.kube/config && \
sed -i "s%server:.*%server: https://127.0.0.1:8443%" ${HOME}/.kube/config
if [[ $? != 0 ]]
then
echo "ERROR: failed to patch ${HOME}/.kube/config"
exit -1
fi


# we don't need to worry about cleaning up this connection,
# because the last step of any GH action is to remove the target VM itself.
echo "awscli_access_minikube: creating a ssh tunnel to ${ipaddr}"
nohup ssh -N -L 0.0.0.0:8443:${serverip}:8443 -i ${HOME}/.ssh/aws.pem ubuntu@${ipaddr} &
sleep 5

# test
echo "awscli_access_minikube: testing kubectl"
export KUBECONFIG=${HOME}/.kube/config
kubectl get nodes > /dev/null 2>&1
if [[ $? != 0 ]]
then
echo "ERROR: kubectl failed to access minikube on ${ipaddr}."
exit -1
fi
local t1=$(date +%s)
echo "awscli_access_minikube: SUCCESS after $((t1-t0)) seconds."
return 0
}