ci-setup-anvil-bm-vm

#!/bin/bash

# ENTRY POINT (mix)

set -e

. $HOME/ci-tools/ci-setup-anvil-vm-env

if [ -n "${NODE_NAME}" ]; then
 echo "Delay build by 180 seconds to allow Jenkins to cancel the job"
 sleep 180
fi

echo "Destroying VMs (if any)"

for i in $ALLNODES $CACHENODES; do
 destroy_vm $i
done

$HOME/ci-tools/ci-setup-anvil-bm-vm-cache $BUILD none none

echo "Creating nodes disks"

cd "$DEPLOYDIR/snapshots"
rm -rf *.qcow2 *.iso
for i in $ALLNODES; do
 qemu-img create -b "$DEPLOYDIR/cache/${BUILD}-latest.qcow2" -F qcow2 -f qcow2 "$i-vol.qcow2" 200G
 qemu-img create -f qcow2 "$i-vol2.qcow2" 200G
done

echo "Generating an-test-deployX kickstart files/isos"

# 1 to 5, see anvil-ci-proxy:/etc/named/ci.alteeve.com
for i in $(seq 1 5); do
 rm -rf "$DEPLOYDIR/configs/ks${i}"
 mkdir "$DEPLOYDIR/configs/ks${i}"
 cat "$TEMPLATESDIR/ks.cfg" | \
  sed \
   -e "s#@NODE@#${i}#g" \
   -e "s#@SSHKEY@#${SSHKEY}#g" \
  > "$DEPLOYDIR/configs/ks${i}/ks.cfg"
 rm -rf "$DEPLOYDIR/snapshots/deploy${i}.iso"
 genisoimage -o "$DEPLOYDIR/snapshots/deploy${i}.iso" -V OEMDRV -r -J "$DEPLOYDIR/configs/ks${i}"
done

cd "$DEPLOYDIR/configs"

# ifn ip network is defined in ci-tools/ci-setup-anvil-bm
# and ci-tools/anvil-config-templates/ network templates

ipbase=2
for i in $ALLNODES; do
 IFNIP=192.168.122.$ipbase
 cat "$TEMPLATESDIR/anvil_striker_net_config.cfg" | \
  sed \
   -e "s#@IFNIP@#$IFNIP#g" \
   > "$DEPLOYDIR/configs/${i}_net_config.cfg"
  ipbase=$((ipbase + 1))
done

echo "Generating cloud-init configs"
for i in $ALLNODES; do
 case $i in
  *striker*)
   ANVILPKG="striker"
  ;;
  *a01n*)
   ANVILPKG="node"
  ;;
  *dr*)
   ANVILPKG="dr"
  ;;
 esac
 cat "$TEMPLATESDIR/anvil_cloud_init_${BUILD}.cfg" | \
  sed \
   -e "s#@HOSTNAME@#${i}#g" \
   -e "s#@BUILD@#${BUILD}#g" \
   -e "s#@SSHKEY@#${SSHKEY}#g" \
   -e "s#@KEY@#${KEY}#g" \
   -e "s#@ORG@#${ORG}#g" \
   -e "s#@AKEY@#${AKEY}#g" \
   -e "s#@PRNUM@#${PRNUM}#g" \
   -e "s#@PRREPO@#${PRREPO}#g" \
   -e "s#@ANVIL@#${ANVIL}#g" \
   -e "s#@ANVILDEPS@#${ANVILDEPS}#g" \
   -e "s#@ALTEEVE@#${ALTEEVE}#g" \
   -e "s#@ALTEEVESTAGING@#${ALTEEVESTAGING}#g" \
   -e "s#@ANVILPKG@# - anvil-${ANVILPKG}#g" \
   > "$DEPLOYDIR/configs/${i}_cloud_init.cfg"
done

echo "Generating cloud-init metadata and seed images"
for i in $ALLNODES; do
  rm -f "$DEPLOYDIR/snapshots/${i}-seed.img"
  echo "instance-id: $(uuidgen)" > "$DEPLOYDIR/configs/${i}-metadata"
  cloud-localds -v --network-config="$DEPLOYDIR/configs/${i}_net_config.cfg" "$DEPLOYDIR/snapshots/${i}-seed.img" "$DEPLOYDIR/configs/${i}_cloud_init.cfg" "$DEPLOYDIR/configs/${i}-metadata"
done

echo "Creating VMs"
# those have to match the ports down for Simengine bmc/storage
vmportbase=20100
monportbase=20200
storageportbase=20300

for i in $ALLNODES; do
 netconf="--network network:ifn1_bridge1,model=e1000 --network network:ifn1_bridge1,model=e1000 --network network:bcn1_bridge1,model=e1000 --network network:bcn1_bridge1,model=e1000"
 case $i in
  *striker*)
   cpu=2
   mem=8192
   qemu_command_list=()
  ;;
  *)
   cpu=8
   mem=32768
   netconf="$netconf --network network:sn1_bridge1,model=e1000 --network network:sn1_bridge1,model=e1000"
   qemu_command_list=(
     "-chardev socket,id=ipmi0,host=localhost,port=$vmportbase,reconnect=2"
     "-device ipmi-bmc-extern,id=bmc0,chardev=ipmi0"
     "-device isa-ipmi-bt,bmc=bmc0"
     "-serial mon:tcp::$monportbase,server,telnet,nowait"
     "-chardev socket,id=simengine-storage-tcp,host=localhost,port=$storageportbase,reconnect=2"
     "-device virtio-serial,bus=pcie.0,addr=11"
     "-device virtserialport,chardev=simengine-storage-tcp,name=systems.cdot.simengine.storage.net"
   )
   vmportbase=$((vmportbase + 1))
   monportbase=$((monportbase + 1))
   storageportbase=$((storageportbase + 1))
  ;;
 esac
 virt-install --name ${i} \
  --virt-type kvm --memory $mem --vcpus $cpu \
  --boot hd,menu=on \
  --disk path="$DEPLOYDIR/snapshots/${i}-seed.img",device=cdrom \
  --disk path="$DEPLOYDIR/snapshots/$i-vol.qcow2",device=disk \
  --disk path="$DEPLOYDIR/snapshots/$i-vol2.qcow2",device=disk \
  --graphics vnc \
  --os-variant $KVMVAR \
  $netconf \
  --qemu-commandline="${qemu_command_list[*]}" \
  --noautoconsole
done

echo "Waiting for nodes and cloud-init to complete full init"

bootdone=no
expected=0
error=0

for i in $ALLNODES; do
 expected=$((expected + 1))
done

while [ "$bootdone" = "no" ]; do
 actual=0
 for i in $ALLNODES; do
  status=$(ssh $i cloud-init status 2>/dev/null | awk '{print $NF}')
  echo "cloud-init status node $i: $status"
  if [ "$status" = "error" ]; then
   error=1
   ALLNODES=$(echo $ALLNODES | sed -e 's#'$i'##g')
   expected=$((expected - 1))
   echo "ERROR cloud init log for node $i:"
   ssh $i cat /var/log/cloud-init-output.log
   echo "---------------------------"
  fi
  if [ "$status" = "done" ]; then
   actual=$((actual + 1))
  fi
 done
 echo "$(date) actual: $actual expected: $expected"
 if [ "$actual" = "$expected" ]; then
  bootdone=yes
 else
  sleep 10
 fi
done

if [ "$error" = "1" ]; then
 exit 1
fi

$HOME/ci-tools/ci-setup-anvil-simengine

echo "Generating anvil striker auto config"

cp "$TEMPLATESDIR/anvil_striker_auto.cfg" "$DEPLOYDIR/configs/anvil_striker_auto.cfg"
if [ "$DEPLOYDR" = "yes" ]; then
 cat "$TEMPLATESDIR/anvil_striker_auto_dr.cfg" >> "$DEPLOYDIR/configs/anvil_striker_auto.cfg"
 runtestopts="-r"
else
 unset DRNODES
fi

for i in $ALLNODES; do
 ifaces=5
 case $i in
  *striker*)
   ifaces=3
  ;;
 esac
 for x in $(seq 0 $ifaces); do
  mac="$(ssh $i ifconfig eth$x | grep ether | awk '{print $2}')"
  sed -i -e 's#@'$i'-eth'$x'@#'$mac'#g' "$DEPLOYDIR/configs/anvil_striker_auto.cfg"
 done
done

rm -rf "$DEPLOYDIR/anvil-test-suite"
cd "$DEPLOYDIR"
git clone git@github.com:Alteeve/anvil-test-suite.git
cd anvil-test-suite
set +e
./run-tests $runtestopts -t $TESTS -b $BREAKPOINT
ret=$?
cd $HOME/ci-tools

# collecting all build info
rm -rf $DEPLOYDIR/logs/
mkdir -p $DEPLOYDIR/logs/$BUILD_NUMBER/simengine

cp /var/log/simengine/info.log $DEPLOYDIR/logs/$BUILD_NUMBER/simengine/simengine-daemon.log
tar zcp -f $DEPLOYDIR/logs/$BUILD_NUMBER/simengine/process_logs.tar.gz /tmp/simengine/

cp $DEPLOYDIR/configs/anvil_striker_auto.cfg $DEPLOYDIR/logs/$BUILD_NUMBER/anvil_striker_auto.cfg

if [ "$ret" != "0" ]; then
 for i in $STRIKERS; do
  ssh $i striker-collect-debug -vv --log-secure --output-file /root/debug.tar.bz2
  scp ${i}:/root/debug.tar.bz2 $DEPLOYDIR/logs/$BUILD_NUMBER/${i}_debug.tar.bz2
 done

 for i in $ALLNODES; do
  ssh $i anvil-manage-daemons --disable --now
 done
fi

cd $DEPLOYDIR/logs/
tar zcp -f $BUILD_NUMBER.tar.gz $BUILD_NUMBER/
scp $BUILD_NUMBER.tar.gz anvil-ci-proxy:/var/www/html/testing-logs/$BUILD_NUMBER.tar.gz
ssh anvil-ci-proxy "find /var/www/html/testing-logs/ -maxdepth 1 -mtime +30 -exec rm -rf {} \;" 
cd -
rm -rf $DEPLOYDIR/logs

echo "Complete anvil logs available here for download: http://anvil-ci-repo.ci.alteeve.com/testing-logs/$BUILD_NUMBER.tar.gz"

if [ -n "$KEY" ]; then
 for i in $ALLNODES; do
  ssh $i subscription-manager unregister || true
 done
fi

exit $ret