From ed04689bc79e457addda116f1c880ffac154c22b Mon Sep 17 00:00:00 2001 From: Stefan Scherer Date: Wed, 8 Mar 2017 22:18:43 +0100 Subject: [PATCH] Update the build steps and installed versions --- .gitignore | 1 + .travis.yml | 7 +- Dockerfile | 26 +------ Makefile | 2 +- README.md | 4 +- builder/build.sh | 74 +++++++++++-------- builder/chroot-script.sh | 23 ++---- .../spec/hypriotos-image/base/release_spec.rb | 2 +- .../hypriotos-image/docker-compose_spec.rb | 4 +- .../hypriotos-image/docker-machine_spec.rb | 15 +--- .../spec/hypriotos-image/docker_spec.rb | 62 ++++++++++++---- builder/test/os-release_spec.rb | 4 +- builder/test/spec_helper.rb | 2 +- versions.config | 15 ++++ 14 files changed, 131 insertions(+), 110 deletions(-) create mode 100644 versions.config diff --git a/.gitignore b/.gitignore index d664d2d..fa53109 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.img *.img.zip +*.img.zip.sha256 *.tar.gz .vagrant/ diff --git a/.travis.yml b/.travis.yml index 2014837..c4d1d08 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,10 +11,11 @@ branches: - /^v\d.*$/ deploy: provider: releases - api_key: - secure: ZxbGrH2fjRBSFrCy3Q7kqOBZs/qeZ8QYXz3ZrbGBaoZlZuz9lZbrpBxwcaUbruyKvxx/J7iNeVBuSobz36K+qWQCe5OxOKkg14KzY0EOE6dLeIXWkTYX2ywYGgvlpzEhooly5KC6eJu5i3jyWjpUCBYOndZrAj4Zokx5v8hBQoLwf0LZB/KRQx7YAuz3f8jOkI881e+ic2rV+SNGPbocyNpcUZEBxeH2tsM+tBui7dThtSXzdc4QsJXnrsMrd+UAlrWqG+DTM+f+o9MWQbIOvY28pwHg2cdaIpd6+Y8tt1v+oT1nWThsaKQRYrk8wzi3SjoHTanaVuBa9jGaW4zzzl7zjGn385WN5FCwPoO3FTi+xOOzpWwucASXjs5LtQkgYfObk4Eo0LRdrjpfQZTlIzmoXElYSWWhhvQQh1gpagG/mCHGv/Vk09DDNHExk+U3KCR7BZISHiJ5F8Km3/1Gm27ewsO5cA8aGL6C6AwzynB7hkQgGvbnYr8MoeI2cfxKAKE1cH3zebdimas1gs/zsTDFIjrmu4bruZYf5V/I4MGsG3B/pwWsdWmRKRdOpylNNM/R791w3Ln/36oDqwQhHj5v7tlrNOqDa6hNg9YuKpOOBZMMHElxmqID0SepJzOZPv2EjFVOV5p7cpPL8z5NTy6mL9ZXpw4pj5s/6UjYzmk= + skip_cleanup: true + api_key: ${GITHUB_TOKEN} file: - - sd-card-odroid-c1-${TRAVIS_TAG}.img.zip + - hypriotos-odroid-c1-${TRAVIS_TAG}.img.zip + - hypriotos-odroid-c1-${TRAVIS_TAG}.img.zip.sha256 on: tags: true repo: hypriot/image-builder-odroid-c1 diff --git a/Dockerfile b/Dockerfile index 67f2044..8148f5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,13 @@ -FROM debian:jessie +FROM hypriot/image-builder:latest -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ - python-pip \ - build-essential \ - libguestfs-tools \ - libncurses5-dev \ - tree \ +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y \ binfmt-support \ qemu \ qemu-user-static \ - debootstrap \ - kpartx \ - lvm2 \ - dosfstools \ - zip \ - unzip \ - pigz \ - awscli \ - ruby \ - ruby-dev \ - shellcheck \ --no-install-recommends && \ rm -rf /var/lib/apt/lists/* -RUN gem update --system && \ - gem install --no-document serverspec && \ - gem install --no-document pry-byebug && \ - gem install --no-document bundler - COPY builder/ /builder/ # build sd card image diff --git a/Makefile b/Makefile index 3797df3..34b115a 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ testshell: build docker run -ti --privileged -v $(shell pwd)/builder:/builder -v $(shell pwd):/workspace -v /boot:/boot -v /lib/modules:/lib/modules image-builder-odroid-c1 bash test: - VERSION=dirty docker run --rm -ti --privileged -v $(shell pwd):/workspace -v /boot:/boot -v /lib/modules:/lib/modules -e VERSION image-builder-odroid-c1 bash -c "unzip /workspace/sd-card-odroid-c1-dirty.img.zip && rspec --format documentation --color /workspace/builder/test/*_spec.rb" + VERSION=dirty docker run --rm -ti --privileged -v $(shell pwd):/workspace -v /boot:/boot -v /lib/modules:/lib/modules -e VERSION image-builder-odroid-c1 bash -c "unzip /workspace/hypriotos-odroid-c1-dirty.img.zip && rspec --format documentation --color /workspace/builder/test/*_spec.rb" shellcheck: build VERSION=dirty docker run --rm -ti -v $(shell pwd):/workspace image-builder-odroid-c1 bash -c 'shellcheck /workspace/builder/*.sh /workspace/builder/files/etc/firstboot.d/*' diff --git a/README.md b/README.md index 9561f38..23dffe4 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ eval $(docker-machine env image-builder-odroid-c1) ### Build the SD card image -From here you can just make the SD card image. The output will be written and compressed to `sd-card-odroid-c1-dirty.img.zip`. +From here you can just make the SD card image. The output will be written and compressed to `hypriotos-odroid-c1-dirty.img.zip`. ```bash make sd-image @@ -55,7 +55,7 @@ make test Now flash the SD card image and boot up a ODROID C1. Run the [Serverspec](http://serverspec.org) integration tests in `builder/test-integration/` folder against your ODROID C1. Set the environment variable `BOARD` to the IP address or host name of your running ODROID C1. ```bash -flash sd-card-odroid-c1-dirty.img.zip +flash hypriotos-odroid-c1-dirty.img.zip BOARD=black-pearl.local make test-integration ``` diff --git a/builder/build.sh b/builder/build.sh index f99aa93..2f52071 100755 --- a/builder/build.sh +++ b/builder/build.sh @@ -1,77 +1,86 @@ #!/bin/bash set -ex # This script should be run only inside of a Docker container -if [ ! -f /.dockerinit ]; then +if [ ! -f /.dockerenv ]; then echo "ERROR: script works only in a Docker container!" exit 1 fi +# get versions for software that needs to be installed +source /workspace/versions.config + ### setting up some important variables to control the build process # where to store our created sd-image file BUILD_RESULT_PATH="/workspace" + +# place to build our sd-image BUILD_PATH="/build" -# where to store our base file system -HYPRIOT_OS_VERSION="v0.7.2" -ROOTFS_TAR="rootfs-armhf-${HYPRIOT_OS_VERSION}.tar.gz" +ROOTFS_TAR="rootfs-armhf-debian-${HYPRIOT_OS_VERSION}.tar.gz" ROOTFS_TAR_PATH="$BUILD_RESULT_PATH/$ROOTFS_TAR" +# Show TRAVSI_TAG in travis builds +echo TRAVIS_TAG="${TRAVIS_TAG}" + # size of root and boot partion ROOT_PARTITION_SIZE="800M" # device specific settings HYPRIOT_IMAGE_VERSION=${VERSION:="dirty"} -HYPRIOT_IMAGE_NAME="sd-card-odroid-c1-${HYPRIOT_IMAGE_VERSION}.img" +HYPRIOT_IMAGE_NAME="hypriotos-odroid-c1-${HYPRIOT_IMAGE_VERSION}.img" IMAGE_ROOTFS_PATH="/image-rootfs.tar.gz" QEMU_ARCH="arm" export HYPRIOT_IMAGE_VERSION -# specific versions of kernel/firmware and docker tools -export KERNEL_VERSION="142-1" -export DOCKER_ENGINE_VERSION="1.10.1-1" -export DOCKER_COMPOSE_VERSION="1.6.0-27" -export DOCKER_MACHINE_VERSION="0.4.1-72" - # create build directory for assembling our image filesystem -rm -rf $BUILD_PATH -mkdir -p $BUILD_PATH +rm -rf ${BUILD_PATH} +mkdir ${BUILD_PATH} # download our base root file system -if [ ! -f $ROOTFS_TAR_PATH ]; then - wget -q -O $ROOTFS_TAR_PATH https://github.com/hypriot/os-rootfs/releases/download/$HYPRIOT_OS_VERSION/$ROOTFS_TAR +if [ ! -f "${ROOTFS_TAR_PATH}" ]; then + wget -q -O "${ROOTFS_TAR_PATH}" "https://github.com/hypriot/os-rootfs/releases/download/${HYPRIOT_OS_VERSION}/${ROOTFS_TAR}" fi +# verify checksum of our root filesystem +echo "${ROOTFS_TAR_CHECKSUM} ${ROOTFS_TAR_PATH}" | sha256sum -c - + # extract root file system -tar -xzf $ROOTFS_TAR_PATH -C $BUILD_PATH +tar xf "${ROOTFS_TAR_PATH}" -C "${BUILD_PATH}" # register qemu-arm with binfmt update-binfmts --enable qemu-$QEMU_ARCH -# set up mount points for pseudo filesystems -mkdir -p $BUILD_PATH/{proc,sys,dev/pts} +# set up mount points for the pseudo filesystems +mkdir -p ${BUILD_PATH}/{proc,sys,dev/pts} -mount -o bind /dev $BUILD_PATH/dev -mount -o bind /dev/pts $BUILD_PATH/dev/pts -mount -t proc none $BUILD_PATH/proc -mount -t sysfs none $BUILD_PATH/sys +mount -o bind /dev ${BUILD_PATH}/dev +mount -o bind /dev/pts ${BUILD_PATH}/dev/pts +mount -t proc none ${BUILD_PATH}/proc +mount -t sysfs none ${BUILD_PATH}/sys -#---modify image--- # modify/add image files directly -cp -R /builder/files/* $BUILD_PATH/ +# e.g. root partition resize script +cp -R /builder/files/* ${BUILD_PATH}/ -# modify image in chroot environment -chroot $BUILD_PATH /bin/bash "$BUILD_RESULT_PATH/$HYPRIOT_IMAGE_NAME.zip" +zip "${BUILD_RESULT_PATH}/${HYPRIOT_IMAGE_NAME}.zip" "${HYPRIOT_IMAGE_NAME}" +cd ${BUILD_RESULT_PATH} && sha256sum "${HYPRIOT_IMAGE_NAME}.zip" > "${HYPRIOT_IMAGE_NAME}.zip.sha256" && cd - # test sd-image that we have built VERSION=${HYPRIOT_IMAGE_VERSION} rspec --format documentation --color /builder/test diff --git a/builder/chroot-script.sh b/builder/chroot-script.sh index 33773de..67613cd 100755 --- a/builder/chroot-script.sh +++ b/builder/chroot-script.sh @@ -24,32 +24,25 @@ apt-get update # install Hypriot packages for using Docker apt-get install -y \ - "docker-hypriot=${DOCKER_ENGINE_VERSION}" \ "docker-compose=${DOCKER_COMPOSE_VERSION}" \ "docker-machine=${DOCKER_MACHINE_VERSION}" -#FIXME: should be handled in .deb package -# setup Docker default configuration for ODROID C1 -rm -f /etc/init.d/docker # we're using a pure systemd init, remove sysvinit script -rm -f /etc/default/docker -# --get upstream config -wget -q -O /etc/default/docker https://github.com/docker/docker/raw/master/contrib/init/sysvinit-debian/docker.default -# --enable aufs by default -sed -i "/#DOCKER_OPTS/a \ -DOCKER_OPTS=\"--storage-driver=aufs -D\"" /etc/default/docker - -#FIXME: should be handled in .deb package -# enable Docker systemd service -systemctl enable docker - # install ODROID kernel apt-get install -y u-boot-tools initramfs-tools +# set up Docker APT repository and install docker-engine package +#TODO: pin package version to ${DOCKER_ENGINE_VERSION} +curl -sSL https://get.docker.com | /bin/sh + # make the kernel package create a copy of the current kernel here touch /boot/uImage apt-get install -y "linux-image-c1=${KERNEL_VERSION}" +# cleanup APT cache and lists +apt-get clean +rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + # set device label and version number echo "HYPRIOT_DEVICE=\"$HYPRIOT_DEVICE\"" >> /etc/os-release echo "HYPRIOT_IMAGE_VERSION=\"$HYPRIOT_IMAGE_VERSION\"" >> /etc/os-release diff --git a/builder/test-integration/spec/hypriotos-image/base/release_spec.rb b/builder/test-integration/spec/hypriotos-image/base/release_spec.rb index 336f330..61069a5 100644 --- a/builder/test-integration/spec/hypriotos-image/base/release_spec.rb +++ b/builder/test-integration/spec/hypriotos-image/base/release_spec.rb @@ -4,7 +4,7 @@ it { should be_file } it { should be_owned_by 'root' } its(:content) { should match 'HYPRIOT_OS="HypriotOS/armhf"' } - its(:content) { should match 'HYPRIOT_OS_VERSION="v0.7.2"' } + its(:content) { should match 'HYPRIOT_OS_VERSION="v1.1.1"' } its(:content) { should match 'HYPRIOT_DEVICE="ODROID C1/C1\+"' } its(:content) { should match 'HYPRIOT_IMAGE_VERSION=' } end diff --git a/builder/test-integration/spec/hypriotos-image/docker-compose_spec.rb b/builder/test-integration/spec/hypriotos-image/docker-compose_spec.rb index abcf38d..25e5fe3 100644 --- a/builder/test-integration/spec/hypriotos-image/docker-compose_spec.rb +++ b/builder/test-integration/spec/hypriotos-image/docker-compose_spec.rb @@ -6,7 +6,7 @@ describe command('dpkg -l docker-compose') do its(:stdout) { should match /ii docker-compose/ } - its(:stdout) { should match /1.6.0-27/ } + its(:stdout) { should match /1.9.0-23/ } its(:exit_status) { should eq 0 } end @@ -17,6 +17,6 @@ end describe command('docker-compose --version') do - its(:stdout) { should match /1.6.0/m } + its(:stdout) { should match /1.9.0/m } its(:exit_status) { should eq 0 } end diff --git a/builder/test-integration/spec/hypriotos-image/docker-machine_spec.rb b/builder/test-integration/spec/hypriotos-image/docker-machine_spec.rb index 3615a0f..8f22afa 100644 --- a/builder/test-integration/spec/hypriotos-image/docker-machine_spec.rb +++ b/builder/test-integration/spec/hypriotos-image/docker-machine_spec.rb @@ -6,7 +6,7 @@ describe command('dpkg -l docker-machine') do its(:stdout) { should match /ii docker-machine/ } - its(:stdout) { should match /0.4.1-72/ } + its(:stdout) { should match /0.9.0-39/ } its(:exit_status) { should eq 0 } end @@ -17,17 +17,6 @@ end describe command('docker-machine --version') do - its(:stdout) { should match /0.4.1/m } - its(:stderr) { should match /^$/ } + its(:stdout) { should match /0.9.0/m } its(:exit_status) { should eq 0 } end - -describe command('docker-machine create --help') do - its(:stdout) { should match /Available drivers:.*hypriot/ } - its(:stdout) { should match /--hypriot-ip-address/ } - its(:stdout) { should match /--hypriot-ssh-key/ } - its(:stdout) { should match /--hypriot-ssh-port/ } - its(:stdout) { should match /--hypriot-ssh-user/ } - its(:stderr) { should match /^$/ } - its(:exit_status) { should eq 0 } -end \ No newline at end of file diff --git a/builder/test-integration/spec/hypriotos-image/docker_spec.rb b/builder/test-integration/spec/hypriotos-image/docker_spec.rb index 4fe6d12..e3bfb55 100644 --- a/builder/test-integration/spec/hypriotos-image/docker_spec.rb +++ b/builder/test-integration/spec/hypriotos-image/docker_spec.rb @@ -1,12 +1,12 @@ require 'spec_helper' -describe package('docker-hypriot') do +describe package('docker-engine') do it { should be_installed } end -describe command('dpkg -l docker-hypriot') do - its(:stdout) { should match /ii docker-hypriot/ } - its(:stdout) { should match /1.10.1-1/ } +describe command('dpkg -l docker-engine') do + its(:stdout) { should match /ii docker-engine/ } + its(:stdout) { should match /17.03.0~ce-0~raspbian-jessie/ } its(:exit_status) { should eq 0 } end @@ -16,49 +16,81 @@ it { should be_owned_by 'root' } end -describe file('/usr/lib/docker/dockerinit') do +describe file('/usr/bin/docker-containerd') do it { should be_file } it { should be_mode 755 } it { should be_owned_by 'root' } end -describe file('/etc/init.d/docker') do - it { should_not exist } +describe file('/usr/bin/docker-containerd-ctr') do + it { should be_file } + it { should be_mode 755 } + it { should be_owned_by 'root' } +end + +describe file('/usr/bin/docker-containerd-shim') do + it { should be_file } + it { should be_mode 755 } + it { should be_owned_by 'root' } +end + +describe file('/usr/bin/docker-runc') do + it { should be_file } + it { should be_mode 755 } + it { should be_owned_by 'root' } +end + +describe file('/lib/systemd/system/docker.socket') do + it { should be_file } + it { should be_mode 644 } + it { should be_owned_by 'root' } +end + +describe file('/var/run/docker.sock') do + it { should be_socket } + it { should be_mode 660 } + it { should be_owned_by 'root' } + it { should be_grouped_into 'docker' } end describe file('/etc/default/docker') do it { should be_file } it { should be_mode 644 } it { should be_owned_by 'root' } - its(:content) { should match /--storage-driver=aufs/ } end describe file('/var/lib/docker') do it { should be_directory } - it { should be_mode 701 } + it { should be_mode 711 } + it { should be_owned_by 'root' } +end + +describe file('/var/lib/docker/overlay2') do + it { should be_directory } + it { should be_mode 700 } it { should be_owned_by 'root' } end describe file('/etc/bash_completion.d/docker') do it { should be_file } - it { should be_mode 755 } + it { should be_mode 644 } it { should be_owned_by 'root' } it { should be_file } end describe command('docker -v') do - its(:stdout) { should match /Docker version 1.10.1, build/ } + its(:stdout) { should match /Docker version 17.03.0-ce, build/ } its(:exit_status) { should eq 0 } end describe command('docker version') do - its(:stdout) { should match /Client:. Version: 1.10.1. API version: 1.22/m } - its(:stdout) { should match /Server:. Version: 1.10.1. API version: 1.22/m } + its(:stdout) { should match /Client:. Version: 17.03.0-ce. API version: 1.26/m } + its(:stdout) { should match /Server:. Version: 17.03.0-ce. API version: 1.26/m } its(:exit_status) { should eq 0 } end describe command('docker info') do - its(:stdout) { should match /Storage Driver: aufs/ } + its(:stdout) { should match /Storage Driver: overlay/ } its(:exit_status) { should eq 0 } end @@ -71,7 +103,7 @@ end describe service('docker') do - #it { should be_enabled } + it { should be_enabled } it { should be_running } end diff --git a/builder/test/os-release_spec.rb b/builder/test/os-release_spec.rb index 4185aa2..734de7a 100644 --- a/builder/test/os-release_spec.rb +++ b/builder/test/os-release_spec.rb @@ -36,8 +36,8 @@ expect(stdout).to contain('HYPRIOT_DEVICE="ODROID C1/C1') end - it "uses os-rootfs version 'HYPRIOT_OS_VERSION=\"v0.7.2\"'" do - expect(stdout).to contain('^HYPRIOT_OS_VERSION="v0.7.2"$') + it "uses os-rootfs version 'HYPRIOT_OS_VERSION=\"v1.1.1\"'" do + expect(stdout).to contain('^HYPRIOT_OS_VERSION="v1.1.1"$') end if ENV.fetch('TRAVIS_TAG','') != '' diff --git a/builder/test/spec_helper.rb b/builder/test/spec_helper.rb index 24460fd..26234ef 100644 --- a/builder/test/spec_helper.rb +++ b/builder/test/spec_helper.rb @@ -2,7 +2,7 @@ set :backend, :exec def image_path - return "sd-card-odroid-c1-#{ENV['VERSION']}.img" + return "hypriotos-odroid-c1-#{ENV['VERSION']}.img" end def run( cmd ) diff --git a/versions.config b/versions.config new file mode 100644 index 0000000..2f5238b --- /dev/null +++ b/versions.config @@ -0,0 +1,15 @@ +# config vars for the root file system +HYPRIOT_OS_VERSION="v1.1.1" +ROOTFS_TAR_CHECKSUM="c7cf61547ab2e588df5f73087ec3d5c3785e9ced59f4fe8e0296c365b1dfd092" + +# name of the ready made raw image for Odroid +RAW_IMAGE="odroid-raw.img" +RAW_IMAGE_VERSION="v0.2.2" +RAW_IMAGE_CHECKSUM="fe9af7686960d5d2dd6c364197e196d751f6adedd7da515ed6521c31e9582f8a" + +# specific versions of kernel/firmware and docker tools +export KERNEL_VERSION="151-1" +export DOCKER_ENGINE_VERSION="17.03.0~ce-0~raspbian-jessie" +export DOCKER_COMPOSE_VERSION="1.9.0-23" +export DOCKER_MACHINE_VERSION="0.9.0-39" +export DEVICE_INIT_VERSION="0.1.8"