From 4660be00948cad3f8b674395f65f4ee55831565a Mon Sep 17 00:00:00 2001 From: wangfei Date: Mon, 5 Dec 2022 09:41:15 +0800 Subject: [PATCH] optimize scripts and add autobuild scripts --- .github/workflows/auto-build-main.yml | 29 ++++++ auto-build-main.sh | 110 +++++++++++++++++++++ context/rootfs/scripts/docker.sh | 10 +- context/rootfs/scripts/init-kube.sh | 8 +- context/rootfs/scripts/init-registry.sh | 26 +++-- context/rootfs/scripts/init.sh | 10 +- context/rootfs/scripts/nvidia-docker.sh | 23 +++-- context/rootfs/scripts/uninstall-docker.sh | 14 +-- context/rootfs/scripts/utils.sh | 6 +- 9 files changed, 202 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/auto-build-main.yml create mode 100644 auto-build-main.sh diff --git a/.github/workflows/auto-build-main.yml b/.github/workflows/auto-build-main.yml new file mode 100644 index 0000000..abacd52 --- /dev/null +++ b/.github/workflows/auto-build-main.yml @@ -0,0 +1,29 @@ +name: Auto build image +on: + issue_comment: + types: + - created +jobs: + issue_comment: + name: Auto build image + if: startswith(github.event.comment.body, '/autobuild') + runs-on: ubuntu-latest + permissions: + issues: write + steps: + + - name: Auto build image + id: autobuild + run: | + commentbody="${{github.event.comment.body}}" + commentbody=$(echo $commentbody | sed "s/\/imagebuild//g") + sudo git clone https://github.com/sealerio/basefs.git && cd basefs + sudo touch autobuild.log && sudo chmod 666 autobuild.log && sudo bash auto-build-main.sh --username="${{secrets.REGISTRY_USERNAME}}" --password="${{secrets.REGISTRY_PASSWORD}}" $commentbody > autobuild.log && cat autobuild.log + echo "::set-output name=info::$(grep 'cri:' autobuild.log))" + + - name: Success Commit + uses: peter-evans/create-or-update-comment@v1 + with: + issue-number: ${{ github.event.issue.number }} + body: | + ${{ steps.autobuild.outputs.info }} \ No newline at end of file diff --git a/auto-build-main.sh b/auto-build-main.sh new file mode 100644 index 0000000..c4a3c8e --- /dev/null +++ b/auto-build-main.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +set -e + +for i in "$@"; do + case $i in + -c=* | --cri=*) + cri="${i#*=}" + if [ "$cri" != "docker" ] && [ "$cri" != "containerd" ]; then + echo "Unsupported container runtime: ${cri}" + exit 1 + fi + shift # past argument=value + ;; + -n=* | --buildName=*) + buildName="${i#*=}" + shift # past argument=value + ;; + --platform=*) + platform="${i#*=}" + shift # past argument=value + ;; + --push) + push="true" + shift # past argument=value + ;; + -p=* | --password=*) + password="${i#*=}" + shift # past argument=value + ;; + -u=* | --username=*) + username="${i#*=}" + shift # past argument=value + ;; + --k8s-version=*) + k8s_version="${i#*=}" + shift # past argument=value + ;; + -h | --help) + echo " +### Options + --k8s-version set the kubernetes k8s_version of the Clusterimage, k8s_version must be greater than 1.13 + -c, --cri cri can be set to docker or containerd between kubernetes 1.20-1.24 versions + -n, --buildName set build image name, default is 'registry.cn-qingdao.aliyuncs.com/sealer-io/kubernetes:${k8s_version}' + --platform set the build mirror platform, the default is linux/amd64,linux/arm64 + --push push clusterimage after building the clusterimage. The image name must contain the full name of the repository, and use -u and -p to specify the username and password. + -u, --username specify the user's username for pushing the Clusterimage + -p, --password specify the user's password for pushing the Clusterimage + -d, --debug show all script logs + -h, --help help for auto build shell scripts" + exit 0 + ;; + -d | --debug) + set -x + shift + ;; + -*) + echo "Unknown option $i" + exit 1 + ;; + *) ;; + + esac +done + +version_compare() { printf '%s\n%s\n' "$2" "$1" | sort -V -C; } ## version_compare $a $b: a>=b + +ARCH=$(case "$(uname -m)" in x86_64) echo -n amd64 ;; aarch64) echo -n arm64 ;; *) echo "unsupported architecture" "$(uname -m)" && exit 1 ;; esac) + +if [ "$k8s_version" = "" ]; then echo "pls use --k8s-version to set Clusterimage kubernetes version" && exit 1; else echo "$k8s_version" | grep "v" || k8s_version="v${k8s_version}"; fi +#cri=$([[ -n "$cri" ]] && echo "$cri" || echo docker) +cri=$( (version_compare "$k8s_version" "v1.24.0" && echo "containerd") || ([[ -n "$cri" ]] && echo "$cri" || echo "docker")) +if [[ -z "$buildName" ]]; then + buildName="docker.io/sealerio/kubernetes:${k8s_version}" + if [[ "$cri" == "containerd" ]] && ! version_compare "$k8s_version" "v1.24.0"; then buildName=${buildName}-containerd; fi +fi +platform=$(if [[ -z "$platform" ]]; then echo "linux/arm64,linux/amd64"; else echo "$platform"; fi) +echo "cri: ${cri}, kubernetes version: ${k8s_version}, build image name: ${buildName}" + +kubeadmApiVersion=$( (version_compare "$k8s_version" "v1.23.0" && echo 'kubeadm.k8s.io\/v1beta3') || (version_compare "$k8s_version" "v1.15.0" && echo 'kubeadm.k8s.io\/v1beta2') || + (version_compare "$k8s_version" "v1.13.0" && echo 'kubeadm.k8s.io\/v1beta1') || (echo "Version must be greater than 1.13: ${k8s_version}" && exit 1)) + +workdir="$(mktemp -d auto-build-XXXXX)" && sudo cp -r context "${workdir}" && cd "${workdir}/context" && sudo cp -rf "${cri}"/* . + +# shellcheck disable=SC1091 +sudo chmod +x version.sh download.sh && export kube_install_version="$k8s_version" && source version.sh +./download.sh "${cri}" + +sudo chmod +x amd64/bin/kube* && sudo chmod +x arm64/bin/kube* +#Download the latest version of sealer +sudo git clone https://github.com/sealerio/sealer && cd sealer && git checkout main && make build-in-docker && cp _output/bin/sealer/linux_amd64/sealer /usr/bin/ && cd .. +sudo sed -i "s/v1.19.8/$k8s_version/g" rootfs/etc/kubeadm.yml ##change k8s_version +if [[ "$cri" == "containerd" ]]; then sudo sed -i "s/\/var\/run\/dockershim.sock/\/run\/containerd\/containerd.sock/g" rootfs/etc/kubeadm.yml; fi +sudo sed -i "s/kubeadm.k8s.io\/v1beta2/$kubeadmApiVersion/g" rootfs/etc/kubeadm.yml +sudo ./"${ARCH}"/bin/kubeadm config images list --config "rootfs/etc/kubeadm.yml" +sudo mkdir manifests +sudo ./"${ARCH}"/bin/kubeadm config images list --config "rootfs/etc/kubeadm.yml" 2>/dev/null | sed "/WARNING/d" >>imageList +if [ "$(sudo ./"${ARCH}"/bin/kubeadm config images list --config rootfs/etc/kubeadm.yml 2>/dev/null | grep -c "coredns/coredns")" -gt 0 ]; then sudo sed -i "s/#imageRepository/imageRepository/g" rootfs/etc/kubeadm.yml; fi +sudo sed -i "s/k8s.gcr.io/sea.hub:5000/g" rootfs/etc/kubeadm.yml +pauseImage=$(./"${ARCH}"/bin/kubeadm config images list --config "rootfs/etc/kubeadm.yml" 2>/dev/null | sed "/WARNING/d" | grep pause) +if [ -f "rootfs/etc/dump-config.toml" ]; then sudo sed -i "s/sea.hub:5000\/pause:3.6/$(echo "$pauseImage" | sed 's/\//\\\//g')/g" rootfs/etc/dump-config.toml; fi +#sudo sed -i "s/v1.19.8/${k8s_version}/g" {arm64,amd64}/etc/Metadata +##linux/arm64,linux/amd64 +sudo sealer build -t "docker.io/sealerio/kubernetes:${k8s_version}" -f Kubefile +if [[ "$push" == "true" ]]; then + if [[ -n "$username" ]] && [[ -n "$password" ]]; then + sudo sealer login "$(echo "docker.io" | cut -d "/" -f1)" -u "${username}" -p "${password}" + fi + sudo sealer push "docker.io/sealerio/kubernetes:${k8s_version}" +fi diff --git a/context/rootfs/scripts/docker.sh b/context/rootfs/scripts/docker.sh index c3aba19..8ede8e2 100644 --- a/context/rootfs/scripts/docker.sh +++ b/context/rootfs/scripts/docker.sh @@ -16,10 +16,13 @@ set -x set -e -scripts_path=$(cd `dirname $0`; pwd) +# shellcheck disable=SC2046 +# shellcheck disable=SC2006 +scripts_path=$(cd `dirname "$0"`; pwd) image_dir="$scripts_path/../images" DOCKER_VERSION="19.03.14-sealer" +# shellcheck disable=SC1091 get_distribution() { lsb_dist="" # Every system that we officially support has /etc/os-release @@ -46,6 +49,7 @@ load_images() { done } +# shellcheck disable=SC2006 check_docker_valid() { if ! docker info 2>&1; then panic "docker is not healthy: $(docker info 2>&1), please check" @@ -58,7 +62,7 @@ check_docker_valid() { } storage=${1:-/var/lib/docker} -mkdir -p $storage +mkdir -p "$storage" if ! utils_command_exists docker; then lsb_dist=$(get_distribution) lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')" @@ -101,8 +105,6 @@ if ! utils_command_exists docker; then systemctl enable docker.service systemctl restart docker.service cp "${scripts_path}"/../etc/daemon.json /etc/docker - mkdir -p /root/.docker/ - cp "${scripts_path}"/../etc/docker-cli-config.json /root/.docker/config.json if [[ -n $1 && -n $2 ]]; then sed -i "s/sea.hub:5000/$2:$3/g" /etc/docker/daemon.json fi diff --git a/context/rootfs/scripts/init-kube.sh b/context/rootfs/scripts/init-kube.sh index 5e9c2d1..82c7836 100644 --- a/context/rootfs/scripts/init-kube.sh +++ b/context/rootfs/scripts/init-kube.sh @@ -1,6 +1,10 @@ #!/bin/bash -scripts_path=$(cd `dirname $0`; pwd) +# shellcheck disable=SC2046 +# shellcheck disable=SC2164 +# shellcheck disable=SC2006 +# shellcheck disable=SC1091 +scripts_path=$(cd `dirname "$0"`; pwd) source "${scripts_path}"/utils.sh set -x @@ -56,4 +60,4 @@ copy_kubelet_service systemctl enable kubelet # nvidia-docker.sh need set kubelet labels, it should be run after kubelet -bash ${scripts_path}/nvidia-docker.sh || exit 1 \ No newline at end of file +bash "${scripts_path}"/nvidia-docker.sh || exit 1 \ No newline at end of file diff --git a/context/rootfs/scripts/init-registry.sh b/context/rootfs/scripts/init-registry.sh index a9918a6..a726ca1 100644 --- a/context/rootfs/scripts/init-registry.sh +++ b/context/rootfs/scripts/init-registry.sh @@ -1,23 +1,37 @@ #!/bin/bash +# Copyright © 2021 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. set -e set -x # prepare registry storage as directory +# shellcheck disable=SC2046 cd $(dirname "$0") +# shellcheck disable=SC2034 REGISTRY_PORT=${1-5000} VOLUME=${2-/var/lib/registry} -REGISTRY_DOMAIN=${3-sea.hub} container=sealer-registry rootfs=$(dirname "$(pwd)") config="$rootfs/etc/registry_config.yml" htpasswd="$rootfs/etc/registry_htpasswd" -certs_dir="$rootfs/certs" image_dir="$rootfs/images" mkdir -p "$VOLUME" || true +# shellcheck disable=SC2106 startRegistry() { n=1 while (( n <= 3 )) @@ -66,17 +80,17 @@ fi regArgs="-d --restart=always \ --net=host \ --name $container \ --v $certs_dir:/certs \ -v $VOLUME:/var/lib/registry \ --e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/$REGISTRY_DOMAIN.crt \ --e REGISTRY_HTTP_TLS_KEY=/certs/$REGISTRY_DOMAIN.key" +-e REGISTRY_HTTP_DEBUG_ADDR=0.0.0.0:5001 \ +-e REGISTRY_HTTP_DEBUG_PROMETHEUS_ENABLED=true" +# shellcheck disable=SC2086 if [ -f $config ]; then sed -i "s/5000/$1/g" $config regArgs="$regArgs \ -v $config:/etc/docker/registry/config.yml" fi - +# shellcheck disable=SC2086 if [ -f $htpasswd ]; then docker run $regArgs \ -v $htpasswd:/htpasswd \ diff --git a/context/rootfs/scripts/init.sh b/context/rootfs/scripts/init.sh index 0fc4c1d..79a6ac8 100644 --- a/context/rootfs/scripts/init.sh +++ b/context/rootfs/scripts/init.sh @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# shellcheck disable=SC2181 STORAGE=${1:-/var/lib/docker} REGISTRY_DOMAIN=${2-sea.hub} REGISTRY_PORT=${3-5000} @@ -20,8 +21,13 @@ REGISTRY_PORT=${3-5000} # Install docker chmod a+x docker.sh #./docker.sh /var/docker/lib sealer.hub 5001 -bash docker.sh ${STORAGE} ${REGISTRY_DOMAIN} $REGISTRY_PORT +bash docker.sh "${STORAGE}" "${REGISTRY_DOMAIN}" "$REGISTRY_PORT" +if [ $? -ne 0 ]; then + exit 1 +fi chmod a+x init-kube.sh - bash init-kube.sh +if [ $? -ne 0 ]; then + exit 1 +fi diff --git a/context/rootfs/scripts/nvidia-docker.sh b/context/rootfs/scripts/nvidia-docker.sh index b1be5ee..8c38792 100644 --- a/context/rootfs/scripts/nvidia-docker.sh +++ b/context/rootfs/scripts/nvidia-docker.sh @@ -1,6 +1,14 @@ #!/bin/bash -scripts_path=$(cd `dirname $0`; pwd) +# shellcheck disable=SC2046 +# shellcheck disable=SC2164 +# shellcheck disable=SC2092 +# shellcheck disable=SC1102 +# shellcheck disable=SC2006 +# shellcheck disable=SC2005 +# shellcheck disable=SC2181 +# shellcheck disable=SC1091 +scripts_path=$(cd `dirname "$0"`; pwd) source "${scripts_path}"/utils.sh set -x @@ -46,7 +54,7 @@ public::nvidia::enable_gpu_device_plugin() { } kube::nvidia::detect_gpu(){ - tar -xvf ${scripts_path}/../tgz/nvidia.tgz -C ${scripts_path}/../rpm/ + tar -xvf "${scripts_path}"/../tgz/nvidia.tgz -C "${scripts_path}"/../rpm/ kube::nvidia::setup_lspci lspci | grep -i nvidia > /dev/null 2>&1 if [[ "$?" == "0" ]]; then @@ -59,7 +67,7 @@ kube::nvidia::setup_lspci(){ return fi utils_info "lspci command not exist, install it" - rpm -ivh --force --nodeps ${RPM_DIR}/pciutils*.rpm + rpm -ivh --force --nodeps "${RPM_DIR}"/pciutils*.rpm if [[ "$?" != "0" ]]; then panic "failed to install pciutils via command (rpm -ivh --force --nodeps ${RPM_DIR}/pciutils*.rpm) in dir ${PWD}, please run it for debug" fi @@ -76,12 +84,13 @@ public::nvidia::install_nvidia_driver(){ public::nvidia::install_nvidia_docker2(){ sleep 3 if `which nvidia-container-runtime > /dev/null 2>&1` && [ $(echo $((docker info | grep nvidia) | wc -l)) -gt 1 ] ; then - utils_info 'nvidia-container-runtime is already insatlled' + utils_info 'nvidia-container-runtime is already installed' return fi # 1. Install nvidia-container-runtime - if ! output=$(rpm -ivh --force --nodeps `ls ${RPM_DIR}/*.rpm` 2>&1);then + # shellcheck disable=SC2046 + if ! output=$(rpm -ivh --force --nodeps `ls "${RPM_DIR}"/*.rpm` 2>&1);then panic "failed to install rpm, output:${output}, maybe your rpm db was broken, please see https://cloudlinux.zendesk.com/hc/en-us/articles/115004075294-Fix-rpmdb-Thread-died-in-Berkeley-DB-library for help" fi @@ -108,9 +117,9 @@ public::nvidia::install_nvidia_docker2(){ # deploy nvidia plugin in static pod public::nvidia::deploy_static_pod() { mkdir -p /etc/kubernetes/manifests - cp -f ${scripts_path}/../statics/nvidia-device-plugin.yml /etc/kubernetes/manifests/nvidia-device-plugin.yml + cp -f "${scripts_path}"/../statics/nvidia-device-plugin.yml /etc/kubernetes/manifests/nvidia-device-plugin.yml - utils_info "nvidia-device-plugin yaml succefully deployed ..." + utils_info "nvidia-device-plugin yaml successfully deployed ..." } public::nvidia::enable_gpu_capability diff --git a/context/rootfs/scripts/uninstall-docker.sh b/context/rootfs/scripts/uninstall-docker.sh index 76bb408..5fc6b7a 100644 --- a/context/rootfs/scripts/uninstall-docker.sh +++ b/context/rootfs/scripts/uninstall-docker.sh @@ -4,11 +4,10 @@ systemctl stop docker ip link delete docker0 type bridge || true rm -rf /lib/systemd/system/docker.service rm -rf /usr/lib/systemd/system/docker.service -rm -rf /etc/docker/daemon.json +rm -rf /etc/docker systemctl daemon-reload rm -f /usr/bin/conntrack -rm -f /usr/bin/kubelet-pre-start.sh rm -f /usr/bin/containerd rm -f /usr/bin/containerd-shim rm -f /usr/bin/containerd-shim-runc-v2 @@ -18,19 +17,10 @@ rm -f /usr/bin/docker rm -f /usr/bin/docker-init rm -f /usr/bin/docker-proxy rm -f /usr/bin/dockerd -rm -f /usr/bin/kubeadm -rm -f /usr/bin/kubectl -rm -f /usr/bin/kubelet rm -f /usr/bin/rootlesskit rm -f /usr/bin/rootlesskit-docker-proxy rm -f /usr/bin/runc rm -f /usr/bin/vpnkit rm -f /usr/bin/containerd-rootless-setuptool.sh rm -f /usr/bin/containerd-rootless.sh -rm -f /usr/bin/nerdctl - -rm -f /etc/sysctl.d/k8s.conf -rm -f /etc/systemd/system/kubelet.service -rm -rf /etc/systemd/system/kubelet.service.d -rm -rf /var/lib/kubelet/ -rm -f /var/lib/kubelet/config.yaml \ No newline at end of file +rm -f /usr/bin/nerdctl \ No newline at end of file diff --git a/context/rootfs/scripts/utils.sh b/context/rootfs/scripts/utils.sh index 1ee33ac..3913279 100644 --- a/context/rootfs/scripts/utils.sh +++ b/context/rootfs/scripts/utils.sh @@ -1,5 +1,9 @@ #!/bin/bash +# shellcheck disable=SC2145 +# shellcheck disable=SC2155 +# shellcheck disable=SC2126 +# shellcheck disable=SC2002 utils_version_ge() { test "$(echo "$@" | tr ' ' '\n' | sort -rV | head -n 1)" == "$1" } @@ -59,7 +63,7 @@ utils_os_env() { elif [ "$anolis" == 1 ];then export OS="Anolis" else - panic "unkown os... exit" + panic "unknown os... exit" fi case "$OS" in