build.img

##
##      Copyright (C) 2012 - 2019 XBian
##
##  Find us at: http://www.xbian.org http://github.com/xbianonpi/xbian
##
##  This Program is free software; you can redistribute it and/or modify
##  it under the terms of the GNU General Public License as published by
##  the Free Software Foundation; either version 2, or (at your option)
##  any later version.
##
##  This Program is distributed in the hope that it will be useful,
##  but WITHOUT ANY WARRANTY; without even the implied warranty of
##  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
##  GNU General Public License for more details.
##
##  You should have received a copy of the GNU General Public License
##  along with XBMC; see the file COPYING.  If not, see
##  <http://www.gnu.org/licenses/>.
##
##

use_debug=no

if [ -e $XBIANROOT/common.functions ]; then
    . $XBIANROOT/common.functions
else
    echo "Wrong setup. Open folder containing xbianonpi/xbian clone and rerun \"./run.me prepare\""
    exit 500
fi

do_wipe()
{
    dir="$1"
    [ ! -e "$dir/working/rootfs" ] || do_run rm -r "$dir/working/rootfs"
    rm -f "$dir/working/*"
}

do_img()
{
    dir=$(readlink -f "$1")
    do_wipe "$dir"
    if [ -f /etc/centos-release ]; then
        yum -y install $config_yum_base 2>/dev/null 1>/dev/null
    else
        [ "$(LC_ALL=C apt-get --version | awk '/^apt/{ print $2 }')" \< "1.1" ] && FORCE="--yes --force-yes" || FORCE="--yes --allow-change-held-packages"
        apt-get install $FORCE -o Acquire::Languages=none -o APT::Install-Suggests=0 -o APT::Install-Recommends=0 $config_deb_base >/dev/null
    fi

    mkdir -p "$dir/working"
    cd "$dir/working"

    rm -fr $IMGNAME; sync; touch $IMGNAME
    #convert size to byte size
    config_img_size=$(echo $config_img_size | sed "s%G% \*1024M%;s%M% \*1024K%;s%K% \*1024%;" | bc)

    [ $config_fstype_root = btrfs ] && [ $config_img_size -lt 1200000000 ] && config_img_size=1200000000
    truncate -s $config_img_size $IMGNAME

    if ! [ -f /etc/centos-release ]; then
        modprobe -q loop
    fi
    parted -s $IMGNAME mklabel msdos

    if [ -z "$config_offset_root" ]; then
        config_offset_root=206848
    fi
    if [ -z "$config_fstype_boot" ]; then
        echo "$config_offset_root,+,83,," | sfdisk -u S -N1 -q $IMGNAME > /dev/null 2>&1
    else
        echo "2048,$((config_offset_root-2048)),b,*," | sfdisk -u S -N1 -q $IMGNAME > /dev/null 2>&1
        echo "$config_offset_root,+,83,," | sfdisk -u S -N2 -q --force $IMGNAME > /dev/null 2>&1
    fi

    [ -d /run ] || mkdir /run

    loopd=$(kpartx -s -l $IMGNAME -av | tail -1 | awk '{print $3}'); loopd=${loopd%%p?}; echo loopd=$loopd > /run/$IMGNAME.loopd
    #### success creating loop device, export already here to make $loopd variable visible for cleanup trap functions ####
    export loopd

    rootpart=2
    case $config_fstype_boot in
        ext2)
            mkfs.ext2 -L xbianboot /dev/mapper/${loopd}p1 2>/dev/null 1>&2
            ;;
        vfat|fat|msdos)
            mkfs.msdos -F 16 -n xbianboot /dev/mapper/${loopd}p1 2>/dev/null 1>&2
            ;;
        '')
            rootpart=1
            ;;
    esac

    case $config_fstype_root in
        ext4|ext2|ext3)
            mkfs.$config_fstype_root -L $config_rootfs_label /dev/mapper/${loopd}p${rootpart} 2>/dev/null 1>&2
            ;;
        btrfs)
            if mkfs.btrfs --help 2>&1 | grep -q '\-O'; then
                mkfs.btrfs -O skinny-metadata -m single -d single -n 16384 -L $config_rootfs_label /dev/mapper/${loopd}p${rootpart} 2>&1
            else
                mkfs.btrfs -m single -d single -n 16384 -L $config_rootfs_label /dev/mapper/${loopd}p${rootpart} 2>&1
            fi
            ;;
        zfs)
            modprobe -q zfs
            zpool create -f -o ashift=12 -O com.sun:auto-snapshot=false -O acltype=posixacl -O xattr=sa -O atime=off -O canmount=off -O compression=lz4 \
                -O normalization=formD -O mountpoint=/ -R $(readlink -m ./rootfs) $config_rootfs_label /dev/mapper/${loopd}p${rootpart} 2>&1
            zfs create -o com.sun:auto-snapshot=true -o canmount=noauto -o mountpoint=/ ${config_rootfs_label}/root 2>&1
            ;;
        *)
            ;;
    esac

#    if [ -n "$config_build_targets" ]; then
#        for t in $config_build_targets; do
#            ( rexp ../env; do_run $config_build_env $config_build_make $t; )
#        done
#    else
#        ( rexp ../env; do_run $config_build_env $config_build_make; )
#    fi

}

do_debootstrap()
{
    dir=$(readlink -f "$1")
    cd "$dir/working"

    if [ -e /run/$IMGNAME.loopd ]; then
        . /run/$IMGNAME.loopd
        export loopd
    else
        do_exit "missing loop/img/other error" 100
    fi

    mkdir -p ./rootfs
    rootpart=2
    [ -n "$config_fstype_boot" ] || rootpart=1

    case $config_fstype_root in
        btrfs)
            mount -t $config_fstype_root $config_fsoptions_root /dev/mapper/${loopd}p${rootpart} ./rootfs
            btrfs sub create ./rootfs/root
            btrfs sub create ./rootfs/root/@
            btrfs sub create ./rootfs/modules
            btrfs sub create ./rootfs/modules/@
            btrfs sub create ./rootfs/home
            btrfs sub create ./rootfs/home/@
            umount ./rootfs
            ;;
        zfs)
            zfs mount ${config_rootfs_label}/root
            zpool set bootfs=${config_rootfs_label}/root ${config_rootfs_label} 2>&1
            zfs create -o com.sun:auto-snapshot=true -o mountpoint=/home ${config_rootfs_label}/home 2>&1
            ;;
        *)
            ;;
    esac

    [ -z "$config_fsoptions_root" ] && [ "$config_fstype_root" = btrfs ] && config_fsoptions_root='-o '

    case $config_fstype_root in
        btrfs)
            mount -t $config_fstype_root $config_fsoptions_root,subvol=root/@ /dev/mapper/${loopd}p${rootpart} ./rootfs
            mkdir ./rootfs/home; mkdir -p ./rootfs/lib/modules
            mount -t $config_fstype_root $config_fsoptions_root,subvol=home/@ /dev/mapper/${loopd}p${rootpart} ./rootfs/home
            ;;
        zfs)
            ;;
        *)
            mount -t $config_fstype_root $config_fsoptions_root /dev/mapper/${loopd}p${rootpart} ./rootfs
            ;;
    esac

    mkdir -p ./rootfs/tmp/tmp && mount -o bind /tmp ./rootfs/tmp/tmp; mkdir -p ./rootfs/var/cache/apt/archives && mount -o bind /tmp ./rootfs/var/cache/apt/archives

    find ./rootfs/tmp -iname \*.deb | xargs rm -f

    mkdir -p ./rootfs/boot
    if ! [ -z $config_fstype_boot ]; then
        mount -t $config_fstype_boot /dev/mapper/${loopd}p1 ./rootfs/boot
    fi

    [ -n "$config_distro_variant" ] && export config_distro_variant="--variant=$config_distro_variant"
    [ -n "$config_include_deb" ] && export config_include_deb="--include=$config_include_deb"
    [ -n "$config_exclude_deb" ] && export config_exclude_deb="--exclude=$config_exclude_deb"

    debootstrap $config_exclude_deb $config_distro_variant $config_include_deb --no-check-gpg --arch $config_distro_arch --foreign $config_distro_name ./rootfs $config_distro_url

    mkdir -p ./rootfs/usr/bin
    if [ -f /etc/centos-release ]; then
        [ -d qemu ] && rm -rf qemu
        mkdir qemu; cd qemu
        wget -q http://security.ubuntu.com/ubuntu/pool/universe/q/qemu/qemu-user-static_2.0.0+dfsg-2ubuntu1.9_amd64.deb
        ar -x *.deb
        tar -xf data.tar.*
        cp usr/bin/qemu-arm-static ../rootfs/usr/bin/qemu-arm-static
        cd ..
        rm -rf qemu
    else
        case "$config_distro_arch" in
            arm64) QEMU="/usr/bin/qemu-aarch64-static /usr/bin/qemu-arm-static" ;;
            *)     QEMU="/usr/bin/qemu-arm-static" ;;
        esac
        cp $QEMU ./rootfs/usr/bin
    fi

    #### hack to remove packages which raspbian considers essential and doesn't want them remove even when excluded and 
    #### another package with it's function is going to be installed (eg sysvinit vs. upstart)
    #### Not needed anymore. Under special condition (see workarounds below) it breaks building image
    #    sed -i 's/sysvinit //' ./rootfs/debootstrap/required; sed -i '/sysvinit /d' ./rootfs/debootstrap/debpaths
    #    sed -i 's/sysvinit-core//' ./rootfs/debootstrap/required; sed -i '/sysvinit-core/d' ./rootfs/debootstrap/debpaths
    #    sed -i 's/systemd-sysv//' ./rootfs/debootstrap/required; sed -i '/systemd-sysv/d' ./rootfs/debootstrap/debpaths

    #### On CentOS the binfmt_misc entry for qemu-arm points
    #### to /usr/bin/qemu-arm instead of /usr/bin/qemu-arm-static
    #### This works cross-platform
    [ -f /proc/sys/fs/binfmt_misc/qemu-arm ] && echo "-1" >/proc/sys/fs/binfmt_misc/qemu-arm 2>/dev/null 1>/dev/null || :
    if ! grep -q /usr/bin/qemu-arm-static /proc/sys/fs/binfmt_misc/qemu-arm; then
        echo ':qemu-arm:M::\x7fELF\x01\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x28\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/qemu-arm-static:' > /proc/sys/fs/binfmt_misc/register || : 
    fi

    chroot ./rootfs debootstrap/debootstrap --second-stage || (
        ### As it was written (for Debian Stretch) we have to do this ugly workaround
        chroot ./rootfs debootstrap/debootstrap --second-stage
    )

    case $config_fstype_root in
        btrfs)
            mount -t $config_fstype_root $config_fsoptions_root,subvol=modules/@ /dev/mapper/${loopd}p${rootpart} ./rootfs/lib/modules
            ;;
        zfs)
            zfs create -o com.sun:auto-snapshot=true -o mountpoint=/lib/modules ${config_rootfs_label}/modules 2>&1
            ;;
    esac

    echo "deb $config_distro_url $config_distro_name main" >> ./rootfs/etc/apt/sources.list
}

do_xbian()
{
    dir=$(readlink -f "$1")

    do_hooks pre-xbiandebs.d

    cd "$dir/working"

    chroot ./rootfs apt-get update

    export DEBIAN_FRONTEND=noninteractive

    ### Since apt-get version > 1.0.* --force-yes option is depreciated we have to use --allow options and gnupg is required for apt-key
    if [ "$(LC_ALL=C chroot ./rootfs apt-get --version | awk '/^apt/{ print $2 }')" \< "1.1" ]; then
        FORCE="--yes --force-yes"
    else
        FORCE="--yes --allow-downgrades --allow-remove-essential --allow-change-held-packages"
        LC_ALL=C chroot ./rootfs apt-get install $FORCE gnupg
    fi

    chroot ./rootfs wget -r -l1 -q --no-parent $config_xbianrepository -P /tmp
    repofile="$(chroot ./rootfs find /tmp -name 'xbian-package-repo*.deb' 2>/dev/null | sort -r | grep -m1 xbian-package-repo)"
    chroot ./rootfs dpkg -i --force-confdef --force-confnew $repofile

    chroot ./rootfs sed -i 's/### //g' /etc/apt/sources.list.d/xbian.list; cp ./rootfs/etc/apt/sources.list.d/xbian.list /tmp/xbian.list

    [ "$config_allow_staging_devel" = yes ] || ( IFS=$'\n'; for l in $(grep devel /tmp/xbian.list); do sed -i "s%$l%###\ $l%" /tmp/xbian.list; done; )
    [ "$config_allow_staging" = yes ] || ( IFS=$'\n'; for l in $(grep staging /tmp/xbian.list); do sed -i "s%$l%###\ $l%" /tmp/xbian.list; done; )
    cp /tmp/xbian.list ./rootfs/etc/apt/sources.list.d/xbian.list

    LC_ALL=C chroot ./rootfs apt-get update 2>/dev/null || ( \
        ### As it was written (for Debian Stretch) apt-get does not work with method mirror://
        case "$config_distro_name" in
            stretch)
                LC_ALL=C chroot ./rootfs sed -i -e 's%mirror://apt.xbian.org/mirror.txt%http://apt.xbian.org/%g' /etc/apt/sources.list.d/xbian.list
                ;;
            *)
                echo "FIXME: apt-get update fails"
                exit 1
                ;;
        esac
        LC_ALL=C chroot ./rootfs apt-get update
    )

    ### Since Debian Buster wie have to make sure that upstart is already installed before
    ### replacing some services with a dummy
    LC_ALL=C chroot ./rootfs apt-get install $FORCE upstart

    LC_ALL=C chroot ./rootfs apt-get upgrade $FORCE

    ### Install fake services. Since Debian Buster modifying PATH variable does not longer work
    ### because apt sets PATH variable to /usr/sbin:/usr/bin:/sbin:/bin
    fpl=''
    for f in initctl invoke-rc.d restart start stop start-stop-daemon service reboot udevadm; do
        PATH=/usr/sbin:/usr/bin:/sbin:/bin fp=$(chroot ./rootfs which $f)
        if [ -n "$fp" ]; then
            chroot ./rootfs mv $fp $fp.orig && chroot ./rootfs ln -s /bin/true $fp
            fpl="$fpl $fp"
        fi
    done

    ### let's settle down apt consistency after debootstrap (without initial install -f will not install
    ### any packages in next loop even with correct dependencies)
    LC_ALL=C chroot ./rootfs apt-get install $FORCE -f
    for i in $(seq 1 3); do
        LC_ALL=C chroot ./rootfs apt-get install $FORCE -o Acquire::Languages=none -o APT::Install-Suggests=0 -o APT::Install-Recommends=0 -o Dir::Cache=/tmp/tmp -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" $config_xbian_deb $config_xbian_base_deb 2>&1 | tee /tmp/apt-get.out | grep --line-buffer -v 'Can not write log, openpty()\|/proc/mounts\|/com/ubuntu/upstart\|/var/run/dbus/system_bus_socket'
        ! grep -q "E: Unable to fetch some" /tmp/apt-get.out && break
    done

    ### make pin settings effective
    LC_ALL=C chroot ./rootfs apt-get upgrade $FORCE

    for fp in $fpl; do
        chroot ./rootfs mv $fp.orig $fp
    done

    ### when all packages has been installed only stable repo is allowed again for stable release
    if ! echo $IMGNAME | grep -qE "devel|staging|test"; then
        ( IFS=$'\n'; for l in $(grep devel ./rootfs/etc/apt/sources.list.d/xbian.list); do echo $l | grep -qv '^### ' && sed -i "s%$l%###\ $l%" ./rootfs/etc/apt/sources.list.d/xbian.list; done; ) || :
        ( IFS=$'\n'; for l in $(grep staging ./rootfs/etc/apt/sources.list.d/xbian.list); do echo $l | grep -qv '^### ' && sed -i "s%$l%###\ $l%" ./rootfs/etc/apt/sources.list.d/xbian.list; done; ) || :
    fi
}

clean_tmp()
{
    set +e

    umfail=1
    while [ $umfail -eq 1 ]; do
        umfail=0
        for d in rootfs/var/cache/apt/archives rootfs/tmp/tmp; do
            if mountpoint -q $XBIANARCHDIR/working/$d; then
                umount $XBIANARCHDIR/working/$d || umfail=1
            fi
        done
        [ $umfail -eq 0 ] || sleep 5
    done

    set -e
    return 0
}

clean_all()
{
    set +e
    [ -n "$loopd" ] || loopd=noexistent

    for d in $(mount | grep "mapper/$loopd\|$dir/working/rootfs" | awk '$5 ~ /btrfs/ {print $3}' 2>/dev/null); do
        mountpoint -q $d && btrfs fi sync $d >/dev/null 2>&1
    done

    umfail=1
    while [ $umfail -eq 1 ]; do
        umfail=0
        for d in $(mount | grep "mapper/$loopd\|$dir/working/rootfs" | awk '{print $3}' | sort -r); do
            if [ $(mountpoint -q $d; echo $?) -eq 0 ] && [ "$d" != "/" ]; then
                umount $d || umfail=1
            fi
        done
        [ $umfail -eq 0 ] || sleep 5
    done

    [ "$config_fstype_root" != zfs ] || zpool export ${config_rootfs_label} 2>/dev/null

    set -e
    return 0
}

clean_loopd()
{
    set +e

    cd $XBIANARCHDIR/working
    kpartx -dv $IMGNAME >/dev/null 2>&1
    rm -f /run/$IMGNAME.loopd

    set -e
    return 0
}

do_finish()
{
    dir=$(readlink -f "$1")

    if [ -e /run/$IMGNAME.loopd ]; then
        . /run/$IMGNAME.loopd
        export loopd
    fi

    cd "$dir/../.."

    do_hooks pre-finish.d

    clean_tmp

    chroot "$dir/working/rootfs" /usr/local/sbin/all-clean >/dev/null 2>&1 || :

    cd "$dir/working"

    rm -f rootfs/usr/bin/qemu-*-static

    df -h | grep /working/

    clean_all

    clean_loopd
}

do_post()
{
    export config_img_size=$(echo $config_img_size | sed "s%G% \*1024M%;s%M% \*1024K%;s%K% \*1024%;" | bc)
    dir=$(readlink -f "$1")
    mkdir -p "$dir/working/rootfs"

    do_hooks post-img.d

    [ -e "$dir/working/newimgname" ] && { . "$dir/working/newimgname"; rm -f "$dir/working/newimgname"; }

    if [ -n "$config_img_compress" ]; then
        echo "compressing $IMGNAME. this may take a while."
        do_run $config_img_compress "$dir/working/$IMGNAME"
        (
            IFS='
'
            for f in $(ls -1 $dir/working/$IMGNAME*); do
                echo "building sha512 for $(basename $f)"
                do_run sha512sum "$f" | awk '{print $1}' > $(echo "$f" | sed "s/.gz/.sha512/g")
            done
        )
    fi
    mv "$dir"/working/$IMGNAME* "$dir"
}

# ----------------------

[ $(id -u) -ne 0 ] && do_exit "for img creation please run again with root privileges"

GETOPT=$(getopt \
        --longoptions=action:,arch:,config_source_refresh:,help,list,debug \
        --options=a:m:hld \
        -- "$@" ) \
        || exit 128

export XBIANPKGDIR="$(pwd)"
if [ -e $XBIANPKGDIR/build.~lock ]; then
    echo "another process running... or stale lock exists at $XBIANPKGDIR/build.~lock"
    exit 100
fi

for dir in $(find build/ -mindepth 1 -maxdepth 1 -type d | sort); do
    export XBIANARCHDIR=$(readlink -f "$dir")

    unset $(env | grep config_ | awk -F'=' '{print $1}')
    result=0

    [ -e build/config ] && rexp build/config
    [ -e "$dir/config" ] && rexp "$dir/config"

    eval set -- ${GETOPT}

    while [ "$#" -gt '0' ]; do
        case "$1" in
            (-l|--list)
                printf "Local Name: %s\n" "${dir#*build/}"
                continue 2
                ;;
            (--action|-a)
                opt_action=$2
                shift 2
                ;;
            (--arch|-m)
                opt_arch=$2
                shift 2
                ;;
            (--config_*)
                param=$1
                param="${param##--}=$2"
                do_run export $param
                shift 2
                ;;
            (--help|-h)
                do_show_help
                exit 0
                ;;
            (--debug|-d)
                use_debug=yes
                shift
                ;;
            *)
                shift 1
                ;;
        esac
    done

    [ $use_debug = no ] || set -x

    [ -n "$opt_arch" -a "$opt_arch" != $(basename "$dir") ] && continue
    [ -z "$opt_arch" ] && opt_arch=$(basename "$dir")

    export config_platform=${opt_arch%%-*}
    export config_platform_branch=$opt_arch
    export config_branch=${opt_arch##*-}

    case ${config_distro_arch} in
        arm64) export config_arch="${config_distro_arch}_" ;;
        *)     export config_arch='' ;;
    esac
    export IMGNAME=${config_img_name}_$(date +%Y.%m.%d)_${config_arch}$(basename $dir | sed "s/-64//g").img

    touch $XBIANPKGDIR/build.~lock
    trap "kill_gitbuilder; sync; clean_tmp; clean_all; clean_loopd; rm -f $XBIANPKGDIR/build.~lock" EXIT TERM

    git_preloader

    case $opt_action in
        img)
            ( do_img "$dir" )
            ;;
        debootstrap)
            ( do_debootstrap "$dir" )
            ;;
        xbian)
            ( do_xbian "$dir" )
            ;;
        finish)
            ( do_finish "$dir" )
            ;;
        post)
            ( do_post "$dir" )
            ;;
        ""|build)
            dr=$(pwd)
            for act in do_img do_debootstrap do_xbian do_finish do_post; do
                ( $act "$dir" )
            done
            ;;
        *)
            ;;
    esac
done

printf "\n-------------------------------\nFINISHED OK\n-------------------------------\n"