diff --git a/.mergify.yml b/.mergify.yml index 936482e97..8cdf97fb8 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -6,13 +6,11 @@ queue_rules: - status-success="spelling" - status-success="python format" - status-success="python lint" - - status-success="bionic - gcc-8,distcheck" - - status-success="bionic - clang-6.0" - - status-success="bionic - test-install" + - status-success="bookworm - gcc-8,distcheck" + - status-success="bookworm - clang-6.0" + - status-success="bookworm - test-install" - status-success="focal" - - status-success="el7" - status-success="el8" - - status-success="fedora33" - status-success="fedora34" - status-success="coverage" - label="merge-when-passing" diff --git a/src/test/docker-deploy.sh b/src/test/docker-deploy.sh index c76e60db6..d1cd10ddf 100755 --- a/src/test/docker-deploy.sh +++ b/src/test/docker-deploy.sh @@ -20,8 +20,8 @@ echo $DOCKER_PASSWORD | docker login -u "$DOCKER_USERNAME" --password-stdin log "docker push ${DOCKER_TAG}" docker push ${DOCKER_TAG} -# If this is the bionic build, then also tag without image name: -if echo "$DOCKER_TAG" | grep -q "bionic"; then +# If this is the jammy build, then also tag without image name: +if echo "$DOCKER_TAG" | grep -q "jammy"; then t="${DOCKER_REPO}:${GITHUB_TAG:-latest}" log "docker push ${t}" docker tag "$DOCKER_TAG" ${t} && docker push ${t} diff --git a/src/test/docker/bookworm/Dockerfile b/src/test/docker/bookworm/Dockerfile new file mode 100644 index 000000000..3f8fefe98 --- /dev/null +++ b/src/test/docker/bookworm/Dockerfile @@ -0,0 +1,28 @@ +FROM fluxrm/flux-core:bookworm + +ARG USER=flux +ARG UID=1000 + +# Install extra buildrequires for flux-sched: +RUN sudo apt-get update +RUN sudo apt-get -qq install -y --no-install-recommends \ + libboost-graph-dev \ + libboost-system-dev \ + libboost-filesystem-dev \ + libboost-regex-dev \ + python3-yaml \ + libyaml-cpp-dev \ + libedit-dev + +# Add configured user to image with sudo access: +# +RUN \ + if test "$USER" != "flux"; then \ + sudo groupadd -g $UID $USER \ + && sudo useradd -g $USER -u $UID -d /home/$USER -m $USER \ + && sudo sh -c "printf \"$USER ALL= NOPASSWD: ALL\\n\" >> /etc/sudoers" \ + && sudo adduser $USER sudo ; \ + fi + +USER $USER +WORKDIR /home/$USER diff --git a/src/test/docker/docker-run-checks.sh b/src/test/docker/docker-run-checks.sh index 2582726d8..4f83a353c 100755 --- a/src/test/docker/docker-run-checks.sh +++ b/src/test/docker/docker-run-checks.sh @@ -10,12 +10,13 @@ PROJECT=flux-sched BASE_DOCKER_REPO=fluxrm/flux-core -IMAGE=bionic +WORKDIR=/usr/src +IMAGE=jammy JOBS=2 MOUNT_HOME_ARGS="--volume=$HOME:/home/$USER -e HOME" if test "$PROJECT" = "flux-core"; then - FLUX_SECURITY_VERSION=0.4.0 + FLUX_SECURITY_VERSION=0.9.0 POISON=t fi @@ -24,8 +25,8 @@ declare -r prog=${0##*/} die() { echo -e "$prog: $@"; exit 1; } # -declare -r long_opts="help,quiet,interactive,image:,flux-security-version:,jobs:,no-cache,no-home,distcheck,tag:,build-directory:,install-only,no-poison,recheck,inception" -declare -r short_opts="hqIdi:S:j:t:D:Pr" +declare -r long_opts="help,quiet,interactive,image:,flux-security-version:,jobs:,no-cache,no-home,distcheck,tag:,build-directory:,install-only,no-poison,recheck,unit-test-only,inception,platform:,workdir:,system" +declare -r short_opts="hqIdi:S:j:t:D:Prup:" declare -r usage=" Usage: $prog [OPTIONS] -- [CONFIGURE_ARGS...]\n\ Build docker image for CI builds, then run tests inside the new\n\ @@ -39,15 +40,19 @@ Options:\n\ --no-home Skip mounting the host home directory\n\ --install-only Skip make check, only make install\n\ --inception Run tests as flux jobs\n\ + --system Run under system instance\n\ -q, --quiet Add --quiet to docker-build\n\ -t, --tag=TAG If checks succeed, tag image as NAME\n\ -i, --image=NAME Use base docker image NAME (default=$IMAGE)\n\ + -p, --platform=NAME Run on alternate platform (if supported)\n\ -S, --flux-security-version=N Install flux-security vers N (default=$FLUX_SECURITY_VERSION)\n -j, --jobs=N Value for make -j (default=$JOBS)\n -d, --distcheck Run 'make distcheck' instead of 'make check'\n\ -r, --recheck Run 'make recheck' after failure\n\ + -u, --unit-test-only Only run unit tests\n\ -P, --no-poison Do not install poison libflux and flux(1)\n\ -D, --build-directory=DIRNAME Name of a subdir to build in, will be made\n\ + --workdir=PATH Use PATH as working directory for build\n\ -I, --interactive Instead of running ci build, run docker\n\ image with interactive shell.\n\ " @@ -55,10 +60,10 @@ Options:\n\ # check if running in OSX if [[ "$(uname)" == "Darwin" ]]; then # BSD getopt - GETOPTS=`/usr/bin/getopt $short_opts -- $*` + GETOPTS=`getopt $short_opts -- $*` else # GNU getopt - GETOPTS=`/usr/bin/getopt -u -o $short_opts -l $long_opts -n $prog -- $@` + GETOPTS=`getopt -u -o $short_opts -l $long_opts -n $prog -- $@` if [[ $? != 0 ]]; then die "$usage" fi @@ -69,16 +74,21 @@ while true; do -h|--help) echo -ne "$usage"; exit 0 ;; -q|--quiet) QUIET="--quiet"; shift ;; -i|--image) IMAGE="$2"; shift 2 ;; + -p|--platform) PLATFORM="--platform=$2"; shift 2 ;; -S|--flux-security-version) FLUX_SECURITY_VERSION="$2"; shift 2 ;; -j|--jobs) JOBS="$2"; shift 2 ;; -I|--interactive) INTERACTIVE="/bin/bash"; shift ;; -d|--distcheck) DISTCHECK=t; shift ;; -r|--recheck) RECHECK=t; shift ;; + -u|--unit-test-only) UNIT_TEST_ONLY=t; shift ;; -D|--build-directory) BUILD_DIR="$2"; shift 2 ;; + --build-arg) BUILD_ARG=" --build-arg $2" shift 2 ;; + --workdir) WORKDIR="$2"; shift 2 ;; --no-cache) NO_CACHE="--no-cache"; shift ;; --no-home) MOUNT_HOME_ARGS=""; shift ;; --install-only) INSTALL_ONLY=t; shift ;; --inception) INCEPTION=t; shift ;; + --system) SYSTEM=t; shift ;; -P|--no-poison) POISON=0; shift ;; -t|--tag) TAG="$2"; shift 2 ;; --) shift; break; ;; @@ -90,10 +100,16 @@ TOP=$(git rev-parse --show-toplevel 2>&1) \ || die "not inside $PROJECT git repository!" which docker >/dev/null \ || die "unable to find a docker binary" +if docker buildx >/dev/null 2>&1; then + DOCKER_BUILD="docker buildx build --load" +else + DOCKER_BUILD="docker build" +fi # distcheck incompatible with some configure args if test "$DISTCHECK" = "t"; then test "$RECHECK" = "t" && die "--recheck not allowed with --distcheck" + test "$SYSTEM" = "t" && die "--system not allowed with --distcheck" for arg in "$@"; do case $arg in --sysconfdir=*|systemdsystemunitdir=*) @@ -117,7 +133,8 @@ else fi checks_group "Building image $IMAGE for user $USER $(id -u) group=$(id -g)" \ - docker build \ + ${DOCKER_BUILD} \ + ${PLATFORM} \ ${NO_CACHE} \ ${QUIET} \ --build-arg BASE_IMAGE=$IMAGE \ @@ -126,6 +143,7 @@ checks_group "Building image $IMAGE for user $USER $(id -u) group=$(id -g)" \ --build-arg UID=$(id -u) \ --build-arg GID=$(id -g) \ --build-arg FLUX_SECURITY_VERSION=$FLUX_SECURITY_VERSION \ + ${BUILD_ARG:- } \ -t ${BUILD_IMAGE} \ ${DOCKERFILE} \ || die "docker build failed" @@ -133,21 +151,25 @@ checks_group "Building image $IMAGE for user $USER $(id -u) group=$(id -g)" \ if [[ -n "$MOUNT_HOME_ARGS" ]]; then echo "mounting $HOME as /home/$USER" fi -echo "mounting $TOP as /usr/src" +echo "mounting $TOP as $WORKDIR" +export PLATFORM export PROJECT export POISON export INCEPTION export JOBS export DISTCHECK export RECHECK +export UNIT_TEST_ONLY export BUILD_DIR +export COVERAGE export chain_lint if [[ "$INSTALL_ONLY" == "t" ]]; then docker run --rm \ - --workdir=/usr/src \ - --volume=$TOP:/usr/src \ + --workdir=$WORKDIR \ + --volume=$TOP:$WORKDIR \ + ${PLATFORM} \ ${BUILD_IMAGE} \ sh -c "./autogen.sh && ./configure --prefix=/usr --sysconfdir=/etc \ @@ -156,13 +178,17 @@ if [[ "$INSTALL_ONLY" == "t" ]]; then --with-flux-security \ --enable-caliper && make clean && - make -j${JOBS}" \ - || (docker rm tmp.$$; die "docker run of 'make install' failed") + make -j${JOBS}" + RC=$? + docker rm tmp.$$ + test $RC -ne 0 && die "docker run of 'make install' failed" else docker run --rm \ - --workdir=/usr/src \ - --volume=$TOP:/usr/src \ + --workdir=$WORKDIR \ + --volume=$TOP:$WORKDIR \ + ${PLATFORM} \ $MOUNT_HOME_ARGS \ + -e PLATFORM \ -e CC \ -e CXX \ -e LDFLAGS \ @@ -176,13 +202,13 @@ else -e CPPCHECK \ -e DISTCHECK \ -e RECHECK \ + -e UNIT_TEST_ONLY \ -e chain_lint \ -e JOBS \ -e USER \ -e PROJECT \ -e CI \ -e TAP_DRIVER_QUIET \ - -e TEST_CHECK_PREREQS \ -e FLUX_TEST_TIMEOUT \ -e FLUX_TEST_SIZE_MAX \ -e PYTHON_VERSION \ @@ -208,9 +234,10 @@ if test -n "$TAG"; then # Re-run 'make install' in fresh image, otherwise we get all # the context from the build above docker run --name=tmp.$$ \ - --workdir=/usr/src/${BUILD_DIR} \ - --volume=$TOP:/usr/src \ + --workdir=${WORKDIR}/${BUILD_DIR} \ + --volume=$TOP:${WORKDIR} \ --user="root" \ + ${PLATFORM} \ ${BUILD_IMAGE} \ sh -c "make install && \ userdel $USER" \ diff --git a/src/test/docker/jammy/Dockerfile b/src/test/docker/jammy/Dockerfile new file mode 100644 index 000000000..97d18e60b --- /dev/null +++ b/src/test/docker/jammy/Dockerfile @@ -0,0 +1,28 @@ +FROM fluxrm/flux-core:jammy + +ARG USER=flux +ARG UID=1000 + +# Install extra buildrequires for flux-sched: +RUN sudo apt-get update +RUN sudo apt-get -qq install -y --no-install-recommends \ + libboost-graph-dev \ + libboost-system-dev \ + libboost-filesystem-dev \ + libboost-regex-dev \ + python3-yaml \ + libyaml-cpp-dev \ + libedit-dev + +# Add configured user to image with sudo access: +# +RUN \ + if test "$USER" != "flux"; then \ + sudo groupadd -g $UID $USER \ + && sudo useradd -g $USER -u $UID -d /home/$USER -m $USER \ + && sudo sh -c "printf \"$USER ALL= NOPASSWD: ALL\\n\" >> /etc/sudoers" \ + && sudo adduser $USER sudo ; \ + fi + +USER $USER +WORKDIR /home/$USER diff --git a/src/test/generate-matrix.py b/src/test/generate-matrix.py index 99a3ea8a3..512ab406f 100755 --- a/src/test/generate-matrix.py +++ b/src/test/generate-matrix.py @@ -45,7 +45,7 @@ def create_docker_tag(self, image, env, command): def add_build( self, name=None, - image="bionic", + image="bookworm", args=default_args, jobs=4, env=None, @@ -79,6 +79,11 @@ def add_build( command += f" -- {args}" + # TODO : remove this when the boost issue is dealt with + if env.get("CC", "gcc").find("clang") < 0: + cppflags = env.get("CPPFLAGS", "") + " -Wno-error=maybe-uninitialized" + env["CPPFLAGS"] = cppflags + self.matrix.append( { "name": name, @@ -102,49 +107,52 @@ def __str__(self): matrix = BuildMatrix() -# Ubuntu: gcc-8, distcheck +# Debian: gcc-12, distcheck matrix.add_build( - name="bionic - gcc-8,distcheck", + name="bookworm - gcc-12,distcheck", env=dict( - CC="gcc-8", - CXX="g++-8", + CC="gcc-12", + CXX="g++-12", DISTCHECK="t", ), args="--prefix=/usr", ) -# Ubuntu: coverage +# Debian: coverage matrix.add_build(name="coverage", coverage=True, jobs=2) -# Ubuntu: py3.7,clang-6.0 +# Debian: py3.7,clang-6.0 matrix.add_build( - name="bionic - clang-6.0", + name="bookworm - clang-15", env=dict( - CC="clang-6.0", - CXX="clang++-6.0", + CC="clang-15", + CXX="clang++-15", + CFLAGS="-O2 -gdwarf-4", chain_lint="t", TEST_CHECK_PREREQS="t", ), + args='CXXFLAGS="-gdwarf-4"', ) -# Ubuntu: TEST_INSTALL +# Debian: TEST_INSTALL matrix.add_build( - name="bionic - test-install", + name="bookworm - test-install", env=dict(TEST_INSTALL="t"), docker_tag=True, ) -# Ubuntu 20.04: py3.8 +# Ubuntu: TEST_INSTALL matrix.add_build( - name="focal", - image="focal", + name="jammy - test-install", + image="jammy", + env=dict(TEST_INSTALL="t"), docker_tag=True, ) -# RHEL7 clone +# Ubuntu 20.04: py3.8 matrix.add_build( - name="el7", - image="el7", + name="focal", + image="focal", docker_tag=True, ) @@ -155,13 +163,6 @@ def __str__(self): docker_tag=True, ) -# Fedora33 -matrix.add_build( - name="fedora33", - image="fedora33", - docker_tag=True, -) - # Fedora34 matrix.add_build( name="fedora34",