From 3906b7adbc58d83d0eeafaa459331cfbccf25220 Mon Sep 17 00:00:00 2001 From: lkrombeenc <58769033+lkrombeenc@users.noreply.github.com> Date: Wed, 8 Jul 2020 11:25:22 +0200 Subject: [PATCH] Separate and optimize pipeline build steps; Add coverage to test step (#265) * Separate and optimize pipeline build steps; Add coverage to test step * Move src copy out of base image; remove env extensions * Use a dymanic tag for the base image * Define base tag and fix test step * Remove wrong slash * Exclude build from coverage * WIP test filter coverage * Use ninja for testing * Use correct ninja package * Optimize coverage files; Add more threads for dp3 build --- .gitlab-ci.yml | 35 +++++++++++++++++++++++++---------- CMakeLists.txt | 2 +- docker/ubuntu_20_04_base | 23 +++++++++++++++++++++++ docker/ubuntu_20_04_lofar | 35 ++++++++--------------------------- docker/ubuntu_20_04_nolofar | 22 ---------------------- 5 files changed, 57 insertions(+), 60 deletions(-) create mode 100644 docker/ubuntu_20_04_base delete mode 100644 docker/ubuntu_20_04_nolofar diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8d035164..42181e00 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,22 +1,35 @@ stages: + - prepare - build - deploy-doc -.env: - before_script: - - export DOCKER_IMAGE=`echo $CI_PROJECT_NAME | awk '{print tolower($0)}'`:$CI_COMMIT_SHORT_SHA +build_base: + stage: prepare + script: + - docker build --tag dppp_base:${CI_COMMIT_SHORT_SHA} -f ./docker/ubuntu_20_04_base . -build_lofar: +build-lofar: stage: build - extends: .env + dependencies: + - build_base script: - - docker build --tag ${DOCKER_IMAGE}_lofar -f ./docker/ubuntu_20_04_lofar . + - docker build --build-arg BASE_TAG=${CI_COMMIT_SHORT_SHA} --tag dppp_lofar:${CI_COMMIT_SHORT_SHA} -f ./docker/ubuntu_20_04_lofar . -build_nolofar: +build-test: stage: build - extends: .env + dependencies: + - build_base + image: dppp_base:${CI_COMMIT_SHORT_SHA} + before_script: + - apt-get update + - apt-get -y install python3-pip ninja-build + - pip3 install gcovr script: - - docker build --tag ${DOCKER_IMAGE}_nolofar -f ./docker/ubuntu_20_04_nolofar . + - mkdir build + - cd build + - cmake -G Ninja -DCMAKE_CXX_FLAGS="-coverage" -DCMAKE_EXE_LINKER_FLAGS="-coverage" .. + - ninja test + - gcovr -r . -f '/builds/RD/DP3/.*' -e '.*/CompilerIdCXX/.*' -e '.*/test/.*' -e '.*/build/.*' build-doc: stage: build @@ -35,6 +48,8 @@ build-doc: deploy-doc: stage: deploy-doc + dependencies: + - build-doc before_script: - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - eval $(ssh-agent -s) @@ -46,4 +61,4 @@ deploy-doc: - scp -r build/docs/* citt@dop288:DP3/ only: refs: - - master + - master \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index bbdb1494..e3395837 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED YES) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_CUSTOM}") - +include(CTest) enable_testing() find_package(HDF5 COMPONENTS C CXX REQUIRED) diff --git a/docker/ubuntu_20_04_base b/docker/ubuntu_20_04_base new file mode 100644 index 00000000..46e9c132 --- /dev/null +++ b/docker/ubuntu_20_04_base @@ -0,0 +1,23 @@ +FROM ubuntu:20.04 + +RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \ + apt-get install -y \ + libboost-python-dev libcfitsio-dev python3-dev python3-numpy cmake \ + build-essential libhdf5-serial-dev libarmadillo-dev \ + libboost-filesystem-dev libboost-system-dev libboost-date-time-dev \ + libboost-program-options-dev libboost-test-dev \ + libxml2-dev libpng-dev pkg-config \ + libgtkmm-3.0-dev git libfftw3-dev \ + gfortran flex bison wcslib-dev \ + libboost-numpy-dev liblua5.3-dev \ + casacore-dev aoflagger-dev \ +# Build latest IDG master from source + && mkdir /idg && cd /idg && git clone https://gitlab.com/astron-idg/idg.git src \ + && mkdir build && cd build \ + && cmake -DCMAKE_INSTALL_PREFIX=/usr ../src \ + && make install -j2 \ +# Build LOFARBeam + && mkdir /lofarbeam && cd /lofarbeam && git clone https://github.com/lofar-astron/LOFARBeam.git src \ + && mkdir build && cd build \ + && cmake -DCMAKE_INSTALL_PREFIX=/usr ../src -DPYTHON_EXECUTABLE=/usr/bin/python3 \ + && make install -j2 \ No newline at end of file diff --git a/docker/ubuntu_20_04_lofar b/docker/ubuntu_20_04_lofar index ea33bb99..c1bcff88 100644 --- a/docker/ubuntu_20_04_lofar +++ b/docker/ubuntu_20_04_lofar @@ -1,30 +1,11 @@ -FROM ubuntu:20.04 - -RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \ - apt-get install -y \ - libboost-python-dev libcfitsio-dev python3-dev python3-numpy cmake \ - build-essential libhdf5-serial-dev libarmadillo-dev \ - libboost-filesystem-dev libboost-system-dev libboost-date-time-dev \ - libboost-program-options-dev libboost-test-dev \ - libxml2-dev libpng-dev pkg-config \ - libgtkmm-3.0-dev git libfftw3-dev \ - gfortran flex bison wcslib-dev \ - libboost-numpy-dev liblua5.3-dev \ - casacore-dev aoflagger-dev - -# Build latest IDG master from source -RUN mkdir /idg && cd /idg && git clone https://gitlab.com/astron-idg/idg.git src -RUN mkdir /idg/build && cd /idg/build && cmake -DCMAKE_INSTALL_PREFIX=/usr ../src && make install -j2 - -# Build LOFARBeam -RUN mkdir /lofarbeam && cd /lofarbeam && git clone https://github.com/lofar-astron/LOFARBeam.git src -RUN mkdir /lofarbeam/build && cd /lofarbeam/build && cmake -DCMAKE_INSTALL_PREFIX=/usr ../src -DPYTHON_EXECUTABLE=/usr/bin/python3 && make install -j2 +ARG BASE_TAG=latest +FROM dppp_base:$BASE_TAG ADD . /src -WORKDIR /src -RUN mkdir /build && cd /build && cmake ../src -RUN cd /build && make -j2 -RUN cd /build && make install -j2 -RUN DPPP -RUN cd /build && CTEST_OUTPUT_ON_FAILURE=1 make test -j2 +# Build DPPP +RUN mkdir /build && cd /build \ + && cmake ../src \ + && make -j4 \ + && make install -j4 \ + && DPPP \ No newline at end of file diff --git a/docker/ubuntu_20_04_nolofar b/docker/ubuntu_20_04_nolofar deleted file mode 100644 index 0ae9a7bd..00000000 --- a/docker/ubuntu_20_04_nolofar +++ /dev/null @@ -1,22 +0,0 @@ -FROM ubuntu:20.04 - -RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \ - apt-get install -y \ - libboost-python-dev libcfitsio-dev python3-dev python3-numpy cmake \ - build-essential libhdf5-serial-dev libarmadillo-dev \ - libboost-filesystem-dev libboost-system-dev libboost-date-time-dev \ - libboost-program-options-dev libboost-test-dev \ - libxml2-dev libpng-dev pkg-config \ - libgtkmm-3.0-dev git libfftw3-dev \ - gfortran flex bison wcslib-dev \ - libboost-numpy-dev liblua5.3-dev \ - casacore-dev aoflagger-dev - -ADD . /src -WORKDIR /src - -RUN mkdir /build && cd /build && cmake ../src -RUN cd /build && make -j2 -RUN cd /build && make install -j2 -RUN DPPP -RUN cd /build && CTEST_OUTPUT_ON_FAILURE=1 make test -j2