Separate and optimize pipeline build steps; Add coverage to test step (…

…#265)

* Separate and optimize pipeline build steps; Add coverage to test step

* Move src copy out of base image; remove env extensions

* Use a dymanic tag for the base image

* Define base tag and fix test step

* Remove wrong slash

* Exclude build from coverage

* WIP test filter coverage

* Use ninja for testing

* Use correct ninja package

* Optimize coverage files; Add more threads for dp3 build

.gitlab-ci.yml

@@ -1,22 +1,35 @@
 stages:
+  - prepare
   - build
   - deploy-doc
 
-.env:
-  before_script:
-  - export DOCKER_IMAGE=`echo $CI_PROJECT_NAME | awk '{print tolower($0)}'`:$CI_COMMIT_SHORT_SHA
+build_base:
+  stage: prepare
+  script:
+  - docker build --tag dppp_base:${CI_COMMIT_SHORT_SHA} -f ./docker/ubuntu_20_04_base .
 
-build_lofar:
+build-lofar:
   stage: build
-  extends: .env
+  dependencies: 
+    - build_base
   script:
-  - docker build --tag ${DOCKER_IMAGE}_lofar -f ./docker/ubuntu_20_04_lofar .
+  - docker build --build-arg BASE_TAG=${CI_COMMIT_SHORT_SHA} --tag dppp_lofar:${CI_COMMIT_SHORT_SHA} -f ./docker/ubuntu_20_04_lofar .
 
-build_nolofar:
+build-test:
   stage: build
-  extends: .env
+  dependencies: 
+    - build_base
+  image: dppp_base:${CI_COMMIT_SHORT_SHA}
+  before_script:
+    - apt-get update
+    - apt-get -y install python3-pip ninja-build
+    - pip3 install gcovr
   script:
-  - docker build --tag ${DOCKER_IMAGE}_nolofar -f ./docker/ubuntu_20_04_nolofar .
+    - mkdir build
+    - cd build
+    - cmake -G Ninja -DCMAKE_CXX_FLAGS="-coverage" -DCMAKE_EXE_LINKER_FLAGS="-coverage" ..
+    - ninja test
+    - gcovr -r . -f '/builds/RD/DP3/.*' -e '.*/CompilerIdCXX/.*' -e '.*/test/.*' -e '.*/build/.*'
 
 build-doc:
   stage: build
@@ -35,6 +48,8 @@ build-doc:
 
 deploy-doc:
   stage: deploy-doc
+  dependencies: 
+    - build-doc
   before_script:
     - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
     - eval $(ssh-agent -s)
@@ -46,4 +61,4 @@ deploy-doc:
     - scp -r build/docs/* citt@dop288:DP3/
   only:
     refs:
-      - master
+      - master

CMakeLists.txt

@@ -16,7 +16,7 @@ set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_STANDARD_REQUIRED YES)
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_CUSTOM}")
-
+include(CTest)
 enable_testing()
 
 find_package(HDF5 COMPONENTS C CXX REQUIRED)

docker/ubuntu_20_04_base

@@ -0,0 +1,23 @@
+FROM ubuntu:20.04
+
+RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \
+    apt-get install -y \
+    libboost-python-dev libcfitsio-dev python3-dev python3-numpy cmake \
+    build-essential libhdf5-serial-dev libarmadillo-dev \
+    libboost-filesystem-dev libboost-system-dev libboost-date-time-dev \
+    libboost-program-options-dev libboost-test-dev \
+    libxml2-dev libpng-dev pkg-config \
+    libgtkmm-3.0-dev git libfftw3-dev \
+    gfortran flex bison wcslib-dev \
+    libboost-numpy-dev liblua5.3-dev \
+    casacore-dev aoflagger-dev \
+# Build latest IDG master from source
+    && mkdir /idg && cd /idg && git clone https://gitlab.com/astron-idg/idg.git src \
+    && mkdir build && cd build \
+    && cmake -DCMAKE_INSTALL_PREFIX=/usr ../src \
+    && make install -j2 \
+# Build LOFARBeam
+    && mkdir /lofarbeam && cd /lofarbeam && git clone https://github.com/lofar-astron/LOFARBeam.git src \
+    && mkdir build && cd build \
+    && cmake -DCMAKE_INSTALL_PREFIX=/usr ../src -DPYTHON_EXECUTABLE=/usr/bin/python3 \
+    && make install -j2

docker/ubuntu_20_04_lofar

@@ -1,30 +1,11 @@
-FROM ubuntu:20.04
-
-RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \
-    apt-get install -y \
-    libboost-python-dev libcfitsio-dev python3-dev python3-numpy cmake \
-    build-essential libhdf5-serial-dev libarmadillo-dev \
-    libboost-filesystem-dev libboost-system-dev libboost-date-time-dev \
-    libboost-program-options-dev libboost-test-dev \
-    libxml2-dev libpng-dev pkg-config \
-    libgtkmm-3.0-dev git libfftw3-dev \
-    gfortran flex bison wcslib-dev \
-    libboost-numpy-dev liblua5.3-dev \
-    casacore-dev aoflagger-dev
-
-# Build latest IDG master from source
-RUN mkdir /idg && cd /idg && git clone https://gitlab.com/astron-idg/idg.git src
-RUN mkdir /idg/build && cd /idg/build && cmake -DCMAKE_INSTALL_PREFIX=/usr ../src && make install -j2
-
-# Build LOFARBeam
-RUN mkdir /lofarbeam && cd /lofarbeam && git clone https://github.com/lofar-astron/LOFARBeam.git src
-RUN mkdir /lofarbeam/build && cd /lofarbeam/build && cmake -DCMAKE_INSTALL_PREFIX=/usr ../src -DPYTHON_EXECUTABLE=/usr/bin/python3 && make install -j2
+ARG BASE_TAG=latest
+FROM dppp_base:$BASE_TAG
 
 ADD . /src
-WORKDIR /src
 
-RUN mkdir /build && cd /build && cmake ../src
-RUN cd /build && make -j2
-RUN cd /build && make install -j2
-RUN DPPP
-RUN cd /build && CTEST_OUTPUT_ON_FAILURE=1 make test -j2
+# Build DPPP
+RUN mkdir /build && cd /build \
+    && cmake ../src \
+    && make -j4 \
+    && make install -j4 \
+    && DPPP