From 0e29938683f4fd644e6dfe7aa15226d4d97a6e50 Mon Sep 17 00:00:00 2001 From: Maik Nijhuis Date: Thu, 3 Dec 2020 11:26:01 +0100 Subject: [PATCH] Add ubuntu 18.04 build, for checking compatibility. Use old python cmake commands for old cmake versions. --- .gitlab-ci.yml | 13 ++++- CMakeLists.txt | 22 +++++++-- README.md | 2 +- docker/ubuntu_18_04 | 49 +++++++++++++++++++ docker/ubuntu_18_04_lofar | 37 -------------- docker/ubuntu_18_04_nolofar | 29 ----------- .../{ubuntu_20_04_lofar => ubuntu_20_04_dppp} | 0 7 files changed, 79 insertions(+), 73 deletions(-) create mode 100644 docker/ubuntu_18_04 delete mode 100644 docker/ubuntu_18_04_lofar delete mode 100644 docker/ubuntu_18_04_nolofar rename docker/{ubuntu_20_04_lofar => ubuntu_20_04_dppp} (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 298f4524..6f496ecb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,6 +16,15 @@ build-base: - docker build --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_BRANCH -f ./docker/ubuntu_20_04_base . - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_BRANCH +build-ubuntu18: + stage: build + needs: [] + script: + - docker build -f ./docker/ubuntu_18_04 . + only: + variables: + - $CI_SERVER_HOST == "git.astron.nl" + build-no-idg: stage: build needs: ["build-base"] @@ -46,14 +55,14 @@ build-debug: paths: - build -build-lofar-release: +build-release: stage: build image: docker:stable services: - docker:dind needs: ["build-base"] script: - - docker build --build-arg BASE_TAG=$CI_REGISTRY_IMAGE:$CI_COMMIT_BRANCH --tag dppp_lofar:${CI_COMMIT_SHORT_SHA} -f ./docker/ubuntu_20_04_lofar . + - docker build --build-arg BASE_TAG=$CI_REGISTRY_IMAGE:$CI_COMMIT_BRANCH --tag dppp_lofar:${CI_COMMIT_SHORT_SHA} -f ./docker/ubuntu_20_04_dppp . build-doc: stage: build diff --git a/CMakeLists.txt b/CMakeLists.txt index 965837d6..87ce1306 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,11 @@ # FindHDF5 uses NATIVE_COMMAND in separate_arguments, which requires # CMake 3.9. cmake_minimum_required(VERSION 3.9) -cmake_policy(SET CMP0074 NEW) + +# CMake >= 3.19.1 gives a warning when this policy is not 'NEW'. +if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.19.1") + cmake_policy(SET CMP0074 NEW) +endif() # Set version number and project name. set(DP3_VERSION 5.0.0) @@ -85,14 +89,24 @@ endif(IDGAPI_LIBRARIES AND IDGAPI_INCLUDE_DIRS) find_package(CFITSIO REQUIRED) include_directories(${CFITSIO_INCLUDE_DIRS}) -find_package(Python3 COMPONENTS Interpreter Development) -message(STATUS "Using python version ${Python3_VERSION}") +if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.12") + find_package(Python3 COMPONENTS Interpreter Development) + set(BOOSTPY ${Python3_VERSION_MAJOR}${Python3_VERSION_MINOR}) +else() # Use old, deprecated means of detecting python. + find_package(PythonInterp 3 REQUIRED) + find_package(PythonLibs 3 REQUIRED) + set(Python3_EXECUTABLE ${PYTHON_EXECUTABLE}) + set(Python3_VERSION ${PYTHON_VERSION_STRING}) + set(Python3_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS}) + set(Python3_LIBRARIES ${PYTHON_LIBRARIES}) + set(BOOSTPY "") +endif() +message(STATUS "Using python version ${Python3_VERSION}") include_directories(${Python3_INCLUDE_DIRS}) #Prevent accidentally finding old BoostConfig.cmake file from casapy set(Boost_NO_BOOST_CMAKE ON) -set(BOOSTPY ${Python3_VERSION_MAJOR}${Python3_VERSION_MINOR}) find_package(Boost COMPONENTS date_time filesystem python${BOOSTPY} numpy${BOOSTPY} program_options system unit_test_framework REQUIRED) include_directories(${Boost_INCLUDE_DIR}) diff --git a/README.md b/README.md index d911a32f..603a8064 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The DPPP documentation can be found at: https://www.astron.nl/citt/DP3 This repository is a continuation of the one at svn.astron.nl/LOFAR. In particular, it has branched off at LOFAR Release 3.2 (Sept 2018). The version of DP3 that is in the ASTRON repository is no longer maintained. ## Installation -Some non-standard dependencies of this project are: armadillo, boost, boost-python, casacore, hdf5, aoflagger, and EveryBeam. See the Dockerfiles [`docker/ubuntu_20_04_base`](docker/ubuntu_20_04_base) and [`docker/ubuntu_20_04_lofar`](docker/ubuntu_20_04_lofar) as examples. +Some non-standard dependencies of this project are: armadillo, boost, boost-python, casacore, hdf5, aoflagger, and EveryBeam. See the Dockerfiles [`docker/ubuntu_20_04_base`](docker/ubuntu_20_04_base) and [`docker/ubuntu_20_04_dppp`](docker/ubuntu_20_04_dppp) as examples. Typical installation commands: ``` diff --git a/docker/ubuntu_18_04 b/docker/ubuntu_18_04 new file mode 100644 index 00000000..bbf80d77 --- /dev/null +++ b/docker/ubuntu_18_04 @@ -0,0 +1,49 @@ +FROM ubuntu:18.04 + +ENV EVERYBEAM_VERSION_TAG=v0.1.1 +ENV IDG_VERSION_TAG=0.8 + +RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \ + apt-get install -y \ + libboost-python-dev libcfitsio-dev python3-dev python3-numpy cmake wget \ + build-essential libhdf5-serial-dev libarmadillo-dev \ + libboost-filesystem-dev libboost-system-dev libboost-date-time-dev \ + libboost-program-options-dev libboost-test-dev \ + libxml2-dev libpng-dev pkg-config \ + libgtkmm-3.0-dev git libfftw3-dev \ + gfortran flex bison wcslib-dev \ + libboost-numpy-dev liblua5.3-dev \ + casacore-dev casacore-tools pybind11-dev python3-pip ninja-build \ +# Build aoflagger3. +# Use make, since ninja in Ubuntu 18 does not support Fortran. + && mkdir /aoflagger && cd /aoflagger \ + && git clone https://gitlab.com/aroffringa/aoflagger.git src \ + && mkdir build && cd build \ + && cmake -DCMAKE_INSTALL_PREFIX=/usr ../src \ + && make install -j$(($(nproc) / 2 > 0 ? $(nproc) / 2:1)) \ + && cd / && rm -rf aoflagger \ +# Build IDG + && mkdir /idg && cd /idg \ + && git clone https://git.astron.nl/RD/idg.git src \ + && ( cd src/ && git checkout ${IDG_VERSION_TAG} ) \ + && mkdir build && cd build \ + && cmake -G Ninja -DCMAKE_INSTALL_PREFIX=/usr ../src \ + && ninja install \ + && cd / && rm -rf idg \ +# Build EveryBeam + && mkdir /everybeam && cd /everybeam \ + && git clone https://git.astron.nl/RD/EveryBeam.git src \ + && ( cd src/ && git checkout ${EVERYBEAM_VERSION_TAG} ) \ + && mkdir build && cd build \ + && cmake -G Ninja -DCMAKE_INSTALL_PREFIX=/usr ../src -DPYTHON_EXECUTABLE=/usr/bin/python3 \ + && ninja install \ + && cd / && rm -rf everybeam + +ADD . /src + +# Build DPPP +RUN mkdir /build && cd /build \ + && cmake -G Ninja ../src \ + && ninja install \ + && DPPP \ + && rm -r /src /build diff --git a/docker/ubuntu_18_04_lofar b/docker/ubuntu_18_04_lofar deleted file mode 100644 index edf3d284..00000000 --- a/docker/ubuntu_18_04_lofar +++ /dev/null @@ -1,37 +0,0 @@ -FROM ubuntu - -RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \ - apt-get install -y \ - libboost-python-dev libcfitsio-dev python3-dev python3-numpy cmake \ - build-essential libhdf5-serial-dev libarmadillo-dev \ - libboost-filesystem-dev libboost-system-dev libboost-date-time-dev \ - libboost-program-options-dev libboost-test-dev \ - libxml2-dev libpng-dev pkg-config \ - libgtkmm-3.0-dev git libfftw3-dev \ - gfortran flex bison wcslib-dev \ - libboost-numpy-dev liblua5.3-dev - -# Build casacore from source -RUN mkdir /casacore && cd /casacore && git clone -b v3.2.0 https://github.com/casacore/casacore.git src -RUN mkdir /casacore/build && cd /casacore/build && cmake -DBUILD_PYTHON3="ON" -DBUILD_PYTHON="OFF" -DUSE_HDF5="ON" -DCXX11="ON" -DBUILD_TESTING="OFF" -DCMAKE_INSTALL_PREFIX=/usr ../src && make install -j2 - -# Get aoflagger (don't use -j2 -- travis can't handle it) -RUN mkdir /aoflagger && cd /aoflagger && git clone git://git.code.sf.net/p/aoflagger/code src -RUN mkdir /aoflagger/build && cd /aoflagger/build && cmake -DCMAKE_INSTALL_PREFIX=/usr ../src && make install - -# Build latest IDG master from source -RUN mkdir /idg && cd /idg && git clone https://gitlab.com/astron-idg/idg.git src -RUN mkdir /idg/build && cd /idg/build && cmake -DCMAKE_INSTALL_PREFIX=/usr ../src && make install -j2 - -# Build EveryBeam -RUN mkdir /everybeam && cd /everybeam && git clone https://git.astron.nl/RD/EveryBeam.git src -RUN mkdir /everybeam/build && cd /everybeam/build && cmake -DCMAKE_INSTALL_PREFIX=/usr ../src -DPYTHON_EXECUTABLE=/usr/bin/python3 && make install -j2 - -ADD . /src -WORKDIR /src - -RUN mkdir /build && cd /build && cmake ../src -RUN cd /build && make -j2 -RUN cd /build && make install -j2 -RUN DPPP -RUN cd /build && CTEST_OUTPUT_ON_FAILURE=1 make test -j2 diff --git a/docker/ubuntu_18_04_nolofar b/docker/ubuntu_18_04_nolofar deleted file mode 100644 index aa8091d1..00000000 --- a/docker/ubuntu_18_04_nolofar +++ /dev/null @@ -1,29 +0,0 @@ -FROM ubuntu:latest - -RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && \ - apt-get install -y \ - libboost-python-dev libcfitsio-dev python3-dev python3-numpy cmake \ - build-essential libhdf5-serial-dev libarmadillo-dev \ - libboost-filesystem-dev libboost-system-dev libboost-date-time-dev \ - libboost-program-options-dev libboost-test-dev \ - libxml2-dev libpng-dev pkg-config \ - libgtkmm-3.0-dev git libfftw3-dev \ - gfortran flex bison wcslib-dev \ - libboost-numpy-dev liblua5.3-dev - -# Build casacore from source -RUN mkdir /casacore && cd /casacore && git clone -b v3.2.0 https://github.com/casacore/casacore.git src -RUN mkdir /casacore/build && cd /casacore/build && cmake -DBUILD_PYTHON3="ON" -DBUILD_PYTHON="OFF" -DUSE_HDF5="ON" -DCXX11="ON" -DBUILD_TESTING="OFF" -DCMAKE_INSTALL_PREFIX=/usr ../src && make install -j2 - -# Get aoflagger (don't use -j2 -- travis can't handle it) -RUN mkdir /aoflagger && cd /aoflagger && git clone git://git.code.sf.net/p/aoflagger/code src -RUN mkdir /aoflagger/build && cd /aoflagger/build && cmake -DCMAKE_INSTALL_PREFIX=/usr ../src && make install - -ADD . /src -WORKDIR /src - -RUN mkdir /build && cd /build && cmake ../src -RUN cd /build && make -j2 -RUN cd /build && make install -j2 -RUN DPPP -RUN cd /build && CTEST_OUTPUT_ON_FAILURE=1 make test -j2 diff --git a/docker/ubuntu_20_04_lofar b/docker/ubuntu_20_04_dppp similarity index 100% rename from docker/ubuntu_20_04_lofar rename to docker/ubuntu_20_04_dppp