.gitlab-ci.common.yml

# Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
# SPDX-License-Identifier: GPL-3.0-or-later

workflow:
  rules:
    # don't create a pipeline if its a commit pipeline, on a branch and that branch has open merge requests.
    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS
      when: never
    - when: always

variables:
  GIT_SUBMODULE_STRATEGY: recursive

.failable:
  retry:
    max: 1
    when:
      - runner_system_failure
      - stuck_or_timeout_failure

stages:
  - versioning
  - prepare
  - build
  - linting
  - test
  - publish
  - pages

include: .gitlab-ci.wheels.yml

# The 'IMAGE' variables allow reusing docker images between different pipelines.
# See https://confluence.skatelescope.org/display/SE/Caching+Docker+images+using+GitLab+CI+registry
versioning:
  stage: versioning
  image: bitnami/git
  script:
    # Unshallowing ensures that 'git log' works
    - git fetch --unshallow
    - echo BASE_IMAGE_2004=${CI_REGISTRY_IMAGE}/base_ubuntu20:$(git log -n 1 --pretty=format:%H -- docker/ubuntu_20_04_base) > versions.env
    - echo BASE_IMAGE_2204=${CI_REGISTRY_IMAGE}/base_ubuntu22:$(git log -n 1 --pretty=format:%H -- docker/ubuntu_22_04_base) >> versions.env
    - echo BASE_IMAGE_2404=${CI_REGISTRY_IMAGE}/base_ubuntu24:$(git log -n 1 --pretty=format:%H -- docker/ubuntu_24_04_base) >> versions.env
    - cat versions.env
  artifacts:
    reports:
      dotenv: versions.env

.prepare:
  extends: [".failable",".dind"]
  stage: prepare
  needs: ["versioning"]
  image: docker:20.10
  before_script:
    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
  script:
    - |
      if ! docker manifest inspect $DOCKER_IMAGE > /dev/null || [ "$BUILD_DOCKER_IMAGE" = "1" ]; then
        if [ "$BUILD_DOCKER_IMAGE" = "1" ]; then
          DOCKER_CACHE_PARAMETERS="--no-cache"
        else
          DOCKER_CACHE_PARAMETERS=""
        fi
        docker build $DOCKER_BUILD_ARG ${DOCKER_CACHE_PARAMETERS} --tag $DOCKER_IMAGE -f $DOCKER_FILE .
        docker push $DOCKER_IMAGE
      fi
  # Skip the job if there are no changes to the Docker file. This shortcut only
  # works for push and merge request jobs.
  # A manual pipeline run will thus create missing docker images.
  #
  # This is disabled since the detections of the changes by GitLab seems flaky.
  # TODO(AST-887) Reenable this to test whether it's no longer flaky.
  #
  #rules:
  #  - changes:
  #    - $DOCKER_FILE

# Create and push the base image to the gitlab registry, if it does not exist.
prepare-base-2004:
  extends: .prepare
  variables:
    DOCKER_IMAGE: $BASE_IMAGE_2004
    DOCKER_FILE: ./docker/ubuntu_20_04_base

prepare-base-2204:
  extends: .prepare
  variables:
    DOCKER_IMAGE: $BASE_IMAGE_2204
    DOCKER_FILE: ./docker/ubuntu_22_04_base

prepare-base-2404:
  extends: .prepare
  variables:
    DOCKER_IMAGE: $BASE_IMAGE_2404
    DOCKER_FILE: ./docker/ubuntu_24_04_base

# Template for basic build jobs.
.build-basic:
  extends: .failable
  stage: build
  script:
    - mkdir build && cd build
    - cmake ${CMAKE_FLAGS} -G Ninja ..
    - ninja -j4 install
    - DP3

# Build (and run) DP3 on Ubuntu 20, ensuring backward compatiblity.
# This build does not include IDG, since IDG no longer supports Ubuntu 20.
# This build also tests building DP3 with the TARGET_CPU option.
build-2004-no-idg-target-cpu:
  extends: .build-basic
  needs: ["versioning", "prepare-base-2004"]
  image: $BASE_IMAGE_2004
  variables:
    CMAKE_FLAGS: -DTARGET_CPU=haswell

# Template for jobs that depend on prepare-base-2204.
.needs-base-2204:
  needs: ["versioning", "prepare-base-2204"]
  image: $BASE_IMAGE_2204

# Template for jobs that depend on prepare-base-2404.
.needs-base-2404:
  needs: ["versioning", "prepare-base-2404"]
  image: $BASE_IMAGE_2404

# Build (and run) DP3 on Ubuntu 24, ensuring compatibility with new systems.
# This build also tests building DP3 with the PORTABLE option.
build-2404-portable:
  extends: [".needs-base-2404", ".build-basic"]
  variables:
    # Enable tests and thereby check if they also compile on new systems.
    CMAKE_FLAGS: -DBUILD_TESTING=On -DPORTABLE=On

build-doc-2204:
  stage: build
  extends: [".failable",".needs-base-2204"]
  before_script:
    # Install here since pytest and these items aren't compatible
    - pip3 install autosemver jsonschema2rst sphinx sphinx-rtd-theme myst_parser
    # Patch to make 'doc' settings not be parsed as code. Make sure to apply this patch also in .readthedocs.yml!
    - patch -p0 /usr/local/lib/python3.10/dist-packages/jsonschema2rst/rst_utils.py < docs/rst_utils.py.patch
  script:
    - mkdir build && cd build
    - cmake -G Ninja ../docs
    - ninja doc userdoc
  artifacts: # Only for master the docs are published; for branches it may be useful to browse the artifacts
    paths:
    - build/docs

build-package-2204:
  stage: build
  extends: [".failable",".needs-base-2204"]
  script:
    - mkdir dp3_package
    - mkdir build && cd build
    - git fetch --unshallow # We need to unshallow for the tags (setting GIT_DEPTH is not good enough)
    - cmake -DBUILD_PACKAGES=On ..
    - make -j4
    - make package
    - mv $(ls -1 *.deb) ../dp3_package/
  artifacts:
    paths:
    - dp3_package/
  rules:
    # The package is built only during a merge_request_event, a merge to master,
    # or when the pipeline is triggered by a tag event.
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_COMMIT_BRANCH == "master"'
    - if: '$CI_COMMIT_TAG'
    - if: '$UPLOAD_PACKAGE'

linting-2204:
  stage: linting
  extends: [".failable",".needs-base-2204"]
  script:
    - ./scripts/run-format.sh

test-2204:
  stage: test
  extends: [".failable",".needs-base-2204"]
  script:
    - mkdir build && cd build
    - cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=On ..
    - ninja -j4
    - ctest --output-on-failure --no-tests=error -j$(nproc)
  artifacts:
    paths:
      - build/unittest*.xml
      - build/pytest_*.xml
    reports:
      junit:
        - build/unittest*.xml
        - build/pytest_*.xml

# Use a separate job for collecting code coverage, which uses a Release build,
# since collecting code coverage in a Debug build results in low performance.
# Especially -fprofile-update=atomic makes the tests slow.
# (A single Debug+coverage job takes longer than the total runtime of
#  the current test-2204 and coverage-2204 jobs. It may even time out.)
# The code coverage figures include both unit and integration tests.
coverage-2204:
  stage: test
  extends: [".failable",".needs-base-2204"]
  script:
    - mkdir build && cd build
    - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=On -DCMAKE_CXX_FLAGS="-coverage -fprofile-update=atomic" -DCMAKE_EXE_LINKER_FLAGS="-coverage" ..
    - ninja -j4
    - ctest --output-on-failure --no-tests=error -j$(nproc)
    # Collect coverage in text, xml and json formats.
    - gcovr -j$(nproc) -r ../ -e '.*/external/.*' -e '_deps/.*' -e '.*/test/.*' -e '.*/CompilerIdCXX/.*' --txt --xml coverage.xml --json coverage.json .
  coverage: /^TOTAL.*\s+(\d+\%)$/
  artifacts:
    paths:
      # The pages and collect-metrics jobs need coverage files. See .gitlab-ci.ska.yml.
      - build/coverage.xml
      - build/coverage.json
      - build/unittest*.xml
      - build/pytest_*.xml
    reports:
      junit:
        - build/unittest*.xml
        - build/pytest_*.xml
      coverage_report:
        coverage_format: cobertura
        path: build/coverage.xml

.sanitizer-2404:
  stage: test
  extends: [".failable",".needs-base-2404"]
  script:
    - mkdir build && cd build
    - cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DBUILD_TESTING=On -DPORTABLE=Off -DCMAKE_CXX_FLAGS="${CXX_FLAGS}" ..
    - ninja -j4
    # Don't run slow tests. The overhead of the sanitizer causes time outs.
    - ctest --output-on-failure --no-tests=error -j$(nproc) -L unit -LE slow
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
      when: always
      allow_failure: false
    - when: manual
      allow_failure: true

unit-test-address-sanitizer-2404:
  extends: [".sanitizer-2404",".dind"]
  variables:
    CXX_FLAGS: -fsanitize=address
    # Ignore the leaks in third-party libraries.
    LSAN_OPTIONS: suppressions=../ci/address_sanitizer_suppressions.txt

# There are NULL pointer issues in Casacore which blocks testing on CI.
unit-test-undefined-behaviour-sanitizer-2404:
  extends: .sanitizer-2404
  variables:
    CXX_FLAGS: -fsanitize=undefined -fno-sanitize=null -fno-sanitize-recover

deploy-package-2204:
  stage: publish
  needs: ["versioning","build-package-2204"]
  image: $BASE_IMAGE_2204
  script:
    - pip3 install aptly-api-client
    - chmod -R 700 external/schaap-packaging
    - cd dp3_package
    - export FILES=$(ls -1 $PWD/*.deb)
    - echo UPLOADING files $FILES
    # The following example command must be executed first or else the update will fail because there is no repository
    # create_repo.py -a amd64 -c testing -d bionic --gpg-key ${GPG_KEY} --gpg-passphrase ${GPG_PASS} schaap
    - ../external/schaap-packaging/update_repo.py --cleanup -d bionic --gpg-key ${GPG_KEY} --gpg-passphrase ${GPG_PASS} schaap ${FILES}
  rules:
    # Only run on master because GPG_KEY and GPG_PASS are protected and therefore only available on protected branches.
    - if: '$CI_COMMIT_BRANCH != "master"'
      when: never
    - if: '$UPLOAD_PACKAGE'