.gitlab-ci.yml

variables:
  CRAN_MIRROR: "https://cloud.r-project.org"
  _R_CHECK_FORCE_SUGGESTS_: "FALSE"

stages:
  - dependencies
  - build
  - test
  - integration
  - deploy

.artifacts-template: &artifacts
  artifacts:
    expire_in: 2 weeks
    when: always
    paths:
      - bus

mirror-packages: # download all recursive dependencies of data.table suggests and integration suggests from inst/tests/tests-DESCRIPTION
  stage: dependencies
  tags:
    - linux
  image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev
  cache:
    paths:
    - bus/$CI_BUILD_NAME/cran
  variables:
    R_BIN_VERSION: "3.5"
    R_DEVEL_BIN_VERSION: "3.6"
  script:
    - echo 'source("ci.R")' >> .Rprofile
    - mkdir -p bus/$CI_BUILD_NAME/cran/src/contrib
    # mirror R dependencies: source, win.binary
    - Rscript -e 'mirror.packages(dcf.dependencies(c("DESCRIPTION","inst/tests/tests-DESCRIPTION"), "all"), repos=c(Sys.getenv("CRAN_MIRROR"), dcf.repos("inst/tests/tests-DESCRIPTION")), repodir="bus/mirror-packages/cran")'
    - Rscript -e 'sapply(simplify=FALSE, setNames(nm=Sys.getenv(c("R_BIN_VERSION","R_DEVEL_BIN_VERSION"))), function(binary.ver) mirror.packages(type="win.binary", dcf.dependencies("DESCRIPTION", "all"), repos=Sys.getenv("CRAN_MIRROR"), repodir="bus/mirror-packages/cran", binary.ver=binary.ver))'
  <<: *artifacts

build: # build data.table sources as tar.gz archive
  stage: build
  tags:
    - linux
  image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
  dependencies:
  - mirror-packages
  script:
    - Rscript -e 'install.packages("knitr", repos=file.path("file:",normalizePath("bus/mirror-packages/cran")))'
    - rm -r bus
    - echo "Revision:" $CI_BUILD_REF >> ./DESCRIPTION
    - R CMD build .
    - mkdir -p bus/$CI_BUILD_NAME/cran/src/contrib
    - mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME/cran/src/contrib/.
    - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/build/cran"), fields="Revision", addFiles=TRUE)'
  <<: *artifacts

.test-copy-src: &copy-src
  - &copy-src
    cp $(ls -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head -n 1) .

.test-move-src: &move-src
  - &move-src
    mkdir -p bus/$CI_BUILD_NAME && mv $(ls -1t data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME

.test-cleanup-src: &cleanup-src
  - &cleanup-src
    rm $(ls -1t data.table_*.tar.gz | head -n 1)

.test-template: &test
  stage: test
  dependencies:
  - mirror-packages
  - build
  <<: *artifacts

.test-linux-template: &test-linux
  <<: *test
  tags:
    - linux

.test-windows-template: &test-windows
  <<: *test
  tags:
    - windows
  before_script:
    - export PATH="/c/$R_DIR/bin:/c/Rtools/bin:$PATH"
    - rm -rf /tmp/$R_DIR/library && mkdir -p /tmp/$R_DIR/library
    - export R_LIBS_USER="/tmp/$R_DIR/library"

.test-macosx-template: &test-macosx
  <<: *test
  tags:
    - macosx

test-r-release: #  most comprehensive tests, force all suggests, also integration tests
  <<: *test-linux
  image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
  variables: # unlike CRAN
    _R_CHECK_CRAN_INCOMING_: "FALSE"
    _R_CHECK_CRAN_INCOMING_REMOTE_: "FALSE"
    _R_CHECK_FORCE_SUGGESTS_: "TRUE"
    OPENBLAS_MAIN_FREE: "1"
    TEST_DATA_TABLE_WITH_OTHER_PACKAGES: "TRUE"
  script:
    - Rscript -e 'source("ci.R"); install.packages(dcf.dependencies(c("DESCRIPTION","inst/tests/tests-DESCRIPTION"), which="all"))'
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check $(ls -1t data.table_*.tar.gz | head -n 1)
    - *cleanup-src

test-r-release-vanilla: # minimal installation, no suggested deps, no vignettes or manuals, measure memory
  <<: *test-linux
  image: registry.gitlab.com/jangorecki/dockerfiles/r-base-dev
  variables:
    TEST_DATA_TABLE_MEMTEST: "TRUE"
  script:
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check --no-manual --ignore-vignettes $(ls -1t data.table_*.tar.gz | head -n 1)
    - *cleanup-src

test-r-release-cran: # as CRAN
  <<: *test-linux
  image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
  variables:
    _R_CHECK_CRAN_INCOMING_: "TRUE"
    _R_CHECK_CRAN_INCOMING_REMOTE_: "TRUE"
  script:
    - Rscript -e 'source("ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="most"))'
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
    - *cleanup-src

test-r-devel-cran: # R-devel as CRAN
  <<: *test-linux
  image: registry.gitlab.com/jangorecki/dockerfiles/r-devel
  allow_failure: true
  variables:
    _R_CHECK_CRAN_INCOMING_: "TRUE"
    _R_CHECK_CRAN_INCOMING_REMOTE_: "TRUE"
  script:
    - Rscript -e 'source("ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="most"))'
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
    - *cleanup-src

test-r-3.1.0-cran: # test stated dependency on R 3.1.0 as CRAN
  <<: *test-linux
  image: registry.gitlab.com/jangorecki/dockerfiles/r-3.1.0
  variables:
    _R_CHECK_CRAN_INCOMING_: "TRUE"
    _R_CHECK_CRAN_INCOMING_REMOTE_: "TRUE"
  script:
    - Rscript -e 'source("ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="most"))'
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check --as-cran $(ls -1t data.table_*.tar.gz | head -n 1)
    - *cleanup-src

test-r-release-windows: # windows test and build binaries
  <<: *test-windows
  variables:
    R_BIN_VERSION: "3.5"
    R_DIR: "R-3.5.0"
  script:
    - Rscript -e "source('ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='all'))"
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check --no-manual $(ls -1t data.table_*.tar.gz | head -n 1)
    - R CMD INSTALL --build $(ls -1t data.table_*.tar.gz | head -n 1)
    - mkdir -p cran/bin/windows/contrib/$R_BIN_VERSION
    - mv $(ls -1t data.table_*.zip | head -n 1) cran/bin/windows/contrib/$R_BIN_VERSION
    - *cleanup-src

test-r-devel-windows: # R-devel on windows
  <<: *test-windows
  variables:
    R_BIN_VERSION: "3.6"
    R_DIR: "R-devel"
    TEST_DATA_TABLE_MEMTEST: "TRUE"
  allow_failure: true
  script:
    - Rscript -e "source('ci.R'); install.packages(dcf.dependencies('DESCRIPTION', which='all'))"
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check --no-manual --ignore-vignettes $(ls -1t data.table_*.tar.gz | head -n 1)
    - R CMD INSTALL --build $(ls -1t data.table_*.tar.gz | head -n 1)
    - mkdir -p cran/bin/windows/contrib/$R_BIN_VERSION
    - mv $(ls -1t data.table_*.zip | head -n 1) cran/bin/windows/contrib/$R_BIN_VERSION
    - *cleanup-src

.test-r-release-macosx: # macosx test and build binaries
  <<: *test-macosx
  variables:
    R_BIN_VERSION: "3.5"
  script:
    - Rscript -e 'source("ci.R"); install.packages(dcf.dependencies("DESCRIPTION", which="all"))'
    - *copy-src
    - rm -r bus
    - *move-src
    - cd bus/$CI_BUILD_NAME
    - R CMD check $(ls -1t data.table_*.tar.gz | head -n 1)
    - R CMD INSTALL --build $(ls -1t data.table_*.tar.gz | head -n 1)
    - mkdir -p cran/bin/macosx/el-capitan/contrib/$R_BIN_VERSION
    - mv $(ls -1t data.table_*.tgz | head -n 1) cran/bin/macosx/el-capitan/contrib/$R_BIN_VERSION
    - *cleanup-src

integration: # merging all artifacts to produce single R repository and summaries
  stage: integration
  image: registry.gitlab.com/jangorecki/dockerfiles/r-builder
  tags:
    - linux
  only:
    - master
  dependencies:
  - mirror-packages
  - build
  - test-r-release
  - test-r-release-cran
  - test-r-devel-cran
  - test-r-release-vanilla
  - test-r-3.1.0-cran
  - test-r-release-windows
  - test-r-devel-windows
  #- test-r-release-macosx
  variables:
    R_BIN_VERSION: "3.5"
    R_DEVEL_BIN_VERSION: "3.6"
  script:
    - echo 'source("ci.R"); source("publish.R")' >> .Rprofile
    # list of available test-* jobs dynamically based on bus/test-* directories
    - Rscript -e 'cat("\ntest.jobs <- c(\n"); cat(paste0("  \"",list.files("bus",pattern="^test-"),"\" = \"data.table\""), sep=",\n"); cat(")\n")' >> .Rprofile
    - Rscript -e 'sapply(names(test.jobs), check.test, pkg="data.table", simplify=FALSE)'
    - mkdir -p bus/$CI_BUILD_NAME
    # delete any existing non-dev version of data.table
    - rm -f bus/mirror-packages/cran/src/contrib/data.table_*.tar.gz
    - rm -f bus/mirror-packages/cran/bin/windows/contrib/$R_BIN_VERSION/data.table_*.zip
    - rm -f bus/mirror-packages/cran/bin/windows/contrib/$R_DEVEL_BIN_VERSION/data.table_*.zip
    #- rm -f bus/mirror-packages/cran/bin/macosx/el-capitan/contrib/$R_BIN_VERSION/data.table_*.tgz
    #- rm -f bus/mirror-packages/cran/bin/macosx/el-capitan/contrib/$R_DEVEL_BIN_VERSION/data.table_*.tgz
    # merge mirror-packages and R devel packages
    - mv bus/mirror-packages/cran bus/$CI_BUILD_NAME/
    # publish package sources
    - mkdir -p bus/$CI_BUILD_NAME/cran/library bus/$CI_BUILD_NAME/cran/doc
    - mv $(ls -1t bus/build/cran/src/contrib/data.table_*.tar.gz | head -n 1) bus/$CI_BUILD_NAME/cran/src/contrib
    - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="source"), type="source", fields="Revision", addFiles=TRUE)'
    # publish binaries
    - Rscript -e 'move.bin("test-r-release-windows", Sys.getenv("R_BIN_VERSION"), os.type="windows")'
    - Rscript -e 'move.bin("test-r-devel-windows", Sys.getenv("R_DEVEL_BIN_VERSION"), os.type="windows", silent=TRUE)'
    - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="win.binary", ver=Sys.getenv("R_BIN_VERSION")), type="win.binary", fields="Revision", addFiles=TRUE)'
    - Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="win.binary", ver=Sys.getenv("R_DEVEL_BIN_VERSION")), type="win.binary", fields="Revision", addFiles=TRUE)'
    #- Rscript -e 'move.bin("test-r-release-macosx", Sys.getenv("R_BIN_VERSION"), os.type="macosx")'
    #- Rscript -e 'move.bin("test-r-devel-macosx", Sys.getenv("R_DEVEL_BIN_VERSION"), os.type="macosx", silent=TRUE)'
    #- Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="mac.binary.el-capitan", ver=Sys.getenv("R_BIN_VERSION")), type="mac.binary.el-capitan", fields="Revision", addFiles=TRUE)'
    #- Rscript -e 'tools::write_PACKAGES(contrib.url("bus/integration/cran", type="mac.binary.el-capitan", ver=Sys.getenv("R_DEVEL_BIN_VERSION")), type="mac.binary.el-capitan", fields="Revision", addFiles=TRUE)'
    # install all pkgs to render html and double check successful installation of all devel packages
    - mkdir -p /tmp/opencran/library /tmp/opencran/doc/html
    - Rscript -e 'install.packages("data.table", dependencies=TRUE, lib="/tmp/opencran/library", repos=file.path("file:",normalizePath("bus/integration/cran")), INSTALL_opts="--html", quiet=TRUE)'
    - Rscript -e 'packageVersion("data.table", lib.loc="/tmp/opencran/library")'
    # CRAN style web/CRAN_web.css
    - wget -q -P bus/integration/cran/web https://cran.r-project.org/web/CRAN_web.css
    # web/packages/$pkg/index.html
    - Rscript -e 'sapply(rownames(installed.packages(lib.loc="/tmp/opencran/library", priority="NA")), package.index, lib.loc="/tmp/opencran/library")'
    # R docs, html, css, icons
    - Rscript -e 'doc.copy(repodir="/tmp/opencran")'
    # Update packages.html, rewrite file:/ to relative path
    - Rscript -e 'setwd("/tmp/opencran/doc/html"); make.packages.html(lib.loc="../../library", docdir="/tmp/opencran/doc"); tmp<-readLines(f<-"/tmp/opencran/doc/html/packages.html"); writeLines(gsub("file:///../../library","../../library", tmp, fixed=TRUE), f)'
    - mv /tmp/opencran/doc bus/integration/cran/
    # library html manual, vignettes
    - Rscript -e 'lib.copy(lib.from="/tmp/opencran/library")'
    # web/checks/$pkg/$job: 00install.out, 00check.log, *.Rout, memtest.csv, memtest.png
    - Rscript -e 'sapply(names(test.jobs), check.copy, simplify=FALSE)'
    # web/packages/$pkg/$pkg.pdf
    - Rscript -e 'pdf.copy("data.table", "test-r-release")'
    # web/checks/check_results_$pkg.html
    - Rscript -e 'check.index("data.table", names(test.jobs))'
    # cleanup artifacts from other jobs
    - mkdir tmpbus
    - mv bus/$CI_BUILD_NAME tmpbus
    - rm -r bus
    - mv tmpbus bus
  <<: *artifacts

.docker-template: &docker
  stage: deploy
  tags:
    - linux
  image: docker
  services:
  - docker:dind
  dependencies:
  - build
  before_script:
    - sed "s/SRC_IMAGE_NAME/$SRC_IMAGE_NAME/" < Dockerfile.in > Dockerfile
    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
  script:
    - docker build --pull -t "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" -f Dockerfile .
    - docker run --rm "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" Rscript -e 'cat(R.version.string, "\ndata.table revision", read.dcf(system.file("DESCRIPTION", package="data.table"), fields="Revision")[[1L]], "\n"); require(data.table); test.data.table()'
    - docker push "$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG"

docker-r-release: # publish docker image of data.table on R-release
  only:
    - master
  variables:
    SRC_IMAGE_NAME: "r-base-dev"
    IMAGE_NAME: "r-release"
    IMAGE_TAG: "latest"
  <<: *docker

docker-r-release-builder: # publish on R-release and OS dependencies for building Rmd vignettes
  only:
    - master
  variables:
    SRC_IMAGE_NAME: "r-builder"
    IMAGE_NAME: "r-release-builder"
    IMAGE_TAG: "latest"
  <<: *docker

docker-r-devel: # publish docker image of data.table on R-devel
  only:
    - master
  variables:
    SRC_IMAGE_NAME: "r-devel"
    IMAGE_NAME: "r-devel"
    IMAGE_TAG: "latest"
  <<: *docker

docker-tags: # publish only on tagged commits, we use tags for version
  only:
    - tags
  variables:
    SRC_IMAGE_NAME: "r-base-dev"
    IMAGE_NAME: "r-release"
    IMAGE_TAG: $CI_COMMIT_TAG
  <<: *docker

pages: # publish R repository, test jobs summaries, html documentation of all packages in repo
  stage: deploy
  environment: production
  tags:
    - linux
  only:
    - master
  image: ubuntu
  dependencies:
  - integration
  script:
    - mkdir -p public
    - cp -r bus/integration/cran/* public
    - cat public/src/contrib/PACKAGES
  artifacts: # publish when no failure
    expire_in: 2 weeks
    paths:
      - public