mrc-5027: pbs driver

.github/workflows/check-linux.yaml

@@ -0,0 +1,54 @@
+# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
+# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+
+name: check-linux
+
+jobs:
+  check-linux:
+    runs-on: ${{ matrix.config.os }}
+
+    name: ${{ matrix.config.os }} (${{ matrix.config.r }})
+
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - {os: ubuntu-latest,   r: 'devel', http-user-agent: 'release'}
+          - {os: ubuntu-latest,   r: 'release'}
+          - {os: ubuntu-latest,   r: 'oldrel-1'}
+
+    # Shorter timeout to prevent mac builders hanging for 6 hours!
+    timeout-minutes: 30
+
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+      R_KEEP_PKG_SOURCE: yes
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: r-lib/actions/setup-pandoc@v2
+
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          r-version: ${{ matrix.config.r }}
+          http-user-agent: ${{ matrix.config.http-user-agent }}
+          use-public-rspm: true
+
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          extra-packages: |
+            any::rcmdcheck
+            local::../..
+          needs: check
+          working-directory: drivers/linux
+
+      - uses: r-lib/actions/check-r-package@v2
+        with:
+          upload-snapshots: true
+          working-directory: drivers/linux

drivers/linux/.Rbuildignore

@@ -0,0 +1,23 @@
+^scripts$
+^Makefile$
+^README\.Rmd$
+^\.travis\.yml$
+^docs$
+^\.lintr$
+^tests/testthat/.*\.o$
+^tests/testthat/.*\.so$
+^tests/testthat/.*\.dll$
+\.dylib$
+^appveyor\.yml$
+^docker$
+^\.hadolint\.yaml$
+\.valgrind_ignore$
+^scripts$
+\.gcda$
+\.gcno$
+^pkgdown$
+^LICENSE\.md$
+^buildkite$
+^\.covrignore$
+^\.github$
+\.*gcov$

drivers/linux/.github/.gitignore

@@ -0,0 +1 @@
+*.html

drivers/linux/.github/workflows/R-CMD-check.yaml

@@ -0,0 +1,52 @@
+# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
+# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+
+name: R-CMD-check
+
+jobs:
+  R-CMD-check:
+    runs-on: ${{ matrix.config.os }}
+
+    name: ${{ matrix.config.os }} (${{ matrix.config.r }})
+
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - {os: macos-latest,   r: 'release'}
+          - {os: windows-latest, r: 'release'}
+          - {os: ubuntu-latest,   r: 'devel', http-user-agent: 'release'}
+          - {os: ubuntu-latest,   r: 'release'}
+          - {os: ubuntu-latest,   r: 'oldrel-1'}
+
+    # Shorter timeout to prevent mac builders hanging for 6 hours!
+    timeout-minutes: 30
+
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+      R_KEEP_PKG_SOURCE: yes
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: r-lib/actions/setup-pandoc@v2
+
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          r-version: ${{ matrix.config.r }}
+          http-user-agent: ${{ matrix.config.http-user-agent }}
+          use-public-rspm: true
+
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          extra-packages: any::rcmdcheck
+          needs: check
+
+      - uses: r-lib/actions/check-r-package@v2
+        with:
+          upload-snapshots: true

drivers/linux/.github/workflows/pkgdown.yaml

@@ -0,0 +1,46 @@
+# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
+# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+  release:
+    types: [published]
+  workflow_dispatch:
+
+name: pkgdown
+
+jobs:
+  pkgdown:
+    runs-on: ubuntu-latest
+    # Only restrict concurrency for non-PR jobs
+    concurrency:
+      group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }}
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: r-lib/actions/setup-pandoc@v2
+
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          use-public-rspm: true
+
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          extra-packages: any::pkgdown, local::.
+          needs: website
+
+      - name: Build site
+        run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE)
+        shell: Rscript {0}
+
+      - name: Deploy to GitHub pages 🚀
+        if: github.event_name != 'pull_request'
+        uses: JamesIves/[email protected]
+        with:
+          clean: false
+          branch: gh-pages
+          folder: docs

drivers/linux/.github/workflows/test-coverage.yaml

@@ -0,0 +1,50 @@
+# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
+# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+
+name: test-coverage
+
+jobs:
+  test-coverage:
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          use-public-rspm: true
+
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          extra-packages: any::covr
+          needs: coverage
+
+      - name: Test coverage
+        run: |
+          covr::codecov(
+            quiet = FALSE,
+            clean = FALSE,
+            install_path = file.path(Sys.getenv("RUNNER_TEMP"), "package")
+          )
+        shell: Rscript {0}
+
+      - name: Show testthat output
+        if: always()
+        run: |
+          ## --------------------------------------------------------------------
+          find ${{ runner.temp }}/package -name 'testthat.Rout*' -exec cat '{}' \; || true
+        shell: bash
+
+      - name: Upload test results
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-test-failures
+          path: ${{ runner.temp }}/package

drivers/linux/.gitignore

@@ -0,0 +1,17 @@
+.Rproj.user
+.Rhistory
+.RData
+.Ruserdata
+.DS_Store
+TODO.md
+docs
+*.o
+*.so
+*.dll
+*.dylib
+*.gcda
+*.gcno
+*.gcov
+.valgrind_ignore
+inst/doc
+pkgdown

drivers/linux/.lintr

@@ -0,0 +1,7 @@
+linters: with_defaults(
+    object_length_linter = NULL,
+    object_usage_linter = NULL,
+    todo_comment_linter = NULL,
+    cyclocomp_linter = NULL
+    )
+exclusions: list("tests/testthat.R", "R/cpp11.R")

drivers/linux/DESCRIPTION

@@ -0,0 +1,24 @@
+Package: hipercow.linux
+Title: Generic Linux Support for 'hipercow'
+Version: 0.1.0
+Authors@R: c(person("Rich", "FitzJohn", role = c("aut", "cre"),
+                    email = "[email protected]"),
+             person("Wes", "Hinsley", role = "aut"),
+             person("Imperial College of Science, Technology and Medicine",
+                    role = "cph"))
+Description: Driver for using any Linux cluster (running pbs or
+    slurm).  Specific cluster drivers will be needed to set up your
+    cluster, but these will be tiny packages that mostly provide a
+    small amount of configuration and additional tooling.
+License: MIT + file LICENSE
+Encoding: UTF-8
+Roxygen: list(markdown = TRUE)
+RoxygenNote: 7.3.1
+URL: https://github.com/mrc-ide/hipercow, https://mrc-ide.github.io/hipercow
+BugReports: https://github.com/mrc-ide/hipercow/issues
+Imports:
+    cli,
+    hipercow
+Suggests:
+    testthat (>= 3.0.0)
+Config/testthat/edition: 3

drivers/linux/LICENSE

@@ -0,0 +1,2 @@
+YEAR: 2021
+COPYRIGHT HOLDER: Imperial College of Science, Technology and Medicine

drivers/linux/Makefile

@@ -0,0 +1,34 @@
+PACKAGE := $(shell grep '^Package:' DESCRIPTION | sed -E 's/^Package:[[:space:]]+//')
+RSCRIPT = Rscript --no-init-file
+
+all:
+	${RSCRIPT} -e 'pkgbuild::compile_dll()'
+
+test:
+	${RSCRIPT} -e 'devtools::test()'
+
+roxygen:
+	@mkdir -p man
+	${RSCRIPT} -e "devtools::document()"
+
+install:
+	R CMD INSTALL .
+
+build:
+	R CMD build .
+
+README.md: README.Rmd
+	Rscript -e 'devtools::load_all(); knitr::knit("README.Rmd")'
+	sed -i.bak 's/[[:space:]]*$$//' README.md
+	rm -f [email protected]
+
+check:
+	_R_CHECK_CRAN_INCOMING_=FALSE make check_all
+
+check_all:
+	${RSCRIPT} -e "rcmdcheck::rcmdcheck(args = c('--as-cran', '--no-manual'))"
+
+clean:
+	rm -f src/*.o src/*.so src/*.gcda src/*.gcno src/*.gcov
+
+.PHONY: clean all test document install

drivers/linux/NAMESPACE

@@ -0,0 +1,2 @@
+# Generated by roxygen2: do not edit by hand
+

drivers/linux/R/batch.R

@@ -0,0 +1,73 @@
+write_batch_task_run <- function(task_id, config, resources, path_root) {
+  data <- template_data(config, resources, path_root)
+  data$task_id <- task_id
+  template_name <- sprintf("%s/task_run.sh", config$manager)
+  str <- glue_whisker(read_template(template_name), data)
+  path <- file.path(path_root, "hipercow", "tasks", task_id, BATCH_RUN)
+  writeLines(str, path)
+  path
+
+}
+
+
+read_template <- function(name) {
+  read_lines(hipercow_linux_file(sprintf("templates/%s", name)))
+}
+
+
+template_data <- function(config, resources, path_root) {
+  ## Semicolon delimited list on windows; see "Managing libraries" in
+  ## https://cran.r-project.org/doc/manuals/r-release/R-admin.html
+  hipercow_library <- paste(config$path_lib, path_bootstrap(config), sep = ":")
+
+  ## TODO: default walltime should come from the configuration
+  walltime <- walltime_with_default(resources, "01:00:00")
+  ## TODO: treatment of cores is quite hard, especially where cores =
+  ## Inf wants to select a single node.  There's also issues around
+  ## "mpinodes" and issues around the complete lack of any sort of
+  ## usable documentation.
+  cores <- resources$cores
+  stopifnot(is.finite(cores))
+
+  memory <- memory_with_default(resources, "4")
+
+  ## TODO: we're going to need to query this periodically and find the
+  ## best match.  Running 'module avail R/' gets us most of the way
+  ## there but we need this in machine readable format and also limit
+  ## to those in the "prod" set..  The prod module will be ict
+  ## specific so that needs to go into the configuration too.
+  r_module <- "R/4.3.2-gfbf-2023a"
+
+  list(
+    hostname = hostname(),
+    date = as.character(Sys.time()),
+    hipercow_version = hipercow_version(),
+    r_module = r_module,
+    walltime = walltime,
+    cores = cores,
+    memory = memory,
+    hipercow_library = hipercow_library)
+}
+
+
+
+walltime_with_default <- function(resources, default) {
+  if (is.null(resources$max_runtime)) {
+    return(default)
+  }
+  mins <- resources$max_runtime %% 60
+  hours <- resources$max_runtime %/% 60
+  sprintf("%02d:02d:00", hours, mins)
+}
+
+
+memory_with_default <- function(resources, default) {
+  if (!is.null(resources$memory_per_node)) {
+    return(resources$memory_per_node)
+  }
+  if (!is.null(resources$memory_per_process)) {
+    stopifnot(is.finite(resources$cores))
+    return(resources$memory_per_process * resources$cores)
+  }
+  default
+}