Merge pull request #127 from mrc-ide/mrc-5384

Support and docs for stan

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: hipercow
 Title: High Performance Computing
-Version: 1.0.22
+Version: 1.0.23
 Authors@R: c(person("Rich", "FitzJohn", role = c("aut", "cre"),
                     email = "[email protected]"),
              person("Wes", "Hinsley", role = "aut"),

R/parallel.R

@@ -19,7 +19,7 @@
 ##' `hipercow_resources`.
 ##'
 ##' For example, you could request 32 cores with `hipercow_resources`,
-##' and then calll `hipercow_parallel` with `cores_per_process = 4`,
+##' and then call `hipercow_parallel` with `cores_per_process = 4`,
 ##' and hipercow will create a local cluster with 8 processes, each
 ##' of which reporting `4` cores if that process calls
 ##' `hipercow_parallel_get_cores`.

R/task-purge.R

@@ -23,13 +23,13 @@
 ##'
 ##' # Consequences of deletion
 ##'
-##' A nonexhaustive list:
+##' A non-exhaustive list:
 ##'
 ##' * If you delete a task that is part of a [task_retry] chain, then
 ##'   all tasks (both upstream and downstream in that chain) are
 ##'   deleted
 ##' * Once we support task dependencies (mrc-4797), deleting tasks
-##'   will mark any unrun dependent task as impossible, or perhaps
+##'   will mark any not-yet-run dependent task as impossible, or perhaps
 ##'   delete it too, or prevent you from deleting the task; we've not
 ##'   decided yet
 ##' * You may have a bundle that references a task that you delete, in

R/zzz.R

@@ -3,7 +3,7 @@ cache <- new.env(parent = emptyenv())
 ##' The default set of environment variables used for all tasks.
 ##'
 ##' These are chosen to provide the best general experience. The default
-##' variables may be overriden globally by setting the hipercow.default_envvars
+##' variables may be overridden globally by setting the hipercow.default_envvars
 ##' option, or on a per-task and variable basis by assigning a different value
 ##' to the environment variable in question.
 ##'

README.md

@@ -22,6 +22,8 @@ This package supercedes didehpc (2015-2023).
 * Having problems? Check the [troubleshooting guide](https://mrc-ide.github.io/hipercow/articles/troubleshooting.html).
 * Check that everything is ok to go:
   - [Windows (wpia-hn)](https://mrc-ide.github.io/hipercow/articles/windows.html#does-it-work)
+* [All long-form documentation](https://mrc-ide.github.io/hipercow/articles/), available on the hipercow website
+* [Reference documentation for each function](https://mrc-ide.github.io/hipercow/reference/index.html)
 
 ## Issues
 

_pkgdown.yml

@@ -11,7 +11,12 @@ articles:
       - packages
       - parallel
       - troubleshooting
+
+  - title: Details
+    navbar: Details
+    contents:
       - details
+      - stan
 
   - title: Clusters
     navbar: Clusters

drivers/windows/DESCRIPTION

@@ -1,6 +1,6 @@
 Package: hipercow.windows
 Title: DIDE HPC Support for Windows
-Version: 1.0.22
+Version: 1.0.23
 Authors@R: c(person("Rich", "FitzJohn", role = c("aut", "cre"),
                     email = "[email protected]"),
              person("Wes", "Hinsley", role = "aut"),

drivers/windows/R/stan.R

@@ -0,0 +1,17 @@
+cmdstan_install <- function(cmdstan_version = NULL,
+                            cmdstanr_use_remotes = FALSE,
+                            overwrite = FALSE,
+                            root = NULL) {
+  path_script <- "hipercow/install-cmdstan.R"
+  path_root <- hipercow:::hipercow_root(root)$path$root
+  path_script_abs <- file.path(path_root, path_script)
+
+  data <- list(cmdstan_version = deparse1(cmdstan_version),
+               cmdstanr_use_remotes = deparse1(cmdstanr_use_remotes),
+               overwrite = deparse1(overwrite))
+  template <- read_template("stan.R")
+  writelines_if_different(glue_whisker(template, data),
+                          path_script_abs)
+
+  hipercow::hipercow_provision("script", script = path_script, root = root)
+}

drivers/windows/R/zzz.R

@@ -25,5 +25,5 @@ cache <- new.env(parent = emptyenv())
 
 
 DEFAULT_ENVVARS <- hipercow::hipercow_envvars( # nolint
-  "CMDSTAN" = "I:/cmdstan",
-  "CMDSTANR_USE_R_TOOLS" = "TRUE")
+  "CMDSTAN" = "I:/cmdstan/cmdstan-2.35.0",
+  "CMDSTANR_USE_RTOOLS" = "TRUE")

drivers/windows/inst/templates/stan.R

@@ -0,0 +1,22 @@
+message("Installing most recent version of 'cmdstanr' package")
+local({
+  lib <- tempfile()
+  loadNamespace("remotes")
+  dir.create(lib, FALSE, TRUE)
+  .libPaths(lib)
+  if ({{cmdstanr_use_remotes}}) {
+    remotes::install_gituhb("stan-dev/cmdstanr", lib = lib)
+  } else {
+    install.packages("cmdstanr",
+                     repos = c("https://stan-dev.r-universe.dev",
+                               "https://cloud.r-project.org"),
+                     lib = lib)
+  }
+  loadNamespace("cmdstanr")
+})
+Sys.setenv(CMDSTANR_USE_RTOOLS = "TRUE",
+           RTOOLS44_HOME = "I:/rtools/rtools44")
+dir.create("I:/cmdstan", FALSE, TRUE)
+cmdstanr::install_cmdstan(dir = "I:/cmdstan",
+                          version = {{cmdstan_version}},
+                          overwrite = {{overwrite}})

inst/WORDLIST

@@ -1,22 +1,33 @@
 BitWarden
+BuildQueue
 CRAN's
+CmdStan
 DIDE
 DIDE's
 HPC
 ICT
 InfiniBand
+JDK
+OpenJDK
 OpenMP
+PBS
 R's
 RCS
 RStudio
+RTools
 Rmd
 Rtools
+StackOverflow
+SystemRoot
 ZScaler
 alloc
+azul
+backends
 callr
 chmod
 cli
 cli's
+cmdstanr
 codecov
 com
 conan
@@ -37,14 +48,19 @@ lockfile
 macOS
 mountname
 mountpoint
+mrc
 pkgdepends
 redis
 redux
 reestablished
 renv
 rfiglet
 rlang
+rrq
+rstan
 src
+stan
+standalone
 std
 supercedes
 th

vignettes/administration.Rmd

@@ -60,19 +60,26 @@ It is assumed that you have the development version installed locally yourself,
 
 # Recreating the vignettes
 
-These need to be run in a network share so set an environment variable like
+These need to be run in a network share so set an environment variable:-
 
 ```
 HIPERCOW_VIGNETTE_ROOT=/path/to/share
 ```
 
+or on Windows, a mapped drive, such as
+
+```
+HIPERCOW_VIGNETTE_ROOT=Q:/hipercow_vignettes
+```
+
 in your `.Renviron` indicating where we should work.  We'll make lots of directories here.
 
 Each vignette can be built by running (ideally in a fresh session with the working directory as the package root)
 
 ```r
 knitr::knit("vignettes_src/windows.Rmd", "vignettes/windows.Rmd")
 knitr::knit("vignettes_src/packages.Rmd", "vignettes/packages.Rmd")
+knitr::knit("vignettes_src/stan.Rmd", "vignettes/stan.Rmd")
 ```
 
 which generates a new `windows.Rmd` file within `vignettes/` that contains no runnable code and so can be safely run on CI.
@@ -105,7 +112,7 @@ echo Using RTOOLS43_HOME = %RTOOLS43_HOME%
 echo Using JAVA_HOME = %JAVA_HOME%
 ```
 
-These get copied to `C:\Windows` on each cluster node, using the HPC Cluster Manager. You simply select the nodes, right click, "Run Commmand", and look in the history for previous copy commands, to copy the batch files in to %SystemRoot% - hence they are always in the path on every node. You also need to edit the permissions to allow everyone to read the file, by running this on each cluster node:-
+These get copied to `C:\Windows` on each cluster node, using the HPC Cluster Manager. You simply select the nodes, right click, "Run Command", and look in the history for previous copy commands, to copy the batch files in to %SystemRoot% - hence they are always in the path on every node. You also need to edit the permissions to allow everyone to read the file, by running this on each cluster node:-
 
 ```
 cacls %SYSTEMROOT%\setr64_*.bat /e /p everyone:r
@@ -136,3 +143,28 @@ This varies each time, and especially between different major R versions. But es
 * Unzip into `I:\Java`, and then rename the folder it makes it to a tidier version number, such as `21.0.2`
 * Update `I:\Java\latest.txt` to contain that new version string.
 * JAVA_HOME will then point to the root of the Java JDK, and `rJava` and `xlsx` will work happily.
+
+## stan
+
+See `vignette("stan")` for general comments about stan.
+
+To update the installation of `CmdStan` on the `I:/` (and generally available), run:
+
+```r
+hipercow::hipercow_init(driver = "windows")
+hipercow.windows:::cmdstan_install()
+```
+
+Users should not run this themselves into their home directories as the installation is about 1GB
+
+There are influential environment variables set at the driver level which prevent stan from breaking our Rtools installation.  These are
+
+```{echo = FALSE}
+hipercow.windows:::DEFAULT_ENVVARS[c("name", "value")]
+```
+
+For details see:
+
+* https://github.com/stan-dev/cmdstanr/issues/979
+* https://github.com/stan-dev/cmdstanr/pull/978
+* https://github.com/stan-dev/cmdstanr/pull/980

vignettes/details.Rmd

@@ -41,7 +41,7 @@ A number, being the maximum size object (in bytes) that we will save when creati
 
 ### `hipercow.default_envvars`
 
-A `hipercow_envvars` object, with environment variables that will be added to every task created. Hipercow sets a few environment variables by default to improve the user experience. These variables can be overriden per-task by passing a `envvars` parameter when creating tasks, or globally by defining this option.
+A `hipercow_envvars` object, with environment variables that will be added to every task created. Hipercow sets a few environment variables by default to improve the user experience. These variables can be overridden per-task by passing a `envvars` parameter when creating tasks, or globally by defining this option.
 
 Environment variables for a task are computed by collecting (in increasing order of preference):