Update Dockerqueue with improvements of master (#145)

* robustify doJobCollection

* new backports

* Update Travis (#111)

* selectively import backports

* fix travis

* removed remotes from desc

* Warn about factors in addExperiments (#112)

* fix appveyor

* relative path handling

* improved docs for reduceResults

* paths

* added tibble to suggests to check xrefs

* improve getLogFiles

* fix logs

* fixed visibility in shared object

* Add support for `partition` in slurm template (#118)

Agreed. Thanks for the PR.

* fix testJob for win?

* simplified path lookup

* added some tests

* use %chin%

* added some comments

* docs

* docs

* fixed a bug where time stamps where rounded to seconds

* fixed printing of remaining runtimes

* fixed error message

* use withr

* warn instead of stop on the master if not all depenencies are satisfied (fixes #122)

* fix travis

* removeExperiments returns ids invisibly

* fix heuristic of getJobPars

* improved message

* always use winslash

* file.path/winslash

* vignettes

* updated README [ci skip]

* updated README [ci skip]

* docs

* depend on withr 2.0.0

* Job names (#124)

* updated NEWS

* increase version number for CRAN release

* vignette and docs

* vignette

* docs

* NEWS formating

* docs

* increase version number after CRAN release

* improved docs of reduceResults (fixes #116)

* fix for #135

* added test, fixes #135

* fix test

* README

* small fix in printer

* read-only mode (#137)

* ClusterFunctions rework: Do not swallow system errors (#138)

* added update routine for job.name

* NEWS

* improved control over sleep duration; nodename in cfSlurm

* Wait for jobs rework (#140)

* wait for brewed file before submit

* version, docs

* increased version after release

* updated travis

* fix links in vignette (#142)

* version 0.9.6

* added test for algo caching

* test for PM

Author: jakob-r

.Rbuildignore

@@ -4,11 +4,11 @@
 \.swp$
 ^\.ignore$
 ^\.editorconfig$
-^.travis.yml$
+^\.travis\.yml$
 ^man-roxygen$
 ^appveyor\.yml$
 ^.*\.Rproj$
 ^\.Rproj\.user$
 ^docs$
 ^paper$
-^_pkgdown.yml$
+^_pkgdown\.yml$

.gitignore

@@ -1,5 +1,8 @@
 .DS_Store
-inst/doc
-src/*.so
-src/*.o
-vignettes/*.html
+/inst/doc
+/src/*.so
+/src/*.o
+/vignettes/*.html
+/vignettes/*.pdf
+/.Rproj.user
+/batchtools.Rproj

.travis.yml

@@ -3,13 +3,23 @@ sudo: false
 language: r
 cache: packages
 
+env:
+  - _R_CHECK_LENGTH_1_CONDITION_=true
+
+addons:
+  apt:
+    packages:
+      - libopenmpi-dev
+      - openmpi-bin
+
 r:
   - oldrel
   - release
   - devel
 
 r_packages:
   - covr
+  - Rmpi
 
 after_success:
   - if [[ "${TRAVIS_R_VERSION_STRING}" == "release" ]]; then Rscript -e 'covr::coveralls()'; fi

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: batchtools
 Title: Tools for Computation on Batch Systems
-Version: 0.9.3-9000
+Version: 0.9.6-9000
 Authors@R: c(
     person("Michel", "Lang", NULL, "[email protected]", role = c("cre", "aut")),
     person("Bernd", "Bischl", NULL, "[email protected]", role = "aut"),
@@ -28,18 +28,19 @@ Depends:
     R (>= 3.0.0),
     data.table (>= 1.9.8)
 Imports:
-    backports (>= 1.0.4),
+    backports (>= 1.1.0),
     base64url (>= 1.1),
     brew,
-    checkmate (>= 1.8.2),
+    checkmate (>= 1.8.3),
     digest (>= 0.6.9),
     parallel,
     progress (>= 1.1.1),
     R6,
     rappdirs,
     stats,
     stringi,
-    utils
+    utils,
+    withr (>= 2.0.0)
 Suggests:
     debugme,
     e1071,
@@ -51,6 +52,7 @@ Suggests:
     rmarkdown,
     rpart,
     snow,
-    testthat
+    testthat,
+    tibble
 VignetteBuilder: knitr
 RoxygenNote: 6.0.1

NAMESPACE

@@ -61,6 +61,7 @@ export(findTagged)
 export(getAlgorithmIds)
 export(getDefaultRegistry)
 export(getErrorMessages)
+export(getJobNames)
 export(getJobPars)
 export(getJobResources)
 export(getJobStatus)
@@ -109,6 +110,7 @@ export(runHook)
 export(runOSCommand)
 export(saveRegistry)
 export(setDefaultRegistry)
+export(setJobNames)
 export(showLog)
 export(sjoin)
 export(submitJobs)
@@ -118,7 +120,6 @@ export(syncRegistry)
 export(testJob)
 export(ujoin)
 export(waitForJobs)
-import(backports)
 import(checkmate)
 import(data.table)
 import(stringi)
@@ -133,6 +134,10 @@ importFrom(rappdirs,user_config_dir)
 importFrom(stats,pexp)
 importFrom(stats,predict)
 importFrom(stats,runif)
+importFrom(withr,local_dir)
+importFrom(withr,local_options)
+importFrom(withr,with_dir)
+importFrom(withr,with_seed)
 useDynLib(batchtools,c_binpack)
 useDynLib(batchtools,c_lpt)
 useDynLib(batchtools,count_not_missing)

NEWS.md

@@ -1,12 +1,41 @@
+# batchtools 0.9.6
+
+* Fixed a bug where the wrong problem was retrieved from the cache. This was only triggered for chunked jobs in
+  combination with an `ExperimentRegistry`.
+
+# batchtools 0.9.5
+
+* Added a missing routine to upgrade registries created with batchtools prior to v0.9.3.
+* Fixed a bug where the registry could not be synced if jobs failed during initialization (#135).
+* The sleep duration for `waitForJobs()` and `submitJobs()` can now be set via the configuration file.
+* A new heuristic will try to detect if the registry has been altered by a simultaneously running R session.
+  If this is detected, the registry in the current session will be set to a read-only state.
+* `waitForJobs()` has been reworked to allow control over the heuristic to detect expired jobs.
+  Jobs are treated as expired if they have been submitted but are not detected on the system for `expire.after` iterations
+  (default 3 iterations, before 1 iteration).
+* New argument `writeable` for `loadRegistry()` to allow loading registries explicitly as read-only.
+* Removed argument `update.paths` from `loadRegistry()`.
+  Paths are always updated, but the registry on the file system remains unchanged unless loaded in read-write mode.
+* `ClusterFunctionsSlurm` now come with an experimental nodename argument. If set, all communication with the master is
+  handled via SSH which effectively allows you to submit jobs from your local machine instead of the head node.
+  Note that mounting the file system (e.g., via SSHFS) is mandatory.
+
 # batchtools 0.9.4
 
 * Fixed handling of `file.dir` with special chars like whitespace.
+* All backward slashes will now be converted to forward slashes on windows.
 * Fixed order of arguments in `findExperiments()` (argument `ids` is now first).
 * Removed code to upgrade registries created with versions prior to v0.9.0 (first CRAN release).
+* `addExperiments()` now warns if a design is passed as `data.frame` with factor columns and `stringsAsFactors` is `TRUE`.
+* Added functions `setJobNames()` and `getJobNames()` to control the name of jobs on batch systems.
+  Templates should be adapted to use `job.name` instead of `job.hash` for naming.
+* Argument `flatten` of `getJobResources()`, `getJobPars()` and `getJobTable()` is deprecated and will be removed.
+  Future versions of the functions will behave like `flatten` is set to `FALSE` explicitly.
+  Single resources/parameters must be extracted manually (or with `tidyr::unnest()`).
 
 # batchtools 0.9.3
 
-* Running jobs now are also included while querying for status "started". This affects `findStarted()`, `findNotStarted` and `getStatus()`.
+* Running jobs now are also included while querying for status "started". This affects `findStarted()`, `findNotStarted()` and `getStatus()`.
 * `findExperiments()` now performs an exact string match (instead of matching substrings) for patterns specified via `prob.name` and `algo.name`.
   For substring matching, use `prob.pattern` or `algo.pattern`, respectively.
 * Changed arguments for `reduceResultsDataTable()`
@@ -37,11 +66,11 @@
 * Fixed handling of `NULL` results in `reduceResultsList()`
 * Fixed key lookup heuristic join functions.
 * Fixed a bug where `getJobTable()` returned `difftimes` with the wrong unit (e.g., in minutes instead of seconds).
-* Deactivated swap allocation for `clusterFunctionsDocker()`.
+* Deactivated swap allocation for `ClusterFunctionsDocker`.
 * The package is now more patient while communicating with the scheduler or file system by using a timeout-based approach.
   This should make the package more reliable and robust under heavy load.
 
 # batchtools 0.9.0
 
 Initial CRAN release.
-See this [vignette](https://mllg.github.io/batchtools/articles/v01_Migration) for a brief comparison with [BatchJobs](https://cran.r-project.org/package=BatchJobs)/[BatchExperiments](https://cran.r-project.org/package=BatchExperiments).
+See the vignette for a brief comparison with [BatchJobs](https://cran.r-project.org/package=BatchJobs)/[BatchExperiments](https://cran.r-project.org/package=BatchExperiments).

R/Algorithm.R

@@ -45,7 +45,7 @@ addAlgorithm = function(name, fun = NULL, reg = getDefaultRegistry())  {
 
   info("Adding algorithm '%s'", name)
   algo = setClasses(list(fun = fun, name = name), "Algorithm")
-  writeRDS(algo, file = getAlgorithmURI(reg$file.dir, name))
+  writeRDS(algo, file = getAlgorithmURI(reg, name))
   reg$defs$algorithm = addlevel(reg$defs$algorithm, name)
   saveRegistry(reg)
   invisible(algo)
@@ -64,7 +64,7 @@ removeAlgorithms = function(name, reg = getDefaultRegistry()) {
     job.ids = filter(def.ids, reg$status, "job.id")
 
     info("Removing Algorithm '%s' and %i corresponding jobs ...", nn, nrow(job.ids))
-    file.remove.safely(getAlgorithmURI(reg$file.dir, nn))
+    file.remove.safely(getAlgorithmURI(reg, nn))
     reg$defs = reg$defs[!def.ids]
     reg$status = reg$status[!job.ids]
     reg$defs$algorithm = rmlevel(reg$defs$algorithm, nn)
@@ -80,7 +80,3 @@ getAlgorithmIds = function(reg = getDefaultRegistry()) {
   assertExperimentRegistry(reg)
   levels(reg$defs$algorithm)
 }
-
-getAlgorithmURI = function(file.dir, name) {
-  file.path(file.dir, "algorithms", mangle(name))
-}

R/ExperimentRegistry.R

@@ -52,8 +52,8 @@ makeExperimentRegistry = function(file.dir = "registry", work.dir = getwd(), con
   reg = makeRegistry(file.dir = file.dir, work.dir = work.dir, conf.file = conf.file,
     packages = packages, namespaces = namespaces, source = source, load = load, seed = seed, make.default = make.default)
 
-  dir.create(file.path(reg$file.dir, "problems"))
-  dir.create(file.path(reg$file.dir, "algorithms"))
+  dir.create(fp(reg$file.dir, "problems"))
+  dir.create(fp(reg$file.dir, "algorithms"))
 
   reg$status$repl = integer(0L)
   reg$defs$problem = factor(character(0L))
@@ -68,13 +68,14 @@ makeExperimentRegistry = function(file.dir = "registry", work.dir = getwd(), con
 #' @export
 print.ExperimentRegistry = function(x, ...) {
   cat("Experiment Registry\n")
-  catf("  Name      : %s", x$cluster.functions$name)
+  catf("  Backend   : %s", x$cluster.functions$name)
   catf("  File dir  : %s", x$file.dir)
   catf("  Work dir  : %s", x$work.dir)
   catf("  Jobs      : %i", nrow(x$status))
   catf("  Problems  : %i", nlevels(x$defs$problem))
   catf("  Algorithms: %i", nlevels(x$defs$algorithm))
   catf("  Seed      : %i", x$seed)
+  catf("  Writeable : %s", x$writeable)
 }
 
 assertExperimentRegistry = function(reg, writeable = FALSE, sync = FALSE, running.ok = TRUE) {

R/Export.R

@@ -37,11 +37,11 @@ batchExport = function(export = list(), unexport = character(0L), reg = getDefau
   assertList(export, names = "named")
   assertCharacter(unexport, any.missing = FALSE, min.chars = 1L)
 
-  path = file.path(reg$file.dir, "exports")
+  path = fp(reg$file.dir, "exports")
 
   if (length(export) > 0L) {
     nn = names(export)
-    fn = file.path(path, mangle(nn))
+    fn = fp(path, mangle(nn))
     found = file.exists(fn)
     if (any(!found))
       info("Exporting new objects: '%s' ...", stri_flatten(nn[!found], "','"))
@@ -51,13 +51,13 @@ batchExport = function(export = list(), unexport = character(0L), reg = getDefau
   }
 
   if (length(unexport) > 0L) {
-    fn = file.path(path, mangle(unexport))
+    fn = fp(path, mangle(unexport))
     found = file.exists(fn)
     if (any(found))
       info("Un-exporting exported objects: '%s' ...", stri_flatten(unexport[found], "','"))
     file.remove.safely(fn[found])
   }
 
   fns = list.files(path, pattern = "\\.rds")
-  invisible(data.table(name = unmangle(fns), uri = file.path(path, fns)))
+  invisible(data.table(name = unmangle(fns), uri = fp(path, fns)))
 }