refactor quartonotebook (#8623)

* refactor quartonotebook

* Update meta.yml

* Add tests for existence of parametrized `meta`

* Add `meta` as implicit parameter

* Always use implicit parameters

Always include `meta`, `cpus` and `artifact_dir` as implicit parameters,
as they are always useful regardless of whether the user has additional
parameters for their notebook. This means the `use_parameters` input
boolean can be removed.

Also, remove the `input_dir` implicit variable, which is not needed for
Quarto but used in the original R Markdown / Jupyter notebooks in the
test data.

* Fix `meta.yml`

* Remove optional output on artifacts and params, and use artifact_dir value

* Fix linting

* Fix artifacts optional state. There may be no files in dir

* Update module outputs

* Reintroduce notebook as output so user can continue working on copy

---------

Co-authored-by: Erik Fasterius <[email protected]>

Author: mahesh-panchal

modules/nf-core/quartonotebook/main.nf

@@ -1,5 +1,3 @@
-include { dumpParamsYaml ; indentCodeBlock } from "./parametrize"
-
 // NB: You'll likely want to override this with a container containing all
 // required dependencies for your analyses. Or use wave to build the container
 // for you from the environment.yml You'll at least need Quarto itself,
@@ -15,58 +13,45 @@ process QUARTONOTEBOOK {
 
     input:
     tuple val(meta), path(notebook)
-    val parameters
+    val(parameters)
     path input_files
     path extensions
 
     output:
-    tuple val(meta), path("*.html"), emit: html
-    tuple val(meta), path("${notebook}"), emit: notebook
-    tuple val(meta), path("artifacts/*"), emit: artifacts, optional: true
-    tuple val(meta), path("params.yml"), emit: params_yaml, optional: true
-    tuple val(meta), path("_extensions"), emit: extensions, optional: true
-    path "versions.yml", emit: versions
+    tuple val(meta), path("*.html")                               , emit: html
+    tuple val(meta), path(notebook)                               , emit: notebook
+    tuple val(meta), path("params.yml")                           , emit: params_yaml
+    tuple val(meta), path("${notebook_parameters.artifact_dir}/*"), emit: artifacts  , optional: true
+    tuple val(meta), path("_extensions")                          , emit: extensions , optional: true
+    path "versions.yml"                                           , emit: versions
 
     when:
     task.ext.when == null || task.ext.when
 
     script:
     def args = task.ext.args ?: ''
     def prefix = task.ext.prefix ?: "${meta.id}"
-    def parametrize = task.ext.parametrize == null ? true : task.ext.parametrize
-    def implicit_params = task.ext.implicit_params == null ? true : task.ext.implicit_params
-    def meta_params = task.ext.meta_params == null ? true : task.ext.meta_params
-
-    // Dump parameters to yaml file.
-    // Using a YAML file over using the CLI params because
+    // Implicit parameters can be overwritten by supplying a value with parameters
+    notebook_parameters = [
+        meta: meta,
+        cpus: task.cpus,
+        artifact_dir: "artifacts",
+    ] + (parameters ?: [:])
+    // Parse parameters through a YAML file, which is better than CLI because:
     //  - No issue with escaping
     //  - Allows passing nested maps instead of just single values
     //  - Allows running with the language-agnostic `--execute-params`
-    def params_cmd = ""
-    def render_args = ""
-    if (parametrize) {
-        nb_params = [:]
-        if (implicit_params) {
-            nb_params["cpus"] = task.cpus
-            nb_params["artifact_dir"] = "artifacts"
-            nb_params["input_dir"] = "./"
-        }
-        if (meta_params) {
-            nb_params["meta"] = meta
-        }
-        nb_params += parameters
-        params_cmd = dumpParamsYaml(nb_params)
-        render_args = "--execute-params params.yml"
-    }
-    ( params_cmd ?
-    """
-    # Dump .params.yml heredoc (if applicable)
-    ${indentCodeBlock(params_cmd, 4)}
-    """ : "")
-    <<
+    def yamlBuilder = new groovy.yaml.YamlBuilder()
+    yamlBuilder(notebook_parameters)
+    def yaml_content = yamlBuilder.toString().tokenize('\n').join("\n    ")
     """
+    # Dump parameters to yaml file
+    cat <<- END_YAML_PARAMS > params.yml
+    ${yaml_content}
+    END_YAML_PARAMS
+
     # Create output directory
-    mkdir artifacts
+    mkdir "${notebook_parameters.artifact_dir}"
 
     # Set environment variables needed for Quarto rendering
     export XDG_CACHE_HOME="./.xdg_cache_home"
@@ -89,8 +74,8 @@ process QUARTONOTEBOOK {
     # Render notebook
     quarto render \\
         ${notebook} \\
-        ${render_args} \\
         ${args} \\
+        --execute-params params.yml \\
         --output ${prefix}.html
 
     cat <<-END_VERSIONS > versions.yml
@@ -102,6 +87,12 @@ process QUARTONOTEBOOK {
 
     stub:
     def prefix = task.ext.prefix ?: "${meta.id}"
+    // Implicit parameters can be overwritten by supplying a value with parameters
+    notebook_parameters = [
+        meta: meta,
+        cpus: task.cpus,
+        artifact_dir: "artifacts",
+    ] + (parameters ?: [:])
     """
     # Fix Quarto for Apptainer (see https://community.seqera.io/t/confusion-over-why-a-tool-works-in-docker-but-fails-in-singularity-when-the-installation-doesnt-differ-i-e-using-wave-micromamba/1244)
     # Note: This is needed in the stub for `quarto -v` to work.
@@ -113,6 +104,7 @@ process QUARTONOTEBOOK {
     set -u
 
     touch ${prefix}.html
+    touch params.yml
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":

modules/nf-core/quartonotebook/meta.yml

@@ -64,29 +64,29 @@ output:
           description: |
             Groovy Map containing sample information
             e.g. `[ id:'sample1', single_end:false ]`.
-      - ${notebook}:
+      - notebook:
           type: file
-          description: The original, un-rendered notebook.
-          pattern: "*.[qmd,ipynb,rmd]"
-  - artifacts:
+          description: The Quarto notebook that was rendered. Allows user to continue working on the notebook.
+          pattern: "*.{qmd}"
+  - params_yaml:
       - meta:
           type: map
           description: |
             Groovy Map containing sample information
             e.g. `[ id:'sample1', single_end:false ]`.
-      - artifacts/*:
+      - params.yml:
           type: file
-          description: Artifacts generated during report rendering.
+          description: Parameters used during report rendering.
           pattern: "*"
-  - params_yaml:
+  - artifacts:
       - meta:
           type: map
           description: |
             Groovy Map containing sample information
             e.g. `[ id:'sample1', single_end:false ]`.
-      - params.yml:
+      - ${notebook_parameters.artifact_dir}/*:
           type: file
-          description: Parameters used during report rendering.
+          description: Artifacts generated during report rendering.
           pattern: "*"
   - extensions:
       - meta:

modules/nf-core/quartonotebook/parametrize.nf

@@ -10,8 +10,6 @@ nextflow_process {
 
     test("test notebook - [qmd:r]") {
 
-        config "./no-parametrization.config"
-
         when {
             process {
                 """
@@ -42,16 +40,14 @@ nextflow_process {
 
     test("test notebook - [qmd:python]") {
 
-        config "./no-parametrization.config"
-
         when {
             process {
                 """
                 input[0] = [
                     [ id:'test' ], // meta map
                     file(params.modules_testdata_base_path + 'generic/notebooks/quarto/quarto_python.qmd', checkIfExists: true) // Notebook
                 ]
-                input[1] = [] // Parameters
+                input[1] = [:] // Parameters
                 input[2] = [] // Input files
                 input[3] = [] // Extensions
                 """
@@ -74,16 +70,14 @@ nextflow_process {
 
     test("test notebook - parametrized - [qmd:r]") {
 
-        config "./with-parametrization.config"
-
         when {
             process {
                 """
                 input[0] = [
                     [ id:'test' ], // meta map
                     file(params.modules_testdata_base_path + 'generic/notebooks/quarto/quarto_r.qmd', checkIfExists: true) // Notebook
                 ]
-                input[1] = [input_filename: "hello.txt", n_iter: 12] // parameters
+                input[1] = [input_filename: "hello.txt", n_iter: 12] // Parameters
                 input[2] = file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true) // Input files
                 input[3] = [] // Extensions
                 """
@@ -99,23 +93,22 @@ nextflow_process {
                     process.out.params_yaml
                 ).match() },
                 { assert path(process.out.html[0][1]).readLines().any { it.contains('Hello World 1') } },
+                { assert path(process.out.params_yaml[0][1]).readLines().any { it.contains('meta:') } },
             )
         }
 
     }
 
     test("test notebook - parametrized - [qmd:python]") {
 
-        config "./with-parametrization.config"
-
         when {
             process {
                 """
                 input[0] = [
                     [ id:'test' ], // meta map
                     file(params.modules_testdata_base_path + 'generic/notebooks/quarto/quarto_python.qmd', checkIfExists: true) // Notebook
                 ]
-                input[1] = [input_filename: "hello.txt", n_iter: 12] // parameters
+                input[1] = [input_filename: "hello.txt", n_iter: 12] // Parameters
                 input[2] = file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true) // Input files
                 input[3] = [] // Extensions
                 """
@@ -131,23 +124,22 @@ nextflow_process {
                     process.out.params_yaml
                 ).match() },
                 { assert path(process.out.html[0][1]).readLines().any { it.contains('Hello World 1') } },
+                { assert path(process.out.params_yaml[0][1]).readLines().any { it.contains('meta:') } },
             )
         }
 
     }
 
     test("test notebook - parametrized - [rmd]") {
 
-        config "./with-parametrization.config"
-
         when {
             process {
                 """
                 input[0] = [
                     [ id:'test' ], // meta map
                     file(params.modules_testdata_base_path + 'generic/notebooks/rmarkdown/rmarkdown_notebook.Rmd', checkIfExists: true) // notebook
                 ]
-                input[1] = [input_filename: "hello.txt", n_iter: 12] // parameters
+                input[1] = [input_filename: "hello.txt", n_iter: 12] // Parameters
                 input[2] = file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true) // Input files
                 input[3] = [] // Extensions
                 """
@@ -163,23 +155,22 @@ nextflow_process {
                     process.out.params_yaml
                 ).match() },
                 { assert path(process.out.html[0][1]).readLines().any { it.contains('Hello World 1') } },
+                { assert path(process.out.params_yaml[0][1]).readLines().any { it.contains('meta:') } },
             )
         }
 
     }
 
     test("test notebook - parametrized - [ipynb]") {
 
-        config "./with-parametrization.config"
-
         when {
             process {
                 """
                 input[0] = [
                     [ id:'test' ], // meta map
                     file(params.modules_testdata_base_path + 'generic/notebooks/jupyter/ipython_notebook.ipynb', checkIfExists: true) // notebook
                 ]
-                input[1] = [input_filename: "hello.txt", n_iter: 12] // parameters
+                input[1] = [input_filename: "hello.txt", n_iter: 12] // Parameters
                 input[2] = file(params.modules_testdata_base_path + 'generic/txt/hello.txt', checkIfExists: true) // Input files
                 input[3] = [] // Extensions
                 """
@@ -195,15 +186,14 @@ nextflow_process {
                     process.out.params_yaml
                 ).match() },
                 { assert path(process.out.html[0][1]).readLines().any { it.contains('Hello World') } },
+                { assert path(process.out.params_yaml[0][1]).readLines().any { it.contains('meta:') } },
             )
         }
 
     }
 
     test("test notebook - stub - [qmd:r]") {
 
-        config "./no-parametrization.config"
-
         options "-stub"
 
         when {