diff --git a/docs/en/docs/side_quests/debugging/index.md b/docs/en/docs/side_quests/debugging/index.md index fcd2362b87..5e8608a4f7 100644 --- a/docs/en/docs/side_quests/debugging/index.md +++ b/docs/en/docs/side_quests/debugging/index.md @@ -2415,7 +2415,7 @@ Now it's time to put the systematic debugging approach into practice. The workfl ``` **Fix:** Take the output from the previous process ```groovy linenums="88" - handleFiles(heavyProcess.out) + file_ch = handleFiles(heavy_ch) ``` With that, the whole workflow should run. @@ -2439,7 +2439,6 @@ Now it's time to put the systematic debugging approach into practice. The workfl * Process with input/output mismatch */ process processFiles { - publishDir "${params.output}/processed", mode: 'copy' input: tuple val(sample_id), path(input_file) @@ -2458,7 +2457,6 @@ Now it's time to put the systematic debugging approach into practice. The workfl * Process with resource issues */ process heavyProcess { - publishDir "${params.output}/heavy", mode: 'copy' time '100 s' @@ -2481,7 +2479,6 @@ Now it's time to put the systematic debugging approach into practice. The workfl * Process with file handling issues */ process handleFiles { - publishDir "${params.output}/files", mode: 'copy' input: path input_file @@ -2501,7 +2498,7 @@ Now it's time to put the systematic debugging approach into practice. The workfl * Main workflow with channel issues */ workflow { - + main: // Channel with incorrect usage input_ch = channel .fromPath(params.input) @@ -2512,7 +2509,24 @@ Now it's time to put the systematic debugging approach into practice. The workfl heavy_ch = heavyProcess(input_ch.map{it[0]}) - handleFiles(heavyProcess.out) + file_ch = handleFiles(heavy_ch) + + publish: + processed = processed_ch + heavy = heavy_ch + files = file_ch + } + + output { + processed { + path 'processed' + } + heavy { + path 'heavy' + } + files { + path 'files' + } } ``` diff --git a/docs/en/docs/side_quests/dev_environment/index.md b/docs/en/docs/side_quests/dev_environment/index.md index 7d38fc2466..66f5ce3e1b 100644 --- a/docs/en/docs/side_quests/dev_environment/index.md +++ b/docs/en/docs/side_quests/dev_environment/index.md @@ -283,7 +283,6 @@ Efficient navigation is crucial when working with complex workflows spanning mul ```groovy title="basic_workflow.nf" linenums="3" process FASTQC { tag "${sample_id}" - publishDir "${params.output_dir}/fastqc", mode: 'copy' input: tuple val(sample_id), path(reads) @@ -417,13 +416,13 @@ This is invaluable when: Sometimes you need to find where specific patterns are used across your entire project. Press `Ctrl/Cmd+Shift+F` to open the search panel. -Try searching for `publishDir` across the workspace: +Try searching for `container` across the workspace: ![Project search](img/project_search.png) -This shows you every file that uses publish directories, helping you: +This shows you every file that uses the container directive, helping you: -- Understand output organization patterns +- Understand which processes use containers - Find examples of specific directives - Ensure consistency across modules diff --git a/docs/en/docs/side_quests/essential_scripting_patterns/index.md b/docs/en/docs/side_quests/essential_scripting_patterns/index.md index 9f50dc568b..6bb402ec7a 100644 --- a/docs/en/docs/side_quests/essential_scripting_patterns/index.md +++ b/docs/en/docs/side_quests/essential_scripting_patterns/index.md @@ -896,7 +896,7 @@ Fix this by adding conditional logic to the `FASTP` process `script:` block. An === "After" - ```groovy title="main.nf" linenums="10" hl_lines="3-27" + ```groovy title="main.nf" linenums="10" hl_lines="2-26" script: // Simple single-end vs paired-end detection def is_single = reads instanceof List ? reads.size() == 1 : true @@ -1001,8 +1001,6 @@ Take a look a the module file `modules/generate_report.nf`: ```groovy title="modules/generate_report.nf" linenums="1" process GENERATE_REPORT { - publishDir 'results/reports', mode: 'copy' - input: tuple val(meta), path(reads) @@ -1378,7 +1376,7 @@ cat work/48/6db0c9e9d8aa65e4bb4936cd3bd59e/.command.run | grep "docker run" You should see something like: ```bash title="docker command" - docker run -i --cpu-shares 4096 --memory 2048m -e "NXF_TASK_WORKDIR" -v /workspaces/training/side-quests/essential_scripting_patterns:/workspaces/training/side-quests/essential_scripting_patterns -w "$NXF_TASK_WORKDIR" --name $NXF_BOXID community.wave.seqera.io/library/fastp:0.24.0--62c97b06e8447690 /bin/bash -ue /workspaces/training/side-quests/essential_scripting_patterns/work/48/6db0c9e9d8aa65e4bb4936cd3bd59e/.command.sh + docker run -i --cpu-shares 2048 --memory 2048m -e "NXF_TASK_WORKDIR" -v /workspaces/training/side-quests/essential_scripting_patterns:/workspaces/training/side-quests/essential_scripting_patterns -w "$NXF_TASK_WORKDIR" --name $NXF_BOXID community.wave.seqera.io/library/fastp:0.24.0--62c97b06e8447690 /bin/bash -ue /workspaces/training/side-quests/essential_scripting_patterns/work/48/6db0c9e9d8aa65e4bb4936cd3bd59e/.command.sh ``` In this example we've chosen an example that requested 2 CPUs (`--cpu-shares 2048`), because it was a high-depth sample, but you should see different CPU allocations depending on the sample depth. Try this for the other tasks as well. @@ -1393,7 +1391,7 @@ Another powerful pattern is using `task.attempt` for retry strategies. To show w process FASTP { container 'community.wave.seqera.io/library/fastp:0.24.0--62c97b06e8447690' - cpus { meta.depth > 40000000 ? 4 : 2 } + cpus { meta.depth > 40000000 ? 2 : 1 } memory 1.GB input: @@ -1406,7 +1404,7 @@ Another powerful pattern is using `task.attempt` for retry strategies. To show w process FASTP { container 'community.wave.seqera.io/library/fastp:0.24.0--62c97b06e8447690' - cpus { meta.depth > 40000000 ? 4 : 2 } + cpus { meta.depth > 40000000 ? 2 : 1 } memory 2.GB input: @@ -1432,7 +1430,7 @@ nextflow run main.nf Detecting adapter sequence for read1... No adapter detected for read1 - .command.sh: line 7: 101 Killed fastp --in1 SAMPLE_002_S2_L001_R1_001.fastq --out1 sample_002_trimmed.fastq.gz --json sample_002.fastp.json --html sample_002.fastp.html --thread 2 + .command.sh: line 7: 101 Killed fastp --in1 SAMPLE_002_S2_L001_R1_001.fastq --out1 sample_002_trimmed.fastq.gz --json sample_002.fastp.json --html sample_002.fastp.html --thread 1 ``` This indicates that the process was killed for exceeding memory limits. @@ -1447,7 +1445,7 @@ To make our workflow more robust, we can implement a retry strategy that increas process FASTP { container 'community.wave.seqera.io/library/fastp:0.24.0--62c97b06e8447690' - cpus { meta.depth > 40000000 ? 4 : 2 } + cpus { meta.depth > 40000000 ? 2 : 1 } memory { 1.GB * task.attempt } errorStrategy 'retry' maxRetries 2 @@ -1462,7 +1460,7 @@ To make our workflow more robust, we can implement a retry strategy that increas process FASTP { container 'community.wave.seqera.io/library/fastp:0.24.0--62c97b06e8447690' - cpus { meta.depth > 40000000 ? 4 : 2 } + cpus { meta.depth > 40000000 ? 2 : 1 } memory 2.GB input: diff --git a/docs/en/docs/side_quests/metadata/index.md b/docs/en/docs/side_quests/metadata/index.md index 904e4444f3..2a4448cfb4 100644 --- a/docs/en/docs/side_quests/metadata/index.md +++ b/docs/en/docs/side_quests/metadata/index.md @@ -133,9 +133,16 @@ Open the `main.nf` workflow file to examine the workflow stub we're giving you a #!/usr/bin/env nextflow workflow { - + main: ch_datasheet = channel.fromPath("./data/datasheet.csv") + publish: + cowpy_art = channel.empty() +} + +output { + cowpy_art { + } } ``` @@ -151,24 +158,16 @@ Make the following changes to add a `splitCsv()` operation to the channel constr === "After" - ```groovy title="main.nf" linenums="3" hl_lines="4-5" - workflow { - + ```groovy title="main.nf" linenums="5" hl_lines="3-4" ch_datasheet = channel.fromPath("./data/datasheet.csv") .splitCsv(header: true) .view() - - } ``` === "Before" - ```groovy title="main.nf" linenums="3" - workflow { - + ```groovy title="main.nf" linenums="5" ch_datasheet = channel.fromPath("./data/datasheet.csv") - - } ``` Note that we're using the `header: true` option to tell Nextflow to read the first row of the CSV file as the header row. @@ -254,29 +253,21 @@ Make the following edits to the workflow: === "After" - ```groovy title="main.nf" linenums="3" hl_lines="5-7" - workflow { - + ```groovy title="main.nf" linenums="5" hl_lines="4-6" ch_datasheet = channel.fromPath("./data/datasheet.csv") .splitCsv(header: true) .map{ row -> row.character } .view() - - } ``` === "Before" - ```groovy title="main.nf" linenums="3" - workflow { - + ```groovy title="main.nf" linenums="5" ch_datasheet = channel.fromPath("./data/datasheet.csv") .splitCsv(header: true) .view() - - } ``` Now run the workflow again: @@ -887,8 +878,6 @@ You can open the module file to examine its code: // Generate ASCII art with cowpy process COWPY { - publishDir "results/", mode: 'copy' - container 'community.wave.seqera.io/library/cowpy:1.1.5--3db457ae1977a273' input: @@ -1074,7 +1063,7 @@ This confirms we're able to access the file and the character for each element i #### 3.2.3. Call the `COWPY` process -Now let's put it all together and actually call the `COWPY` process on the `ch_languages` channel. +Now we can put it all together and actually call the `COWPY` process on the `ch_languages` channel. In the main workflow, make the following code changes: @@ -1086,18 +1075,23 @@ In the main workflow, make the following code changes: ch_languages.map { meta, file -> file }, ch_languages.map { meta, file -> meta.character } ) + + publish: + cowpy_art = COWPY.out ``` === "Before" ```groovy title="main.nf" linenums="34" // Temporary: access the file and character - ch_languages.map { meta, file -> [file, meta.character] } - .view() + ch_languages.map { meta, file -> file }.view { file -> "File: " + file } + ch_languages.map { meta, file -> meta.character }.view { character -> "Character: " + character } + + publish: + cowpy_art = channel.empty() ``` -You see we simply copy the two map operations (minus the `.view()` statements) as the inputs to the process call. -Just make sure you don't forget the comma between them! +We replaced the temporary view operations with the actual `COWPY` process call, and updated the `publish:` section to wire up `COWPY.out` for publishing. It's a bit clunky, but we'll see how to make that better in the next section. @@ -1270,18 +1264,24 @@ Make the following edits to the main workflow: === "After" ```groovy title="main.nf" linenums="34" hl_lines="2" - // Run cowpy to generate ASCII art - COWPY(ch_languages) + // Run cowpy to generate ASCII art + COWPY(ch_languages) + + publish: + cowpy_art = COWPY.out ``` === "Before" ```groovy title="main.nf" linenums="34" hl_lines="3-4" - // Run cowpy to generate ASCII art - COWPY( - ch_languages.map { meta, file -> file }, - ch_languages.map { meta, file -> meta.character } - ) + // Run cowpy to generate ASCII art + COWPY( + ch_languages.map { meta, file -> file }, + ch_languages.map { meta, file -> meta.character } + ) + + publish: + cowpy_art = COWPY.out ``` That simplifies the call significantly! diff --git a/docs/en/docs/side_quests/nf_test/index.md b/docs/en/docs/side_quests/nf_test/index.md index 12ab2b7d0a..f8c29a7cbf 100644 --- a/docs/en/docs/side_quests/nf_test/index.md +++ b/docs/en/docs/side_quests/nf_test/index.md @@ -119,8 +119,6 @@ You can see the full workflow code below. */ process sayHello { - publishDir 'results', mode: 'copy' - input: val greeting @@ -138,8 +136,6 @@ You can see the full workflow code below. */ process convertToUpper { - publishDir 'results', mode: 'copy' - input: path input_file @@ -153,7 +149,7 @@ You can see the full workflow code below. } workflow { - + main: // create a channel for inputs from a CSV file greeting_ch = channel.fromPath(params.input_file).splitCsv().flatten() @@ -162,6 +158,17 @@ You can see the full workflow code below. // convert the greeting to uppercase convertToUpper(sayHello.out) + + publish: + greetings = sayHello.out + upper_greetings = convertToUpper.out + } + + output { + greetings { + } + upper_greetings { + } } ``` diff --git a/docs/en/docs/side_quests/working_with_files/index.md b/docs/en/docs/side_quests/working_with_files/index.md index d2bdaa3287..47cb245c71 100644 --- a/docs/en/docs/side_quests/working_with_files/index.md +++ b/docs/en/docs/side_quests/working_with_files/index.md @@ -1568,18 +1568,14 @@ Make the following edit to the workflow: You can open the module file to examine its code: ```groovy title="modules/analyze_reads.nf - process example" linenums="1" -#!/usr/bin/env nextflow - process ANALYZE_READS { tag { meta.id } - publishDir { "results/${meta.id}" }, mode: 'copy' - input: tuple val(meta), path(files) output: - tuple val(meta.id), path("${meta.id}_stats.txt") + tuple val(meta), path("${meta.id}_stats.txt") script: """ @@ -1596,9 +1592,8 @@ process ANALYZE_READS { !!! note - The `tag` and `publishDir` directives use closure syntax (`{ ... }`) instead of string interpolation (`"${...}"`). - This is because these directives reference input variables (`meta`) that aren't available until runtime. - The closure syntax defers evaluation until the process actually runs. + The `tag` directive uses closure syntax (`{ ... }`) because it references input variables (`meta`) that aren't available until the process runs. + The closure defers evaluation until runtime. !!! note @@ -1689,20 +1684,71 @@ Now let's actually call the `ANALYZE_READS` process on the `ch_samples` channel. In the main workflow, make the following code changes: +1. Add a `main:` label at the top of the workflow block (before the existing channel code) +2. Replace the temporary `.view()` call with the process call +3. Add a `publish:` section to the workflow +4. Add an `output {}` block after the workflow + === "After" - ```groovy title="main.nf" linenums="23" + ```groovy title="main.nf" linenums="5" hl_lines="2 5 7 14 17 18 20 21 24 25 26 27 28" + workflow { + main: + // Load files with channel.fromFilePairs + ch_files = channel.fromFilePairs('data/patientA_rep1_normal_R{1,2}_001.fastq.gz') + ch_samples = ch_files.map { id, files -> + def (sample, replicate, type, readNum) = id.tokenize('_') + tuple( + [ + id: sample, + replicate: replicate.replace('rep', ''), + type: type + ], + files + ) + } + // Run the analysis ANALYZE_READS(ch_samples) + + publish: + analysis_results = ANALYZE_READS.out + } + + output { + analysis_results { + path { meta, file -> "${meta.id}" } + } + } ``` === "Before" - ```groovy title="main.nf" linenums="23" + ```groovy title="main.nf" linenums="5" + workflow { + // Load files with channel.fromFilePairs + ch_files = channel.fromFilePairs('data/patientA_rep1_normal_R{1,2}_001.fastq.gz') + ch_files.map { id, files -> + def (sample, replicate, type, readNum) = id.tokenize('_') + [ + [ + id: sample, + replicate: replicate.replace('rep', ''), + type: type + ], + files + ] + } + .set { ch_samples } + // Temporary: peek into ch_samples ch_samples.view() + } ``` +The `publish:` section declares which process outputs to publish, and the `output {}` block configures where and how they are published. +The `path` closure in the `output {}` block receives each output element and determines the subdirectory structure. Here we use `meta.id` to organize results by patient. + Let's run this: ```bash @@ -1720,7 +1766,7 @@ nextflow run main.nf [b5/110360] process > ANALYZE_READS (patientA) [100%] 1 of 1 ✔ ``` -This process is set up to publish its outputs to a `results` directory, so have a look in there. +The outputs are published to a `results` directory, so have a look in there. ??? abstract "Directory and file contents" @@ -1809,18 +1855,22 @@ We are overwriting the output file each time. Since we have access to the patient metadata, we can use it to make the published files unique by including differentiating metadata, either in the directory structure or in the filenames themselves. -Make the following change to the workflow: +Make the following change to the `output {}` block: === "After" - ```groovy title="modules/analyze_reads.nf" linenums="6" - publishDir { "results/${meta.type}/${meta.id}/${meta.replicate}" }, mode: 'copy' + ```groovy title="main.nf" hl_lines="3" + analysis_results { + path { meta, file -> "${meta.type}/${meta.id}/${meta.replicate}" } + } ``` === "Before" - ```groovy title="modules/analyze_reads.nf" linenums="6" - publishDir { "results/${meta.id}" }, mode: 'copy' + ```groovy title="main.nf" hl_lines="3" + analysis_results { + path { meta, file -> "${meta.id}" } + } ``` Here we show the option of using additional directory levels to account for sample types and replicates, but you could experiment with doing it at the filename level as well. @@ -1888,7 +1938,7 @@ You can learn more about this in the [Metadata and meta maps](../metadata/) side ### Takeaway -- The `publishDir` directive can organize outputs based on metadata values +- The `output {}` block can organize outputs based on metadata values using dynamic path closures - Metadata in tuples enables structured organization of results - This approach creates maintainable workflows with clear data provenance - Processes can take tuples of metadata and files as input @@ -1996,24 +2046,23 @@ Applying these techniques in your own work will enable you to build more efficie ch_pairs = channel.fromFilePairs('data/*_R{1,2}_001.fastq.gz') ``` -6. **Using File Operations in Processes:** We integrated file operations into Nextflow processes with proper input handling, using `publishDir` to organize outputs based on metadata. +6. **Using File Operations in Processes:** We integrated file operations into Nextflow processes with proper input handling, using the `output {}` block to organize outputs based on metadata. - Associate a meta map with the process inputs ```groovy ch_files = channel.fromFilePairs('data/patientA_rep1_normal_R{1,2}_001.fastq.gz') - ch_files.map { id, files -> + ch_samples = ch_files.map { id, files -> def (sample, replicate, type, readNum) = id.tokenize('_') - [ + tuple( [ id: sample, replicate: replicate.replace('rep', ''), type: type ], - files - ] + files + ) } - .set { ch_samples } ANALYZE_READS(ch_samples) ``` @@ -2021,7 +2070,11 @@ Applying these techniques in your own work will enable you to build more efficie - Organize outputs based on metadata ```groovy - publishDir { "results/${meta.type}/${meta.id}/${meta.replicate}" }, mode: 'copy' + output { + analysis_results { + path { meta, file -> "${meta.type}/${meta.id}/${meta.replicate}" } + } + } ``` ### Additional resources diff --git a/side-quests/debugging/buggy_workflow.nf b/side-quests/debugging/buggy_workflow.nf index 577427cf55..c4e1f76f3a 100644 --- a/side-quests/debugging/buggy_workflow.nf +++ b/side-quests/debugging/buggy_workflow.nf @@ -15,7 +15,6 @@ params{ * Process with input/output mismatch */ process processFiles { - publishDir "${params.output}/processed", mode: 'copy' input: tuple val(sample_id), path(input_file) @@ -33,7 +32,6 @@ process processFiles { * Process with resource issues */ process heavyProcess { - publishDir "${params.output}/heavy", mode: 'copy' time '1 ms' @@ -56,7 +54,6 @@ process heavyProcess { * Process with file handling issues */ process handleFiles { - publishDir "${params.output}/files", mode: 'copy' input: path input_file @@ -76,7 +73,7 @@ process handleFiles { * Main workflow with channel issues */ workflow { - + main: // Channel with incorrect usage input_ch = channel .fromPath(params.input) @@ -89,4 +86,21 @@ workflow { file_ch = channel.fromPath("*.txt") handleFiles(file_ch) + + publish: + processed = processFiles.out + heavy = heavyProcess.out + files = handleFiles.out +} + +output { + processed { + path 'processed' + } + heavy { + path 'heavy' + } + files { + path 'files' + } } diff --git a/side-quests/debugging/nextflow.config b/side-quests/debugging/nextflow.config index 76a590cac8..b01c0cf7ec 100644 --- a/side-quests/debugging/nextflow.config +++ b/side-quests/debugging/nextflow.config @@ -7,6 +7,10 @@ params { publish_dir_mode = 'copy' } +// Workflow output settings +outputDir = params.output +workflow.output.mode = 'copy' + // Execution profiles profiles { test { diff --git a/side-quests/essential_scripting_patterns/modules/generate_report.nf b/side-quests/essential_scripting_patterns/modules/generate_report.nf index 4bf7a1d786..eead40d97e 100644 --- a/side-quests/essential_scripting_patterns/modules/generate_report.nf +++ b/side-quests/essential_scripting_patterns/modules/generate_report.nf @@ -1,7 +1,5 @@ process GENERATE_REPORT { - publishDir 'results/reports', mode: 'copy' - input: tuple val(meta), path(reads) diff --git a/side-quests/ide_features/basic_workflow.nf b/side-quests/ide_features/basic_workflow.nf index 8609f29574..b1c6886f29 100644 --- a/side-quests/ide_features/basic_workflow.nf +++ b/side-quests/ide_features/basic_workflow.nf @@ -2,7 +2,6 @@ process FASTQC { tag "${sample_id}" - publishDir "${params.output_dir}/fastqc", mode: 'copy' input: tuple val(sample_id), path(reads) @@ -33,6 +32,7 @@ params{ * Main workflow demonstrating module usage and navigation */ workflow { + main: // Create input channel from CSV file ch_input = channel.fromPath(params.input, checkIfExists: true) .splitCsv(header: true) @@ -47,4 +47,17 @@ workflow { // View outputs FASTQC.out.html.view { "FastQC report: ${it}" } FASTQC.out.zip.view { "FastQC data: ${it}" } + + publish: + fastqc_html = FASTQC.out.html + fastqc_zip = FASTQC.out.zip +} + +output { + fastqc_html { + path 'fastqc' + } + fastqc_zip { + path 'fastqc' + } } diff --git a/side-quests/ide_features/complex_workflow.nf b/side-quests/ide_features/complex_workflow.nf index ee298d707f..5fcce03ed8 100644 --- a/side-quests/ide_features/complex_workflow.nf +++ b/side-quests/ide_features/complex_workflow.nf @@ -10,7 +10,6 @@ include { MULTIQC } from './modules/utils' process TRIM_GALORE { tag "${sample_id}" - publishDir "${params.output}/trimmed", mode: 'copy' input: tuple val(sample_id), path(reads) @@ -26,7 +25,6 @@ process TRIM_GALORE { process FEATURECOUNTS { tag "${sample_id}" - publishDir "${params.output}/counts", mode: 'copy' input: tuple val(sample_id), path(bam), path(gtf) @@ -65,11 +63,15 @@ workflow RNASEQ_PIPELINE { MULTIQC(qc_files.collect()) emit: + fastqc = FASTQC.out + trimmed = TRIM_GALORE.out + alignments = STAR_ALIGN.out counts = FEATURECOUNTS.out qc_report = MULTIQC.out } workflow { + main: // Input channels ch_reads = channel.fromPath(params.input) .splitCsv(header: true) @@ -85,4 +87,29 @@ workflow { RNASEQ_PIPELINE.out.counts .collectFile(name: 'expression_matrix.txt', sort: true) .view { "Expression matrix created: ${it}" } + + publish: + fastqc = RNASEQ_PIPELINE.out.fastqc + trimmed = RNASEQ_PIPELINE.out.trimmed + alignments = RNASEQ_PIPELINE.out.alignments + counts = RNASEQ_PIPELINE.out.counts + qc_report = RNASEQ_PIPELINE.out.qc_report +} + +output { + fastqc { + path 'fastqc' + } + trimmed { + path 'trimmed' + } + alignments { + path 'alignments' + } + counts { + path 'counts' + } + qc_report { + path 'multiqc' + } } diff --git a/side-quests/ide_features/modules/fastqc.nf b/side-quests/ide_features/modules/fastqc.nf index 32ba9c45ed..e04eea8bff 100644 --- a/side-quests/ide_features/modules/fastqc.nf +++ b/side-quests/ide_features/modules/fastqc.nf @@ -1,6 +1,5 @@ process FASTQC { tag "${sample_id}" - publishDir "${params.output_dir}/fastqc", mode: 'copy' // Container directive for reproducibility container 'biocontainers/fastqc:v0.11.9_cv8' diff --git a/side-quests/ide_features/modules/star.nf b/side-quests/ide_features/modules/star.nf index a83c3bafd7..58a609f4ee 100644 --- a/side-quests/ide_features/modules/star.nf +++ b/side-quests/ide_features/modules/star.nf @@ -1,6 +1,5 @@ process STAR_ALIGN { tag "${sample_id}" - publishDir "${params.output}/alignments", mode: 'copy' container 'biocontainers/star:2.7.10a_cv1' cpus 8 memory '32.GB' diff --git a/side-quests/ide_features/modules/utils.nf b/side-quests/ide_features/modules/utils.nf index f6de00ebb6..0887fa4557 100644 --- a/side-quests/ide_features/modules/utils.nf +++ b/side-quests/ide_features/modules/utils.nf @@ -1,5 +1,4 @@ process MULTIQC { - publishDir "${params.output}/multiqc", mode: 'copy' container 'biocontainers/multiqc:1.13_cv1' cpus 2 memory '4.GB' @@ -28,7 +27,6 @@ process MULTIQC { } process CUSTOM_DUMPSOFTWAREVERSIONS { - publishDir "${params.output}/pipeline_info", mode: 'copy' input: path versions diff --git a/side-quests/ide_features/nextflow.config b/side-quests/ide_features/nextflow.config index 57d0de3184..5ea8d1f2fc 100644 --- a/side-quests/ide_features/nextflow.config +++ b/side-quests/ide_features/nextflow.config @@ -2,5 +2,6 @@ * Nextflow Configuration for IDE Features Training */ - docker.enabled = true + +workflow.output.mode = 'copy' diff --git a/side-quests/metadata/main.nf b/side-quests/metadata/main.nf index bc330698be..88d4cc26d2 100644 --- a/side-quests/metadata/main.nf +++ b/side-quests/metadata/main.nf @@ -1,7 +1,14 @@ #!/usr/bin/env nextflow workflow { - + main: ch_datasheet = channel.fromPath("./data/datasheet.csv") + publish: + cowpy_art = channel.empty() +} + +output { + cowpy_art { + } } diff --git a/side-quests/metadata/modules/cowpy.nf b/side-quests/metadata/modules/cowpy.nf index 283ca8e2bd..41eef3ad96 100644 --- a/side-quests/metadata/modules/cowpy.nf +++ b/side-quests/metadata/modules/cowpy.nf @@ -1,8 +1,6 @@ // Generate ASCII art with cowpy process COWPY { - publishDir "results/", mode: 'copy' - container 'community.wave.seqera.io/library/cowpy:1.1.5--3db457ae1977a273' input: diff --git a/side-quests/nf-test/main.nf b/side-quests/nf-test/main.nf index 387c0f849c..b6b65284c2 100644 --- a/side-quests/nf-test/main.nf +++ b/side-quests/nf-test/main.nf @@ -10,8 +10,6 @@ params.input_file = "greetings.csv" */ process sayHello { - publishDir 'results', mode: 'copy' - input: val greeting @@ -29,8 +27,6 @@ process sayHello { */ process convertToUpper { - publishDir 'results', mode: 'copy' - input: path input_file @@ -44,7 +40,7 @@ process convertToUpper { } workflow { - + main: // create a channel for inputs from a CSV file greeting_ch = channel.fromPath(params.input_file).splitCsv().flatten() @@ -53,4 +49,15 @@ workflow { // convert the greeting to uppercase convertToUpper(sayHello.out) + + publish: + greetings = sayHello.out + upper_greetings = convertToUpper.out +} + +output { + greetings { + } + upper_greetings { + } } diff --git a/side-quests/nf-test/nextflow.config b/side-quests/nf-test/nextflow.config new file mode 100644 index 0000000000..571ff27d19 --- /dev/null +++ b/side-quests/nf-test/nextflow.config @@ -0,0 +1,2 @@ +outputDir = 'results' +workflow.output.mode = 'copy' diff --git a/side-quests/solutions/debugging/buggy_workflow.nf b/side-quests/solutions/debugging/buggy_workflow.nf index 172fc6892f..30c848940c 100644 --- a/side-quests/solutions/debugging/buggy_workflow.nf +++ b/side-quests/solutions/debugging/buggy_workflow.nf @@ -15,7 +15,6 @@ params{ * Process with input/output mismatch */ process processFiles { - publishDir "${params.output}/processed", mode: 'copy' input: tuple val(sample_id), path(input_file) @@ -37,7 +36,6 @@ process processFiles { * Process with resource issues */ process heavyProcess { - publishDir "${params.output}/heavy", mode: 'copy' // Fixed: Increase execution time to avoid timeout time '100 s' @@ -61,7 +59,6 @@ process heavyProcess { * Process with file handling issues */ process handleFiles { - publishDir "${params.output}/files", mode: 'copy' input: path input_file @@ -82,7 +79,7 @@ process handleFiles { * Main workflow with channel issues */ workflow { - + main: // Channel with incorrect usage input_ch = channel.fromPath(params.input) .splitCsv(header: true) @@ -96,5 +93,22 @@ workflow { // BUG: Incorrect channel usage // Fixed: take output from second process - handleFiles(heavyProcess.out) + file_ch = handleFiles(heavy_ch) + + publish: + processed = processed_ch + heavy = heavy_ch + files = file_ch +} + +output { + processed { + path 'processed' + } + heavy { + path 'heavy' + } + files { + path 'files' + } } diff --git a/side-quests/solutions/essential_scripting_patterns/main.nf b/side-quests/solutions/essential_scripting_patterns/main.nf index a5c59ff76e..319899f727 100644 --- a/side-quests/solutions/essential_scripting_patterns/main.nf +++ b/side-quests/solutions/essential_scripting_patterns/main.nf @@ -47,6 +47,7 @@ def separateMetadata(row) { } workflow { + main: validateInputs() ch_samples = channel.fromPath(params.input) @@ -87,4 +88,13 @@ workflow { println("Error: ${workflow.errorMessage}") } } + + publish: + reports = GENERATE_REPORT.out +} + +output { + reports { + path 'reports' + } } diff --git a/side-quests/solutions/essential_scripting_patterns/modules/generate_report.nf b/side-quests/solutions/essential_scripting_patterns/modules/generate_report.nf index fca06dad2a..bd079e61a7 100644 --- a/side-quests/solutions/essential_scripting_patterns/modules/generate_report.nf +++ b/side-quests/solutions/essential_scripting_patterns/modules/generate_report.nf @@ -1,7 +1,5 @@ process GENERATE_REPORT { - publishDir 'results/reports', mode: 'copy' - input: tuple val(meta), path(reads) diff --git a/side-quests/solutions/essential_scripting_patterns/nextflow.config b/side-quests/solutions/essential_scripting_patterns/nextflow.config index b57f70c8c2..298c383a7d 100644 --- a/side-quests/solutions/essential_scripting_patterns/nextflow.config +++ b/side-quests/solutions/essential_scripting_patterns/nextflow.config @@ -1,3 +1,6 @@ // Nextflow configuration for Groovy Essentials side quest docker.enabled = true + +outputDir = 'results' +workflow.output.mode = 'copy' diff --git a/side-quests/solutions/metadata/3.2/main.nf b/side-quests/solutions/metadata/3.2/main.nf index 0273c610dd..88c81bb571 100644 --- a/side-quests/solutions/metadata/3.2/main.nf +++ b/side-quests/solutions/metadata/3.2/main.nf @@ -4,7 +4,7 @@ include { IDENTIFY_LANGUAGE } from './modules/langid.nf' include { COWPY } from './modules/cowpy.nf' workflow { - + main: ch_datasheet = channel.fromPath("./data/datasheet.csv") .splitCsv(header: true) .map { row -> @@ -38,4 +38,12 @@ workflow { ch_languages.map { meta, file -> file }, ch_languages.map { meta, file -> meta.character }, ) + + publish: + cowpy_art = COWPY.out +} + +output { + cowpy_art { + } } diff --git a/side-quests/solutions/metadata/3.2/modules/cowpy.nf b/side-quests/solutions/metadata/3.2/modules/cowpy.nf index 6a502fd80b..ca7c1ae108 100644 --- a/side-quests/solutions/metadata/3.2/modules/cowpy.nf +++ b/side-quests/solutions/metadata/3.2/modules/cowpy.nf @@ -3,8 +3,6 @@ */ process COWPY { - publishDir "results/", mode: 'copy' - container 'community.wave.seqera.io/library/cowpy:1.1.5--3db457ae1977a273' input: diff --git a/side-quests/solutions/metadata/3.2/nextflow.config b/side-quests/solutions/metadata/3.2/nextflow.config new file mode 100644 index 0000000000..d3b5e01dd3 --- /dev/null +++ b/side-quests/solutions/metadata/3.2/nextflow.config @@ -0,0 +1,4 @@ +docker.enabled = true + +outputDir = 'results' +workflow.output.mode = 'copy' diff --git a/side-quests/solutions/metadata/3.3/main.nf b/side-quests/solutions/metadata/3.3/main.nf index f7b4f72c00..bccdb30d5a 100644 --- a/side-quests/solutions/metadata/3.3/main.nf +++ b/side-quests/solutions/metadata/3.3/main.nf @@ -4,7 +4,7 @@ include { IDENTIFY_LANGUAGE } from './modules/langid.nf' include { COWPY } from './modules/cowpy.nf' workflow { - + main: ch_datasheet = channel.fromPath("./data/datasheet.csv") .splitCsv(header: true) .map { row -> @@ -35,4 +35,12 @@ workflow { // Run cowpy to generate ASCII art COWPY(ch_languages) + + publish: + cowpy_art = COWPY.out +} + +output { + cowpy_art { + } } diff --git a/side-quests/solutions/metadata/3.3/modules/cowpy.nf b/side-quests/solutions/metadata/3.3/modules/cowpy.nf index cf1258eb5f..a855ab1fad 100644 --- a/side-quests/solutions/metadata/3.3/modules/cowpy.nf +++ b/side-quests/solutions/metadata/3.3/modules/cowpy.nf @@ -3,8 +3,6 @@ */ process COWPY { - publishDir "results/", mode: 'copy' - container 'community.wave.seqera.io/library/cowpy:1.1.5--3db457ae1977a273' input: diff --git a/side-quests/solutions/metadata/3.3/nextflow.config b/side-quests/solutions/metadata/3.3/nextflow.config new file mode 100644 index 0000000000..d3b5e01dd3 --- /dev/null +++ b/side-quests/solutions/metadata/3.3/nextflow.config @@ -0,0 +1,4 @@ +docker.enabled = true + +outputDir = 'results' +workflow.output.mode = 'copy' diff --git a/side-quests/solutions/workflows_of_workflows/main.nf b/side-quests/solutions/workflows_of_workflows/main.nf index fd266fe85e..fe0b5eff89 100644 --- a/side-quests/solutions/workflows_of_workflows/main.nf +++ b/side-quests/solutions/workflows_of_workflows/main.nf @@ -2,6 +2,7 @@ include { GREETING_WORKFLOW } from './workflows/greeting' include { TRANSFORM_WORKFLOW } from './workflows/transform' workflow { + main: names = channel.of('Alice', 'Bob', 'Charlie') // Run the greeting workflow @@ -13,4 +14,18 @@ workflow { // View results TRANSFORM_WORKFLOW.out.upper.view { "Uppercase: ${it}" } TRANSFORM_WORKFLOW.out.reversed.view { "Reversed: ${it}" } + + publish: + greetings = GREETING_WORKFLOW.out.greetings + upper = TRANSFORM_WORKFLOW.out.upper + reversed = TRANSFORM_WORKFLOW.out.reversed +} + +output { + greetings { + } + upper { + } + reversed { + } } diff --git a/side-quests/solutions/workflows_of_workflows/modules/reverse_text.nf b/side-quests/solutions/workflows_of_workflows/modules/reverse_text.nf index 2eb0576c8c..62d7b6cc90 100644 --- a/side-quests/solutions/workflows_of_workflows/modules/reverse_text.nf +++ b/side-quests/solutions/workflows_of_workflows/modules/reverse_text.nf @@ -2,8 +2,6 @@ * Use a text manipulation tool to reverse the text in a file */ process REVERSE_TEXT { - publishDir 'results', mode: 'copy' - tag "reversing ${input_file}" input: diff --git a/side-quests/solutions/workflows_of_workflows/modules/say_hello.nf b/side-quests/solutions/workflows_of_workflows/modules/say_hello.nf index 22222fadb6..b771259cfa 100644 --- a/side-quests/solutions/workflows_of_workflows/modules/say_hello.nf +++ b/side-quests/solutions/workflows_of_workflows/modules/say_hello.nf @@ -2,8 +2,6 @@ * Use echo to print 'Hello World!' to a file */ process SAY_HELLO { - publishDir 'results', mode: 'copy' - tag "greeting ${name}" input: diff --git a/side-quests/solutions/workflows_of_workflows/modules/say_hello_upper.nf b/side-quests/solutions/workflows_of_workflows/modules/say_hello_upper.nf index 4090885999..9e949e231c 100644 --- a/side-quests/solutions/workflows_of_workflows/modules/say_hello_upper.nf +++ b/side-quests/solutions/workflows_of_workflows/modules/say_hello_upper.nf @@ -2,8 +2,6 @@ * Use a text replacement tool to convert the greeting to uppercase */ process SAY_HELLO_UPPER { - publishDir 'results', mode: 'copy' - tag "converting ${input_file}" input: diff --git a/side-quests/solutions/workflows_of_workflows/nextflow.config b/side-quests/solutions/workflows_of_workflows/nextflow.config new file mode 100644 index 0000000000..571ff27d19 --- /dev/null +++ b/side-quests/solutions/workflows_of_workflows/nextflow.config @@ -0,0 +1,2 @@ +outputDir = 'results' +workflow.output.mode = 'copy' diff --git a/side-quests/solutions/working_with_files/6/main.nf b/side-quests/solutions/working_with_files/6/main.nf index f8177238e2..2d3e2f2cae 100644 --- a/side-quests/solutions/working_with_files/6/main.nf +++ b/side-quests/solutions/working_with_files/6/main.nf @@ -3,23 +3,30 @@ include { ANALYZE_READS } from './modules/analyze_reads.nf' workflow { - + main: // Load files with channel.fromFilePairs ch_files = channel.fromFilePairs('data/*_R{1,2}_001.fastq.gz') - ch_files - .map { id, files -> + ch_samples = ch_files.map { id, files -> def (sample, replicate, type, readNum) = id.tokenize('_') - [ + tuple( [ id: sample, replicate: replicate.replace('rep', ''), type: type, ], files, - ] + ) } - .set { ch_samples } // Run the analysis ANALYZE_READS(ch_samples) + + publish: + analysis_results = ANALYZE_READS.out +} + +output { + analysis_results { + path { meta, file -> "${meta.type}/${meta.id}/${meta.replicate}" } + } } diff --git a/side-quests/solutions/working_with_files/6/modules/analyze_reads.nf b/side-quests/solutions/working_with_files/6/modules/analyze_reads.nf index 6784701c3f..fd982b99cf 100644 --- a/side-quests/solutions/working_with_files/6/modules/analyze_reads.nf +++ b/side-quests/solutions/working_with_files/6/modules/analyze_reads.nf @@ -1,13 +1,11 @@ process ANALYZE_READS { tag { meta.id } - publishDir { "results/${meta.type}/${meta.id}/${meta.replicate}" }, mode: 'copy' - input: tuple val(meta), path(files) output: - tuple val(meta.id), path("${meta.id}_stats.txt") + tuple val(meta), path("${meta.id}_stats.txt") script: """ diff --git a/side-quests/solutions/working_with_files/6/nextflow.config b/side-quests/solutions/working_with_files/6/nextflow.config new file mode 100644 index 0000000000..571ff27d19 --- /dev/null +++ b/side-quests/solutions/working_with_files/6/nextflow.config @@ -0,0 +1,2 @@ +outputDir = 'results' +workflow.output.mode = 'copy' diff --git a/side-quests/workflows_of_workflows/modules/reverse_text.nf b/side-quests/workflows_of_workflows/modules/reverse_text.nf index 2eb0576c8c..62d7b6cc90 100644 --- a/side-quests/workflows_of_workflows/modules/reverse_text.nf +++ b/side-quests/workflows_of_workflows/modules/reverse_text.nf @@ -2,8 +2,6 @@ * Use a text manipulation tool to reverse the text in a file */ process REVERSE_TEXT { - publishDir 'results', mode: 'copy' - tag "reversing ${input_file}" input: diff --git a/side-quests/workflows_of_workflows/modules/say_hello.nf b/side-quests/workflows_of_workflows/modules/say_hello.nf index 22222fadb6..b771259cfa 100644 --- a/side-quests/workflows_of_workflows/modules/say_hello.nf +++ b/side-quests/workflows_of_workflows/modules/say_hello.nf @@ -2,8 +2,6 @@ * Use echo to print 'Hello World!' to a file */ process SAY_HELLO { - publishDir 'results', mode: 'copy' - tag "greeting ${name}" input: diff --git a/side-quests/workflows_of_workflows/modules/say_hello_upper.nf b/side-quests/workflows_of_workflows/modules/say_hello_upper.nf index 4090885999..9e949e231c 100644 --- a/side-quests/workflows_of_workflows/modules/say_hello_upper.nf +++ b/side-quests/workflows_of_workflows/modules/say_hello_upper.nf @@ -2,8 +2,6 @@ * Use a text replacement tool to convert the greeting to uppercase */ process SAY_HELLO_UPPER { - publishDir 'results', mode: 'copy' - tag "converting ${input_file}" input: diff --git a/side-quests/working_with_files/modules/analyze_reads.nf b/side-quests/working_with_files/modules/analyze_reads.nf index e175088a90..fd982b99cf 100644 --- a/side-quests/working_with_files/modules/analyze_reads.nf +++ b/side-quests/working_with_files/modules/analyze_reads.nf @@ -1,13 +1,11 @@ process ANALYZE_READS { tag { meta.id } - publishDir { "results/${meta.id}" }, mode: 'copy' - input: tuple val(meta), path(files) output: - tuple val(meta.id), path("${meta.id}_stats.txt") + tuple val(meta), path("${meta.id}_stats.txt") script: """