From bb44d5b79eed1791ebf143cd9a5818b0a18b7b25 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Thu, 28 Apr 2022 11:54:21 +0200 Subject: [PATCH 001/145] ADD MIRGENEDB COMMANDS Add parameters to allow the use of MirGeneDB as database in the smranseq pipeline. The paths to the files are still temporary. --- nextflow.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nextflow.config b/nextflow.config index c5fa807d..a6a277c7 100644 --- a/nextflow.config +++ b/nextflow.config @@ -26,6 +26,11 @@ params { mirtrace_protocol = 'illumina' mature = "https://mirbase.org/ftp/CURRENT/mature.fa.gz" hairpin = "https://mirbase.org/ftp/CURRENT/hairpin.fa.gz" + MirGeneDB = false + MirGeneDB_mature = "https://mirgenedb.org/fasta/ALL?mat=1" + MirGeneDB_hairpin = "https://mirgenedb.org/static/data/ALL/ALL-pre.fas" + MirGeneDB_gff = "https://mirgenedb.org/gff/ALL?sort=pos&all=1" + MirGeneDB_species = null // Trimming options clip_r1 = 0 From ed3e850d43d4529c3d7804934ecd2154f66226aa Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Thu, 28 Apr 2022 13:55:18 +0200 Subject: [PATCH 002/145] ASSIGN NEW MIRNA DB FILES Added a if statement to assign the new mature and hairpin fasta files if MirGeneDB is to be used in the analysis. --- workflows/smrnaseq.nf | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index ff8669bf..8079d4da 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -51,8 +51,13 @@ ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multi // // SUBWORKFLOW: Consisting of a mix of local and nf-core/modules // -if (params.mature) { reference_mature = file(params.mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mature}" } -if (params.hairpin) { reference_hairpin = file(params.hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.hairpin}" } +if (!params.MirGeneDB) { + if (params.mature) { reference_mature = file(params.mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mature}" } + if (params.hairpin) { reference_hairpin = file(params.hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.hairpin}" } +} else { + if (params.MirGeneDR_mature) { reference_mature = file(params.MirGeneDB_mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mature}" } + if (params.MirGeneDB_hairpin) { reference_hairpin = file(params.MirGeneDB_hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.hairpin}" } +} include { INPUT_CHECK } from '../subworkflows/local/input_check' include { FASTQC_TRIMGALORE } from '../subworkflows/nf-core/fastqc_trimgalore' From 9aad3485f88268cee1c294441d26ae0d49443249 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Thu, 28 Apr 2022 14:15:01 +0200 Subject: [PATCH 003/145] DISTINGUISH BETWEEN SPECIES IN PARSE_FASTA_MIRNA Added the option to use the mirGeneDB species to parse the fasta files as different styles in species naming result in mis-parsed files. --- modules/local/parse_fasta_mirna.nf | 2 ++ nextflow.config | 10 +++++----- workflows/smrnaseq.nf | 8 ++++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/modules/local/parse_fasta_mirna.nf b/modules/local/parse_fasta_mirna.nf index 8b4c21f8..717bf07f 100644 --- a/modules/local/parse_fasta_mirna.nf +++ b/modules/local/parse_fasta_mirna.nf @@ -9,6 +9,8 @@ process PARSE_FASTA_MIRNA { input: path fasta + if (!params.mirGeneDB) {species = params.mirtrace_species} else {species = params.mirGeneDB_species} + output: path '*_igenome.fa', emit: parsed_fasta path "versions.yml", emit: versions diff --git a/nextflow.config b/nextflow.config index a6a277c7..7f88fc01 100644 --- a/nextflow.config +++ b/nextflow.config @@ -26,11 +26,11 @@ params { mirtrace_protocol = 'illumina' mature = "https://mirbase.org/ftp/CURRENT/mature.fa.gz" hairpin = "https://mirbase.org/ftp/CURRENT/hairpin.fa.gz" - MirGeneDB = false - MirGeneDB_mature = "https://mirgenedb.org/fasta/ALL?mat=1" - MirGeneDB_hairpin = "https://mirgenedb.org/static/data/ALL/ALL-pre.fas" - MirGeneDB_gff = "https://mirgenedb.org/gff/ALL?sort=pos&all=1" - MirGeneDB_species = null + mirGeneDB = false + mirGeneDB_mature = "https://mirgenedb.org/fasta/ALL?mat=1" + mirGeneDB_hairpin = "https://mirgenedb.org/static/data/ALL/ALL-pre.fas" + mirGeneDB_gff = "https://mirgenedb.org/gff/ALL?sort=pos&all=1" + mirGeneDB_species = null // Trimming options clip_r1 = 0 diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 8079d4da..24c786f5 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -25,7 +25,7 @@ if (!params.mirtrace_species){ } // Genome options bt_index_from_species = params.genome ? params.genomes[ params.genome ].bowtie ?: false : false -bt_index = params.bt_indices ?: bt_index_from_species +bt_index = params.bt_indices ?: bt_index_from_species mirtrace_species_from_species = params.genome ? params.genomes[ params.genome ].mirtrace_species ?: false : false mirtrace_species = params.mirtrace_species ?: mirtrace_species_from_species fasta_from_species = params.genome ? params.genomes[ params.genome ].fasta ?: false : false @@ -51,12 +51,12 @@ ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multi // // SUBWORKFLOW: Consisting of a mix of local and nf-core/modules // -if (!params.MirGeneDB) { +if (!params.mirGeneDB) { if (params.mature) { reference_mature = file(params.mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mature}" } if (params.hairpin) { reference_hairpin = file(params.hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.hairpin}" } } else { - if (params.MirGeneDR_mature) { reference_mature = file(params.MirGeneDB_mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mature}" } - if (params.MirGeneDB_hairpin) { reference_hairpin = file(params.MirGeneDB_hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.hairpin}" } + if (params.mirGeneDR_mature) { reference_mature = file(params.mirGeneDB_mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mature}" } + if (params.mirGeneDB_hairpin) { reference_hairpin = file(params.mirGeneDB_hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.hairpin}" } } include { INPUT_CHECK } from '../subworkflows/local/input_check' From b760c5ab56bdaf02b224df6f226a12896a913faa Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Thu, 28 Apr 2022 14:54:53 +0200 Subject: [PATCH 004/145] FIX TYPO --- workflows/smrnaseq.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 24c786f5..3f52a5e5 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -55,7 +55,7 @@ if (!params.mirGeneDB) { if (params.mature) { reference_mature = file(params.mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mature}" } if (params.hairpin) { reference_hairpin = file(params.hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.hairpin}" } } else { - if (params.mirGeneDR_mature) { reference_mature = file(params.mirGeneDB_mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mature}" } + if (params.mirGeneDB_mature) { reference_mature = file(params.mirGeneDB_mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mature}" } if (params.mirGeneDB_hairpin) { reference_hairpin = file(params.mirGeneDB_hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.hairpin}" } } From 17bffbae540fdba2629c82e32386d3747ae08488 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Tue, 10 May 2022 10:17:17 +0200 Subject: [PATCH 005/145] ADD LATEST CHANGES Commit of the latest testing changes --- conf/test.config | 2 +- modules/local/parse_fasta_mirna.nf | 2 +- nextflow.config | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conf/test.config b/conf/test.config index 36333670..78c9b953 100644 --- a/conf/test.config +++ b/conf/test.config @@ -26,7 +26,7 @@ params { mature = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/mature.fa' hairpin = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/hairpin.fa' mirna_gtf = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/hsa.gff3' - mirtrace_species = "hsa" + mirtrace_species = "Hsa" skip_mirdeep = true } diff --git a/modules/local/parse_fasta_mirna.nf b/modules/local/parse_fasta_mirna.nf index 717bf07f..a471b246 100644 --- a/modules/local/parse_fasta_mirna.nf +++ b/modules/local/parse_fasta_mirna.nf @@ -9,7 +9,7 @@ process PARSE_FASTA_MIRNA { input: path fasta - if (!params.mirGeneDB) {species = params.mirtrace_species} else {species = params.mirGeneDB_species} + //if (!params.mirGeneDB) {params.species = params.mirtrace_species} else {params.species = params.mirGeneDB_species} output: path '*_igenome.fa', emit: parsed_fasta diff --git a/nextflow.config b/nextflow.config index 7f88fc01..97190f2d 100644 --- a/nextflow.config +++ b/nextflow.config @@ -27,9 +27,9 @@ params { mature = "https://mirbase.org/ftp/CURRENT/mature.fa.gz" hairpin = "https://mirbase.org/ftp/CURRENT/hairpin.fa.gz" mirGeneDB = false - mirGeneDB_mature = "https://mirgenedb.org/fasta/ALL?mat=1" - mirGeneDB_hairpin = "https://mirgenedb.org/static/data/ALL/ALL-pre.fas" - mirGeneDB_gff = "https://mirgenedb.org/gff/ALL?sort=pos&all=1" + mirGeneDB_mature = "/Users/chriskub/Downloads/ALL-mat.fas" + mirGeneDB_hairpin = "/Users/chriskub/Downloads/ALL-pre.fas" + mirGeneDB_gff = "/Users/chriskub/Downloads/ALL.gff" mirGeneDB_species = null // Trimming options From 0939a32a5a2098da5e3af1261505645b14a73053 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Wed, 11 May 2022 17:10:05 +0200 Subject: [PATCH 006/145] ADD NEW PARAMETERS AND CHECKS Enable the use of the mirGeneDB gff file in the mirtop analysis and replace the mirna_gtf file with it internally. --- conf/test.config | 2 +- workflows/smrnaseq.nf | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/conf/test.config b/conf/test.config index 78c9b953..36333670 100644 --- a/conf/test.config +++ b/conf/test.config @@ -26,7 +26,7 @@ params { mature = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/mature.fa' hairpin = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/hairpin.fa' mirna_gtf = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/hsa.gff3' - mirtrace_species = "Hsa" + mirtrace_species = "hsa" skip_mirdeep = true } diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 3f52a5e5..c2a7a1e6 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -55,8 +55,9 @@ if (!params.mirGeneDB) { if (params.mature) { reference_mature = file(params.mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mature}" } if (params.hairpin) { reference_hairpin = file(params.hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.hairpin}" } } else { - if (params.mirGeneDB_mature) { reference_mature = file(params.mirGeneDB_mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mature}" } - if (params.mirGeneDB_hairpin) { reference_hairpin = file(params.mirGeneDB_hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.hairpin}" } + if (params.mirGeneDB_mature) { reference_mature = file(params.mirGeneDB_mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mirGeneDB_mature}" } + if (params.mirGeneDB_hairpin) { reference_hairpin = file(params.mirGeneDB_hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.mirGeneDB_hairpin}" } + if (params.mirGeneDB_gff) { mirna_gtf = file(params.mirGeneDB_gff, checkIfExists: true) } else { exit 1, "MirGeneDB gff file not found: ${params.mirGeneDB_gff}"} } include { INPUT_CHECK } from '../subworkflows/local/input_check' From 48fb232c2799627309f7d66e0c68bac18a0b2bf8 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Thu, 12 May 2022 09:31:28 +0200 Subject: [PATCH 007/145] ENABLE THE USE OF MIRGENEDB SPECIES Added a new variable that either takes the mirGeneDB or the mirtrace species to filter the fasta files. --- modules/local/parse_fasta_mirna.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/local/parse_fasta_mirna.nf b/modules/local/parse_fasta_mirna.nf index a471b246..ee8a954d 100644 --- a/modules/local/parse_fasta_mirna.nf +++ b/modules/local/parse_fasta_mirna.nf @@ -9,7 +9,7 @@ process PARSE_FASTA_MIRNA { input: path fasta - //if (!params.mirGeneDB) {params.species = params.mirtrace_species} else {params.species = params.mirGeneDB_species} + if (!params.mirGeneDB) {params.filterSpecies = params.mirtrace_species} else {params.filterSpecies = params.mirGeneDB_species} output: path '*_igenome.fa', emit: parsed_fasta @@ -29,7 +29,7 @@ process PARSE_FASTA_MIRNA { # TODO perl -ane 's/[ybkmrsw]/N/ig;print;' \${FASTA}_parsed_tmp.fa > \${FASTA}_parsed.fa sed -i 's/\s.*//' \${FASTA}_parsed.fa - seqkit grep -r --pattern \".*${params.mirtrace_species}-.*\" \${FASTA}_parsed.fa > \${FASTA}_sps.fa + seqkit grep -r --pattern \".*${params.filterSpecies}-.*\" \${FASTA}_parsed.fa > \${FASTA}_sps.fa seqkit seq --rna2dna \${FASTA}_sps.fa > \${FASTA}_igenome.fa cat <<-END_VERSIONS > versions.yml From 1a08eb9b828906a6c287bc0bbf3feee2b3e29089 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Thu, 12 May 2022 11:14:51 +0200 Subject: [PATCH 008/145] ADD NEW OPTIONS Add the new options and their description to the schema. --- nextflow_schema.json | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/nextflow_schema.json b/nextflow_schema.json index 027f1b37..b6337d19 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -62,12 +62,23 @@ "fa_icon": "fas fa-book", "help_text": "If using a reference genome configured in the pipeline using iGenomes, use this parameter to give the ID for the reference. This is then used to build the full paths for all required reference genome files e.g. `--genome GRCh38`. \n\nSee the [nf-core website docs](https://nf-co.re/usage/reference_genomes) for more details." }, + "mirGeneDB": { + "type": "boolean", + "description": "Boolean wether mirGeneDB should be used instead of miRBase", + "help_text": "This allows you to use mirGeneDB instead of miRBase as the database. \n Note that you will need to set the additional flags `--mirGeneDB_species`, `--mirGeneDB_gff`, `--mirGeneDB_mature` and `--mirGeneDB_hairpin`", + "default": "false" + }, "mirtrace_species": { "type": "string", "description": "Species for miRTrace.", "help_text": "This is automatically set when using `--genome`. Example values: `hsa`, `mmu`...\n Note that mirTrace relies on miRBase for its species reference. See available references [here](https://mirbase.org/ftp/CURRENT/genomes/).", "fa_icon": "fas fa-journal-whills" }, + "mirGeneDB_species": { + "type": "string", + "description": "Species of mirGeneDB.", + "help_text": "This replaces the value of `--mirtrace_species` if `--mirGeneDB` is used. \n Note the difference in case for species names used in MirGeneDB and miRBase." + }, "fasta": { "type": "string", "fa_icon": "fas fa-font", @@ -80,6 +91,11 @@ "help_text": "miRBase `.gff3` file, typically downloaded from [`https://mirbase.org/ftp/CURRENT/genomes/`](https://mirbase.org/ftp/CURRENT/genomes/)\n\nIf using iGenomes with `--genome` this file will be downloaded from miRBase automatically during the pipeline run.\n\n", "fa_icon": "fas fa-address-book" }, + "mirGeneDB_gff": { + "type": "string", + "description": "GFF/GTF file with coordinates positions of precursor and miRNAs.", + "help_text": "mirGeneDB `.gff3` file, typically downloaded from [`https://mirgenedb.org/download`]. This replaces the value of --mirna_gff if --mirGeneDB is used." + }, "mature": { "type": "string", "description": "Path to FASTA file with mature miRNAs.", @@ -87,6 +103,11 @@ "help_text": "Typically this will be the `mature.fa` file from miRBase. Can be given either as a plain text `.fa` file or a compressed `.gz` file.\n\nDefaults to the current miRBase release URL, from which the file will be downloaded.", "default": "https://mirbase.org/ftp/CURRENT/mature.fa.gz" }, + "mirGeneDB_mature": { + "type": "string", + "description": "Path to FASTA file with mirGeneDB mature miRNAs.", + "help_text": "This file needs to be downloaded from [`https://mirgenedb.org/download`]. Can be given either as a plain text `.fa` file or a compressed `.gz` file." + }, "hairpin": { "type": "string", "description": "Path to FASTA file with miRNAs precursors.", @@ -94,6 +115,11 @@ "help_text": "Typically this will be the `mature.fa` file from miRBase. Can be given either as a plain text `.fa` file or a compressed `.gz` file.\n\nDefaults to the current miRBase release URL, from which the file will be downloaded.", "default": "https://mirbase.org/ftp/CURRENT/hairpin.fa.gz" }, + "mirGeneDB_hairpin": { + "type": "string", + "description": "Path to FASTA file with miRNAs precursors.", + "help_text": "This file needs to be downloaded from [`https://mirgenedb.org/download`]. Can be given either as a plain text `.fa` file or a compressed `.gz` file.\nNote that mirGeneDB does not have a dedicated hairpin file. The equivalent is the `Precursor sequences`." + }, "bt_indices": { "type": "string", "description": "Path to a Bowtie 1 index directory", From 8e0ba87867e03c020d61d3f34badc12a198385cf Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Thu, 12 May 2022 11:16:18 +0200 Subject: [PATCH 009/145] MOVE VARIABLE DEFINITION Moved the devinition of the filterSpecies variable to the smrnaseq workflow instead of the individual modules. --- modules/local/mirtop_quant.nf | 8 +++++--- modules/local/parse_fasta_mirna.nf | 2 +- workflows/smrnaseq.nf | 2 ++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/modules/local/mirtop_quant.nf b/modules/local/mirtop_quant.nf index 07643e99..72c3ba74 100644 --- a/modules/local/mirtop_quant.nf +++ b/modules/local/mirtop_quant.nf @@ -11,6 +11,8 @@ process MIRTOP_QUANT { path hairpin path gtf + //if (!params.mirGeneDB) {params.filterSpecies = params.mirtrace_species} else {params.filterSpecies = params.mirGeneDB_species} + output: path "mirtop/mirtop.gff" path "mirtop/mirtop.tsv" , emit: mirtop_table @@ -20,9 +22,9 @@ process MIRTOP_QUANT { script: """ - mirtop gff --hairpin $hairpin --gtf $gtf -o mirtop --sps $params.mirtrace_species ./bams/* - mirtop counts --hairpin $hairpin --gtf $gtf -o mirtop --sps $params.mirtrace_species --add-extra --gff mirtop/mirtop.gff - mirtop export --format isomir --hairpin $hairpin --gtf $gtf --sps $params.mirtrace_species -o mirtop mirtop/mirtop.gff + mirtop gff --hairpin $hairpin --gtf $gtf -o mirtop --sps $params.filterSpecies ./bams/* + mirtop counts --hairpin $hairpin --gtf $gtf -o mirtop --sps $params.filterSpecies --add-extra --gff mirtop/mirtop.gff + mirtop export --format isomir --hairpin $hairpin --gtf $gtf --sps $params.filterSpecies -o mirtop mirtop/mirtop.gff mirtop stats mirtop/mirtop.gff --out mirtop/stats mv mirtop/stats/mirtop_stats.log mirtop/stats/full_mirtop_stats.log diff --git a/modules/local/parse_fasta_mirna.nf b/modules/local/parse_fasta_mirna.nf index ee8a954d..18b51066 100644 --- a/modules/local/parse_fasta_mirna.nf +++ b/modules/local/parse_fasta_mirna.nf @@ -9,7 +9,7 @@ process PARSE_FASTA_MIRNA { input: path fasta - if (!params.mirGeneDB) {params.filterSpecies = params.mirtrace_species} else {params.filterSpecies = params.mirGeneDB_species} + //if (!params.mirGeneDB) {params.filterSpecies = params.mirtrace_species} else {params.filterSpecies = params.mirGeneDB_species} output: path '*_igenome.fa', emit: parsed_fasta diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index c2a7a1e6..f59c85bc 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -54,10 +54,12 @@ ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multi if (!params.mirGeneDB) { if (params.mature) { reference_mature = file(params.mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mature}" } if (params.hairpin) { reference_hairpin = file(params.hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.hairpin}" } + params.filterSpecies = params.mirtrace_species } else { if (params.mirGeneDB_mature) { reference_mature = file(params.mirGeneDB_mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mirGeneDB_mature}" } if (params.mirGeneDB_hairpin) { reference_hairpin = file(params.mirGeneDB_hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.mirGeneDB_hairpin}" } if (params.mirGeneDB_gff) { mirna_gtf = file(params.mirGeneDB_gff, checkIfExists: true) } else { exit 1, "MirGeneDB gff file not found: ${params.mirGeneDB_gff}"} + params.filterSpecies = params.mirGeneDB_species } include { INPUT_CHECK } from '../subworkflows/local/input_check' From 26cb509de2d7dacf73f0148f1c95315ffde500a3 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Thu, 12 May 2022 13:20:57 +0200 Subject: [PATCH 010/145] UPDATE DOCUMENTATION Added the decoumentation of new parameters and features to the appropriate files. --- CHANGELOG.md | 17 ++++++++++++----- README.md | 8 ++++---- docs/output.md | 2 +- docs/usage.md | 13 +++++++++++-- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1b6875e..615b42f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,14 +21,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Other enhancements & fixes - [#134](https://github.com/nf-core/smrnaseq/issues/134) - Fixed colSum of zero issues for edgeR_miRBase.R script +- [#55](https://github.com/nf-core/smrnaseq/issues/12) - Enabled the use of `MirGeneDB` as an alternative database insted of `miRBase` ### Parameters -| Old parameter | New parameter | -| -------------------- | ---------------- | -| `--conda` | `--enable_conda` | -| `--clusterOptions` | | -| `--publish_dir_mode` | | +| Old parameter | New parameter | +| -------------------- | --------------------- | +| `--conda` | `--enable_conda` | +| `--clusterOptions` | | +| `--publish_dir_mode` | | +| | `--mirGeneDB` | +| | `--mirGeneDB_species` | +| | `--mirGeneDB_gff` | +| | `--mirGeneDB_mature` | +| | `--mirGeneDB_hairpin` | + > **NB:** Parameter has been **updated** if both old and new parameter information is present. > **NB:** Parameter has been **added** if just the new parameter information is present. diff --git a/README.md b/README.md index 6d3efa52..aec34f05 100644 --- a/README.md +++ b/README.md @@ -34,13 +34,13 @@ On release, automated continuous integration tests run the pipeline on a full-si 2. Adapter trimming ([`Trim Galore!`](https://www.bioinformatics.babraham.ac.uk/projects/trim_galore/)) 1. Insert Size calculation 2. Collapse reads ([`seqcluster`](https://seqcluster.readthedocs.io/mirna_annotation.html#processing-of-reads)) -3. Alignment against miRBase mature miRNA ([`Bowtie1`](http://bowtie-bio.sourceforge.net/index.shtml)) -4. Alignment against miRBase hairpin +3. Alignment against miRBase or MirGeneDB mature miRNA ([`Bowtie1`](http://bowtie-bio.sourceforge.net/index.shtml)) +4. Alignment against miRBase or MirGeneDB hairpin 1. Unaligned reads from step 3 ([`Bowtie1`](http://bowtie-bio.sourceforge.net/index.shtml)) 2. Collapsed reads from step 2.2 ([`Bowtie1`](http://bowtie-bio.sourceforge.net/index.shtml)) -5. Post-alignment processing of miRBase hairpin +5. Post-alignment processing of miRBase, or MirGeneDB hairpin 1. Basic statistics from step 3 and step 4.1 ([`SAMtools`](https://sourceforge.net/projects/samtools/files/samtools/)) - 2. Analysis on miRBase hairpin counts ([`edgeR`](https://bioconductor.org/packages/release/bioc/html/edgeR.html)) + 2. Analysis on miRBase, or MirGeneDB hairpin counts ([`edgeR`](https://bioconductor.org/packages/release/bioc/html/edgeR.html)) - TMM normalization and a table of top expression hairpin - MDS plot clustering samples - Heatmap of sample similarities diff --git a/docs/output.md b/docs/output.md index ce1f8347..f3a23da8 100644 --- a/docs/output.md +++ b/docs/output.md @@ -60,7 +60,7 @@ This is an example of the output we can get: ## Bowtie -[Bowtie](http://bowtie-bio.sourceforge.net/index.shtml) is used for mapping adapter trimmed reads against the mature miRNAs and miRNA precursors (hairpins) in [miRBase](http://www.mirbase.org/). +[Bowtie](http://bowtie-bio.sourceforge.net/index.shtml) is used for mapping adapter trimmed reads against the mature miRNAs and miRNA precursors (hairpins) of the chosen database [miRBase](http://www.mirbase.org/) or [MirGeneDB](https://mirgenedb.org/). **Output directory: `results/samtools`** diff --git a/docs/usage.md b/docs/usage.md index f1304605..c85c2a4f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -16,16 +16,25 @@ This option indicates the experimental protocol used for the sample preparation. - 'cats': adapter (`GATCGGAAGAGCACACGTCTG), clip_r1(`3) - 'custom' (where the ser can indicate the `three_prime_adapter`, `clip_r1` and three_prime_clip_r1`) -### `mirtrace_species` +### `mirtrace_species or mirGeneDB_species` -It should point to the 3-letter species name used by `miRBase`. +It should point to the 3-letter species name used by `miRBase`, or `MirGeneDB`. Note the difference in case for the two databases. ### miRNA related files +Different parameters can be set for the two supported datbases. By default `miRBase` will be used with the parameters below. + - `mirna_gtf`: If not supplied by the user, then `mirna_gtf` will point to the latest GFF3 file in miRbase: `https://mirbase.org/ftp/CURRENT/genomes/${params.mirtrace_species}.gff3` - `mature`: points to the FASTA file of mature miRNA sequences. `https://mirbase.org/ftp/CURRENT/mature.fa.gz` - `hairpin`: points to the FASTA file of precursor miRNA sequences. `https://mirbase.org/ftp/CURRENT/hairpin.fa.gz` +If `MirGeneDB` should be used instead it needs to be specified using `--mirGeneDB` and use the parameters below . + +- `mirGeneDB_gff`: The data can not be downloaded automatically, thus the user needs to supply the gff file for either his species, or all species downloaded from `https://mirgenedb.org/download`. The total set will automatically be subsetted to the species specified with `mirGeneDB_species`. +- `mirGeneDB_mature`: points to the FASTA file of mature miRNA sequences. Download from `https://mirgenedb.org/download`. +- `mirGeneDB_hairpin`: points to the FASTA file of precursor miRNA sequences. Download from `https://mirgenedb.org/download`. Note that `MirGeneDB` does not have a dedicated `hairpin` file, but the `Precursor sequences` are to be used. + + ### Genome - `fasta`: the reference genome FASTA file From c78e2ca504ffb6078b11ceed5c0b35f595e0119f Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Tue, 17 May 2022 14:49:20 +0200 Subject: [PATCH 011/145] ADD CONTAMINANT FILTERING MODULES This commit adds three modules to filter conatminants from the smRNA-Seq data. Using databses provided by the user the reads are filtered for rrna, trna, cdna, ncrna, and pirna. The provided files are filtered to remove miRNAs, indexed and used as reference to map the reads to using bowtie2. Reads that map to a contaminant database are removed from the sample. The process is optional and only those contaminants will be filtered for which a reference database has been provided. --- modules/local/blat_mirna.nf | 59 ++++++++++++++++++++++++ modules/local/bowtie_contaminants.nf | 26 +++++++++++ modules/local/bowtie_map_contaminants.nf | 40 ++++++++++++++++ 3 files changed, 125 insertions(+) create mode 100644 modules/local/blat_mirna.nf create mode 100644 modules/local/bowtie_contaminants.nf create mode 100644 modules/local/bowtie_map_contaminants.nf diff --git a/modules/local/blat_mirna.nf b/modules/local/blat_mirna.nf new file mode 100644 index 00000000..bc0398a5 --- /dev/null +++ b/modules/local/blat_mirna.nf @@ -0,0 +1,59 @@ +process BLAT_MIRNA { + tag "$fasta" + label 'process_medium' + + conda (params.enable_conda ? 'bioconda::blat=36' : null) + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/blat:36--0' : + 'quay.io/biocontainers/blat:36--0' }" + + input: + val db_type + path mirna + path contaminants + + + output: + path 'filtered.fa' , emit: filtered_set + path "versions.yml" , emit: versions + + script: + + if ( db_type == "cdna" ) + """ + echo $db_type + awk '/^>/ { x=index(\$6, "transcript_biotype:miRNA") } { if(!x) print }' $contaminants > subset.fa + blat -out=blast8 $mirna subset.fa /dev/stdout | awk 'BEGIN{FS="\t"}{if(\$11 < 1e-5)print \$1;}' | uniq > mirnahit.txt + awk 'BEGIN { while((getline<"mirnahit.txt")>0) l[">"\$1]=1 } /^>/ {x = l[\$1]} {if(!x) print }' subset.fa > filtered.fa + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + blat: \$(echo \$(blat) | grep Standalone | awk '{ if (match(\$0,/[0-9]*[0-9]/,m)) print m[0] }') + END_VERSIONS + """ + + else if ( db_type == "ncrna" ) + """ + echo $db_type + awk '/^>/ { x=(index(\$6, "transcript_biotype:rRNA") || index(\$6, "transcript_biotype:miRNA")) } { if(!x) print }' $contaminants > subset.fa + blat -out=blast8 $mirna subset.fa /dev/stdout | awk 'BEGIN{FS="\t"}{if(\$11 < 1e-5)print \$1;}' | uniq > mirnahit.txt + awk 'BEGIN { while((getline<"mirnahit.txt")>0) l[">"\$1]=1 } /^>/ {x = l[\$1]} {if(!x) print }' subset.fa > filtered.fa + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + blat: \$(echo \$(blat) | grep Standalone | awk '{ if (match(\$0,/[0-9]*[0-9]/,m)) print m[0] }') + END_VERSIONS + """ + + else + """ + echo $db_type + blat -out=blast8 $mirna $contaminants /dev/stdout | awk 'BEGIN{FS="\t"}{if(\$11 < 1e-5)print \$1;}' | uniq > mirnahit.txt + awk 'BEGIN { while((getline<"mirnahit.txt")>0) l[">"\$1]=1 } /^>/ {x = l[\$1]} {if(!x) print }' $contaminants > filtered.fa + cat <<-END_VERSIONS > versions.yml + "${task.process}": + blat: \$(echo \$(blat) | grep Standalone | awk '{ if (match(\$0,/[0-9]*[0-9]/,m)) print m[0] }') + END_VERSIONS + """ + +} \ No newline at end of file diff --git a/modules/local/bowtie_contaminants.nf b/modules/local/bowtie_contaminants.nf new file mode 100644 index 00000000..562ed30a --- /dev/null +++ b/modules/local/bowtie_contaminants.nf @@ -0,0 +1,26 @@ +process INDEX_CONTAMINANTS { + label 'process_medium' + + conda (params.enable_conda ? 'bowtie2=2.4.5' : null) + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/bowtie2:2.4.5--py39hd2f7db1_2' : + 'quay.io/biocontainers/bowtie2:2.4.5--py36hfca12d5_2' }" + + input: + path fasta + + output: + path 'fasta_bidx*' , emit: bt_indices + path "versions.yml", emit: versions + + script: + """ + bowtie2-build ${fasta} fasta_bidx --threads ${task.cpus} + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + bowtie2: \$(echo \$(bowtie2 --version 2>&1) | sed 's/^.*bowtie2-align-s version //; s/ .*\$//') + END_VERSIONS + """ + +} \ No newline at end of file diff --git a/modules/local/bowtie_map_contaminants.nf b/modules/local/bowtie_map_contaminants.nf new file mode 100644 index 00000000..5330f111 --- /dev/null +++ b/modules/local/bowtie_map_contaminants.nf @@ -0,0 +1,40 @@ +process BOWTIE_MAP_CONTAMINANTS { +// tag "$meta.id" + label 'process_medium' + + conda (params.enable_conda ? 'bowtie2=2.4.5' : null) + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/bowtie2:2.4.5--py39hd2f7db1_2' : + 'quay.io/biocontainers/bowtie2:2.4.5--py36hfca12d5_2' }" + + input: + tuple val(meta), path(reads) + path index + + output: + tuple val(meta), path("*sam") , emit: bam + tuple val(meta), path('*.filter.unmapped.contaminant.fastq'), emit: unmapped + path "versions.yml" , emit: versions + + script: + def index_base = index.toString().tokenize(' ')[0].tokenize('.')[0] + """ + bowtie2 \\ + --threads ${task.cpus} \\ + --very-sensitive-local \\ + -k 1 \\ + -x $index_base \\ + --un ${meta.id}.filter.unmapped.contaminant.fastq \\ + ${reads} \\ + -S ${meta.id}.filter.contaminant.sam > ${meta.id}.contaminant_bowtie.log 2>&1 + + # extracting number of reads from bowtie logs + awk 'BEGIN{tot=0} {if(NR==4 || NR == 5){tot += \$1}} END {print tot }' ${meta.id}.contaminant_bowtie.log | tr -d , > ${meta.id}.contaminants.txt + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + bowtie2: \$(echo \$(bowtie2 --version 2>&1) | sed 's/^.*bowtie2-align-s version //; s/ .*\$//' | tr -d '\0') + END_VERSIONS + """ + +} \ No newline at end of file From 74190d7e24cfaa3ec733fee8274b5ce08b8fd18a Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Tue, 17 May 2022 15:00:05 +0200 Subject: [PATCH 012/145] ADD CONTAMINANT FILTER SUBWORKFLOW Adds a workflow to run the contamination filtering processes introduced by an earlier commit. --- subworkflows/local/contaminant_filter.nf | 139 +++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 subworkflows/local/contaminant_filter.nf diff --git a/subworkflows/local/contaminant_filter.nf b/subworkflows/local/contaminant_filter.nf new file mode 100644 index 00000000..ffcd6697 --- /dev/null +++ b/subworkflows/local/contaminant_filter.nf @@ -0,0 +1,139 @@ +// +// Filter contamination by rrna, trna, cdna, ncma, pirna +// + +include { BLAT_MIRNA as BLAT_CDNA + BLAT_MIRNA as BLAT_NCRNA + BLAT_MIRNA as BLAT_PIRNA } from '../../modules/local/blat_mirna' + +include { INDEX_CONTAMINANTS as INDEX_RRNA + INDEX_CONTAMINANTS as INDEX_TRNA + INDEX_CONTAMINANTS as INDEX_CDNA + INDEX_CONTAMINANTS as INDEX_NCRNA + INDEX_CONTAMINANTS as INDEX_PIRNA } from '../../modules/local/bowtie_contaminants' + +include { BOWTIE_MAP_CONTAMINANTS as MAP_RRNA + BOWTIE_MAP_CONTAMINANTS as MAP_TRNA + BOWTIE_MAP_CONTAMINANTS as MAP_CDNA + BOWTIE_MAP_CONTAMINANTS as MAP_NCRNA + BOWTIE_MAP_CONTAMINANTS as MAP_PIRNA } from '../../modules/local/bowtie_map_contaminants' + +workflow CONTAMINANT_FILTER { + take: + mirna + rrna + trna + cdna + ncrna + pirna + reads // channel: [ val(meta), [ reads ] ] + + main: + + ch_versions = Channel.empty() + + rrna_reads = reads + + reads + .map { add_suffix(it, "rrna") } + .dump (tag:'rrna') + .set { rrna_reads } + + + if (params.rrna) { + // Index DB and filter $reads emit: $rrna_reads + INDEX_RRNA ( rrna ) + ch_versions = ch_versions.mix(INDEX_RRNA.out.versions) + MAP_RRNA ( rrna_reads, INDEX_RRNA.out.bt_indices ) + ch_versions = ch_versions.mix(MAP_RRNA.out.versions) + MAP_RRNA.out.unmapped + .map { add_suffix(it, "rrna") } + .dump (tag:'rrna') + .set { rrna_reads } + } + + rrna_reads + .map { add_suffix(it, "rrna") } + .dump (tag:'rrna') + .set { trna_reads } + + if (params.trna) { + // Index DB and filter $rrna_reads emit: $trna_reads + INDEX_TRNA ( trna ) + ch_versions = ch_versions.mix(INDEX_TRNA.out.versions) + MAP_TRNA ( rrna_reads, INDEX_TRNA.out.bt_indices) + ch_versions = ch_versions.mix(MAP_TRNA.out.versions) + MAP_TRNA.out.unmapped + .map { add_suffix(it, "trna") } + .dump (tag:'trna') + .set { trna_reads } + } + + trna_reads + .map { add_suffix(it, "cdna") } + .dump (tag:'cdna') + .set { cdna_reads } + + + if (params.cdna) { + BLAT_CDNA ( 'cdna', mirna, cdna ) + ch_versions = ch_versions.mix(BLAT_CDNA.out.versions) + INDEX_CDNA ( BLAT_CDNA.out.filtered_set ) + ch_versions = ch_versions.mix(INDEX_CDNA.out.versions) + MAP_CDNA ( trna_reads, INDEX_CDNA.out.bt_indices ) + ch_versions = ch_versions.mix(MAP_CDNA.out.versions) + MAP_CDNA.out.unmapped + .map { add_suffix(it, "cdna") } + .dump (tag:'cdna') + .set { cdna_reads } + } + + cdna_reads + .map { add_suffix(it, "ncrna") } + .dump (tag:'ncrna') + .set { ncrna_reads } + + if (params.ncrna) { + BLAT_NCRNA ( 'ncrna', mirna, ncrna ) + ch_versions = ch_versions.mix(BLAT_NCRNA.out.versions) + INDEX_NCRNA ( BLAT_NCRNA.out.filtered_set ) + ch_versions = ch_versions.mix(INDEX_NCRNA.out.versions) + MAP_NCRNA ( cdna_reads, INDEX_NCRNA.out.bt_indices ) + ch_versions = ch_versions.mix(MAP_NCRNA.out.versions) + MAP_NCRNA.out.unmapped + .map { add_suffix(it, "ncrna") } + .dump (tag:'ncrna') + .set { ncrna_reads } + } + + ncrna_reads + .map { add_suffix(it, "pirna") } + .dump (tag:'pirna') + .set { pirna_reads } + + if (params.pirna) { + BLAT_PIRNA ( 'other', mirna, pirna ) + ch_versions = ch_versions.mix(BLAT_PIRNA.out.versions) + INDEX_PIRNA ( BLAT_PIRNA.out.filtered_set ) + ch_versions = ch_versions.mix(INDEX_PIRNA.out.versions) + MAP_PIRNA (ncrna_reads, INDEX_PIRNA.out.bt_indices ) + ch_versions = ch_versions.mix(MAP_PIRNA.out.versions) + MAP_PIRNA.out.unmapped + .map { add_suffix(it, "pirna") } + .dump (tag:'pirna') + .set { pirna_reads } + } + + + emit: + filtered_reads = pirna_reads + versions = ch_versions +} + +def add_suffix(row, suffix) { + def meta = [:] + meta.id = "${row[0].id}_${suffix}" + def array = [] + array = [ meta, row[1] ] + return array +} \ No newline at end of file From 13f7136ee93fbea145852b297cbbe9650f7eb13a Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Tue, 17 May 2022 15:01:54 +0200 Subject: [PATCH 013/145] IMPLEMENT CONTAMINATION FILTERING Adds the contamination filtering subworkflow into the existing pipeline. --- workflows/smrnaseq.nf | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index ff8669bf..7fa0557e 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -1,3 +1,11 @@ +params.rrna = '/Users/chriskub/software/Q2687/Q2687/02_testdata/01_BI_data/contaminants/rRNACollection.fas' +params.trna = '' +params.cdna = '/Users/chriskub/software/Q2687/Q2687/02_testdata/01_BI_data/contaminants/Homo_sapiens.GRCh38.cdna.all.fa' +params.ncrna = '/Users/chriskub/software/Q2687/Q2687/02_testdata/01_BI_data/contaminants/Homo_sapiens.GRCh38.ncrna.fa' +params.pirna = '' +params.filter_contamination = 'true' + + /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ VALIDATE INPUTS @@ -54,12 +62,13 @@ ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multi if (params.mature) { reference_mature = file(params.mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mature}" } if (params.hairpin) { reference_hairpin = file(params.hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.hairpin}" } -include { INPUT_CHECK } from '../subworkflows/local/input_check' -include { FASTQC_TRIMGALORE } from '../subworkflows/nf-core/fastqc_trimgalore' -include { MIRNA_QUANT } from '../subworkflows/local/mirna_quant' -include { GENOME_QUANT } from '../subworkflows/local/genome_quant' -include { MIRTRACE } from '../subworkflows/local/mirtrace' -include { MIRDEEP2 } from '../subworkflows/local/mirdeep2' +include { INPUT_CHECK } from '../subworkflows/local/input_check' +include { FASTQC_TRIMGALORE } from '../subworkflows/nf-core/fastqc_trimgalore' +include { CONTAMINANT_FILTER } from '../subworkflows/local/contaminant_filter' +include { MIRNA_QUANT } from '../subworkflows/local/mirna_quant' +include { GENOME_QUANT } from '../subworkflows/local/genome_quant' +include { MIRTRACE } from '../subworkflows/local/mirtrace' +include { MIRDEEP2 } from '../subworkflows/local/mirdeep2' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -141,6 +150,24 @@ workflow SMRNASEQ { ch_versions = ch_versions.mix(FASTQC_TRIMGALORE.out.versions) reads_for_mirna = FASTQC_TRIMGALORE.out.reads + // + // SUBWORKFLOW: remove contaminants from reads + // + if (params.filter_contamination){ + CONTAMINANT_FILTER ( + reference_hairpin, + params.rrna, + params.trna, + params.cdna, + params.ncrna, + params.pirna, + FASTQC_TRIMGALORE.out.reads + ) + + reads_for_mirna = CONTAMINANT_FILTER.out.filtered_reads + ch_versions = ch_versions.mix(CONTAMINANT_FILTER.out.versions) + } + MIRNA_QUANT ( reference_mature, reference_hairpin, From 430e86492b46556abbfbfdd406ced3d716b97817 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Thu, 19 May 2022 08:19:38 +0200 Subject: [PATCH 014/145] ADD FILTER_STATS PROCESS The added process collects the contamination filtering statistics and combines them to be added to the MultiQC output. --- modules/local/filter_stats.nf | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 modules/local/filter_stats.nf diff --git a/modules/local/filter_stats.nf b/modules/local/filter_stats.nf new file mode 100644 index 00000000..a2208312 --- /dev/null +++ b/modules/local/filter_stats.nf @@ -0,0 +1,23 @@ +process FILTER_STATS { +// tag "$meta.id" + label 'process_medium' + + conda (params.enable_conda ? 'bowtie2=2.4.5' : null) + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/bowtie2:2.4.5--py39hd2f7db1_2' : + 'quay.io/biocontainers/bowtie2:2.4.5--py36hfca12d5_2' }" + + input: + path reads + file stats + + output: + path "*_mqc.yaml" , emit: stats + + script: + """ + echo filtered* + touch test_mqc.yaml + """ + +} \ No newline at end of file From f83a58a91f0f84ad66ef9d2aef4879b67987e97a Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Thu, 19 May 2022 08:25:20 +0200 Subject: [PATCH 015/145] ADD FILTER STATISTICS Added the FILTER_STATS step and the necessary alterations to hand over the data to the new process. --- modules/local/bowtie_map_contaminants.nf | 4 +++- subworkflows/local/contaminant_filter.nf | 21 ++++++++++++++++----- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/modules/local/bowtie_map_contaminants.nf b/modules/local/bowtie_map_contaminants.nf index 5330f111..5abf3fb8 100644 --- a/modules/local/bowtie_map_contaminants.nf +++ b/modules/local/bowtie_map_contaminants.nf @@ -10,11 +10,13 @@ process BOWTIE_MAP_CONTAMINANTS { input: tuple val(meta), path(reads) path index + val contaminant_type output: tuple val(meta), path("*sam") , emit: bam tuple val(meta), path('*.filter.unmapped.contaminant.fastq'), emit: unmapped path "versions.yml" , emit: versions + path "filtered.*.stats" , emit: stats script: def index_base = index.toString().tokenize(' ')[0].tokenize('.')[0] @@ -29,7 +31,7 @@ process BOWTIE_MAP_CONTAMINANTS { -S ${meta.id}.filter.contaminant.sam > ${meta.id}.contaminant_bowtie.log 2>&1 # extracting number of reads from bowtie logs - awk 'BEGIN{tot=0} {if(NR==4 || NR == 5){tot += \$1}} END {print tot }' ${meta.id}.contaminant_bowtie.log | tr -d , > ${meta.id}.contaminants.txt + awk -v type=${contaminant_type} 'BEGIN{tot=0} {if(NR==4 || NR == 5){tot += \$1}} END {print type": "tot }' ${meta.id}.contaminant_bowtie.log | tr -d , > filtered.${meta.id}.stats cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/subworkflows/local/contaminant_filter.nf b/subworkflows/local/contaminant_filter.nf index ffcd6697..a5192e29 100644 --- a/subworkflows/local/contaminant_filter.nf +++ b/subworkflows/local/contaminant_filter.nf @@ -18,6 +18,8 @@ include { BOWTIE_MAP_CONTAMINANTS as MAP_RRNA BOWTIE_MAP_CONTAMINANTS as MAP_NCRNA BOWTIE_MAP_CONTAMINANTS as MAP_PIRNA } from '../../modules/local/bowtie_map_contaminants' +include { FILTER_STATS } from '../../modules/local/filter_stats' + workflow CONTAMINANT_FILTER { take: mirna @@ -31,6 +33,7 @@ workflow CONTAMINANT_FILTER { main: ch_versions = Channel.empty() + ch_filter_stats = Channel.empty() rrna_reads = reads @@ -44,8 +47,9 @@ workflow CONTAMINANT_FILTER { // Index DB and filter $reads emit: $rrna_reads INDEX_RRNA ( rrna ) ch_versions = ch_versions.mix(INDEX_RRNA.out.versions) - MAP_RRNA ( rrna_reads, INDEX_RRNA.out.bt_indices ) + MAP_RRNA ( rrna_reads, INDEX_RRNA.out.bt_indices, '"rRNA"' ) ch_versions = ch_versions.mix(MAP_RRNA.out.versions) + ch_filter_stats = ch_filter_stats.mix(MAP_RRNA.out.stats.ifEmpty(null)) MAP_RRNA.out.unmapped .map { add_suffix(it, "rrna") } .dump (tag:'rrna') @@ -61,8 +65,9 @@ workflow CONTAMINANT_FILTER { // Index DB and filter $rrna_reads emit: $trna_reads INDEX_TRNA ( trna ) ch_versions = ch_versions.mix(INDEX_TRNA.out.versions) - MAP_TRNA ( rrna_reads, INDEX_TRNA.out.bt_indices) + MAP_TRNA ( rrna_reads, INDEX_TRNA.out.bt_indices, '"tRNA"') ch_versions = ch_versions.mix(MAP_TRNA.out.versions) + ch_filter_stats = ch_filter_stats.mix(MAP_TRNA.out.stats.ifEmpty(null)) MAP_TRNA.out.unmapped .map { add_suffix(it, "trna") } .dump (tag:'trna') @@ -80,8 +85,9 @@ workflow CONTAMINANT_FILTER { ch_versions = ch_versions.mix(BLAT_CDNA.out.versions) INDEX_CDNA ( BLAT_CDNA.out.filtered_set ) ch_versions = ch_versions.mix(INDEX_CDNA.out.versions) - MAP_CDNA ( trna_reads, INDEX_CDNA.out.bt_indices ) + MAP_CDNA ( trna_reads, INDEX_CDNA.out.bt_indices, '"cDNA"' ) ch_versions = ch_versions.mix(MAP_CDNA.out.versions) + ch_filter_stats = ch_filter_stats.mix(MAP_CDNA.out.stats.ifEmpty(null)) MAP_CDNA.out.unmapped .map { add_suffix(it, "cdna") } .dump (tag:'cdna') @@ -98,8 +104,9 @@ workflow CONTAMINANT_FILTER { ch_versions = ch_versions.mix(BLAT_NCRNA.out.versions) INDEX_NCRNA ( BLAT_NCRNA.out.filtered_set ) ch_versions = ch_versions.mix(INDEX_NCRNA.out.versions) - MAP_NCRNA ( cdna_reads, INDEX_NCRNA.out.bt_indices ) + MAP_NCRNA ( cdna_reads, INDEX_NCRNA.out.bt_indices, '"ncRNA"' ) ch_versions = ch_versions.mix(MAP_NCRNA.out.versions) + ch_filter_stats = ch_filter_stats.mix(MAP_NCRNA.out.stats.ifEmpty(null)) MAP_NCRNA.out.unmapped .map { add_suffix(it, "ncrna") } .dump (tag:'ncrna') @@ -116,18 +123,22 @@ workflow CONTAMINANT_FILTER { ch_versions = ch_versions.mix(BLAT_PIRNA.out.versions) INDEX_PIRNA ( BLAT_PIRNA.out.filtered_set ) ch_versions = ch_versions.mix(INDEX_PIRNA.out.versions) - MAP_PIRNA (ncrna_reads, INDEX_PIRNA.out.bt_indices ) + MAP_PIRNA (ncrna_reads, INDEX_PIRNA.out.bt_indices, '"piRNA"' ) ch_versions = ch_versions.mix(MAP_PIRNA.out.versions) + ch_filter_stats = ch_filter_stats.mix(MAP_PIRNA.out.stats.ifEmpty(null)) MAP_PIRNA.out.unmapped .map { add_suffix(it, "pirna") } .dump (tag:'pirna') .set { pirna_reads } } + FILTER_STATS ( pirna_reads, ch_filter_stats.collect() ) + emit: filtered_reads = pirna_reads versions = ch_versions + filter_stats = FILTER_STATS.out.stats } def add_suffix(row, suffix) { From 964b503fde9d0ad5a6e26ad0f71c85d0d6c9270d Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Thu, 19 May 2022 14:51:33 +0200 Subject: [PATCH 016/145] STAGE STATS FILES Enables the process to access all required stats files of each sample for processing. --- modules/local/filter_stats.nf | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/local/filter_stats.nf b/modules/local/filter_stats.nf index a2208312..85d9b94f 100644 --- a/modules/local/filter_stats.nf +++ b/modules/local/filter_stats.nf @@ -8,16 +8,18 @@ process FILTER_STATS { 'quay.io/biocontainers/bowtie2:2.4.5--py36hfca12d5_2' }" input: - path reads - file stats + tuple val(meta), path(reads) + path stats_files output: path "*_mqc.yaml" , emit: stats script: """ - echo filtered* - touch test_mqc.yaml + for file in ./filtered.${meta.id}_*.stats + do + echo \$file + done """ } \ No newline at end of file From 59102f269abcc6726317e9bdcf5fbe85c71fb2cf Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Thu, 19 May 2022 14:53:22 +0200 Subject: [PATCH 017/145] CHANGE FILE NAMES Removed the appended filter tags and changed the naming of the individual stats files to be better accessible downstream --- modules/local/bowtie_map_contaminants.nf | 2 +- subworkflows/local/contaminant_filter.nf | 40 ++++++++++++------------ 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/modules/local/bowtie_map_contaminants.nf b/modules/local/bowtie_map_contaminants.nf index 5abf3fb8..8f9ee952 100644 --- a/modules/local/bowtie_map_contaminants.nf +++ b/modules/local/bowtie_map_contaminants.nf @@ -31,7 +31,7 @@ process BOWTIE_MAP_CONTAMINANTS { -S ${meta.id}.filter.contaminant.sam > ${meta.id}.contaminant_bowtie.log 2>&1 # extracting number of reads from bowtie logs - awk -v type=${contaminant_type} 'BEGIN{tot=0} {if(NR==4 || NR == 5){tot += \$1}} END {print type": "tot }' ${meta.id}.contaminant_bowtie.log | tr -d , > filtered.${meta.id}.stats + awk -v type=${contaminant_type} 'BEGIN{tot=0} {if(NR==4 || NR == 5){tot += \$1}} END {print type": "tot }' ${meta.id}.contaminant_bowtie.log | tr -d , > filtered.${meta.id}_${contaminant_type}.stats cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/subworkflows/local/contaminant_filter.nf b/subworkflows/local/contaminant_filter.nf index a5192e29..71b11655 100644 --- a/subworkflows/local/contaminant_filter.nf +++ b/subworkflows/local/contaminant_filter.nf @@ -38,8 +38,8 @@ workflow CONTAMINANT_FILTER { rrna_reads = reads reads - .map { add_suffix(it, "rrna") } - .dump (tag:'rrna') + .map {} + .dump () .set { rrna_reads } @@ -51,14 +51,14 @@ workflow CONTAMINANT_FILTER { ch_versions = ch_versions.mix(MAP_RRNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_RRNA.out.stats.ifEmpty(null)) MAP_RRNA.out.unmapped - .map { add_suffix(it, "rrna") } - .dump (tag:'rrna') + .map {} + .dump () .set { rrna_reads } } rrna_reads - .map { add_suffix(it, "rrna") } - .dump (tag:'rrna') + .map {} + .dump () .set { trna_reads } if (params.trna) { @@ -69,14 +69,14 @@ workflow CONTAMINANT_FILTER { ch_versions = ch_versions.mix(MAP_TRNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_TRNA.out.stats.ifEmpty(null)) MAP_TRNA.out.unmapped - .map { add_suffix(it, "trna") } - .dump (tag:'trna') + .map {} + .dump () .set { trna_reads } } trna_reads - .map { add_suffix(it, "cdna") } - .dump (tag:'cdna') + .map {} + .dump () .set { cdna_reads } @@ -89,14 +89,14 @@ workflow CONTAMINANT_FILTER { ch_versions = ch_versions.mix(MAP_CDNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_CDNA.out.stats.ifEmpty(null)) MAP_CDNA.out.unmapped - .map { add_suffix(it, "cdna") } - .dump (tag:'cdna') + .map {} + .dump () .set { cdna_reads } } cdna_reads - .map { add_suffix(it, "ncrna") } - .dump (tag:'ncrna') + .map {} + .dump () .set { ncrna_reads } if (params.ncrna) { @@ -108,14 +108,14 @@ workflow CONTAMINANT_FILTER { ch_versions = ch_versions.mix(MAP_NCRNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_NCRNA.out.stats.ifEmpty(null)) MAP_NCRNA.out.unmapped - .map { add_suffix(it, "ncrna") } - .dump (tag:'ncrna') + .map {} + .dump () .set { ncrna_reads } } ncrna_reads - .map { add_suffix(it, "pirna") } - .dump (tag:'pirna') + .map {} + .dump () .set { pirna_reads } if (params.pirna) { @@ -127,8 +127,8 @@ workflow CONTAMINANT_FILTER { ch_versions = ch_versions.mix(MAP_PIRNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_PIRNA.out.stats.ifEmpty(null)) MAP_PIRNA.out.unmapped - .map { add_suffix(it, "pirna") } - .dump (tag:'pirna') + .map {} + .dump () .set { pirna_reads } } From c09e628b02b2903d1fce91db421a16b33ae21b89 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Fri, 20 May 2022 11:56:11 +0200 Subject: [PATCH 018/145] ADD MULTIQC OUTPUT Altered the statistics output to be in line with the format for custom multiQC plots. In addition the number of remaining reads is now included in the statistics. --- modules/local/filter_stats.nf | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/local/filter_stats.nf b/modules/local/filter_stats.nf index 85d9b94f..506829bf 100644 --- a/modules/local/filter_stats.nf +++ b/modules/local/filter_stats.nf @@ -13,13 +13,15 @@ process FILTER_STATS { output: path "*_mqc.yaml" , emit: stats + tuple val(meta), path('*.filtered.fastq') , emit: reads script: """ - for file in ./filtered.${meta.id}_*.stats - do - echo \$file - done + readnumber=\$(wc -l ${reads} | awk '{ print \$1/4 }') + cat ./filtered.${meta.id}_*.stats | \\ + tr '\n' ', ' | \\ + awk -v sample=${meta.id} -v readnumber=\$readnumber '{ print "id: \\"my_pca_section\\"\\nsection_name: \\"Contamination Filtering\\"\\ndescription: \\"This plot shows the amount of reads filtered by contaminant type.\\"\\nplot_type: \\"bargraph\\"\\npconfig:\\n id: \\"contamination_filter_plot\\"\\n title: \\"Contamination Plot\\"\\n ylab: \\"Number of reads\\"\\ndata:\\n "sample": {"\$0"\\"remaining reads\\": "readnumber"}" }' > ${meta.id}.contamination_mqc.yaml + cat ${reads} > ${meta.id}.filtered.fastq """ } \ No newline at end of file From 497f009cfd1f0b9dda6e3712989f41cf8a4cbdd4 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Fri, 20 May 2022 11:58:08 +0200 Subject: [PATCH 019/145] CHANGE OUTPUT FILENAME Changed the output file name and adjusted the awk output to be readable by multiQC. --- modules/local/bowtie_map_contaminants.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/local/bowtie_map_contaminants.nf b/modules/local/bowtie_map_contaminants.nf index 8f9ee952..04dff4a3 100644 --- a/modules/local/bowtie_map_contaminants.nf +++ b/modules/local/bowtie_map_contaminants.nf @@ -26,12 +26,12 @@ process BOWTIE_MAP_CONTAMINANTS { --very-sensitive-local \\ -k 1 \\ -x $index_base \\ - --un ${meta.id}.filter.unmapped.contaminant.fastq \\ + --un ${meta.id}.${contaminant_type}.filter.unmapped.contaminant.fastq \\ ${reads} \\ -S ${meta.id}.filter.contaminant.sam > ${meta.id}.contaminant_bowtie.log 2>&1 # extracting number of reads from bowtie logs - awk -v type=${contaminant_type} 'BEGIN{tot=0} {if(NR==4 || NR == 5){tot += \$1}} END {print type": "tot }' ${meta.id}.contaminant_bowtie.log | tr -d , > filtered.${meta.id}_${contaminant_type}.stats + awk -v type=${contaminant_type} 'BEGIN{tot=0} {if(NR==4 || NR == 5){tot += \$1}} END {print "\\""type"\\": "tot }' ${meta.id}.contaminant_bowtie.log | tr -d , > filtered.${meta.id}_${contaminant_type}.stats cat <<-END_VERSIONS > versions.yml "${task.process}": From 44fdffbd13094d70659aad632cafdf67136859d2 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Fri, 20 May 2022 11:59:36 +0200 Subject: [PATCH 020/145] ADD OPTIONAL CUSTOM FILTER Added an additional filter step that can be used to filter with a custom database in addition to the other databases. --- subworkflows/local/contaminant_filter.nf | 91 ++++++++++-------------- 1 file changed, 39 insertions(+), 52 deletions(-) diff --git a/subworkflows/local/contaminant_filter.nf b/subworkflows/local/contaminant_filter.nf index 71b11655..14a51f47 100644 --- a/subworkflows/local/contaminant_filter.nf +++ b/subworkflows/local/contaminant_filter.nf @@ -2,21 +2,24 @@ // Filter contamination by rrna, trna, cdna, ncma, pirna // -include { BLAT_MIRNA as BLAT_CDNA - BLAT_MIRNA as BLAT_NCRNA - BLAT_MIRNA as BLAT_PIRNA } from '../../modules/local/blat_mirna' +include { BLAT_MIRNA as BLAT_CDNA + BLAT_MIRNA as BLAT_NCRNA + BLAT_MIRNA as BLAT_PIRNA + BLAT_MIRNA as BLAT_OTHER } from '../../modules/local/blat_mirna' include { INDEX_CONTAMINANTS as INDEX_RRNA INDEX_CONTAMINANTS as INDEX_TRNA INDEX_CONTAMINANTS as INDEX_CDNA INDEX_CONTAMINANTS as INDEX_NCRNA - INDEX_CONTAMINANTS as INDEX_PIRNA } from '../../modules/local/bowtie_contaminants' + INDEX_CONTAMINANTS as INDEX_PIRNA + INDEX_CONTAMINANTS as INDEX_OTHER } from '../../modules/local/bowtie_contaminants' include { BOWTIE_MAP_CONTAMINANTS as MAP_RRNA BOWTIE_MAP_CONTAMINANTS as MAP_TRNA BOWTIE_MAP_CONTAMINANTS as MAP_CDNA BOWTIE_MAP_CONTAMINANTS as MAP_NCRNA - BOWTIE_MAP_CONTAMINANTS as MAP_PIRNA } from '../../modules/local/bowtie_map_contaminants' + BOWTIE_MAP_CONTAMINANTS as MAP_PIRNA + BOWTIE_MAP_CONTAMINANTS as MAP_OTHER } from '../../modules/local/bowtie_map_contaminants' include { FILTER_STATS } from '../../modules/local/filter_stats' @@ -28,6 +31,7 @@ workflow CONTAMINANT_FILTER { cdna ncrna pirna + other reads // channel: [ val(meta), [ reads ] ] main: @@ -37,47 +41,32 @@ workflow CONTAMINANT_FILTER { rrna_reads = reads - reads - .map {} - .dump () - .set { rrna_reads } + reads.set { rrna_reads } if (params.rrna) { // Index DB and filter $reads emit: $rrna_reads INDEX_RRNA ( rrna ) ch_versions = ch_versions.mix(INDEX_RRNA.out.versions) - MAP_RRNA ( rrna_reads, INDEX_RRNA.out.bt_indices, '"rRNA"' ) + MAP_RRNA ( reads, INDEX_RRNA.out.bt_indices, 'rRNA' ) ch_versions = ch_versions.mix(MAP_RRNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_RRNA.out.stats.ifEmpty(null)) - MAP_RRNA.out.unmapped - .map {} - .dump () - .set { rrna_reads } + MAP_RRNA.out.unmapped.set { rrna_reads } } - rrna_reads - .map {} - .dump () - .set { trna_reads } + rrna_reads.set { trna_reads } if (params.trna) { // Index DB and filter $rrna_reads emit: $trna_reads INDEX_TRNA ( trna ) ch_versions = ch_versions.mix(INDEX_TRNA.out.versions) - MAP_TRNA ( rrna_reads, INDEX_TRNA.out.bt_indices, '"tRNA"') + MAP_TRNA ( rrna_reads, INDEX_TRNA.out.bt_indices, 'tRNA') ch_versions = ch_versions.mix(MAP_TRNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_TRNA.out.stats.ifEmpty(null)) - MAP_TRNA.out.unmapped - .map {} - .dump () - .set { trna_reads } + MAP_TRNA.out.unmapped.set { trna_reads } } - trna_reads - .map {} - .dump () - .set { cdna_reads } + trna_reads.set { cdna_reads } if (params.cdna) { @@ -85,58 +74,56 @@ workflow CONTAMINANT_FILTER { ch_versions = ch_versions.mix(BLAT_CDNA.out.versions) INDEX_CDNA ( BLAT_CDNA.out.filtered_set ) ch_versions = ch_versions.mix(INDEX_CDNA.out.versions) - MAP_CDNA ( trna_reads, INDEX_CDNA.out.bt_indices, '"cDNA"' ) + MAP_CDNA ( trna_reads, INDEX_CDNA.out.bt_indices, 'cDNA' ) ch_versions = ch_versions.mix(MAP_CDNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_CDNA.out.stats.ifEmpty(null)) - MAP_CDNA.out.unmapped - .map {} - .dump () - .set { cdna_reads } + MAP_CDNA.out.unmapped.set { cdna_reads } } - cdna_reads - .map {} - .dump () - .set { ncrna_reads } + cdna_reads.set { ncrna_reads } if (params.ncrna) { BLAT_NCRNA ( 'ncrna', mirna, ncrna ) ch_versions = ch_versions.mix(BLAT_NCRNA.out.versions) INDEX_NCRNA ( BLAT_NCRNA.out.filtered_set ) ch_versions = ch_versions.mix(INDEX_NCRNA.out.versions) - MAP_NCRNA ( cdna_reads, INDEX_NCRNA.out.bt_indices, '"ncRNA"' ) + MAP_NCRNA ( cdna_reads, INDEX_NCRNA.out.bt_indices, 'ncRNA' ) ch_versions = ch_versions.mix(MAP_NCRNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_NCRNA.out.stats.ifEmpty(null)) - MAP_NCRNA.out.unmapped - .map {} - .dump () - .set { ncrna_reads } + MAP_NCRNA.out.unmapped.set { ncrna_reads } } - ncrna_reads - .map {} - .dump () - .set { pirna_reads } + ncrna_reads.set { pirna_reads } if (params.pirna) { BLAT_PIRNA ( 'other', mirna, pirna ) ch_versions = ch_versions.mix(BLAT_PIRNA.out.versions) INDEX_PIRNA ( BLAT_PIRNA.out.filtered_set ) ch_versions = ch_versions.mix(INDEX_PIRNA.out.versions) - MAP_PIRNA (ncrna_reads, INDEX_PIRNA.out.bt_indices, '"piRNA"' ) + MAP_PIRNA (ncrna_reads, INDEX_PIRNA.out.bt_indices, 'piRNA' ) ch_versions = ch_versions.mix(MAP_PIRNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_PIRNA.out.stats.ifEmpty(null)) - MAP_PIRNA.out.unmapped - .map {} - .dump () - .set { pirna_reads } + MAP_PIRNA.out.unmapped.set { pirna_reads } } - FILTER_STATS ( pirna_reads, ch_filter_stats.collect() ) + pirna_reads.set { other_cont_reads } + + if (other) { + BLAT_OTHER ( 'other', mirna, other) + ch_versions = ch_versions.mix(BLAT_OTHER.out.versions) + INDEX_OTHER ( BLAT_OTHER.out.filtered_set ) + ch_versions = ch_versions.mix(INDEX_OTHER.out.versions) + MAP_OTHER (ncrna_reads, INDEX_OTHER.out.bt_indices, 'other' ) + ch_versions = ch_versions.mix(MAP_OTHER.out.versions) + ch_filter_stats = ch_filter_stats.mix(MAP_OTHER.out.stats.ifEmpty(null)) + MAP_OTHER.out.unmapped.set { other_cont_reads } + } + + FILTER_STATS ( other_cont_reads, ch_filter_stats.collect() ) emit: - filtered_reads = pirna_reads + filtered_reads = FILTER_STATS.out.reads versions = ch_versions filter_stats = FILTER_STATS.out.stats } From 9ede40136e08dfa2b254b3aaaed52d1c1406d01a Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Fri, 20 May 2022 12:01:40 +0200 Subject: [PATCH 021/145] ADD MULTIQC OUTPUT Added the contamination filtering statistics to the multiqc output. --- workflows/smrnaseq.nf | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 7fa0557e..433f8450 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -1,11 +1,3 @@ -params.rrna = '/Users/chriskub/software/Q2687/Q2687/02_testdata/01_BI_data/contaminants/rRNACollection.fas' -params.trna = '' -params.cdna = '/Users/chriskub/software/Q2687/Q2687/02_testdata/01_BI_data/contaminants/Homo_sapiens.GRCh38.cdna.all.fa' -params.ncrna = '/Users/chriskub/software/Q2687/Q2687/02_testdata/01_BI_data/contaminants/Homo_sapiens.GRCh38.ncrna.fa' -params.pirna = '' -params.filter_contamination = 'true' - - /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ VALIDATE INPUTS @@ -161,6 +153,7 @@ workflow SMRNASEQ { params.cdna, params.ncrna, params.pirna, + params.other_contamination FASTQC_TRIMGALORE.out.reads ) @@ -214,6 +207,7 @@ workflow SMRNASEQ { ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(FASTQC_TRIMGALORE.out.fastqc_zip.collect{it[1]}.ifEmpty([])) + ch_multiqc_files = ch_multiqc_files.mix(CONTAMINANT_FILTER.out.filter_stats.collect{it[1]}.ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.mature_stats.collect({it[1]}).ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.hairpin_stats.collect({it[1]}).ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(genome_stats.collect({it[1]}).ifEmpty([])) From dac4e06d16a2955545ea28059e672c8ee97f8dd9 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Fri, 20 May 2022 12:02:54 +0200 Subject: [PATCH 022/145] FIX TYPO --- subworkflows/local/contaminant_filter.nf | 8 -------- 1 file changed, 8 deletions(-) diff --git a/subworkflows/local/contaminant_filter.nf b/subworkflows/local/contaminant_filter.nf index 14a51f47..6ac18267 100644 --- a/subworkflows/local/contaminant_filter.nf +++ b/subworkflows/local/contaminant_filter.nf @@ -126,12 +126,4 @@ workflow CONTAMINANT_FILTER { filtered_reads = FILTER_STATS.out.reads versions = ch_versions filter_stats = FILTER_STATS.out.stats -} - -def add_suffix(row, suffix) { - def meta = [:] - meta.id = "${row[0].id}_${suffix}" - def array = [] - array = [ meta, row[1] ] - return array } \ No newline at end of file From cc9a02c157c66bb59d87394950faee046722dc09 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Fri, 20 May 2022 13:27:43 +0200 Subject: [PATCH 023/145] FIX TYPO --- workflows/smrnaseq.nf | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 433f8450..80b481c9 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -1,3 +1,12 @@ +params.rrna = '/Users/chriskub/software/Q2687/Q2687/02_testdata/01_BI_data/contaminants/rRNACollection.fas' +params.trna = '/Users/chriskub/software/Q2687/Q2687/02_testdata/01_BI_data/contaminants/hg38-tRNAs.fa' +params.cdna = '/Users/chriskub/software/Q2687/Q2687/02_testdata/01_BI_data/contaminants/hsa.cdna.10000.fa' +params.ncrna = '/Users/chriskub/software/Q2687/Q2687/02_testdata/01_BI_data/contaminants/hsa.ncrna.10000.fa' +params.pirna = '/Users/chriskub/software/Q2687/Q2687/02_testdata/01_BI_data/contaminants/hsa.pirna.10000.fa' +params.other_contamination = '' +params.filter_contamination = 'true' + + /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ VALIDATE INPUTS @@ -153,7 +162,7 @@ workflow SMRNASEQ { params.cdna, params.ncrna, params.pirna, - params.other_contamination + params.other_contamination, FASTQC_TRIMGALORE.out.reads ) From afc4c883640a0bc4d610e20cbc1934bb561a112a Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Fri, 20 May 2022 13:32:05 +0200 Subject: [PATCH 024/145] ADD CONTAMINATION FILTERING OPTIONS Added the options and default settings to filter contamination from the supplied sequence files. By default contamination filtering is disabled. --- nextflow.config | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nextflow.config b/nextflow.config index c5fa807d..74f451f0 100644 --- a/nextflow.config +++ b/nextflow.config @@ -40,6 +40,15 @@ params { save_reference = false trim_galore_max_length = 40 + // Contamination filtering + rrna = null + trna = null + cdna = null + ncrna = null + pirna = null + other_contamination = null + filter_contamination = 'false' + // MultiQC options multiqc_config = null multiqc_title = null From 6ffd2c707cd8142aa66537c9bd23db32388d9fd1 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Fri, 20 May 2022 13:43:59 +0200 Subject: [PATCH 025/145] ADD CONTAMINATION FILTERING OPTIONS Added the new options to the nextflow_schema. --- nextflow_schema.json | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/nextflow_schema.json b/nextflow_schema.json index 027f1b37..43d6210b 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -168,6 +168,43 @@ } } }, + "contamination_filtering": { + "title": "Contamination filter options", + "type": "object", + "description": "Options to remove contamination from the reads.", + "fa_icon": "fas fa-cut", + "properties": { + "filter_contamination": { + "type": "boolean", + "default": false, + "description": "Enables the contamination filtering." + }, + "rrna": { + "type": "file-path", + "description": "Path to the rRNA fasta file to be used as contamination database." + }, + "trna": { + "type": "file-path", + "description": "Path to the tRNA fasta file to be used as contamination database." + }, + "cdna": { + "type": "file-path", + "description": "Path to the cDNA fasta file to be used as contamination database." + }, + "ncrna": { + "type": "file-path", + "description": "Path to the ncRNA fasta file to be used as contamination database." + }, + "pirna": { + "type": "file-path", + "description": "Path to the piRNA fasta file to be used as contamination database." + }, + "other_contamination": { + "type": "file-path", + "description": "Path to an additional fasta file to be used as contamination database." + } + } + } "skipping_pipeline_steps": { "title": "Skipping pipeline steps", "type": "object", From 80ca57a9423de41f39ea8e301af2f83ba6e25a24 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Fri, 20 May 2022 13:49:26 +0200 Subject: [PATCH 026/145] FIX TYPO --- nextflow_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 43d6210b..987fc88a 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -204,7 +204,7 @@ "description": "Path to an additional fasta file to be used as contamination database." } } - } + }, "skipping_pipeline_steps": { "title": "Skipping pipeline steps", "type": "object", From f6fb3793ffd0d5b7e33871b5c67d9df6ffa27e81 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Fri, 20 May 2022 13:51:09 +0200 Subject: [PATCH 027/145] ADD FORMAT --- nextflow_schema.json | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 987fc88a..277700f6 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -180,27 +180,33 @@ "description": "Enables the contamination filtering." }, "rrna": { - "type": "file-path", + "type": "string", + "format": "file-path", "description": "Path to the rRNA fasta file to be used as contamination database." }, "trna": { - "type": "file-path", + "type": "string", + "format": "file-path", "description": "Path to the tRNA fasta file to be used as contamination database." }, "cdna": { - "type": "file-path", + "type": "string", + "format": "file-path", "description": "Path to the cDNA fasta file to be used as contamination database." }, "ncrna": { - "type": "file-path", + "type": "string", + "format": "file-path", "description": "Path to the ncRNA fasta file to be used as contamination database." }, "pirna": { - "type": "file-path", + "type": "string", + "format": "file-path", "description": "Path to the piRNA fasta file to be used as contamination database." }, "other_contamination": { - "type": "file-path", + "type": "string", + "format": "file-path", "description": "Path to an additional fasta file to be used as contamination database." } } From f76f8a676129c281adcc633299bee92682d0d186 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Fri, 20 May 2022 14:21:11 +0200 Subject: [PATCH 028/145] CHANGE OUTPUT OF FILTERING TO GZIP Changed the output of the contamination step to be gzipped. --- modules/local/filter_stats.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/local/filter_stats.nf b/modules/local/filter_stats.nf index 506829bf..441f9827 100644 --- a/modules/local/filter_stats.nf +++ b/modules/local/filter_stats.nf @@ -12,8 +12,8 @@ process FILTER_STATS { path stats_files output: - path "*_mqc.yaml" , emit: stats - tuple val(meta), path('*.filtered.fastq') , emit: reads + path "*_mqc.yaml" , emit: stats + tuple val(meta), path('*.filtered.fastq.gz') , emit: reads script: """ @@ -21,7 +21,7 @@ process FILTER_STATS { cat ./filtered.${meta.id}_*.stats | \\ tr '\n' ', ' | \\ awk -v sample=${meta.id} -v readnumber=\$readnumber '{ print "id: \\"my_pca_section\\"\\nsection_name: \\"Contamination Filtering\\"\\ndescription: \\"This plot shows the amount of reads filtered by contaminant type.\\"\\nplot_type: \\"bargraph\\"\\npconfig:\\n id: \\"contamination_filter_plot\\"\\n title: \\"Contamination Plot\\"\\n ylab: \\"Number of reads\\"\\ndata:\\n "sample": {"\$0"\\"remaining reads\\": "readnumber"}" }' > ${meta.id}.contamination_mqc.yaml - cat ${reads} > ${meta.id}.filtered.fastq + gzip -c ${reads} > ${meta.id}.filtered.fastq.gz """ } \ No newline at end of file From ec889b430c5751eac62e0b4664a303d5f2eff6df Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Fri, 20 May 2022 14:22:36 +0200 Subject: [PATCH 029/145] CHANGE ORDER Moved the enabeling command for contaminaition filtering to the top of hte section. --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 74f451f0..457d4a73 100644 --- a/nextflow.config +++ b/nextflow.config @@ -41,13 +41,13 @@ params { trim_galore_max_length = 40 // Contamination filtering + filter_contamination = 'false' rrna = null trna = null cdna = null ncrna = null pirna = null other_contamination = null - filter_contamination = 'false' // MultiQC options multiqc_config = null From 49b84df359fceb6f152a9530271dbfa1ff03e5fa Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Fri, 20 May 2022 14:56:45 +0200 Subject: [PATCH 030/145] ADJUST MULTIQC INPUT Adjusted the channeluse for the contamination filtering statistics to be used in multiQC. --- modules/local/bowtie_map_contaminants.nf | 2 +- subworkflows/local/contaminant_filter.nf | 2 +- workflows/smrnaseq.nf | 12 ++---------- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/modules/local/bowtie_map_contaminants.nf b/modules/local/bowtie_map_contaminants.nf index 04dff4a3..2bfb2f5d 100644 --- a/modules/local/bowtie_map_contaminants.nf +++ b/modules/local/bowtie_map_contaminants.nf @@ -16,7 +16,7 @@ process BOWTIE_MAP_CONTAMINANTS { tuple val(meta), path("*sam") , emit: bam tuple val(meta), path('*.filter.unmapped.contaminant.fastq'), emit: unmapped path "versions.yml" , emit: versions - path "filtered.*.stats" , emit: stats + path "filtered.*.stats" , emit: stats script: def index_base = index.toString().tokenize(' ')[0].tokenize('.')[0] diff --git a/subworkflows/local/contaminant_filter.nf b/subworkflows/local/contaminant_filter.nf index 6ac18267..848682ea 100644 --- a/subworkflows/local/contaminant_filter.nf +++ b/subworkflows/local/contaminant_filter.nf @@ -38,6 +38,7 @@ workflow CONTAMINANT_FILTER { ch_versions = Channel.empty() ch_filter_stats = Channel.empty() + ch_mqc_results = Channel.empty() rrna_reads = reads @@ -121,7 +122,6 @@ workflow CONTAMINANT_FILTER { FILTER_STATS ( other_cont_reads, ch_filter_stats.collect() ) - emit: filtered_reads = FILTER_STATS.out.reads versions = ch_versions diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 80b481c9..945ca4d8 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -1,12 +1,3 @@ -params.rrna = '/Users/chriskub/software/Q2687/Q2687/02_testdata/01_BI_data/contaminants/rRNACollection.fas' -params.trna = '/Users/chriskub/software/Q2687/Q2687/02_testdata/01_BI_data/contaminants/hg38-tRNAs.fa' -params.cdna = '/Users/chriskub/software/Q2687/Q2687/02_testdata/01_BI_data/contaminants/hsa.cdna.10000.fa' -params.ncrna = '/Users/chriskub/software/Q2687/Q2687/02_testdata/01_BI_data/contaminants/hsa.ncrna.10000.fa' -params.pirna = '/Users/chriskub/software/Q2687/Q2687/02_testdata/01_BI_data/contaminants/hsa.pirna.10000.fa' -params.other_contamination = '' -params.filter_contamination = 'true' - - /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ VALIDATE INPUTS @@ -168,6 +159,7 @@ workflow SMRNASEQ { reads_for_mirna = CONTAMINANT_FILTER.out.filtered_reads ch_versions = ch_versions.mix(CONTAMINANT_FILTER.out.versions) + } MIRNA_QUANT ( @@ -216,7 +208,7 @@ workflow SMRNASEQ { ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(FASTQC_TRIMGALORE.out.fastqc_zip.collect{it[1]}.ifEmpty([])) - ch_multiqc_files = ch_multiqc_files.mix(CONTAMINANT_FILTER.out.filter_stats.collect{it[1]}.ifEmpty([])) + ch_multiqc_files = ch_multiqc_files.mix(CONTAMINANT_FILTER.out.filter_stats.collect().ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.mature_stats.collect({it[1]}).ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.hairpin_stats.collect({it[1]}).ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(genome_stats.collect({it[1]}).ifEmpty([])) From 44ba7fda1c7e0515bfc0f953a1f8027a7d0a56c6 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Fri, 20 May 2022 15:03:50 +0200 Subject: [PATCH 031/145] ADD FILTERING TO ALLOF Added the filtering section to the allOf definition of the schema. --- nextflow_schema.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nextflow_schema.json b/nextflow_schema.json index 277700f6..2b0dce56 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -428,6 +428,9 @@ { "$ref": "#/definitions/trimming_options" }, + { + "$ref": "#/definitions/contamination_filtering" + }, { "$ref": "#/definitions/skipping_pipeline_steps" }, From 70aa6addb6ebfd37a5b9ecde6b17c6abfb1f62c3 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Fri, 20 May 2022 15:05:08 +0200 Subject: [PATCH 032/145] SWITCH TO BOOLEAN --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 457d4a73..519bccfb 100644 --- a/nextflow.config +++ b/nextflow.config @@ -41,7 +41,7 @@ params { trim_galore_max_length = 40 // Contamination filtering - filter_contamination = 'false' + filter_contamination = false rrna = null trna = null cdna = null From c033d0c1f6b22724b5602aff97e74f8e6c3e1deb Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Fri, 20 May 2022 15:13:27 +0200 Subject: [PATCH 033/145] SWITCH CONTAMINATION FILTER STATS TO CHANNEL Changed the statistics output of the contamination filtering to a channel that remains empty if the step is not invoked. --- workflows/smrnaseq.nf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 945ca4d8..34df0a34 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -145,6 +145,7 @@ workflow SMRNASEQ { // // SUBWORKFLOW: remove contaminants from reads // + contamination_stats = Channel.empty() if (params.filter_contamination){ CONTAMINANT_FILTER ( reference_hairpin, @@ -159,6 +160,8 @@ workflow SMRNASEQ { reads_for_mirna = CONTAMINANT_FILTER.out.filtered_reads ch_versions = ch_versions.mix(CONTAMINANT_FILTER.out.versions) + CONTAMINANT_FILTER.out.filter_stats + .set { contamination_stats } } @@ -208,7 +211,7 @@ workflow SMRNASEQ { ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(FASTQC_TRIMGALORE.out.fastqc_zip.collect{it[1]}.ifEmpty([])) - ch_multiqc_files = ch_multiqc_files.mix(CONTAMINANT_FILTER.out.filter_stats.collect().ifEmpty([])) + ch_multiqc_files = ch_multiqc_files.mix(contamination_stats.collect().ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.mature_stats.collect({it[1]}).ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.hairpin_stats.collect({it[1]}).ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(genome_stats.collect({it[1]}).ifEmpty([])) From a61bcc9016915454916e11cf7e577fead2b95939 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Mon, 23 May 2022 11:41:37 +0200 Subject: [PATCH 034/145] ADD DOCUMENTATION Added the necessary documentation for the new features. --- README.md | 15 ++++++++------- docs/output.md | 7 +++++++ docs/usage.md | 10 ++++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 6d3efa52..8214f8c7 100644 --- a/README.md +++ b/README.md @@ -34,24 +34,25 @@ On release, automated continuous integration tests run the pipeline on a full-si 2. Adapter trimming ([`Trim Galore!`](https://www.bioinformatics.babraham.ac.uk/projects/trim_galore/)) 1. Insert Size calculation 2. Collapse reads ([`seqcluster`](https://seqcluster.readthedocs.io/mirna_annotation.html#processing-of-reads)) -3. Alignment against miRBase mature miRNA ([`Bowtie1`](http://bowtie-bio.sourceforge.net/index.shtml)) -4. Alignment against miRBase hairpin +3. Contamination filtering ([`Bowtie2`](http://bowtie-bio.sourceforge.net/bowtie2/index.shtml)) +4. Alignment against miRBase mature miRNA ([`Bowtie1`](http://bowtie-bio.sourceforge.net/index.shtml)) +5. Alignment against miRBase hairpin 1. Unaligned reads from step 3 ([`Bowtie1`](http://bowtie-bio.sourceforge.net/index.shtml)) 2. Collapsed reads from step 2.2 ([`Bowtie1`](http://bowtie-bio.sourceforge.net/index.shtml)) -5. Post-alignment processing of miRBase hairpin +6. Post-alignment processing of miRBase hairpin 1. Basic statistics from step 3 and step 4.1 ([`SAMtools`](https://sourceforge.net/projects/samtools/files/samtools/)) 2. Analysis on miRBase hairpin counts ([`edgeR`](https://bioconductor.org/packages/release/bioc/html/edgeR.html)) - TMM normalization and a table of top expression hairpin - MDS plot clustering samples - Heatmap of sample similarities 3. miRNA and isomiR annotation from step 4.1 ([`mirtop`](https://github.com/miRTop/mirtop)) -6. Alignment against host reference genome ([`Bowtie1`](http://bowtie-bio.sourceforge.net/index.shtml)) +7. Alignment against host reference genome ([`Bowtie1`](http://bowtie-bio.sourceforge.net/index.shtml)) 1. Post-alignment processing of alignment against host reference genome ([`SAMtools`](https://sourceforge.net/projects/samtools/files/samtools/)) -7. Novel miRNAs and known miRNAs discovery ([`MiRDeep2`](https://www.mdc-berlin.de/content/mirdeep2-documentation)) +8. Novel miRNAs and known miRNAs discovery ([`MiRDeep2`](https://www.mdc-berlin.de/content/mirdeep2-documentation)) 1. Mapping against reference genome with the mapper module 2. Known and novel miRNA discovery with the mirdeep2 module -8. miRNA quality control ([`mirtrace`](https://github.com/friedlanderlab/mirtrace)) -9. Present QC for raw read, alignment, and expression results ([`MultiQC`](http://multiqc.info/)) +9. miRNA quality control ([`mirtrace`](https://github.com/friedlanderlab/mirtrace)) +10. Present QC for raw read, alignment, and expression results ([`MultiQC`](http://multiqc.info/)) ## Quick Start diff --git a/docs/output.md b/docs/output.md index ce1f8347..e6b6777d 100644 --- a/docs/output.md +++ b/docs/output.md @@ -14,6 +14,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d - [FastQC](#fastqc) - read quality control - [TrimGalore](#trimgalore) - adapter trimming +- [Bowtie2](#bowtie2) - contamination filtering - [Bowtie](#bowtie) - alignment against mature miRNAs and miRNA precursors (hairpins) - [SAMtools](#samtools) - alignment result processing and feature counting - [edgeR](#edger) - normalization, MDS plot and sample pairwise distance heatmap @@ -58,6 +59,12 @@ This is an example of the output we can get: ![cutadapt](images/cutadapt_plot.png) +## Bowtie2 + +[Bowtie2](http://bowtie-bio.sourceforge.net/bowtie2/index.shtml) is used to align the reads to user defined databases of contaminants. + +MultiQC reports the number of reads that were removed by each of the contaminant databases. + ## Bowtie [Bowtie](http://bowtie-bio.sourceforge.net/index.shtml) is used for mapping adapter trimmed reads against the mature miRNAs and miRNA precursors (hairpins) in [miRBase](http://www.mirbase.org/). diff --git a/docs/usage.md b/docs/usage.md index f1304605..d7aec4f9 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -31,6 +31,16 @@ It should point to the 3-letter species name used by `miRBase`. - `fasta`: the reference genome FASTA file - `bt_indices`: points to the folder containing the `bowtie2` indices for the genome reference specified by `fasta`. **Note:** if the FASTA file in `fasta` is not the same file used to generate the `bowtie2` indices, then the pipeline will fail. +### Contamination filtering + +Contamination filtering of the sequencing reads is optional and can be invoked using `filter_contamination`. FASTA files with contamination sequences to use need to be supplied, otherwise the contamination filtering of the specific type will be omitted. using the following commands: +- `rrna`: Used to supply a FASTA file containing rRNA conatminations sequence. +- `trna`: Used to supply a FASTA file containing tRNA conatminations sequence. e.g. `http://gtrnadb.ucsc.edu/genomes/eukaryota/Hsapi38/hg38-tRNAs.fa` +- `cdna`: Used to supply a FASTA file containing cDNA conatminations sequence. e.g. `ftp://ftp.ensembl.org/pub/release-86/fasta/homo_sapiens/cdna/Homo_sapiens.GRCh38.cdna.all.fa.gz` The FASTA file is first compared to the available miRNA sequences and overlaps are removed. +- `ncrna`: Used to supply a FASTA file containing ncRNA conatminations sequence. e.g. `ftp://ftp.ensembl.org/pub/release-86/fasta/homo_sapiens/ncrna/Homo_sapiens.GRCh38.ncrna.fa.gz` The FASTA file is first compared to the available miRNA sequences and overlaps are removed. +- `pirna`: Used to supply a FASTA file containing piRNA conatminations sequence. e.g. The FASTA file is first compared to the available miRNA sequences and overlaps are removed. +- `other_contamination`: Used to supply an additional filtering set. The FASTA file is first compared to the available miRNA sequences and overlaps are removed. + ## Samplesheet input You will need to create a samplesheet with information about the samples you would like to analyse before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row as shown in the examples below. From e08f55c0e796952a49433de02aa5862ff87bd964 Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Mon, 23 May 2022 12:46:25 +0200 Subject: [PATCH 035/145] UPDATE CHANGELOG Updated the changelog to represent the latst changes. --- CHANGELOG.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1b6875e..86fec715 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,14 +21,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Other enhancements & fixes - [#134](https://github.com/nf-core/smrnaseq/issues/134) - Fixed colSum of zero issues for edgeR_miRBase.R script +- [#113](https://github.com/nf-core/smrnaseq/issues/113) - Added a optional contamination filtering step, including MultiQC plot. ### Parameters -| Old parameter | New parameter | -| -------------------- | ---------------- | -| `--conda` | `--enable_conda` | -| `--clusterOptions` | | -| `--publish_dir_mode` | | +| Old parameter | New parameter | +| -------------------- | ------------------------ | +| `--conda` | `--enable_conda` | +| `--clusterOptions` | | +| `--publish_dir_mode` | | +| | `--contamination_filter` | +| | `--rrna` | +| | `--trna` | +| | `--cdna` | +| | `--ncrna` | +| | `--pirna` | +| | `--other_contamination` | > **NB:** Parameter has been **updated** if both old and new parameter information is present. > **NB:** Parameter has been **added** if just the new parameter information is present. @@ -60,6 +68,8 @@ Note, since the pipeline is now using Nextflow DSL2, each process will be run wi | `pymdown-extensions` | - | - | | `pygments` | - | - | | `r-r.methodss3` | - | - | +| `bowtie2` | - | 2.4.5 | +| `blat` | - | 36 | > **NB:** Dependency has been **updated** if both old and new version information is present. > **NB:** Dependency has been **added** if just the new version information is present. From 47ea9a3796a50c87ccead3a09c98e2f607641c9c Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Tue, 31 May 2022 21:56:44 +0200 Subject: [PATCH 036/145] Bump pipeline version to 2.1.0dev --- CHANGELOG.md | 5 +++++ nextflow.config | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e685803e..44665056 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unpublished Version / DEV] + +### Enhancements & fixes + +### Parameters ## [v2.0.0](https://github.com/nf-core/smrnaseq/releases/tag/2.0.0) - 2022-05-31 Aqua Zinc Chihuahua diff --git a/nextflow.config b/nextflow.config index 1974f835..23d74c15 100644 --- a/nextflow.config +++ b/nextflow.config @@ -193,7 +193,7 @@ manifest { description = 'Small RNA-Seq Best Practice Analysis Pipeline.' mainScript = 'main.nf' nextflowVersion = '!>=21.10.3' - version = '2.0.0' + version = '2.1.0dev' } // Load modules.config for DSL2 module specific options From 4ba142694d4a265c3763d86c48f005354719ac02 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Tue, 31 May 2022 21:59:48 +0200 Subject: [PATCH 037/145] Fix changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44665056..72b58d17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + ## [Unpublished Version / DEV] ### Enhancements & fixes From 9a9439a7a18b2dad5eaf919c367876bcf777f10a Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 10 Jun 2022 12:50:52 +0000 Subject: [PATCH 038/145] [automated] Fix linting with Prettier --- CHANGELOG.md | 15 ++++++++------- docs/usage.md | 1 - 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf70a0dc..f49f2e1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Enhancements & fixes -| Old parameter | New parameter | -| -------------------- | --------------------- | -| | `--mirGeneDB` | -| | `--mirGeneDB_species` | -| | `--mirGeneDB_gff` | -| | `--mirGeneDB_mature` | -| | `--mirGeneDB_hairpin` | +| Old parameter | New parameter | +| ------------- | --------------------- | +| | `--mirGeneDB` | +| | `--mirGeneDB_species` | +| | `--mirGeneDB_gff` | +| | `--mirGeneDB_mature` | +| | `--mirGeneDB_hairpin` | + ### Parameters ## [v2.0.0](https://github.com/nf-core/smrnaseq/releases/tag/2.0.0) - 2022-05-31 Aqua Zinc Chihuahua diff --git a/docs/usage.md b/docs/usage.md index c85c2a4f..c55ad4a0 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -34,7 +34,6 @@ If `MirGeneDB` should be used instead it needs to be specified using `--mirGeneD - `mirGeneDB_mature`: points to the FASTA file of mature miRNA sequences. Download from `https://mirgenedb.org/download`. - `mirGeneDB_hairpin`: points to the FASTA file of precursor miRNA sequences. Download from `https://mirgenedb.org/download`. Note that `MirGeneDB` does not have a dedicated `hairpin` file, but the `Precursor sequences` are to be used. - ### Genome - `fasta`: the reference genome FASTA file From 85fb55a9b4f00e5a3deac454613c350f4f0222d0 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 10 Jun 2022 14:51:39 +0200 Subject: [PATCH 039/145] Update CHANGELOG.md Move changelog up to dev --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f49f2e1d..b12442dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 | | `--mirGeneDB_mature` | | | `--mirGeneDB_hairpin` | +### Other enhancements + +- [#55](https://github.com/nf-core/smrnaseq/issues/12) - Enabled the use of `MirGeneDB` as an alternative database insted of `miRBase` + ### Parameters ## [v2.0.0](https://github.com/nf-core/smrnaseq/releases/tag/2.0.0) - 2022-05-31 Aqua Zinc Chihuahua @@ -34,7 +38,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Other enhancements & fixes - [#134](https://github.com/nf-core/smrnaseq/issues/134) - Fixed colSum of zero issues for edgeR_miRBase.R script -- [#55](https://github.com/nf-core/smrnaseq/issues/12) - Enabled the use of `MirGeneDB` as an alternative database insted of `miRBase` - [#55](https://github.com/lpantano/seqcluster/pull/55) - update seqcluster to fix UMI-detecting bug ### Parameters From 82da317c1927c1b346bf52c9063f909032dd690b Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 10 Jun 2022 12:56:27 +0000 Subject: [PATCH 040/145] [automated] Fix linting with Prettier --- CHANGELOG.md | 39 +++++++++++++++++++-------------------- docs/output.md | 2 +- docs/usage.md | 11 ++++++----- 3 files changed, 26 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3134ddb4..bb1cc06b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,21 +14,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Parameters -| Old parameter | New parameter | -| ------------- | --------------------- | -| | `--mirGeneDB` | -| | `--mirGeneDB_species` | -| | `--mirGeneDB_gff` | -| | `--mirGeneDB_mature` | -| | `--mirGeneDB_hairpin` | -| | `--contamination_filter` | -| | `--rrna` | -| | `--trna` | -| | `--cdna` | -| | `--ncrna` | -| | `--pirna` | -| | `--other_contamination` | - +| Old parameter | New parameter | +| ------------- | ------------------------ | +| | `--mirGeneDB` | +| | `--mirGeneDB_species` | +| | `--mirGeneDB_gff` | +| | `--mirGeneDB_mature` | +| | `--mirGeneDB_hairpin` | +| | `--contamination_filter` | +| | `--rrna` | +| | `--trna` | +| | `--cdna` | +| | `--ncrna` | +| | `--pirna` | +| | `--other_contamination` | ## [v2.0.0](https://github.com/nf-core/smrnaseq/releases/tag/2.0.0) - 2022-05-31 Aqua Zinc Chihuahua @@ -51,11 +50,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Parameters -| Old parameter | New parameter | -| ------------------ | ---------------- | -| `--conda` | `--enable_conda` | -| `--clusterOptions` | | -| `--publish_dir_mode` | | +| Old parameter | New parameter | +| -------------------- | ---------------- | +| `--conda` | `--enable_conda` | +| `--clusterOptions` | | +| `--publish_dir_mode` | | > **NB:** Parameter has been **updated** if both old and new parameter information is present. > **NB:** Parameter has been **added** if just the new parameter information is present. diff --git a/docs/output.md b/docs/output.md index b676eb46..c79f591c 100644 --- a/docs/output.md +++ b/docs/output.md @@ -14,7 +14,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d - [FastQC](#fastqc) - read quality control - [TrimGalore](#trimgalore) - adapter trimming -- [Bowtie2](#bowtie2) - contamination filtering +- [Bowtie2](#bowtie2) - contamination filtering - [Bowtie](#bowtie) - alignment against mature miRNAs and miRNA precursors (hairpins) - [SAMtools](#samtools) - alignment result processing and feature counting - [edgeR](#edger) - normalization, MDS plot and sample pairwise distance heatmap diff --git a/docs/usage.md b/docs/usage.md index 5ea4394a..81fac2c2 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -42,12 +42,13 @@ If `MirGeneDB` should be used instead it needs to be specified using `--mirGeneD ### Contamination filtering Contamination filtering of the sequencing reads is optional and can be invoked using `filter_contamination`. FASTA files with contamination sequences to use need to be supplied, otherwise the contamination filtering of the specific type will be omitted. using the following commands: -- `rrna`: Used to supply a FASTA file containing rRNA conatminations sequence. + +- `rrna`: Used to supply a FASTA file containing rRNA conatminations sequence. - `trna`: Used to supply a FASTA file containing tRNA conatminations sequence. e.g. `http://gtrnadb.ucsc.edu/genomes/eukaryota/Hsapi38/hg38-tRNAs.fa` -- `cdna`: Used to supply a FASTA file containing cDNA conatminations sequence. e.g. `ftp://ftp.ensembl.org/pub/release-86/fasta/homo_sapiens/cdna/Homo_sapiens.GRCh38.cdna.all.fa.gz` The FASTA file is first compared to the available miRNA sequences and overlaps are removed. -- `ncrna`: Used to supply a FASTA file containing ncRNA conatminations sequence. e.g. `ftp://ftp.ensembl.org/pub/release-86/fasta/homo_sapiens/ncrna/Homo_sapiens.GRCh38.ncrna.fa.gz` The FASTA file is first compared to the available miRNA sequences and overlaps are removed. -- `pirna`: Used to supply a FASTA file containing piRNA conatminations sequence. e.g. The FASTA file is first compared to the available miRNA sequences and overlaps are removed. -- `other_contamination`: Used to supply an additional filtering set. The FASTA file is first compared to the available miRNA sequences and overlaps are removed. +- `cdna`: Used to supply a FASTA file containing cDNA conatminations sequence. e.g. `ftp://ftp.ensembl.org/pub/release-86/fasta/homo_sapiens/cdna/Homo_sapiens.GRCh38.cdna.all.fa.gz` The FASTA file is first compared to the available miRNA sequences and overlaps are removed. +- `ncrna`: Used to supply a FASTA file containing ncRNA conatminations sequence. e.g. `ftp://ftp.ensembl.org/pub/release-86/fasta/homo_sapiens/ncrna/Homo_sapiens.GRCh38.ncrna.fa.gz` The FASTA file is first compared to the available miRNA sequences and overlaps are removed. +- `pirna`: Used to supply a FASTA file containing piRNA conatminations sequence. e.g. The FASTA file is first compared to the available miRNA sequences and overlaps are removed. +- `other_contamination`: Used to supply an additional filtering set. The FASTA file is first compared to the available miRNA sequences and overlaps are removed. ## Samplesheet input From e12c165e449d8f8f293a5b11dca8670b2fbf1832 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 10 Jun 2022 16:23:41 +0200 Subject: [PATCH 041/145] Update modules.config Fix #161 --- conf/modules.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index f8f2d5ad..bb28301b 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -108,7 +108,7 @@ if (!params.skip_trimming) { [ path: { "${params.outdir}/trimmed" }, mode: params.publish_dir_mode, - pattern: "*.txt" + pattern: "*.{txt,fastq.gz}" ] ] } From 4ee860e16305114586b2cf9146f04e97b0ae0604 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 10 Jun 2022 16:25:16 +0200 Subject: [PATCH 042/145] Update output.md Adjust docs to match what is the reality. --- docs/output.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/output.md b/docs/output.md index f3a23da8..d9be5f58 100644 --- a/docs/output.md +++ b/docs/output.md @@ -45,7 +45,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes d MultiQC reports the percentage of bases removed by TrimGalore in the _General Statistics_ table, along with a line plot showing where reads were trimmed. -**Output directory: `results/trim_galore`** +**Output directory: `results/trimmed`** Contains FastQ files with quality and adapter trimmed reads for each sample, along with a log file describing the trimming. From 3a07d61ebcb456cdc384e2461465f411966d1f18 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 10 Jun 2022 16:26:38 +0200 Subject: [PATCH 043/145] Update CHANGELOG.md --- CHANGELOG.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b12442dd..028b0142 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Enhancements & fixes +- Trimmed output was not as documented and not correctly published [#161](https://github.com/nf-core/smrnaseq/issues/161) + +### Other enhancements + +- [#55](https://github.com/nf-core/smrnaseq/issues/12) - Enabled the use of `MirGeneDB` as an alternative database insted of `miRBase` + +### Parameters + | Old parameter | New parameter | | ------------- | --------------------- | | | `--mirGeneDB` | @@ -15,11 +23,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 | | `--mirGeneDB_mature` | | | `--mirGeneDB_hairpin` | -### Other enhancements - -- [#55](https://github.com/nf-core/smrnaseq/issues/12) - Enabled the use of `MirGeneDB` as an alternative database insted of `miRBase` - -### Parameters ## [v2.0.0](https://github.com/nf-core/smrnaseq/releases/tag/2.0.0) - 2022-05-31 Aqua Zinc Chihuahua From 39de6ca9873cc8fc9576d118ddbde8f9ec4bed6a Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 10 Jun 2022 17:02:45 +0000 Subject: [PATCH 044/145] [automated] Fix linting with Prettier --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 028b0142..178452e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 | | `--mirGeneDB_mature` | | | `--mirGeneDB_hairpin` | - ## [v2.0.0](https://github.com/nf-core/smrnaseq/releases/tag/2.0.0) - 2022-05-31 Aqua Zinc Chihuahua ### Major enhancements From 6eca5e45c00268aa0aa80eea2ccdfec14ab8effe Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Mon, 13 Jun 2022 09:21:30 +0200 Subject: [PATCH 045/145] ADD FURTHER INFO TO DOCS Added a line to advise the user that the contamination filtering has only been tested with human data and unexpected behaviour can occur when this code with a different species. --- docs/usage.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 81fac2c2..ae3a70ff 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -41,7 +41,9 @@ If `MirGeneDB` should be used instead it needs to be specified using `--mirGeneD ### Contamination filtering -Contamination filtering of the sequencing reads is optional and can be invoked using `filter_contamination`. FASTA files with contamination sequences to use need to be supplied, otherwise the contamination filtering of the specific type will be omitted. using the following commands: +This step has, until now, only been tested for human data. Unexpected behaviour can occur when using it with a different species. + +Contamination filtering of the sequencing reads is optional and can be invoked using `filter_contamination`. FASTA files with contamination sequences to use need to be supplied using the following commands. Otherwise the contamination filtering of the specific type will be omitted. - `rrna`: Used to supply a FASTA file containing rRNA conatminations sequence. - `trna`: Used to supply a FASTA file containing tRNA conatminations sequence. e.g. `http://gtrnadb.ucsc.edu/genomes/eukaryota/Hsapi38/hg38-tRNAs.fa` From 2de18e904bb2cb5e99021b3b5921d46ab34bee3a Mon Sep 17 00:00:00 2001 From: Christian Kubica Date: Mon, 13 Jun 2022 10:55:40 +0200 Subject: [PATCH 046/145] Fix prettier --- modules/local/blat_mirna.nf | 2 +- modules/local/bowtie_contaminants.nf | 4 ++-- modules/local/bowtie_map_contaminants.nf | 3 +-- modules/local/filter_stats.nf | 1 - 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/modules/local/blat_mirna.nf b/modules/local/blat_mirna.nf index bc0398a5..1e8cc229 100644 --- a/modules/local/blat_mirna.nf +++ b/modules/local/blat_mirna.nf @@ -49,7 +49,7 @@ process BLAT_MIRNA { """ echo $db_type blat -out=blast8 $mirna $contaminants /dev/stdout | awk 'BEGIN{FS="\t"}{if(\$11 < 1e-5)print \$1;}' | uniq > mirnahit.txt - awk 'BEGIN { while((getline<"mirnahit.txt")>0) l[">"\$1]=1 } /^>/ {x = l[\$1]} {if(!x) print }' $contaminants > filtered.fa + awk 'BEGIN { while((getline<"mirnahit.txt")>0) l[">"\$1]=1 } /^>/ {x = l[\$1]} {if(!x) print }' $contaminants > filtered.fa cat <<-END_VERSIONS > versions.yml "${task.process}": blat: \$(echo \$(blat) | grep Standalone | awk '{ if (match(\$0,/[0-9]*[0-9]/,m)) print m[0] }') diff --git a/modules/local/bowtie_contaminants.nf b/modules/local/bowtie_contaminants.nf index 562ed30a..e5b624a5 100644 --- a/modules/local/bowtie_contaminants.nf +++ b/modules/local/bowtie_contaminants.nf @@ -10,8 +10,8 @@ process INDEX_CONTAMINANTS { path fasta output: - path 'fasta_bidx*' , emit: bt_indices - path "versions.yml", emit: versions + path 'fasta_bidx*' , emit: bt_indices + path "versions.yml" , emit: versions script: """ diff --git a/modules/local/bowtie_map_contaminants.nf b/modules/local/bowtie_map_contaminants.nf index 2bfb2f5d..92ebcce9 100644 --- a/modules/local/bowtie_map_contaminants.nf +++ b/modules/local/bowtie_map_contaminants.nf @@ -1,5 +1,4 @@ process BOWTIE_MAP_CONTAMINANTS { -// tag "$meta.id" label 'process_medium' conda (params.enable_conda ? 'bowtie2=2.4.5' : null) @@ -8,7 +7,7 @@ process BOWTIE_MAP_CONTAMINANTS { 'quay.io/biocontainers/bowtie2:2.4.5--py36hfca12d5_2' }" input: - tuple val(meta), path(reads) + tuple val(meta), path(reads) path index val contaminant_type diff --git a/modules/local/filter_stats.nf b/modules/local/filter_stats.nf index 441f9827..41b16457 100644 --- a/modules/local/filter_stats.nf +++ b/modules/local/filter_stats.nf @@ -1,5 +1,4 @@ process FILTER_STATS { -// tag "$meta.id" label 'process_medium' conda (params.enable_conda ? 'bowtie2=2.4.5' : null) From ebe378278b363d8bfabfafb8c8d1c7d3d11d6fce Mon Sep 17 00:00:00 2001 From: CKComputomics Date: Fri, 17 Jun 2022 14:52:13 +0200 Subject: [PATCH 047/145] FIX NEWLINES --- modules/local/blat_mirna.nf | 2 +- modules/local/bowtie_contaminants.nf | 2 +- modules/local/bowtie_map_contaminants.nf | 2 +- modules/local/filter_stats.nf | 3 +-- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/modules/local/blat_mirna.nf b/modules/local/blat_mirna.nf index 1e8cc229..b0037565 100644 --- a/modules/local/blat_mirna.nf +++ b/modules/local/blat_mirna.nf @@ -56,4 +56,4 @@ process BLAT_MIRNA { END_VERSIONS """ -} \ No newline at end of file +} diff --git a/modules/local/bowtie_contaminants.nf b/modules/local/bowtie_contaminants.nf index e5b624a5..7a86cd1d 100644 --- a/modules/local/bowtie_contaminants.nf +++ b/modules/local/bowtie_contaminants.nf @@ -23,4 +23,4 @@ process INDEX_CONTAMINANTS { END_VERSIONS """ -} \ No newline at end of file +} diff --git a/modules/local/bowtie_map_contaminants.nf b/modules/local/bowtie_map_contaminants.nf index 92ebcce9..a041e512 100644 --- a/modules/local/bowtie_map_contaminants.nf +++ b/modules/local/bowtie_map_contaminants.nf @@ -38,4 +38,4 @@ process BOWTIE_MAP_CONTAMINANTS { END_VERSIONS """ -} \ No newline at end of file +} diff --git a/modules/local/filter_stats.nf b/modules/local/filter_stats.nf index 41b16457..f0819e99 100644 --- a/modules/local/filter_stats.nf +++ b/modules/local/filter_stats.nf @@ -22,5 +22,4 @@ process FILTER_STATS { awk -v sample=${meta.id} -v readnumber=\$readnumber '{ print "id: \\"my_pca_section\\"\\nsection_name: \\"Contamination Filtering\\"\\ndescription: \\"This plot shows the amount of reads filtered by contaminant type.\\"\\nplot_type: \\"bargraph\\"\\npconfig:\\n id: \\"contamination_filter_plot\\"\\n title: \\"Contamination Plot\\"\\n ylab: \\"Number of reads\\"\\ndata:\\n "sample": {"\$0"\\"remaining reads\\": "readnumber"}" }' > ${meta.id}.contamination_mqc.yaml gzip -c ${reads} > ${meta.id}.filtered.fastq.gz """ - -} \ No newline at end of file +} From 01846bc4b4f17aa3157683167777d0b6d0bf98c5 Mon Sep 17 00:00:00 2001 From: CKComputomics <104438629+CKComputomics@users.noreply.github.com> Date: Fri, 17 Jun 2022 15:36:34 +0200 Subject: [PATCH 048/145] Update docs/output.md Co-authored-by: Alexander Peltzer --- docs/output.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/output.md b/docs/output.md index f0b9eec1..83de2bbe 100644 --- a/docs/output.md +++ b/docs/output.md @@ -61,7 +61,7 @@ This is an example of the output we can get: ## Bowtie2 -[Bowtie2](http://bowtie-bio.sourceforge.net/bowtie2/index.shtml) is used to align the reads to user defined databases of contaminants. +[Bowtie2](http://bowtie-bio.sourceforge.net/bowtie2/index.shtml) is used to align the reads to user-defined databasesof contaminants. MultiQC reports the number of reads that were removed by each of the contaminant databases. From 858099c48e455d4360a2afa052977958f4b9d47d Mon Sep 17 00:00:00 2001 From: CKComputomics <104438629+CKComputomics@users.noreply.github.com> Date: Fri, 17 Jun 2022 15:36:44 +0200 Subject: [PATCH 049/145] Update docs/usage.md Co-authored-by: Alexander Peltzer --- docs/usage.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index ae3a70ff..8e4015ba 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -45,11 +45,11 @@ This step has, until now, only been tested for human data. Unexpected behaviour Contamination filtering of the sequencing reads is optional and can be invoked using `filter_contamination`. FASTA files with contamination sequences to use need to be supplied using the following commands. Otherwise the contamination filtering of the specific type will be omitted. -- `rrna`: Used to supply a FASTA file containing rRNA conatminations sequence. -- `trna`: Used to supply a FASTA file containing tRNA conatminations sequence. e.g. `http://gtrnadb.ucsc.edu/genomes/eukaryota/Hsapi38/hg38-tRNAs.fa` -- `cdna`: Used to supply a FASTA file containing cDNA conatminations sequence. e.g. `ftp://ftp.ensembl.org/pub/release-86/fasta/homo_sapiens/cdna/Homo_sapiens.GRCh38.cdna.all.fa.gz` The FASTA file is first compared to the available miRNA sequences and overlaps are removed. -- `ncrna`: Used to supply a FASTA file containing ncRNA conatminations sequence. e.g. `ftp://ftp.ensembl.org/pub/release-86/fasta/homo_sapiens/ncrna/Homo_sapiens.GRCh38.ncrna.fa.gz` The FASTA file is first compared to the available miRNA sequences and overlaps are removed. -- `pirna`: Used to supply a FASTA file containing piRNA conatminations sequence. e.g. The FASTA file is first compared to the available miRNA sequences and overlaps are removed. +- `rrna`: Used to supply a FASTA file containing rRNA contamination sequence. +- `trna`: Used to supply a FASTA file containing tRNA contamination sequence. e.g. `http://gtrnadb.ucsc.edu/genomes/eukaryota/Hsapi38/hg38-tRNAs.fa` +- `cdna`: Used to supply a FASTA file containing cDNA contamination sequence. e.g. `ftp://ftp.ensembl.org/pub/release-86/fasta/homo_sapiens/cdna/Homo_sapiens.GRCh38.cdna.all.fa.gz` The FASTA file is first compared to the available miRNA sequences and overlaps are removed. +- `ncrna`: Used to supply a FASTA file containing ncRNA contamination sequence. e.g. `ftp://ftp.ensembl.org/pub/release-86/fasta/homo_sapiens/ncrna/Homo_sapiens.GRCh38.ncrna.fa.gz` The FASTA file is first compared to the available miRNA sequences and overlaps are removed. +- `pirna`: Used to supply a FASTA file containing piRNA contamination sequence. e.g. The FASTA file is first compared to the available miRNA sequences and overlaps are removed. - `other_contamination`: Used to supply an additional filtering set. The FASTA file is first compared to the available miRNA sequences and overlaps are removed. ## Samplesheet input From 6395315b2da2e8a373e4a9407c6f139b5c0e9aff Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 17 Jun 2022 15:48:00 +0200 Subject: [PATCH 050/145] Update docs/output.md --- docs/output.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/output.md b/docs/output.md index 83de2bbe..d5b5b1c4 100644 --- a/docs/output.md +++ b/docs/output.md @@ -61,7 +61,7 @@ This is an example of the output we can get: ## Bowtie2 -[Bowtie2](http://bowtie-bio.sourceforge.net/bowtie2/index.shtml) is used to align the reads to user-defined databasesof contaminants. +[Bowtie2](http://bowtie-bio.sourceforge.net/bowtie2/index.shtml) is used to align the reads to user-defined databases of contaminants. MultiQC reports the number of reads that were removed by each of the contaminant databases. From d555becb8fb798d027eeb7b9bdf3b1e1491ae4f3 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Mon, 27 Jun 2022 16:44:16 +0200 Subject: [PATCH 051/145] Collect bowtie index files --- subworkflows/local/genome_quant.nf | 2 +- workflows/smrnaseq.nf | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/subworkflows/local/genome_quant.nf b/subworkflows/local/genome_quant.nf index 54c7675c..8cf812d7 100644 --- a/subworkflows/local/genome_quant.nf +++ b/subworkflows/local/genome_quant.nf @@ -17,7 +17,7 @@ workflow GENOME_QUANT { if (!bt_index){ INDEX_GENOME ( fasta ) - bowtie_indices = INDEX_GENOME.out.bowtie_indices + bowtie_indices = INDEX_GENOME.out.bowtie_indices fasta_formatted = INDEX_GENOME.out.fasta ch_versions = ch_versions.mix(INDEX_GENOME.out.versions) } else { diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index cf6ed597..470bebc0 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -59,7 +59,7 @@ if (!params.mirGeneDB) { } else { if (params.mirGeneDB_mature) { reference_mature = file(params.mirGeneDB_mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mirGeneDB_mature}" } if (params.mirGeneDB_hairpin) { reference_hairpin = file(params.mirGeneDB_hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.mirGeneDB_hairpin}" } - if (params.mirGeneDB_gff) { mirna_gtf = file(params.mirGeneDB_gff, checkIfExists: true) } else { exit 1, "MirGeneDB gff file not found: ${params.mirGeneDB_gff}"} + if (params.mirGeneDB_gff) { mirna_gtf = file(params.mirGeneDB_gff, checkIfExists: true) } else { exit 1, "MirGeneDB gff file not found: ${params.mirGeneDB_gff}"} params.filterSpecies = params.mirGeneDB_species } @@ -169,7 +169,13 @@ workflow SMRNASEQ { ch_versions = ch_versions.mix(GENOME_QUANT.out.versions) if (!params.skip_mirdeep) { - MIRDEEP2 (FASTQC_TRIMGALORE.out.reads, GENOME_QUANT.out.fasta, GENOME_QUANT.out.indices, MIRNA_QUANT.out.fasta_hairpin, MIRNA_QUANT.out.fasta_mature) + MIRDEEP2 ( + FASTQC_TRIMGALORE.out.reads, + GENOME_QUANT.out.fasta, + GENOME_QUANT.out.indices.collect(), + MIRNA_QUANT.out.fasta_hairpin, + MIRNA_QUANT.out.fasta_mature + ) ch_versions = ch_versions.mix(MIRDEEP2.out.versions) } } From 31103e098ebcd99fa0240fd2dbac0db30b92acaa Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Mon, 27 Jun 2022 16:51:17 +0200 Subject: [PATCH 052/145] Update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 178452e2..a23a48bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Enhancements & fixes - Trimmed output was not as documented and not correctly published [#161](https://github.com/nf-core/smrnaseq/issues/161) +- Index files were not collected when `bowtie_indices` was used and thus mapping was failing [#159](https://github.com/nf-core/smrnaseq/issues/159) ### Other enhancements From b2ba538029689a49c9e5f74e20cb84fec8a6c9ca Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 29 Jun 2022 09:55:05 +0200 Subject: [PATCH 053/145] Updated all modules to fix some issues --- modules.json | 16 ++++++++-------- modules/nf-core/modules/cat/fastq/main.nf | 4 ++-- modules/nf-core/modules/multiqc/main.nf | 14 ++++++++++---- modules/nf-core/modules/multiqc/meta.yml | 8 ++++++++ .../nf-core/modules/samtools/flagstat/main.nf | 9 +++++---- .../nf-core/modules/samtools/idxstats/main.nf | 10 ++++++---- modules/nf-core/modules/samtools/index/main.nf | 6 +++--- modules/nf-core/modules/samtools/sort/main.nf | 6 +++--- modules/nf-core/modules/samtools/stats/main.nf | 15 ++++++++------- modules/nf-core/modules/trimgalore/main.nf | 13 ++++++++----- modules/nf-core/modules/trimgalore/meta.yml | 5 +++++ 11 files changed, 66 insertions(+), 40 deletions(-) diff --git a/modules.json b/modules.json index 81208b16..0b772f2e 100644 --- a/modules.json +++ b/modules.json @@ -4,7 +4,7 @@ "repos": { "nf-core/modules": { "cat/fastq": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" + "git_sha": "9aadd9a6d3f5964476582319b3a1c54a3e3fe7c9" }, "custom/dumpsoftwareversions": { "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" @@ -13,25 +13,25 @@ "git_sha": "49b18b1639f4f7104187058866a8fab33332bdfe" }, "multiqc": { - "git_sha": "49b18b1639f4f7104187058866a8fab33332bdfe" + "git_sha": "08376da6843b14c82d84d444784c0b3635bb7fd5" }, "samtools/flagstat": { - "git_sha": "1ad73f1b2abdea9398680d6d20014838135c9a35" + "git_sha": "ecece498f10b47b7c9d06f53a310cea5811b4c5f" }, "samtools/idxstats": { - "git_sha": "1ad73f1b2abdea9398680d6d20014838135c9a35" + "git_sha": "ecece498f10b47b7c9d06f53a310cea5811b4c5f" }, "samtools/index": { - "git_sha": "49b18b1639f4f7104187058866a8fab33332bdfe" + "git_sha": "897c33d5da084b61109500ee44c01da2d3e4e773" }, "samtools/sort": { - "git_sha": "49b18b1639f4f7104187058866a8fab33332bdfe" + "git_sha": "897c33d5da084b61109500ee44c01da2d3e4e773" }, "samtools/stats": { - "git_sha": "49b18b1639f4f7104187058866a8fab33332bdfe" + "git_sha": "ecece498f10b47b7c9d06f53a310cea5811b4c5f" }, "trimgalore": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" + "git_sha": "85ec13ff1fc2196c5a507ea497de468101baabed" } } } diff --git a/modules/nf-core/modules/cat/fastq/main.nf b/modules/nf-core/modules/cat/fastq/main.nf index bf0877c3..b6854895 100644 --- a/modules/nf-core/modules/cat/fastq/main.nf +++ b/modules/nf-core/modules/cat/fastq/main.nf @@ -4,8 +4,8 @@ process CAT_FASTQ { conda (params.enable_conda ? "conda-forge::sed=4.7" : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://containers.biocontainers.pro/s3/SingImgsRepo/biocontainers/v1.2.0_cv1/biocontainers_v1.2.0_cv1.img' : - 'biocontainers/biocontainers:v1.2.0_cv1' }" + 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : + 'ubuntu:20.04' }" input: tuple val(meta), path(reads, stageAs: "input*/*") diff --git a/modules/nf-core/modules/multiqc/main.nf b/modules/nf-core/modules/multiqc/main.nf index ae019dbf..3c3517bf 100644 --- a/modules/nf-core/modules/multiqc/main.nf +++ b/modules/nf-core/modules/multiqc/main.nf @@ -3,11 +3,12 @@ process MULTIQC { conda (params.enable_conda ? 'bioconda::multiqc=1.12' : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.12--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.12--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.13a--pyhdfd78af_1' : + 'quay.io/biocontainers/multiqc:1.13a--pyhdfd78af_1' }" input: - path multiqc_files + path multiqc_files, stageAs: "?/*" + tuple path(multiqc_config), path(multiqc_logo) output: path "*multiqc_report.html", emit: report @@ -20,8 +21,13 @@ process MULTIQC { script: def args = task.ext.args ?: '' + def config = multiqc_config ? "--config $multiqc_config" : '' """ - multiqc -f $args . + multiqc \\ + --force \\ + $config \\ + $args \\ + . cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/modules/multiqc/meta.yml b/modules/nf-core/modules/multiqc/meta.yml index 6fa891ef..bf3a27fe 100644 --- a/modules/nf-core/modules/multiqc/meta.yml +++ b/modules/nf-core/modules/multiqc/meta.yml @@ -17,6 +17,14 @@ input: type: file description: | List of reports / files recognised by MultiQC, for example the html and zip output of FastQC + - multiqc_config: + type: file + description: Config yml for MultiQC + pattern: "*.{yml,yaml}" + - multiqc_logo: + type: file + description: Logo file for MultiQC + pattern: "*.{png}" output: - report: type: file diff --git a/modules/nf-core/modules/samtools/flagstat/main.nf b/modules/nf-core/modules/samtools/flagstat/main.nf index 9e3440ac..03ec2dcf 100644 --- a/modules/nf-core/modules/samtools/flagstat/main.nf +++ b/modules/nf-core/modules/samtools/flagstat/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_FLAGSTAT { tag "$meta.id" label 'process_low' - conda (params.enable_conda ? "bioconda::samtools=1.15" : null) + conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.15--h1170115_1' : - 'quay.io/biocontainers/samtools:1.15--h1170115_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.15.1--h1170115_0' : + 'quay.io/biocontainers/samtools:1.15.1--h1170115_0' }" input: tuple val(meta), path(bam), path(bai) @@ -19,12 +19,13 @@ process SAMTOOLS_FLAGSTAT { script: def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" """ samtools \\ flagstat \\ --threads ${task.cpus-1} \\ $bam \\ - > ${bam}.flagstat + > ${prefix}.flagstat cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/modules/samtools/idxstats/main.nf b/modules/nf-core/modules/samtools/idxstats/main.nf index 7d5cee17..4b245419 100644 --- a/modules/nf-core/modules/samtools/idxstats/main.nf +++ b/modules/nf-core/modules/samtools/idxstats/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_IDXSTATS { tag "$meta.id" label 'process_low' - conda (params.enable_conda ? "bioconda::samtools=1.15" : null) + conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.15--h1170115_1' : - 'quay.io/biocontainers/samtools:1.15--h1170115_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.15.1--h1170115_0' : + 'quay.io/biocontainers/samtools:1.15.1--h1170115_0' }" input: tuple val(meta), path(bam), path(bai) @@ -19,11 +19,13 @@ process SAMTOOLS_IDXSTATS { script: def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ samtools \\ idxstats \\ $bam \\ - > ${bam}.idxstats + > ${prefix}.idxstats cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/modules/samtools/index/main.nf b/modules/nf-core/modules/samtools/index/main.nf index fff6e1b8..e04e63e8 100644 --- a/modules/nf-core/modules/samtools/index/main.nf +++ b/modules/nf-core/modules/samtools/index/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_INDEX { tag "$meta.id" label 'process_low' - conda (params.enable_conda ? "bioconda::samtools=1.15" : null) + conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.15--h1170115_1' : - 'quay.io/biocontainers/samtools:1.15--h1170115_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.15.1--h1170115_0' : + 'quay.io/biocontainers/samtools:1.15.1--h1170115_0' }" input: tuple val(meta), path(input) diff --git a/modules/nf-core/modules/samtools/sort/main.nf b/modules/nf-core/modules/samtools/sort/main.nf index ba46f0c9..b4fc1cbe 100644 --- a/modules/nf-core/modules/samtools/sort/main.nf +++ b/modules/nf-core/modules/samtools/sort/main.nf @@ -2,10 +2,10 @@ process SAMTOOLS_SORT { tag "$meta.id" label 'process_medium' - conda (params.enable_conda ? "bioconda::samtools=1.15" : null) + conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.15--h1170115_1' : - 'quay.io/biocontainers/samtools:1.15--h1170115_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.15.1--h1170115_0' : + 'quay.io/biocontainers/samtools:1.15.1--h1170115_0' }" input: tuple val(meta), path(bam) diff --git a/modules/nf-core/modules/samtools/stats/main.nf b/modules/nf-core/modules/samtools/stats/main.nf index 85cb64f3..c913bc5e 100644 --- a/modules/nf-core/modules/samtools/stats/main.nf +++ b/modules/nf-core/modules/samtools/stats/main.nf @@ -2,13 +2,13 @@ process SAMTOOLS_STATS { tag "$meta.id" label 'process_low' - conda (params.enable_conda ? "bioconda::samtools=1.15" : null) + conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/samtools:1.15--h1170115_1' : - 'quay.io/biocontainers/samtools:1.15--h1170115_1' }" + 'https://depot.galaxyproject.org/singularity/samtools:1.15.1--h1170115_0' : + 'quay.io/biocontainers/samtools:1.15.1--h1170115_0' }" input: - tuple val(meta), path(input), path(input_index) + tuple val(meta), path(bam), path(bai) path fasta output: @@ -20,14 +20,15 @@ process SAMTOOLS_STATS { script: def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" def reference = fasta ? "--reference ${fasta}" : "" """ samtools \\ stats \\ --threads ${task.cpus-1} \\ ${reference} \\ - ${input} \\ - > ${input}.stats + ${bam} \\ + > ${prefix}.stats cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -38,7 +39,7 @@ process SAMTOOLS_STATS { stub: def prefix = task.ext.prefix ?: "${meta.id}" """ - touch ${input}.stats + touch ${prefix}.stats cat <<-END_VERSIONS > versions.yml "${task.process}": diff --git a/modules/nf-core/modules/trimgalore/main.nf b/modules/nf-core/modules/trimgalore/main.nf index 9487c799..3a3fca90 100644 --- a/modules/nf-core/modules/trimgalore/main.nf +++ b/modules/nf-core/modules/trimgalore/main.nf @@ -11,12 +11,13 @@ process TRIMGALORE { tuple val(meta), path(reads) output: - tuple val(meta), path("*.fq.gz") , emit: reads - tuple val(meta), path("*report.txt"), emit: log - path "versions.yml" , emit: versions + tuple val(meta), path("*{trimmed,val}*.fq.gz"), emit: reads + tuple val(meta), path("*report.txt") , emit: log + path "versions.yml" , emit: versions - tuple val(meta), path("*.html"), emit: html optional true - tuple val(meta), path("*.zip") , emit: zip optional true + tuple val(meta), path("*unpaired*.fq.gz") , emit: unpaired, optional: true + tuple val(meta), path("*.html") , emit: html , optional: true + tuple val(meta), path("*.zip") , emit: zip , optional: true when: task.ext.when == null || task.ext.when @@ -52,6 +53,7 @@ process TRIMGALORE { $c_r1 \\ $tpc_r1 \\ ${prefix}.fastq.gz + cat <<-END_VERSIONS > versions.yml "${task.process}": trimgalore: \$(echo \$(trim_galore --version 2>&1) | sed 's/^.*version //; s/Last.*\$//') @@ -73,6 +75,7 @@ process TRIMGALORE { $tpc_r2 \\ ${prefix}_1.fastq.gz \\ ${prefix}_2.fastq.gz + cat <<-END_VERSIONS > versions.yml "${task.process}": trimgalore: \$(echo \$(trim_galore --version 2>&1) | sed 's/^.*version //; s/Last.*\$//') diff --git a/modules/nf-core/modules/trimgalore/meta.yml b/modules/nf-core/modules/trimgalore/meta.yml index e99a8833..439f566d 100644 --- a/modules/nf-core/modules/trimgalore/meta.yml +++ b/modules/nf-core/modules/trimgalore/meta.yml @@ -37,6 +37,11 @@ output: List of input adapter trimmed FastQ files of size 1 and 2 for single-end and paired-end data, respectively. pattern: "*.{fq.gz}" + - unpaired: + type: file + description: | + FastQ files containing unpaired reads from read 1 or read 2 + pattern: "*unpaired*.fq.gz" - html: type: file description: FastQC report (optional) From bdc08fbc1ec35de057fa16f30eb3762ca3058c22 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 29 Jun 2022 09:59:05 +0200 Subject: [PATCH 054/145] Add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41dc14b0..45e6713b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Trimmed output was not as documented and not correctly published [#161](https://github.com/nf-core/smrnaseq/issues/161) - Index files were not collected when `bowtie_indices` was used and thus mapping was failing [#159](https://github.com/nf-core/smrnaseq/issues/159) +- Fixed issue with mirTop and MultiQC by upgrading to MultiQC V1.13dev [#137](https://github.com/nf-core/smrnaseq/issues/137) ### Other enhancements From 0f6bffd4f91950c50a98b57c62cdab45b5ea8668 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 29 Jun 2022 10:23:37 +0200 Subject: [PATCH 055/145] MultiQC module changed to expect 2 inputs --- workflows/smrnaseq.nf | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index e8dfc5fe..07f702d4 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -42,6 +42,7 @@ mirna_gtf = params.mirna_gtf ? params.mirna_gtf : mirna_gtf_from_species ch_multiqc_config = file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config) : Channel.empty() +ch_multiqc_image = file("$projectDir/assets/smrnaseq_logo.png", checkIfExists: true) /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -155,22 +156,22 @@ workflow SMRNASEQ { // contamination_stats = Channel.empty() if (params.filter_contamination){ - CONTAMINANT_FILTER ( + CONTAMINANT_FILTER ( reference_hairpin, - params.rrna, - params.trna, - params.cdna, - params.ncrna, - params.pirna, + params.rrna, + params.trna, + params.cdna, + params.ncrna, + params.pirna, params.other_contamination, - FASTQC_TRIMGALORE.out.reads + FASTQC_TRIMGALORE.out.reads ) - + reads_for_mirna = CONTAMINANT_FILTER.out.filtered_reads ch_versions = ch_versions.mix(CONTAMINANT_FILTER.out.versions) CONTAMINANT_FILTER.out.filter_stats .set { contamination_stats } - + } MIRNA_QUANT ( @@ -233,7 +234,8 @@ workflow SMRNASEQ { ch_multiqc_files = ch_multiqc_files.mix(MIRTRACE.out.results.collect().ifEmpty([])) MULTIQC ( - ch_multiqc_files.collect() + ch_multiqc_files.collect(), + [ch_multiqc_config, ch_multiqc_image] ) multiqc_report = MULTIQC.out.report.toList() From 27575a591f8238a46ade01247fd75ec5682141a8 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 29 Jun 2022 10:40:36 +0200 Subject: [PATCH 056/145] Update workflows/smrnaseq.nf Co-authored-by: Jose Espinosa-Carrasco --- workflows/smrnaseq.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 07f702d4..be455612 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -42,7 +42,7 @@ mirna_gtf = params.mirna_gtf ? params.mirna_gtf : mirna_gtf_from_species ch_multiqc_config = file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config) : Channel.empty() -ch_multiqc_image = file("$projectDir/assets/smrnaseq_logo.png", checkIfExists: true) +ch_multiqc_image = file("$projectDir/assets/smrnaseq_logo.png", checkIfExists: true) /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 9fd93dcf2dc7c14ff08dd7a5feafada66676fb2a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Wed, 29 Jun 2022 10:50:59 +0200 Subject: [PATCH 057/145] Improve bt index handling --- CHANGELOG.md | 1 + modules/local/bowtie_map_contaminants.nf | 4 ++-- modules/local/bowtie_map_mirna.nf | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45e6713b..e8902d23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Trimmed output was not as documented and not correctly published [#161](https://github.com/nf-core/smrnaseq/issues/161) - Index files were not collected when `bowtie_indices` was used and thus mapping was failing [#159](https://github.com/nf-core/smrnaseq/issues/159) - Fixed issue with mirTop and MultiQC by upgrading to MultiQC V1.13dev [#137](https://github.com/nf-core/smrnaseq/issues/137) +- Improved Bowtie index handling which should be more stable now and follow nf-core/modules approach ### Other enhancements diff --git a/modules/local/bowtie_map_contaminants.nf b/modules/local/bowtie_map_contaminants.nf index a041e512..397aae82 100644 --- a/modules/local/bowtie_map_contaminants.nf +++ b/modules/local/bowtie_map_contaminants.nf @@ -18,13 +18,13 @@ process BOWTIE_MAP_CONTAMINANTS { path "filtered.*.stats" , emit: stats script: - def index_base = index.toString().tokenize(' ')[0].tokenize('.')[0] """ + INDEX=`find -L ./ -name "*.3.ebwt" | sed 's/.3.ebwt//'` bowtie2 \\ --threads ${task.cpus} \\ --very-sensitive-local \\ -k 1 \\ - -x $index_base \\ + -x \$INDEX \\ --un ${meta.id}.${contaminant_type}.filter.unmapped.contaminant.fastq \\ ${reads} \\ -S ${meta.id}.filter.contaminant.sam > ${meta.id}.contaminant_bowtie.log 2>&1 diff --git a/modules/local/bowtie_map_mirna.nf b/modules/local/bowtie_map_mirna.nf index 9a1301fb..316f94af 100644 --- a/modules/local/bowtie_map_mirna.nf +++ b/modules/local/bowtie_map_mirna.nf @@ -17,10 +17,10 @@ process BOWTIE_MAP_SEQ { path "versions.yml" , emit: versions script: - def index_base = index.toString().tokenize(' ')[0].tokenize('.')[0] """ + INDEX=`find -L ./ -name "*.3.ebwt" | sed 's/.3.ebwt//'` bowtie \\ - -x $index_base \\ + -x \$INDEX \\ -q <(zcat $reads) \\ -p ${task.cpus} \\ -t \\ From b762e4305b9b1fe115bdf815277210eacc1b8dcb Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 30 Jun 2022 19:20:49 +0000 Subject: [PATCH 058/145] Removed mirtrace_protocol --- CHANGELOG.md | 1 + modules/local/mirtrace.nf | 9 +++------ nextflow.config | 1 - nextflow_schema.json | 8 -------- 4 files changed, 4 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8902d23..c100756d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Index files were not collected when `bowtie_indices` was used and thus mapping was failing [#159](https://github.com/nf-core/smrnaseq/issues/159) - Fixed issue with mirTop and MultiQC by upgrading to MultiQC V1.13dev [#137](https://github.com/nf-core/smrnaseq/issues/137) - Improved Bowtie index handling which should be more stable now and follow nf-core/modules approach +- Removed `mirtrace_protocol` as the parameter was redundant and `params.protocol` is entirely sufficient [#168](https://github.com/nf-core/smrnaseq/issues/168) ### Other enhancements diff --git a/modules/local/mirtrace.nf b/modules/local/mirtrace.nf index 6dd553e9..fd04ff8d 100644 --- a/modules/local/mirtrace.nf +++ b/modules/local/mirtrace.nf @@ -30,11 +30,8 @@ process MIRTRACE_RUN { three_prime_adapter = params.three_prime_adapter } // mirtrace protocol defaults to 'params.protocol' if not set - def mirtrace_protocol = params.mirtrace_protocol - if (!params.mirtrace_protocol){ - mirtrace_protocol = params.protocol - } - def primer = (mirtrace_protocol=="cats") ? " " : " --adapter $three_prime_adapter " + def protocol = params.protocol + def primer = (protocol=="cats") ? " " : " --adapter $three_prime_adapter " def java_mem = '' if(task.memory){ tmem = task.memory.toBytes() @@ -52,7 +49,7 @@ process MIRTRACE_RUN { java $java_mem -jar \$mirtracejar/mirtrace.jar --mirtrace-wrapper-name mirtrace qc \\ --species $params.mirtrace_species \\ $primer \\ - --protocol $mirtrace_protocol \\ + --protocol $protocol \\ --config mirtrace_config \\ --write-fasta \\ --output-dir mirtrace \\ diff --git a/nextflow.config b/nextflow.config index 842645af..fe98b39c 100644 --- a/nextflow.config +++ b/nextflow.config @@ -23,7 +23,6 @@ params { mirna_gtf = null bowtie_indices = null mirtrace_species = null - mirtrace_protocol = 'illumina' mature = "https://mirbase.org/ftp/CURRENT/mature.fa.gz" hairpin = "https://mirbase.org/ftp/CURRENT/hairpin.fa.gz" mirGeneDB = false diff --git a/nextflow_schema.json b/nextflow_schema.json index 040234b3..77faa9ba 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -178,14 +178,6 @@ "fa_icon": "fas fa-text-width", "description": "Sequencing adapter sequence to use for trimming." }, - "mirtrace_protocol": { - "type": "string", - "default": "illumina", - "fa_icon": "fas fa-vial", - "description": "The miRTrace protocol for QC reporting.", - "help_text": "miRTrace can handle four protocols, each with their own primer-read structure. See the protocol descriptions [here](https://github.com/friedlanderlab/mirtrace/blob/master/release-bundle-includes/doc/manual/mirtrace_manual.pdf).", - "enum": ["illumina", "cats", "qiaseq", "nextflex"] - }, "trim_galore_max_length": { "type": "integer", "default": 40, From 6382b441fdc27d0fc192b5b790fea98b93cfe4df Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 14 Jul 2022 10:03:03 +0000 Subject: [PATCH 059/145] Tiny typo fix --- docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 8e4015ba..197e6c80 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -12,7 +12,7 @@ This option indicates the experimental protocol used for the sample preparation. - 'illumina': adapter (`TGGAATTCTCGGGTGCCAAGG`) - 'nextflex': adapter (`TGGAATTCTCGGGTGCCAAGG), clip_r1 (`4`), three_prime_clip_r1 (`4`) -- 'qiaseq': adapter (`AACTGTAGGCACCATCAAT) +- 'qiaseq': adapter (`AACTGTAGGCACCATCAAT`) - 'cats': adapter (`GATCGGAAGAGCACACGTCTG), clip_r1(`3) - 'custom' (where the ser can indicate the `three_prime_adapter`, `clip_r1` and three_prime_clip_r1`) From 35bb9a4a72fe2eb79891a5ab63297d5dcacb95b5 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 14 Jul 2022 13:04:11 +0000 Subject: [PATCH 060/145] Add clearer docs --- docs/usage.md | 4 +++- modules/local/trimgalore.nf | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 197e6c80..475dc9dc 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -14,7 +14,9 @@ This option indicates the experimental protocol used for the sample preparation. - 'nextflex': adapter (`TGGAATTCTCGGGTGCCAAGG), clip_r1 (`4`), three_prime_clip_r1 (`4`) - 'qiaseq': adapter (`AACTGTAGGCACCATCAAT`) - 'cats': adapter (`GATCGGAAGAGCACACGTCTG), clip_r1(`3) -- 'custom' (where the ser can indicate the `three_prime_adapter`, `clip_r1` and three_prime_clip_r1`) +- 'custom' (where the user can indicate the `three_prime_adapter`, `clip_r1` and `three_prime_clip_r1` manually) + +:warning: At least the `custom` protocol has to be specified, otherwise the pipeline won't run. ### `mirtrace_species or mirGeneDB_species` diff --git a/modules/local/trimgalore.nf b/modules/local/trimgalore.nf index 98519584..3dec8f2e 100644 --- a/modules/local/trimgalore.nf +++ b/modules/local/trimgalore.nf @@ -51,12 +51,11 @@ process TRIMGALORE { three_prime_clip_r1 = 0 // three_prime_adapter = "GATCGGAAGAGCACACGTCTG" three_prime_adapter = "AAAAAAAA" - } else { + } else if (params.protocol == "custom"){ //custom protocol clip_r1 = params.clip_r1 three_prime_clip_r1 = params.three_prime_clip_r1 three_prime_adapter = params.three_prime_adapter - protocol = params.protocol } def tg_length = "--length ${params.min_length}" def c_r1 = clip_r1 > 0 ? "--clip_r1 ${clip_r1}" : '' From 354600cdf24044543e1b2dbc0ea18176c19bc72a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 14 Jul 2022 13:26:09 +0000 Subject: [PATCH 061/145] Tiny bugfix for mirtrace --- modules/local/mirtrace.nf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/local/mirtrace.nf b/modules/local/mirtrace.nf index fd04ff8d..ab005045 100644 --- a/modules/local/mirtrace.nf +++ b/modules/local/mirtrace.nf @@ -24,11 +24,10 @@ process MIRTRACE_RUN { three_prime_adapter = "AACTGTAGGCACCATCAAT" } else if (params.protocol == "cats"){ three_prime_adapter = "AAAAAAAA" - } - if (params.three_prime_adapter){ - // to allow replace of 3' primer using one of the previous protocols + } else if (params.protocol == "custom"){ three_prime_adapter = params.three_prime_adapter } + // mirtrace protocol defaults to 'params.protocol' if not set def protocol = params.protocol def primer = (protocol=="cats") ? " " : " --adapter $three_prime_adapter " From 72cd3de0dd93885eb4b7f8a255a2c4e4493de298 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 14 Jul 2022 13:29:12 +0000 Subject: [PATCH 062/145] Another fix --- modules/local/mirtrace.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/local/mirtrace.nf b/modules/local/mirtrace.nf index ab005045..11203124 100644 --- a/modules/local/mirtrace.nf +++ b/modules/local/mirtrace.nf @@ -29,8 +29,8 @@ process MIRTRACE_RUN { } // mirtrace protocol defaults to 'params.protocol' if not set - def protocol = params.protocol def primer = (protocol=="cats") ? " " : " --adapter $three_prime_adapter " + def protocol = (protocol=="custom") ? "" : "--protocol $params.protocol" def java_mem = '' if(task.memory){ tmem = task.memory.toBytes() @@ -48,7 +48,7 @@ process MIRTRACE_RUN { java $java_mem -jar \$mirtracejar/mirtrace.jar --mirtrace-wrapper-name mirtrace qc \\ --species $params.mirtrace_species \\ $primer \\ - --protocol $protocol \\ + $protocol \\ --config mirtrace_config \\ --write-fasta \\ --output-dir mirtrace \\ From 5f4426b8ee95d87f86cb9195b8629a1b2a76f3a7 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 14 Jul 2022 13:34:51 +0000 Subject: [PATCH 063/145] Fix for sure --- modules/local/mirtrace.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/local/mirtrace.nf b/modules/local/mirtrace.nf index 11203124..e893c2e9 100644 --- a/modules/local/mirtrace.nf +++ b/modules/local/mirtrace.nf @@ -29,8 +29,8 @@ process MIRTRACE_RUN { } // mirtrace protocol defaults to 'params.protocol' if not set - def primer = (protocol=="cats") ? " " : " --adapter $three_prime_adapter " - def protocol = (protocol=="custom") ? "" : "--protocol $params.protocol" + def primer = (params.protocol=="cats") ? " " : " --adapter $three_prime_adapter " + def protocol = (params.protocol=="custom") ? " " : "--protocol $params.protocol" def java_mem = '' if(task.memory){ tmem = task.memory.toBytes() From f46c811a9854cef723a89d122ab8b0b8b0349a06 Mon Sep 17 00:00:00 2001 From: Marcel Ribeiro-Dantas Date: Fri, 15 Jul 2022 14:29:12 +0200 Subject: [PATCH 064/145] Remove extra/outdated zenodo shield --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 02689202..4dacd5e3 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![GitHub Actions CI Status](https://github.com/nf-core/smrnaseq/workflows/nf-core%20CI/badge.svg)](https://github.com/nf-core/smrnaseq/actions?query=workflow%3A%22nf-core+CI%22) [![GitHub Actions Linting Status](https://github.com/nf-core/smrnaseq/workflows/nf-core%20linting/badge.svg)](https://github.com/nf-core/smrnaseq/actions?query=workflow%3A%22nf-core+linting%22) [![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?logo=Amazon%20AWS)](https://nf-co.re/smrnaseq/results) -[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8)](https://doi.org/10.5281/zenodo.4956678) +[![DOI](https://zenodo.org/badge/140590861.svg)](https://zenodo.org/badge/latestdoi/140590861) [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?logo=anaconda)](https://docs.conda.io/en/latest/) @@ -15,7 +15,7 @@ [![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?logo=twitter)](https://twitter.com/nf_core) [![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?logo=youtube)](https://www.youtube.com/c/nf-core) -[![DOI](https://zenodo.org/badge/140590861.svg)](https://zenodo.org/badge/latestdoi/140590861) + ## Introduction From f6ad4c38349550f946d058c81db701b943c82d5b Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Fri, 15 Jul 2022 18:41:51 +0000 Subject: [PATCH 065/145] [automated] Fix linting with Prettier --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 4dacd5e3..f7c7dbc0 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,6 @@ [![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?logo=twitter)](https://twitter.com/nf_core) [![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?logo=youtube)](https://www.youtube.com/c/nf-core) - - ## Introduction **nf-core/smrnaseq** is a bioinformatics best-practice analysis pipeline for Small RNA-Seq Best Practice Analysis Pipeline. From 415d7a3d00946a3fd1f166da395bff247aa11f41 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 30 Aug 2022 13:41:33 +0000 Subject: [PATCH 066/145] Template update for nf-core/tools version 2.5 --- .editorconfig | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 3 +- .github/workflows/ci.yml | 23 ++------ .github/workflows/linting.yml | 38 +++++++++++-- CHANGELOG.md | 2 +- CITATION.cff | 56 +++++++++++++++++++ README.md | 21 +++---- assets/email_template.txt | 1 - bin/check_samplesheet.py | 41 +++++++------- conf/base.config | 5 ++ docs/usage.md | 12 ++-- lib/WorkflowMain.groovy | 9 ++- lib/WorkflowSmrnaseq.groovy | 5 +- main.nf | 2 +- modules.json | 22 +++++--- .../templates/dumpsoftwareversions.py | 14 +++-- nextflow.config | 23 +++++++- 17 files changed, 186 insertions(+), 93 deletions(-) create mode 100644 CITATION.cff diff --git a/.editorconfig b/.editorconfig index b6b31907..b78de6e6 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,7 +8,7 @@ trim_trailing_whitespace = true indent_size = 4 indent_style = space -[*.{md,yml,yaml,html,css,scss,js}] +[*.{md,yml,yaml,html,css,scss,js,cff}] indent_size = 2 # These files are edited and tested upstream in nf-core/modules diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d103b361..cf3fb13d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -15,8 +15,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/smrn - [ ] This comment contains a description of changes (with reason). - [ ] If you've fixed a bug or added code that should be tested, add tests! - - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/smrnaseq/tree/master/.github/CONTRIBUTING.md) - - [ ] If necessary, also make a PR on the nf-core/smrnaseq _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. +- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/smrnaseq/tree/master/.github/CONTRIBUTING.md)- [ ] If necessary, also make a PR on the nf-core/smrnaseq _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3638a64b..00c9d511 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,6 @@ on: env: NXF_ANSI_LOG: false - CAPSULE_LOG: none jobs: test: @@ -20,27 +19,17 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Nextflow versions - include: - # Test pipeline minimum Nextflow version - - NXF_VER: "21.10.3" - NXF_EDGE: "" - # Test latest edge release of Nextflow - - NXF_VER: "" - NXF_EDGE: "1" + NXF_VER: + - "21.10.3" + - "latest-everything" steps: - name: Check out pipeline code uses: actions/checkout@v2 - name: Install Nextflow - env: - NXF_VER: ${{ matrix.NXF_VER }} - # Uncomment only if the edge release is more recent than the latest stable release - # See https://github.com/nextflow-io/nextflow/issues/2467 - # NXF_EDGE: ${{ matrix.NXF_EDGE }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + uses: nf-core/setup-nextflow@v1 + with: + version: "${{ matrix.NXF_VER }}" - name: Run pipeline with test data # TODO nf-core: You can customise CI pipeline run tests as required diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 77358dee..8a5ce69b 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -35,6 +35,36 @@ jobs: - name: Run Prettier --check run: prettier --check ${GITHUB_WORKSPACE} + PythonBlack: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Check code lints with Black + uses: psf/black@stable + + # If the above check failed, post a comment on the PR explaining the failure + - name: Post PR comment + if: failure() + uses: mshick/add-pr-comment@v1 + with: + message: | + ## Python linting (`black`) is failing + + To keep the code consistent with lots of contributors, we run automated code consistency checks. + To fix this CI test, please run: + + * Install [`black`](https://black.readthedocs.io/en/stable/): `pip install black` + * Fix formatting errors in your pipeline: `black .` + + Once you push these changes the test should pass, and you can hide this comment :+1: + + We highly recommend setting up Black in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + + Thanks again for your contribution! + repo-token: ${{ secrets.GITHUB_TOKEN }} + allow-repeats: false + nf-core: runs-on: ubuntu-latest steps: @@ -42,15 +72,11 @@ jobs: uses: actions/checkout@v2 - name: Install Nextflow - env: - CAPSULE_LOG: none - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + uses: nf-core/setup-nextflow@v1 - uses: actions/setup-python@v3 with: - python-version: "3.6" + python-version: "3.7" architecture: "x64" - name: Install dependencies diff --git a/CHANGELOG.md b/CHANGELOG.md index 82501771..c54766b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## v2.0.0 - [date] +## v2.1.0dev - [date] Initial release of nf-core/smrnaseq, created with the [nf-core](https://nf-co.re/) template. diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000..4533e2f2 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,56 @@ +cff-version: 1.2.0 +message: "If you use `nf-core tools` in your work, please cite the `nf-core` publication" +authors: + - family-names: Ewels + given-names: Philip + - family-names: Peltzer + given-names: Alexander + - family-names: Fillinger + given-names: Sven + - family-names: Patel + given-names: Harshil + - family-names: Alneberg + given-names: Johannes + - family-names: Wilm + given-names: Andreas + - family-names: Ulysse Garcia + given-names: Maxime + - family-names: Di Tommaso + given-names: Paolo + - family-names: Nahnsen + given-names: Sven +title: "The nf-core framework for community-curated bioinformatics pipelines." +version: 2.4.1 +doi: 10.1038/s41587-020-0439-x +date-released: 2022-05-16 +url: https://github.com/nf-core/tools +prefered-citation: + type: article + authors: + - family-names: Ewels + given-names: Philip + - family-names: Peltzer + given-names: Alexander + - family-names: Fillinger + given-names: Sven + - family-names: Patel + given-names: Harshil + - family-names: Alneberg + given-names: Johannes + - family-names: Wilm + given-names: Andreas + - family-names: Ulysse Garcia + given-names: Maxime + - family-names: Di Tommaso + given-names: Paolo + - family-names: Nahnsen + given-names: Sven + doi: 10.1038/s41587-020-0439-x + journal: nature biotechnology + start: 276 + end: 278 + title: "The nf-core framework for community-curated bioinformatics pipelines." + issue: 3 + volume: 38 + year: 2020 + url: https://dx.doi.org/10.1038/s41587-020-0439-x diff --git a/README.md b/README.md index 1109375e..166d0429 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,14 @@ # ![nf-core/smrnaseq](docs/images/nf-core-smrnaseq_logo_light.png#gh-light-mode-only) ![nf-core/smrnaseq](docs/images/nf-core-smrnaseq_logo_dark.png#gh-dark-mode-only) -[![GitHub Actions CI Status](https://github.com/nf-core/smrnaseq/workflows/nf-core%20CI/badge.svg)](https://github.com/nf-core/smrnaseq/actions?query=workflow%3A%22nf-core+CI%22) -[![GitHub Actions Linting Status](https://github.com/nf-core/smrnaseq/workflows/nf-core%20linting/badge.svg)](https://github.com/nf-core/smrnaseq/actions?query=workflow%3A%22nf-core+linting%22) -[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?logo=Amazon%20AWS)](https://nf-co.re/smrnaseq/results) -[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8)](https://doi.org/10.5281/zenodo.XXXXXXX) +[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/smrnaseq/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A521.10.3-23aa62.svg)](https://www.nextflow.io/) -[![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?logo=anaconda)](https://docs.conda.io/en/latest/) -[![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?logo=docker)](https://www.docker.com/) -[![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg)](https://sylabs.io/docs/) +[![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) +[![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) +[![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) [![Launch on Nextflow Tower](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Nextflow%20Tower-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/nf-core/smrnaseq) -[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23smrnaseq-4A154B?logo=slack)](https://nfcore.slack.com/channels/smrnaseq) -[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?logo=twitter)](https://twitter.com/nf_core) -[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?logo=youtube)](https://www.youtube.com/c/nf-core) +[![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23smrnaseq-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/smrnaseq)[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core)[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) ## Introduction @@ -25,7 +20,7 @@ The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool -On release, automated continuous integration tests run the pipeline on a full-sized dataset on the AWS cloud infrastructure. This ensures that the pipeline runs on AWS, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources. The results obtained from the full-sized test can be viewed on the [nf-core website](https://nf-co.re/smrnaseq/results). +On release, automated continuous integration tests run the pipeline on a full-sized dataset on the AWS cloud infrastructure. This ensures that the pipeline runs on AWS, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources.The results obtained from the full-sized test can be viewed on the [nf-core website](https://nf-co.re/smrnaseq/results). ## Pipeline summary @@ -42,7 +37,7 @@ On release, automated continuous integration tests run the pipeline on a full-si 3. Download the pipeline and test it on a minimal dataset with a single command: - ```console + ```bash nextflow run nf-core/smrnaseq -profile test,YOURPROFILE --outdir ``` @@ -57,7 +52,7 @@ On release, automated continuous integration tests run the pipeline on a full-si - ```console + ```bash nextflow run nf-core/smrnaseq --input samplesheet.csv --outdir --genome GRCh37 -profile ``` diff --git a/assets/email_template.txt b/assets/email_template.txt index 6e8b6393..654e78db 100644 --- a/assets/email_template.txt +++ b/assets/email_template.txt @@ -6,7 +6,6 @@ `._,._,' nf-core/smrnaseq v${version} ---------------------------------------------------- - Run Name: $runName <% if (success){ diff --git a/bin/check_samplesheet.py b/bin/check_samplesheet.py index 3652c63c..9a8b8962 100755 --- a/bin/check_samplesheet.py +++ b/bin/check_samplesheet.py @@ -11,7 +11,6 @@ from collections import Counter from pathlib import Path - logger = logging.getLogger() @@ -79,13 +78,15 @@ def validate_and_transform(self, row): def _validate_sample(self, row): """Assert that the sample name exists and convert spaces to underscores.""" - assert len(row[self._sample_col]) > 0, "Sample input is required." + if len(row[self._sample_col]) <= 0: + raise AssertionError("Sample input is required.") # Sanitize samples slightly. row[self._sample_col] = row[self._sample_col].replace(" ", "_") def _validate_first(self, row): """Assert that the first FASTQ entry is non-empty and has the right format.""" - assert len(row[self._first_col]) > 0, "At least the first FASTQ file is required." + if len(row[self._first_col]) <= 0: + raise AssertionError("At least the first FASTQ file is required.") self._validate_fastq_format(row[self._first_col]) def _validate_second(self, row): @@ -97,36 +98,34 @@ def _validate_pair(self, row): """Assert that read pairs have the same file extension. Report pair status.""" if row[self._first_col] and row[self._second_col]: row[self._single_col] = False - assert ( - Path(row[self._first_col]).suffixes[-2:] == Path(row[self._second_col]).suffixes[-2:] - ), "FASTQ pairs must have the same file extensions." + if Path(row[self._first_col]).suffixes[-2:] != Path(row[self._second_col]).suffixes[-2:]: + raise AssertionError("FASTQ pairs must have the same file extensions.") else: row[self._single_col] = True def _validate_fastq_format(self, filename): """Assert that a given filename has one of the expected FASTQ extensions.""" - assert any(filename.endswith(extension) for extension in self.VALID_FORMATS), ( - f"The FASTQ file has an unrecognized extension: {filename}\n" - f"It should be one of: {', '.join(self.VALID_FORMATS)}" - ) + if not any(filename.endswith(extension) for extension in self.VALID_FORMATS): + raise AssertionError( + f"The FASTQ file has an unrecognized extension: {filename}\n" + f"It should be one of: {', '.join(self.VALID_FORMATS)}" + ) def validate_unique_samples(self): """ Assert that the combination of sample name and FASTQ filename is unique. - In addition to the validation, also rename the sample if more than one sample, - FASTQ file combination exists. + In addition to the validation, also rename all samples to have a suffix of _T{n}, where n is the + number of times the same sample exist, but with different FASTQ files, e.g., multiple runs per experiment. """ - assert len(self._seen) == len(self.modified), "The pair of sample name and FASTQ must be unique." - if len({pair[0] for pair in self._seen}) < len(self._seen): - counts = Counter(pair[0] for pair in self._seen) - seen = Counter() - for row in self.modified: - sample = row[self._sample_col] - seen[sample] += 1 - if counts[sample] > 1: - row[self._sample_col] = f"{sample}_T{seen[sample]}" + if len(self._seen) != len(self.modified): + raise AssertionError("The pair of sample name and FASTQ must be unique.") + seen = Counter() + for row in self.modified: + sample = row[self._sample_col] + seen[sample] += 1 + row[self._sample_col] = f"{sample}_T{seen[sample]}" def read_head(handle, num_lines=10): diff --git a/conf/base.config b/conf/base.config index 23890a7e..6f4a15a6 100644 --- a/conf/base.config +++ b/conf/base.config @@ -26,6 +26,11 @@ process { // adding in your local modules too. // TODO nf-core: Customise requirements for specific processes. // See https://www.nextflow.io/docs/latest/config.html#config-process-selectors + withLabel:process_single { + cpus = { check_max( 1 , 'cpus' ) } + memory = { check_max( 6.GB * task.attempt, 'memory' ) } + time = { check_max( 4.h * task.attempt, 'time' ) } + } withLabel:process_low { cpus = { check_max( 2 * task.attempt, 'cpus' ) } memory = { check_max( 12.GB * task.attempt, 'memory' ) } diff --git a/docs/usage.md b/docs/usage.md index 08a222a9..b75cbc81 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -12,7 +12,7 @@ You will need to create a samplesheet with information about the samples you would like to analyse before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row as shown in the examples below. -```console +```bash --input '[path to samplesheet file]' ``` @@ -56,7 +56,7 @@ An [example samplesheet](../assets/samplesheet.csv) has been provided with the p The typical command for running the pipeline is as follows: -```console +```bash nextflow run nf-core/smrnaseq --input samplesheet.csv --outdir --genome GRCh37 -profile docker ``` @@ -64,9 +64,9 @@ This will launch the pipeline with the `docker` configuration profile. See below Note that the pipeline will create the following files in your working directory: -```console +```bash work # Directory containing the nextflow working files - # Finished results in specified location (defined with --outdir) + # Finished results in specified location (defined with --outdir) .nextflow_log # Log file from Nextflow # Other nextflow hidden files, eg. history of pipeline runs and old logs. ``` @@ -75,7 +75,7 @@ work # Directory containing the nextflow working files When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline: -```console +```bash nextflow pull nf-core/smrnaseq ``` @@ -251,6 +251,6 @@ Some HPC setups also allow you to run nextflow within a cluster job submitted yo In some cases, the Nextflow Java virtual machines can start to request a large amount of memory. We recommend adding the following line to your environment to limit this (typically in `~/.bashrc` or `~./bash_profile`): -```console +```bash NXF_OPTS='-Xms1g -Xmx4g' ``` diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy index 74b5820a..417fde67 100755 --- a/lib/WorkflowMain.groovy +++ b/lib/WorkflowMain.groovy @@ -59,6 +59,7 @@ class WorkflowMain { } // Print parameter summary log to screen + log.info paramsSummaryLog(workflow, params, log) // Check that a -profile or Nextflow config has been provided to run the pipeline @@ -78,17 +79,15 @@ class WorkflowMain { System.exit(1) } } - // // Get attribute from genome config file e.g. fasta // - public static String getGenomeAttribute(params, attribute) { - def val = '' + public static Object getGenomeAttribute(params, attribute) { if (params.genomes && params.genome && params.genomes.containsKey(params.genome)) { if (params.genomes[ params.genome ].containsKey(attribute)) { - val = params.genomes[ params.genome ][ attribute ] + return params.genomes[ params.genome ][ attribute ] } } - return val + return null } } diff --git a/lib/WorkflowSmrnaseq.groovy b/lib/WorkflowSmrnaseq.groovy index f33ecd7e..5bc00333 100755 --- a/lib/WorkflowSmrnaseq.groovy +++ b/lib/WorkflowSmrnaseq.groovy @@ -10,6 +10,7 @@ class WorkflowSmrnaseq { public static void initialise(params, log) { genomeExistsError(params, log) + if (!params.fasta) { log.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." System.exit(1) @@ -41,9 +42,7 @@ class WorkflowSmrnaseq { yaml_file_text += "data: |\n" yaml_file_text += "${summary_section}" return yaml_file_text - } - - // + }// // Exit pipeline if incorrect --genome key provided // private static void genomeExistsError(params, log) { diff --git a/main.nf b/main.nf index 5b057014..a7e6c457 100644 --- a/main.nf +++ b/main.nf @@ -4,7 +4,7 @@ nf-core/smrnaseq ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Github : https://github.com/nf-core/smrnaseq - Website: https://nf-co.re/smrnaseq +Website: https://nf-co.re/smrnaseq Slack : https://nfcore.slack.com/channels/smrnaseq ---------------------------------------------------------------------------------------- */ diff --git a/modules.json b/modules.json index b7e26668..81f0ce77 100644 --- a/modules.json +++ b/modules.json @@ -3,14 +3,20 @@ "homePage": "https://github.com/nf-core/smrnaseq", "repos": { "nf-core/modules": { - "custom/dumpsoftwareversions": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" - }, - "fastqc": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" - }, - "multiqc": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" + "git_url": "https://github.com/nf-core/modules.git", + "modules": { + "custom/dumpsoftwareversions": { + "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", + "branch": "master" + }, + "fastqc": { + "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", + "branch": "master" + }, + "multiqc": { + "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", + "branch": "master" + } } } } diff --git a/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py index d1390392..787bdb7b 100644 --- a/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ b/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py @@ -1,9 +1,10 @@ #!/usr/bin/env python -import yaml import platform from textwrap import dedent +import yaml + def _make_versions_html(versions): html = [ @@ -58,11 +59,12 @@ def _make_versions_html(versions): for process, process_versions in versions_by_process.items(): module = process.split(":")[-1] try: - assert versions_by_module[module] == process_versions, ( - "We assume that software versions are the same between all modules. " - "If you see this error-message it means you discovered an edge-case " - "and should open an issue in nf-core/tools. " - ) + if versions_by_module[module] != process_versions: + raise AssertionError( + "We assume that software versions are the same between all modules. " + "If you see this error-message it means you discovered an edge-case " + "and should open an issue in nf-core/tools. " + ) except KeyError: versions_by_module[module] = process_versions diff --git a/nextflow.config b/nextflow.config index d6c8e23c..e4a50caa 100644 --- a/nextflow.config +++ b/nextflow.config @@ -13,11 +13,11 @@ params { // Input options input = null + // References genome = null igenomes_base = 's3://ngi-igenomes/igenomes' igenomes_ignore = false - // MultiQC options multiqc_config = null multiqc_title = null @@ -37,6 +37,7 @@ params { schema_ignore_params = 'genomes' enable_conda = false + // Config options custom_config_version = 'master' custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" @@ -45,6 +46,7 @@ params { config_profile_url = null config_profile_name = null + // Max resource options // Defaults only, expecting to be overwritten max_memory = '128.GB' @@ -72,6 +74,7 @@ try { // } + profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { @@ -82,6 +85,15 @@ profiles { shifter.enabled = false charliecloud.enabled = false } + mamba { + params.enable_conda = true + conda.useMamba = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } docker { docker.enabled = true docker.userEmulation = true @@ -119,10 +131,16 @@ profiles { podman.enabled = false shifter.enabled = false } + gitpod { + executor.name = 'local' + executor.cpus = 16 + executor.memory = 60.GB + } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } } + // Load igenomes.config if required if (!params.igenomes_ignore) { includeConfig 'conf/igenomes.config' @@ -130,6 +148,7 @@ if (!params.igenomes_ignore) { params.genomes = [:] } + // Export these variables to prevent local Python/R libraries from conflicting with those in the container // The JULIA depot path has been adjusted to a fixed path `/usr/local/share/julia` that needs to be used for packages in the container. // See https://apeltzer.github.io/post/03-julia-lang-nextflow/ for details on that. Once we have a common agreement on where to keep Julia packages, this is adjustable. @@ -169,7 +188,7 @@ manifest { description = 'Small RNA-Seq Best Practice Analysis Pipeline.' mainScript = 'main.nf' nextflowVersion = '!>=21.10.3' - version = '2.0.0' + version = '2.1.0dev' } // Load modules.config for DSL2 module specific options From 37885c5491e82a8fa3afddd37a58315d5091e24a Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 1 Sep 2022 13:33:01 +0000 Subject: [PATCH 067/145] Template update for nf-core/tools version 2.5.1 --- bin/check_samplesheet.py | 9 ++++++--- pyproject.toml | 10 ++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 pyproject.toml diff --git a/bin/check_samplesheet.py b/bin/check_samplesheet.py index 9a8b8962..11b15572 100755 --- a/bin/check_samplesheet.py +++ b/bin/check_samplesheet.py @@ -98,7 +98,9 @@ def _validate_pair(self, row): """Assert that read pairs have the same file extension. Report pair status.""" if row[self._first_col] and row[self._second_col]: row[self._single_col] = False - if Path(row[self._first_col]).suffixes[-2:] != Path(row[self._second_col]).suffixes[-2:]: + first_col_suffix = Path(row[self._first_col]).suffixes[-2:] + second_col_suffix = Path(row[self._second_col]).suffixes[-2:] + if first_col_suffix != second_col_suffix: raise AssertionError("FASTQ pairs must have the same file extensions.") else: row[self._single_col] = True @@ -157,7 +159,7 @@ def sniff_format(handle): handle.seek(0) sniffer = csv.Sniffer() if not sniffer.has_header(peek): - logger.critical(f"The given sample sheet does not appear to contain a header.") + logger.critical("The given sample sheet does not appear to contain a header.") sys.exit(1) dialect = sniffer.sniff(peek) return dialect @@ -195,7 +197,8 @@ def check_samplesheet(file_in, file_out): reader = csv.DictReader(in_handle, dialect=sniff_format(in_handle)) # Validate the existence of the expected header columns. if not required_columns.issubset(reader.fieldnames): - logger.critical(f"The sample sheet **must** contain the column headers: {', '.join(required_columns)}.") + req_cols = ", ".join(required_columns) + logger.critical(f"The sample sheet **must** contain these column headers: {req_cols}.") sys.exit(1) # Validate each row. checker = RowChecker() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..0d62beb6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,10 @@ +# Config file for Python. Mostly used to configure linting of bin/check_samplesheet.py with Black. +# Should be kept the same as nf-core/tools to avoid fighting with template synchronisation. +[tool.black] +line-length = 120 +target_version = ["py37", "py38", "py39", "py310"] + +[tool.isort] +profile = "black" +known_first_party = ["nf_core"] +multi_line_output = 3 From 2a8d9fbd2a30dc3abbfdd87cea9abfa814cac601 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 6 Sep 2022 13:03:00 +0100 Subject: [PATCH 068/145] Use snake_case for mirGeneDB parameters --- CHANGELOG.md | 10 +++++----- docs/usage.md | 12 ++++++------ modules/local/mirtop_quant.nf | 8 ++++---- modules/local/parse_fasta_mirna.nf | 4 ++-- nextflow.config | 10 +++++----- nextflow_schema.json | 26 +++++++++++++------------- workflows/smrnaseq.nf | 12 ++++++------ 7 files changed, 41 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c100756d..ce1f3077 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,11 +22,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 | Old parameter | New parameter | | ------------- | ------------------------ | -| | `--mirGeneDB` | -| | `--mirGeneDB_species` | -| | `--mirGeneDB_gff` | -| | `--mirGeneDB_mature` | -| | `--mirGeneDB_hairpin` | +| | `--mirgenedb` | +| | `--mirgenedb_species` | +| | `--mirgenedb_gff` | +| | `--mirgenedb_mature` | +| | `--mirgenedb_hairpin` | | | `--contamination_filter` | | | `--rrna` | | | `--trna` | diff --git a/docs/usage.md b/docs/usage.md index 475dc9dc..f42023a1 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -18,9 +18,9 @@ This option indicates the experimental protocol used for the sample preparation. :warning: At least the `custom` protocol has to be specified, otherwise the pipeline won't run. -### `mirtrace_species or mirGeneDB_species` +### `mirtrace_species` or `mirgenedb_species` -It should point to the 3-letter species name used by `miRBase`, or `MirGeneDB`. Note the difference in case for the two databases. +It should point to the 3-letter species name used by [miRBase](https://www.mirbase.org/help/genome_summary.shtml) or MirGeneDB. Note the difference in case for the two databases. ### miRNA related files @@ -30,11 +30,11 @@ Different parameters can be set for the two supported datbases. By default `miRB - `mature`: points to the FASTA file of mature miRNA sequences. `https://mirbase.org/ftp/CURRENT/mature.fa.gz` - `hairpin`: points to the FASTA file of precursor miRNA sequences. `https://mirbase.org/ftp/CURRENT/hairpin.fa.gz` -If `MirGeneDB` should be used instead it needs to be specified using `--mirGeneDB` and use the parameters below . +If MirGeneDB should be used instead it needs to be specified using `--mirgenedb` and use the parameters below . -- `mirGeneDB_gff`: The data can not be downloaded automatically, thus the user needs to supply the gff file for either his species, or all species downloaded from `https://mirgenedb.org/download`. The total set will automatically be subsetted to the species specified with `mirGeneDB_species`. -- `mirGeneDB_mature`: points to the FASTA file of mature miRNA sequences. Download from `https://mirgenedb.org/download`. -- `mirGeneDB_hairpin`: points to the FASTA file of precursor miRNA sequences. Download from `https://mirgenedb.org/download`. Note that `MirGeneDB` does not have a dedicated `hairpin` file, but the `Precursor sequences` are to be used. +- `mirgenedb_gff`: The data can not be downloaded automatically, thus the user needs to supply the gff file for either his species, or all species downloaded from `https://mirgenedb.org/download`. The total set will automatically be subsetted to the species specified with `--mirgenedb_species`. +- `mirgenedb_mature`: points to the FASTA file of mature miRNA sequences. Download from `https://mirgenedb.org/download`. +- `mirgenedb_hairpin`: points to the FASTA file of precursor miRNA sequences. Download from `https://mirgenedb.org/download`. Note that MirGeneDB does not have a dedicated `hairpin` file, but the `Precursor sequences` are to be used. ### Genome diff --git a/modules/local/mirtop_quant.nf b/modules/local/mirtop_quant.nf index ffbfc0a0..6f903add 100644 --- a/modules/local/mirtop_quant.nf +++ b/modules/local/mirtop_quant.nf @@ -11,7 +11,7 @@ process MIRTOP_QUANT { path hairpin path gtf - //if (!params.mirGeneDB) {params.filterSpecies = params.mirtrace_species} else {params.filterSpecies = params.mirGeneDB_species} + //if (!params.mirgenedb) {params.filter_species = params.mirtrace_species} else {params.filter_species = params.mirgenedb_species} output: path "mirtop/mirtop.gff" @@ -22,9 +22,9 @@ process MIRTOP_QUANT { script: """ - mirtop gff --hairpin $hairpin --gtf $gtf -o mirtop --sps $params.filterSpecies ./bams/* - mirtop counts --hairpin $hairpin --gtf $gtf -o mirtop --sps $params.filterSpecies --add-extra --gff mirtop/mirtop.gff - mirtop export --format isomir --hairpin $hairpin --gtf $gtf --sps $params.filterSpecies -o mirtop mirtop/mirtop.gff + mirtop gff --hairpin $hairpin --gtf $gtf -o mirtop --sps $params.filter_species ./bams/* + mirtop counts --hairpin $hairpin --gtf $gtf -o mirtop --sps $params.filter_species --add-extra --gff mirtop/mirtop.gff + mirtop export --format isomir --hairpin $hairpin --gtf $gtf --sps $params.filter_species -o mirtop mirtop/mirtop.gff mirtop stats mirtop/mirtop.gff --out mirtop/stats mv mirtop/stats/mirtop_stats.log mirtop/stats/full_mirtop_stats.log diff --git a/modules/local/parse_fasta_mirna.nf b/modules/local/parse_fasta_mirna.nf index 18b51066..27caf42b 100644 --- a/modules/local/parse_fasta_mirna.nf +++ b/modules/local/parse_fasta_mirna.nf @@ -9,7 +9,7 @@ process PARSE_FASTA_MIRNA { input: path fasta - //if (!params.mirGeneDB) {params.filterSpecies = params.mirtrace_species} else {params.filterSpecies = params.mirGeneDB_species} + //if (!params.mirgenedb) {params.filter_species = params.mirtrace_species} else {params.filter_species = params.mirgenedb_species} output: path '*_igenome.fa', emit: parsed_fasta @@ -29,7 +29,7 @@ process PARSE_FASTA_MIRNA { # TODO perl -ane 's/[ybkmrsw]/N/ig;print;' \${FASTA}_parsed_tmp.fa > \${FASTA}_parsed.fa sed -i 's/\s.*//' \${FASTA}_parsed.fa - seqkit grep -r --pattern \".*${params.filterSpecies}-.*\" \${FASTA}_parsed.fa > \${FASTA}_sps.fa + seqkit grep -r --pattern \".*${params.filter_species}-.*\" \${FASTA}_parsed.fa > \${FASTA}_sps.fa seqkit seq --rna2dna \${FASTA}_sps.fa > \${FASTA}_igenome.fa cat <<-END_VERSIONS > versions.yml diff --git a/nextflow.config b/nextflow.config index fe98b39c..08978844 100644 --- a/nextflow.config +++ b/nextflow.config @@ -25,11 +25,11 @@ params { mirtrace_species = null mature = "https://mirbase.org/ftp/CURRENT/mature.fa.gz" hairpin = "https://mirbase.org/ftp/CURRENT/hairpin.fa.gz" - mirGeneDB = false - mirGeneDB_mature = "/Users/chriskub/Downloads/ALL-mat.fas" - mirGeneDB_hairpin = "/Users/chriskub/Downloads/ALL-pre.fas" - mirGeneDB_gff = "/Users/chriskub/Downloads/ALL.gff" - mirGeneDB_species = null + mirgenedb = false + mirgenedb_mature = "/Users/chriskub/Downloads/ALL-mat.fas" + mirgenedb_hairpin = "/Users/chriskub/Downloads/ALL-pre.fas" + mirgenedb_gff = "/Users/chriskub/Downloads/ALL.gff" + mirgenedb_species = null // Trimming options clip_r1 = 0 diff --git a/nextflow_schema.json b/nextflow_schema.json index 77faa9ba..2ca88862 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -62,22 +62,22 @@ "fa_icon": "fas fa-book", "help_text": "If using a reference genome configured in the pipeline using iGenomes, use this parameter to give the ID for the reference. This is then used to build the full paths for all required reference genome files e.g. `--genome GRCh38`. \n\nSee the [nf-core website docs](https://nf-co.re/usage/reference_genomes) for more details." }, - "mirGeneDB": { + "mirgenedb": { "type": "boolean", - "description": "Boolean wether mirGeneDB should be used instead of miRBase", - "help_text": "This allows you to use mirGeneDB instead of miRBase as the database. \n Note that you will need to set the additional flags `--mirGeneDB_species`, `--mirGeneDB_gff`, `--mirGeneDB_mature` and `--mirGeneDB_hairpin`", + "description": "Boolean whether MirGeneDB should be used instead of miRBase", + "help_text": "This allows you to use MirGeneDB instead of miRBase as the database. \n Note that you will need to set the additional flags `--mirgenedb_species`, `--mirgenedb_gff`, `--mirgenedb_mature` and `--mirgenedb_hairpin`", "default": "false" }, "mirtrace_species": { "type": "string", "description": "Species for miRTrace.", - "help_text": "This is automatically set when using `--genome`. Example values: `hsa`, `mmu`...\n Note that mirTrace relies on miRBase for its species reference. See available references [here](https://mirbase.org/ftp/CURRENT/genomes/).", + "help_text": "This is automatically set when using `--genome`. Example values: `hsa`, `mmu`...\n Note that mirTrace relies on miRBase for its species reference. See available references [here](https://www.mirbase.org/help/genome_summary.shtml).", "fa_icon": "fas fa-journal-whills" }, - "mirGeneDB_species": { + "mirgenedb_species": { "type": "string", - "description": "Species of mirGeneDB.", - "help_text": "This replaces the value of `--mirtrace_species` if `--mirGeneDB` is used. \n Note the difference in case for species names used in MirGeneDB and miRBase." + "description": "Species of MirGeneDB.", + "help_text": "This replaces the value of `--mirtrace_species` if `--mirgenedb` is used. \n Note the difference in case for species names used in MirGeneDB and miRBase." }, "fasta": { "type": "string", @@ -91,10 +91,10 @@ "help_text": "miRBase `.gff3` file, typically downloaded from [`https://mirbase.org/ftp/CURRENT/genomes/`](https://mirbase.org/ftp/CURRENT/genomes/)\n\nIf using iGenomes with `--genome` this file will be downloaded from miRBase automatically during the pipeline run.\n\n", "fa_icon": "fas fa-address-book" }, - "mirGeneDB_gff": { + "mirgenedb_gff": { "type": "string", "description": "GFF/GTF file with coordinates positions of precursor and miRNAs.", - "help_text": "mirGeneDB `.gff3` file, typically downloaded from [`https://mirgenedb.org/download`]. This replaces the value of --mirna_gff if --mirGeneDB is used." + "help_text": "MirGeneDB `.gff3` file, typically downloaded from [`https://mirgenedb.org/download`]. This replaces the value of --mirna_gff if --mirgenedb is used." }, "mature": { "type": "string", @@ -103,9 +103,9 @@ "help_text": "Typically this will be the `mature.fa` file from miRBase. Can be given either as a plain text `.fa` file or a compressed `.gz` file.\n\nDefaults to the current miRBase release URL, from which the file will be downloaded.", "default": "https://mirbase.org/ftp/CURRENT/mature.fa.gz" }, - "mirGeneDB_mature": { + "mirgenedb_mature": { "type": "string", - "description": "Path to FASTA file with mirGeneDB mature miRNAs.", + "description": "Path to FASTA file with MirGeneDB mature miRNAs.", "help_text": "This file needs to be downloaded from [`https://mirgenedb.org/download`]. Can be given either as a plain text `.fa` file or a compressed `.gz` file." }, "hairpin": { @@ -115,10 +115,10 @@ "help_text": "Typically this will be the `mature.fa` file from miRBase. Can be given either as a plain text `.fa` file or a compressed `.gz` file.\n\nDefaults to the current miRBase release URL, from which the file will be downloaded.", "default": "https://mirbase.org/ftp/CURRENT/hairpin.fa.gz" }, - "mirGeneDB_hairpin": { + "mirgenedb_hairpin": { "type": "string", "description": "Path to FASTA file with miRNAs precursors.", - "help_text": "This file needs to be downloaded from [`https://mirgenedb.org/download`]. Can be given either as a plain text `.fa` file or a compressed `.gz` file.\nNote that mirGeneDB does not have a dedicated hairpin file. The equivalent is the `Precursor sequences`." + "help_text": "This file needs to be downloaded from [`https://mirgenedb.org/download`]. Can be given either as a plain text `.fa` file or a compressed `.gz` file.\nNote that MirGeneDB does not have a dedicated hairpin file. The equivalent is the `Precursor sequences`." }, "bowtie_indices": { "type": "string", diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index be455612..a32ef4d8 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -53,15 +53,15 @@ ch_multiqc_image = file("$projectDir/assets/smrnaseq_logo.png", checkIfE // // SUBWORKFLOW: Consisting of a mix of local and nf-core/modules // -if (!params.mirGeneDB) { +if (!params.mirgenedb) { if (params.mature) { reference_mature = file(params.mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mature}" } if (params.hairpin) { reference_hairpin = file(params.hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.hairpin}" } - params.filterSpecies = params.mirtrace_species + params.filter_species = params.mirtrace_species } else { - if (params.mirGeneDB_mature) { reference_mature = file(params.mirGeneDB_mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mirGeneDB_mature}" } - if (params.mirGeneDB_hairpin) { reference_hairpin = file(params.mirGeneDB_hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.mirGeneDB_hairpin}" } - if (params.mirGeneDB_gff) { mirna_gtf = file(params.mirGeneDB_gff, checkIfExists: true) } else { exit 1, "MirGeneDB gff file not found: ${params.mirGeneDB_gff}"} - params.filterSpecies = params.mirGeneDB_species + if (params.mirgenedb_mature) { reference_mature = file(params.mirgenedb_mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mirgenedb_mature}" } + if (params.mirgenedb_hairpin) { reference_hairpin = file(params.mirgenedb_hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.mirgenedb_hairpin}" } + if (params.mirgenedb_gff) { mirna_gtf = file(params.mirgenedb_gff, checkIfExists: true) } else { exit 1, "MirGeneDB gff file not found: ${params.mirgenedb_gff}"} + params.filter_species = params.mirgenedb_species } include { INPUT_CHECK } from '../subworkflows/local/input_check' From 4d57a000a015cbe9a5112487fa4a181e58b4cc43 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 6 Sep 2022 13:27:29 +0100 Subject: [PATCH 069/145] Fix nf-core lint --- .github/PULL_REQUEST_TEMPLATE.md | 3 +- .github/workflows/ci.yml | 23 +++-------- .github/workflows/linting.yml | 39 +++++++++++++++--- .nf-core.yml | 9 +--- assets/email_template.txt | 1 - modules.json | 71 +++++++++++++++++++------------- pyproject.toml | 10 +++++ 7 files changed, 93 insertions(+), 63 deletions(-) create mode 100644 pyproject.toml diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d103b361..cf3fb13d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -15,8 +15,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/smrn - [ ] This comment contains a description of changes (with reason). - [ ] If you've fixed a bug or added code that should be tested, add tests! - - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/smrnaseq/tree/master/.github/CONTRIBUTING.md) - - [ ] If necessary, also make a PR on the nf-core/smrnaseq _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. +- [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/smrnaseq/tree/master/.github/CONTRIBUTING.md)- [ ] If necessary, also make a PR on the nf-core/smrnaseq _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31a2a000..265fffee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,6 @@ on: env: NXF_ANSI_LOG: false - CAPSULE_LOG: none jobs: test: @@ -20,27 +19,17 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - # Nextflow versions - include: - # Test pipeline minimum Nextflow version - - NXF_VER: "21.10.3" - NXF_EDGE: "" - # Test latest edge release of Nextflow - - NXF_VER: "" - NXF_EDGE: "1" + NXF_VER: + - "21.10.3" + - "latest-everything" steps: - name: Check out pipeline code uses: actions/checkout@v2 - name: Install Nextflow - env: - NXF_VER: ${{ matrix.NXF_VER }} - # Uncomment only if the edge release is more recent than the latest stable release - # See https://github.com/nextflow-io/nextflow/issues/2467 - # NXF_EDGE: ${{ matrix.NXF_EDGE }} - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + uses: nf-core/setup-nextflow@v1 + with: + version: "${{ matrix.NXF_VER }}" - name: Run pipeline with test data run: | diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 8a092d87..8a5ce69b 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -35,6 +35,36 @@ jobs: - name: Run Prettier --check run: prettier --check ${GITHUB_WORKSPACE} + PythonBlack: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Check code lints with Black + uses: psf/black@stable + + # If the above check failed, post a comment on the PR explaining the failure + - name: Post PR comment + if: failure() + uses: mshick/add-pr-comment@v1 + with: + message: | + ## Python linting (`black`) is failing + + To keep the code consistent with lots of contributors, we run automated code consistency checks. + To fix this CI test, please run: + + * Install [`black`](https://black.readthedocs.io/en/stable/): `pip install black` + * Fix formatting errors in your pipeline: `black .` + + Once you push these changes the test should pass, and you can hide this comment :+1: + + We highly recommend setting up Black in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + + Thanks again for your contribution! + repo-token: ${{ secrets.GITHUB_TOKEN }} + allow-repeats: false + nf-core: runs-on: ubuntu-latest steps: @@ -42,21 +72,18 @@ jobs: uses: actions/checkout@v2 - name: Install Nextflow - env: - CAPSULE_LOG: none - run: | - wget -qO- get.nextflow.io | bash - sudo mv nextflow /usr/local/bin/ + uses: nf-core/setup-nextflow@v1 - uses: actions/setup-python@v3 with: - python-version: "3.6" + python-version: "3.7" architecture: "x64" - name: Install dependencies run: | python -m pip install --upgrade pip pip install nf-core + - name: Run nf-core lint env: GITHUB_COMMENTS_URL: ${{ github.event.pull_request.comments_url }} diff --git a/.nf-core.yml b/.nf-core.yml index 9de90de2..b1a7f0e9 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -1,8 +1 @@ -repository_type: pipeline -lint: - files_unchanged: - - .github/workflows/linting.yml - - .github/workflows/branch.yml - - .github/workflows/ci.yml - - .github/workflows/linting.yml - - .github/workflows/linting_comment.yml +repository_type: pipeline \ No newline at end of file diff --git a/assets/email_template.txt b/assets/email_template.txt index 6e8b6393..654e78db 100644 --- a/assets/email_template.txt +++ b/assets/email_template.txt @@ -6,7 +6,6 @@ `._,._,' nf-core/smrnaseq v${version} ---------------------------------------------------- - Run Name: $runName <% if (success){ diff --git a/modules.json b/modules.json index 0b772f2e..49ac89e7 100644 --- a/modules.json +++ b/modules.json @@ -3,35 +3,48 @@ "homePage": "https://github.com/nf-core/smrnaseq", "repos": { "nf-core/modules": { - "cat/fastq": { - "git_sha": "9aadd9a6d3f5964476582319b3a1c54a3e3fe7c9" - }, - "custom/dumpsoftwareversions": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" - }, - "fastqc": { - "git_sha": "49b18b1639f4f7104187058866a8fab33332bdfe" - }, - "multiqc": { - "git_sha": "08376da6843b14c82d84d444784c0b3635bb7fd5" - }, - "samtools/flagstat": { - "git_sha": "ecece498f10b47b7c9d06f53a310cea5811b4c5f" - }, - "samtools/idxstats": { - "git_sha": "ecece498f10b47b7c9d06f53a310cea5811b4c5f" - }, - "samtools/index": { - "git_sha": "897c33d5da084b61109500ee44c01da2d3e4e773" - }, - "samtools/sort": { - "git_sha": "897c33d5da084b61109500ee44c01da2d3e4e773" - }, - "samtools/stats": { - "git_sha": "ecece498f10b47b7c9d06f53a310cea5811b4c5f" - }, - "trimgalore": { - "git_sha": "85ec13ff1fc2196c5a507ea497de468101baabed" + "git_url": "https://github.com/nf-core/modules.git", + "modules": { + "cat/fastq": { + "branch": "master", + "git_sha": "9aadd9a6d3f5964476582319b3a1c54a3e3fe7c9" + }, + "custom/dumpsoftwareversions": { + "branch": "master", + "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" + }, + "fastqc": { + "branch": "master", + "git_sha": "49b18b1639f4f7104187058866a8fab33332bdfe" + }, + "multiqc": { + "branch": "master", + "git_sha": "08376da6843b14c82d84d444784c0b3635bb7fd5" + }, + "samtools/flagstat": { + "branch": "master", + "git_sha": "ecece498f10b47b7c9d06f53a310cea5811b4c5f" + }, + "samtools/idxstats": { + "branch": "master", + "git_sha": "ecece498f10b47b7c9d06f53a310cea5811b4c5f" + }, + "samtools/index": { + "branch": "master", + "git_sha": "897c33d5da084b61109500ee44c01da2d3e4e773" + }, + "samtools/sort": { + "branch": "master", + "git_sha": "897c33d5da084b61109500ee44c01da2d3e4e773" + }, + "samtools/stats": { + "branch": "master", + "git_sha": "ecece498f10b47b7c9d06f53a310cea5811b4c5f" + }, + "trimgalore": { + "branch": "master", + "git_sha": "85ec13ff1fc2196c5a507ea497de468101baabed" + } } } } diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..0d62beb6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,10 @@ +# Config file for Python. Mostly used to configure linting of bin/check_samplesheet.py with Black. +# Should be kept the same as nf-core/tools to avoid fighting with template synchronisation. +[tool.black] +line-length = 120 +target_version = ["py37", "py38", "py39", "py310"] + +[tool.isort] +profile = "black" +known_first_party = ["nf_core"] +multi_line_output = 3 From cbb18f1d6d8aaae819c5e88635aee0acc8809f12 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 6 Sep 2022 12:35:01 +0000 Subject: [PATCH 070/145] [automated] Fix linting with Prettier --- .nf-core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nf-core.yml b/.nf-core.yml index b1a7f0e9..3805dc81 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -1 +1 @@ -repository_type: pipeline \ No newline at end of file +repository_type: pipeline From ed1ac84e9ce550b82a2f1a408eb28748b1f6bf1a Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 6 Sep 2022 13:40:52 +0100 Subject: [PATCH 071/145] Fix PythonBlack --- bin/check_samplesheet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/check_samplesheet.py b/bin/check_samplesheet.py index 00a4c93a..3ca65774 100755 --- a/bin/check_samplesheet.py +++ b/bin/check_samplesheet.py @@ -79,7 +79,7 @@ def check_samplesheet(file_in, file_out): ) ## Check sample name entries - sample, fastq_1 = lspl[: len(HEADER)] + sample, fastq_1 = lspl[: len(HEADER)] sample = sample.replace(" ", "_") if not sample: print_error("Sample entry has not been specified!", "Line", line) From 3bf9ab72a38b4e94372ed47808c42758c561a7c8 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 6 Sep 2022 13:50:44 +0100 Subject: [PATCH 072/145] Split out CI to parallelise --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 265fffee..d0bd4ac8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,9 @@ jobs: NXF_VER: - "21.10.3" - "latest-everything" + profile: + - "test" + - "test_no_genome" steps: - name: Check out pipeline code uses: actions/checkout@v2 @@ -33,5 +36,4 @@ jobs: - name: Run pipeline with test data run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results - nextflow run ${GITHUB_WORKSPACE} -profile test_no_genome,docker --outdir ./results + nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.profile }},docker --outdir ./results From 05ae7d227c1bd64c009762e70e13faf5bf10cace Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Wed, 7 Sep 2022 16:34:06 +0100 Subject: [PATCH 073/145] Fix linting --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index ad773211..499134b1 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) P. Ewels , C. Wang , R. Hammarén , L. Pantano +Copyright (c) P. Ewels, C. Wang, R. Hammarén, L. Pantano Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From eb60d8556ffcb29ab4e36d801890b90668fa2c86 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 12 Sep 2022 16:07:34 +0200 Subject: [PATCH 074/145] Update Modules before release 2.1.0 --- modules.json | 10 +++---- modules/nf-core/modules/cat/fastq/main.nf | 29 +++++++++++++++++++ .../custom/dumpsoftwareversions/main.nf | 8 ++--- .../templates/dumpsoftwareversions.py | 14 ++++----- modules/nf-core/modules/multiqc/main.nf | 9 +++--- modules/nf-core/modules/multiqc/meta.yml | 7 +++-- .../nf-core/modules/samtools/flagstat/main.nf | 4 +-- .../nf-core/modules/samtools/stats/main.nf | 8 ++--- 8 files changed, 60 insertions(+), 29 deletions(-) diff --git a/modules.json b/modules.json index 49ac89e7..d169ad11 100644 --- a/modules.json +++ b/modules.json @@ -7,11 +7,11 @@ "modules": { "cat/fastq": { "branch": "master", - "git_sha": "9aadd9a6d3f5964476582319b3a1c54a3e3fe7c9" + "git_sha": "b034029b59b1198075da8019074bc02051a6100e" }, "custom/dumpsoftwareversions": { "branch": "master", - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d" + "git_sha": "5e7b1ef9a5a2d9258635bcbf70fcf37dacd1b247" }, "fastqc": { "branch": "master", @@ -19,11 +19,11 @@ }, "multiqc": { "branch": "master", - "git_sha": "08376da6843b14c82d84d444784c0b3635bb7fd5" + "git_sha": "5587389874dac9c9953a2ab6f01d49af81969492" }, "samtools/flagstat": { "branch": "master", - "git_sha": "ecece498f10b47b7c9d06f53a310cea5811b4c5f" + "git_sha": "bbb99cb8d679555cc01c98766de7869f83283545" }, "samtools/idxstats": { "branch": "master", @@ -39,7 +39,7 @@ }, "samtools/stats": { "branch": "master", - "git_sha": "ecece498f10b47b7c9d06f53a310cea5811b4c5f" + "git_sha": "f4eab7945952dc4934224309701a49913ea05ae6" }, "trimgalore": { "branch": "master", diff --git a/modules/nf-core/modules/cat/fastq/main.nf b/modules/nf-core/modules/cat/fastq/main.nf index b6854895..d275f19c 100644 --- a/modules/nf-core/modules/cat/fastq/main.nf +++ b/modules/nf-core/modules/cat/fastq/main.nf @@ -48,4 +48,33 @@ process CAT_FASTQ { """ } } + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + def readList = reads.collect{ it.toString() } + if (meta.single_end) { + if (readList.size > 1) { + """ + touch ${prefix}.merged.fastq.gz + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + cat: \$(echo \$(cat --version 2>&1) | sed 's/^.*coreutils) //; s/ .*\$//') + END_VERSIONS + """ + } + } else { + if (readList.size > 2) { + """ + touch ${prefix}_1.merged.fastq.gz + touch ${prefix}_2.merged.fastq.gz + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + cat: \$(echo \$(cat --version 2>&1) | sed 's/^.*coreutils) //; s/ .*\$//') + END_VERSIONS + """ + } + } + } diff --git a/modules/nf-core/modules/custom/dumpsoftwareversions/main.nf b/modules/nf-core/modules/custom/dumpsoftwareversions/main.nf index 327d5100..34b50b9f 100644 --- a/modules/nf-core/modules/custom/dumpsoftwareversions/main.nf +++ b/modules/nf-core/modules/custom/dumpsoftwareversions/main.nf @@ -1,11 +1,11 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { - label 'process_low' + label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda (params.enable_conda ? "bioconda::multiqc=1.11" : null) + conda (params.enable_conda ? 'bioconda::multiqc=1.13a' : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.11--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.11--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.13a--pyhdfd78af_1' : + 'quay.io/biocontainers/multiqc:1.13a--pyhdfd78af_1' }" input: path versions diff --git a/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py index 787bdb7b..d1390392 100644 --- a/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ b/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py @@ -1,10 +1,9 @@ #!/usr/bin/env python +import yaml import platform from textwrap import dedent -import yaml - def _make_versions_html(versions): html = [ @@ -59,12 +58,11 @@ def _make_versions_html(versions): for process, process_versions in versions_by_process.items(): module = process.split(":")[-1] try: - if versions_by_module[module] != process_versions: - raise AssertionError( - "We assume that software versions are the same between all modules. " - "If you see this error-message it means you discovered an edge-case " - "and should open an issue in nf-core/tools. " - ) + assert versions_by_module[module] == process_versions, ( + "We assume that software versions are the same between all modules. " + "If you see this error-message it means you discovered an edge-case " + "and should open an issue in nf-core/tools. " + ) except KeyError: versions_by_module[module] = process_versions diff --git a/modules/nf-core/modules/multiqc/main.nf b/modules/nf-core/modules/multiqc/main.nf index 3c3517bf..d10dae69 100644 --- a/modules/nf-core/modules/multiqc/main.nf +++ b/modules/nf-core/modules/multiqc/main.nf @@ -1,14 +1,15 @@ process MULTIQC { label 'process_medium' - conda (params.enable_conda ? 'bioconda::multiqc=1.12' : null) + conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.13a--pyhdfd78af_1' : - 'quay.io/biocontainers/multiqc:1.13a--pyhdfd78af_1' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : + 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" - tuple path(multiqc_config), path(multiqc_logo) + path(multiqc_config) + path(multiqc_logo) output: path "*multiqc_report.html", emit: report diff --git a/modules/nf-core/modules/multiqc/meta.yml b/modules/nf-core/modules/multiqc/meta.yml index bf3a27fe..a1029f33 100644 --- a/modules/nf-core/modules/multiqc/meta.yml +++ b/modules/nf-core/modules/multiqc/meta.yml @@ -12,6 +12,7 @@ tools: homepage: https://multiqc.info/ documentation: https://multiqc.info/docs/ licence: ["GPL-3.0-or-later"] + input: - multiqc_files: type: file @@ -19,12 +20,13 @@ input: List of reports / files recognised by MultiQC, for example the html and zip output of FastQC - multiqc_config: type: file - description: Config yml for MultiQC + description: Optional config yml for MultiQC pattern: "*.{yml,yaml}" - multiqc_logo: type: file - description: Logo file for MultiQC + description: Optional logo file for MultiQC pattern: "*.{png}" + output: - report: type: file @@ -46,3 +48,4 @@ authors: - "@abhi18av" - "@bunop" - "@drpatelh" + - "@jfy133" diff --git a/modules/nf-core/modules/samtools/flagstat/main.nf b/modules/nf-core/modules/samtools/flagstat/main.nf index 03ec2dcf..c3152aca 100644 --- a/modules/nf-core/modules/samtools/flagstat/main.nf +++ b/modules/nf-core/modules/samtools/flagstat/main.nf @@ -1,6 +1,6 @@ process SAMTOOLS_FLAGSTAT { tag "$meta.id" - label 'process_low' + label 'process_single' conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? @@ -23,7 +23,7 @@ process SAMTOOLS_FLAGSTAT { """ samtools \\ flagstat \\ - --threads ${task.cpus-1} \\ + --threads ${task.cpus} \\ $bam \\ > ${prefix}.flagstat diff --git a/modules/nf-core/modules/samtools/stats/main.nf b/modules/nf-core/modules/samtools/stats/main.nf index c913bc5e..9b0c3867 100644 --- a/modules/nf-core/modules/samtools/stats/main.nf +++ b/modules/nf-core/modules/samtools/stats/main.nf @@ -1,6 +1,6 @@ process SAMTOOLS_STATS { tag "$meta.id" - label 'process_low' + label 'process_single' conda (params.enable_conda ? "bioconda::samtools=1.15.1" : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? @@ -8,7 +8,7 @@ process SAMTOOLS_STATS { 'quay.io/biocontainers/samtools:1.15.1--h1170115_0' }" input: - tuple val(meta), path(bam), path(bai) + tuple val(meta), path(input), path(input_index) path fasta output: @@ -25,9 +25,9 @@ process SAMTOOLS_STATS { """ samtools \\ stats \\ - --threads ${task.cpus-1} \\ + --threads ${task.cpus} \\ ${reference} \\ - ${bam} \\ + ${input} \\ > ${prefix}.stats cat <<-END_VERSIONS > versions.yml From 3e9fc14ab442cbc48952cb25b405e56860ae3eb0 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 12 Sep 2022 16:11:59 +0200 Subject: [PATCH 075/145] Bugfix for mqc channels --- workflows/smrnaseq.nf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index a32ef4d8..f9675425 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -235,7 +235,8 @@ workflow SMRNASEQ { MULTIQC ( ch_multiqc_files.collect(), - [ch_multiqc_config, ch_multiqc_image] + ch_multiqc_config, + ch_multiqc_image ) multiqc_report = MULTIQC.out.report.toList() From 1b6b8c65e9ddb5d4abc92a42c5bdc1f98f635812 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 13 Sep 2022 15:03:00 +0100 Subject: [PATCH 076/145] Fix empty channel logic for versions --- subworkflows/local/mirna_quant.nf | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/subworkflows/local/mirna_quant.nf b/subworkflows/local/mirna_quant.nf index a851d729..9a8259d4 100644 --- a/subworkflows/local/mirna_quant.nf +++ b/subworkflows/local/mirna_quant.nf @@ -64,7 +64,6 @@ workflow MIRNA_QUANT { .dump (tag:'hsux') .set { reads_hairpin } - BOWTIE_MAP_HAIRPIN ( reads_hairpin, hairpin_bowtie.collect() ) ch_versions = ch_versions.mix(BOWTIE_MAP_HAIRPIN.out.versions) @@ -93,8 +92,10 @@ workflow MIRNA_QUANT { BOWTIE_MAP_SEQCLUSTER ( reads_collapsed, hairpin_bowtie.collect() ) ch_versions = ch_versions.mix(BOWTIE_MAP_SEQCLUSTER.out.versions) + ch_mirtop_logs = Channel.empty() if (params.mirtrace_species){ MIRTOP_QUANT ( BOWTIE_MAP_SEQCLUSTER.out.bam.collect{it[1]}, FORMAT_HAIRPIN.out.formatted_fasta, gtf ) + ch_mirtop_logs = MIRTOP_QUANT.out.logs ch_versions = ch_versions.mix(MIRTOP_QUANT.out.versions) TABLE_MERGE ( MIRTOP_QUANT.out.mirtop_table ) @@ -109,22 +110,16 @@ workflow MIRNA_QUANT { fasta_mature = FORMAT_MATURE.out.formatted_fasta fasta_hairpin = FORMAT_HAIRPIN.out.formatted_fasta unmapped = reads_genome - bowtie_versions = BOWTIE_MAP_MATURE.out.versions - samtools_versions = BAM_STATS_MATURE.out.versions - seqcluster_versions = SEQCLUSTER_SEQUENCES.out.versions - mirtop_versions = MIRTOP_QUANT.out.versions mature_stats = BAM_STATS_MATURE.out.stats hairpin_stats = BAM_STATS_HAIRPIN.out.stats - mirtop_logs = MIRTOP_QUANT.out.logs - merge_versions = TABLE_MERGE.out.versions + mirtop_logs = ch_mirtop_logs versions = ch_versions } - def add_suffix(row, suffix) { def meta = [:] - meta.id = "${row[0].id}_${suffix}" + meta.id = "${row[0].id}_${suffix}" def array = [] array = [ meta, row[1] ] return array From e3f89f11d415781296b9e3ff35d9ac0a24d6c20a Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 13 Sep 2022 15:52:11 +0100 Subject: [PATCH 077/145] Standardise genome parameters and fix bugs --- CHANGELOG.md | 2 +- conf/modules.config | 64 +++++++++++------------- conf/test_full.config | 2 - main.nf | 6 ++- modules/local/bowtie_contaminants.nf | 2 +- modules/local/bowtie_genome.nf | 2 +- modules/local/bowtie_mirna.nf | 2 +- modules/local/mirtop_quant.nf | 9 ++-- modules/local/parse_fasta_mirna.nf | 5 +- nextflow.config | 13 ++--- nextflow_schema.json | 5 +- subworkflows/local/contaminant_filter.nf | 12 ++--- subworkflows/local/genome_quant.nf | 16 +++--- subworkflows/local/mirdeep2.nf | 4 +- subworkflows/local/mirna_quant.nf | 4 +- workflows/smrnaseq.nf | 26 ++++------ 16 files changed, 78 insertions(+), 96 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfe96c92..08ee6520 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[#12](https://github.com/nf-core/smrnaseq/issues/12)] - Enabled the use of `MirGeneDB` as an alternative database insted of `miRBase` - [[#113](https://github.com/nf-core/smrnaseq/issues/113)] - Added a optional contamination filtering step, including MultiQC plot - [[#137](https://github.com/nf-core/smrnaseq/issues/137)] - Fixed issue with mirTop and MultiQC by upgrading to MultiQC V1.13dev -- [[#159](https://github.com/nf-core/smrnaseq/issues/159)] - Index files were not collected when `bowtie_indices` was used and thus mapping was failing +- [[#159](https://github.com/nf-core/smrnaseq/issues/159)] - Index files were not collected when `bowtie_index` was used and thus mapping was failing - [[#161](https://github.com/nf-core/smrnaseq/issues/161)] - Trimmed output was not as documented and not correctly published - [[#168](https://github.com/nf-core/smrnaseq/issues/168)] - Removed `mirtrace_protocol` as the parameter was redundant and `params.protocol` is entirely sufficient - Updated pipeline template to [nf-core/tools 2.5.1](https://github.com/nf-core/tools/releases/tag/2.5.1) diff --git a/conf/modules.config b/conf/modules.config index bb28301b..04d47b37 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -179,47 +179,41 @@ process { } } -def fasta_from_species = false -def fasta = false -fasta_from_species = params.genome ? params.genomes[ params.genome ].fasta ?: false : false -fasta = params.fasta ?: fasta_from_species -if (fasta) { +process { + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:GENOME_QUANT:BAM_SORT_SAMTOOLS:SAMTOOLS_.*' { + ext.prefix = { "${meta.id}.sorted" } + publishDir = [ + path: { "${params.outdir}/samtools" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:GENOME_QUANT:BAM_SORT_SAMTOOLS:BAM_STATS_SAMTOOLS:.*' { + ext.prefix = { "${meta.id}.sorted" } + publishDir = [ + path: { "${params.outdir}/samtools/samtools_stats" }, + mode: params.publish_dir_mode, + pattern: "*.{stats,flagstat,idxstats}" + ] + } + withName: 'NFCORE_SMRNASEQ:SMRNASEQ:GENOME_QUANT:BOWTIE_MAP_.*' { + publishDir = [ + path: { "${params.outdir}/unmapped/fastq" }, + mode: params.publish_dir_mode, + pattern: "unmapped/*.gz" + ] + } +} + +if (!params.skip_mirdeep) { process { - withName: 'NFCORE_SMRNASEQ:SMRNASEQ:GENOME_QUANT:BAM_SORT_SAMTOOLS:SAMTOOLS_.*' { - ext.prefix = { "${meta.id}.sorted" } + withName: 'MIRDEEP2_MAPPER' { publishDir = [ - path: { "${params.outdir}/samtools" }, + path: { "${params.outdir}/mirdeep" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } - withName: 'NFCORE_SMRNASEQ:SMRNASEQ:GENOME_QUANT:BAM_SORT_SAMTOOLS:BAM_STATS_SAMTOOLS:.*' { - ext.prefix = { "${meta.id}.sorted" } - publishDir = [ - path: { "${params.outdir}/samtools/samtools_stats" }, - mode: params.publish_dir_mode, - pattern: "*.{stats,flagstat,idxstats}" - ] - } - withName: 'NFCORE_SMRNASEQ:SMRNASEQ:GENOME_QUANT:BOWTIE_MAP_.*' { - publishDir = [ - path: { "${params.outdir}/unmapped/fastq" }, - mode: params.publish_dir_mode, - pattern: "unmapped/*.gz" - ] - } - } - - if (!params.skip_mirdeep) { - process { - withName: 'MIRDEEP2_MAPPER' { - publishDir = [ - path: { "${params.outdir}/mirdeep" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - } } } diff --git a/conf/test_full.config b/conf/test_full.config index 8d82bef5..cc5ecd92 100644 --- a/conf/test_full.config +++ b/conf/test_full.config @@ -11,8 +11,6 @@ */ params { - max_memory = '12.GB' - max_cpus = 8 config_profile_name = 'Full test profile' config_profile_description = 'Full test dataset to check pipeline function' diff --git a/main.nf b/main.nf index ae52f738..68e88980 100644 --- a/main.nf +++ b/main.nf @@ -4,7 +4,7 @@ nf-core/smrnaseq ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Github : https://github.com/nf-core/smrnaseq -Website: https://nf-co.re/smrnaseq + Website: https://nf-co.re/smrnaseq Slack : https://nfcore.slack.com/channels/smrnaseq ---------------------------------------------------------------------------------------- */ @@ -17,6 +17,10 @@ nextflow.enable.dsl = 2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ +params.fasta = WorkflowMain.getGenomeAttribute(params, 'fasta') +params.mirtrace_species = WorkflowMain.getGenomeAttribute(params, 'mirtrace_species') +params.bowtie_index = WorkflowMain.getGenomeAttribute(params, 'bowtie') + /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ VALIDATE & PRINT PARAMETER SUMMARY diff --git a/modules/local/bowtie_contaminants.nf b/modules/local/bowtie_contaminants.nf index 7a86cd1d..4fd04643 100644 --- a/modules/local/bowtie_contaminants.nf +++ b/modules/local/bowtie_contaminants.nf @@ -10,7 +10,7 @@ process INDEX_CONTAMINANTS { path fasta output: - path 'fasta_bidx*' , emit: bt_indices + path 'fasta_bidx*' , emit: index path "versions.yml" , emit: versions script: diff --git a/modules/local/bowtie_genome.nf b/modules/local/bowtie_genome.nf index 723100db..cdf572f7 100644 --- a/modules/local/bowtie_genome.nf +++ b/modules/local/bowtie_genome.nf @@ -11,7 +11,7 @@ process INDEX_GENOME { path fasta output: - path 'genome*ebwt' , emit: bowtie_indices + path 'genome*ebwt' , emit: index path 'genome.edited.fa', emit: fasta path "versions.yml" , emit: versions diff --git a/modules/local/bowtie_mirna.nf b/modules/local/bowtie_mirna.nf index 1f340cdf..07050538 100644 --- a/modules/local/bowtie_mirna.nf +++ b/modules/local/bowtie_mirna.nf @@ -10,7 +10,7 @@ process INDEX_MIRNA { path fasta output: - path 'fasta_bidx*' , emit: bowtie_indices + path 'fasta_bidx*' , emit: index path "versions.yml", emit: versions script: diff --git a/modules/local/mirtop_quant.nf b/modules/local/mirtop_quant.nf index 6f903add..2ce3e1c6 100644 --- a/modules/local/mirtop_quant.nf +++ b/modules/local/mirtop_quant.nf @@ -11,8 +11,6 @@ process MIRTOP_QUANT { path hairpin path gtf - //if (!params.mirgenedb) {params.filter_species = params.mirtrace_species} else {params.filter_species = params.mirgenedb_species} - output: path "mirtop/mirtop.gff" path "mirtop/mirtop.tsv" , emit: mirtop_table @@ -21,10 +19,11 @@ process MIRTOP_QUANT { path "versions.yml" , emit: versions script: + def filter_species = params.mirgenedb ? params.mirgenedb_species : params.mirtrace_species """ - mirtop gff --hairpin $hairpin --gtf $gtf -o mirtop --sps $params.filter_species ./bams/* - mirtop counts --hairpin $hairpin --gtf $gtf -o mirtop --sps $params.filter_species --add-extra --gff mirtop/mirtop.gff - mirtop export --format isomir --hairpin $hairpin --gtf $gtf --sps $params.filter_species -o mirtop mirtop/mirtop.gff + mirtop gff --hairpin $hairpin --gtf $gtf -o mirtop --sps $filter_species ./bams/* + mirtop counts --hairpin $hairpin --gtf $gtf -o mirtop --sps $filter_species --add-extra --gff mirtop/mirtop.gff + mirtop export --format isomir --hairpin $hairpin --gtf $gtf --sps $filter_species -o mirtop mirtop/mirtop.gff mirtop stats mirtop/mirtop.gff --out mirtop/stats mv mirtop/stats/mirtop_stats.log mirtop/stats/full_mirtop_stats.log diff --git a/modules/local/parse_fasta_mirna.nf b/modules/local/parse_fasta_mirna.nf index 27caf42b..88626bec 100644 --- a/modules/local/parse_fasta_mirna.nf +++ b/modules/local/parse_fasta_mirna.nf @@ -9,13 +9,12 @@ process PARSE_FASTA_MIRNA { input: path fasta - //if (!params.mirgenedb) {params.filter_species = params.mirtrace_species} else {params.filter_species = params.mirgenedb_species} - output: path '*_igenome.fa', emit: parsed_fasta path "versions.yml", emit: versions script: + def filter_species = params.mirgenedb ? params.mirgenedb_species : params.mirtrace_species """ # Uncompress FASTA reference files if necessary FASTA="$fasta" @@ -29,7 +28,7 @@ process PARSE_FASTA_MIRNA { # TODO perl -ane 's/[ybkmrsw]/N/ig;print;' \${FASTA}_parsed_tmp.fa > \${FASTA}_parsed.fa sed -i 's/\s.*//' \${FASTA}_parsed.fa - seqkit grep -r --pattern \".*${params.filter_species}-.*\" \${FASTA}_parsed.fa > \${FASTA}_sps.fa + seqkit grep -r --pattern \".*${filter_species}-.*\" \${FASTA}_parsed.fa > \${FASTA}_sps.fa seqkit seq --rna2dna \${FASTA}_sps.fa > \${FASTA}_igenome.fa cat <<-END_VERSIONS > versions.yml diff --git a/nextflow.config b/nextflow.config index 08de9d63..40795723 100644 --- a/nextflow.config +++ b/nextflow.config @@ -19,21 +19,18 @@ params { genome = null igenomes_base = 's3://ngi-igenomes/igenomes' igenomes_ignore = false - fasta = null mirna_gtf = null - bowtie_indices = null - mirtrace_species = null mature = "https://mirbase.org/ftp/CURRENT/mature.fa.gz" hairpin = "https://mirbase.org/ftp/CURRENT/hairpin.fa.gz" mirgenedb = false - mirgenedb_mature = "/Users/chriskub/Downloads/ALL-mat.fas" - mirgenedb_hairpin = "/Users/chriskub/Downloads/ALL-pre.fas" - mirgenedb_gff = "/Users/chriskub/Downloads/ALL.gff" + mirgenedb_mature = null + mirgenedb_hairpin = null + mirgenedb_gff = null mirgenedb_species = null // Trimming options - clip_r1 = 0 - three_prime_clip_r1 = 0 + clip_r1 = null + three_prime_clip_r1 = null three_prime_adapter = "TGGAATTCTCGGGTGCCAAGG" min_length = 17 skip_qc = false diff --git a/nextflow_schema.json b/nextflow_schema.json index 2ca88862..7d89945c 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -65,8 +65,7 @@ "mirgenedb": { "type": "boolean", "description": "Boolean whether MirGeneDB should be used instead of miRBase", - "help_text": "This allows you to use MirGeneDB instead of miRBase as the database. \n Note that you will need to set the additional flags `--mirgenedb_species`, `--mirgenedb_gff`, `--mirgenedb_mature` and `--mirgenedb_hairpin`", - "default": "false" + "help_text": "This allows you to use MirGeneDB instead of miRBase as the database. \n Note that you will need to set the additional flags `--mirgenedb_species`, `--mirgenedb_gff`, `--mirgenedb_mature` and `--mirgenedb_hairpin`" }, "mirtrace_species": { "type": "string", @@ -120,7 +119,7 @@ "description": "Path to FASTA file with miRNAs precursors.", "help_text": "This file needs to be downloaded from [`https://mirgenedb.org/download`]. Can be given either as a plain text `.fa` file or a compressed `.gz` file.\nNote that MirGeneDB does not have a dedicated hairpin file. The equivalent is the `Precursor sequences`." }, - "bowtie_indices": { + "bowtie_index": { "type": "string", "description": "Path to a Bowtie 1 index directory", "fa_icon": "fas fa-book", diff --git a/subworkflows/local/contaminant_filter.nf b/subworkflows/local/contaminant_filter.nf index 848682ea..4ab180c7 100644 --- a/subworkflows/local/contaminant_filter.nf +++ b/subworkflows/local/contaminant_filter.nf @@ -49,7 +49,7 @@ workflow CONTAMINANT_FILTER { // Index DB and filter $reads emit: $rrna_reads INDEX_RRNA ( rrna ) ch_versions = ch_versions.mix(INDEX_RRNA.out.versions) - MAP_RRNA ( reads, INDEX_RRNA.out.bt_indices, 'rRNA' ) + MAP_RRNA ( reads, INDEX_RRNA.out.index, 'rRNA' ) ch_versions = ch_versions.mix(MAP_RRNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_RRNA.out.stats.ifEmpty(null)) MAP_RRNA.out.unmapped.set { rrna_reads } @@ -61,7 +61,7 @@ workflow CONTAMINANT_FILTER { // Index DB and filter $rrna_reads emit: $trna_reads INDEX_TRNA ( trna ) ch_versions = ch_versions.mix(INDEX_TRNA.out.versions) - MAP_TRNA ( rrna_reads, INDEX_TRNA.out.bt_indices, 'tRNA') + MAP_TRNA ( rrna_reads, INDEX_TRNA.out.index, 'tRNA') ch_versions = ch_versions.mix(MAP_TRNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_TRNA.out.stats.ifEmpty(null)) MAP_TRNA.out.unmapped.set { trna_reads } @@ -75,7 +75,7 @@ workflow CONTAMINANT_FILTER { ch_versions = ch_versions.mix(BLAT_CDNA.out.versions) INDEX_CDNA ( BLAT_CDNA.out.filtered_set ) ch_versions = ch_versions.mix(INDEX_CDNA.out.versions) - MAP_CDNA ( trna_reads, INDEX_CDNA.out.bt_indices, 'cDNA' ) + MAP_CDNA ( trna_reads, INDEX_CDNA.out.index, 'cDNA' ) ch_versions = ch_versions.mix(MAP_CDNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_CDNA.out.stats.ifEmpty(null)) MAP_CDNA.out.unmapped.set { cdna_reads } @@ -88,7 +88,7 @@ workflow CONTAMINANT_FILTER { ch_versions = ch_versions.mix(BLAT_NCRNA.out.versions) INDEX_NCRNA ( BLAT_NCRNA.out.filtered_set ) ch_versions = ch_versions.mix(INDEX_NCRNA.out.versions) - MAP_NCRNA ( cdna_reads, INDEX_NCRNA.out.bt_indices, 'ncRNA' ) + MAP_NCRNA ( cdna_reads, INDEX_NCRNA.out.index, 'ncRNA' ) ch_versions = ch_versions.mix(MAP_NCRNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_NCRNA.out.stats.ifEmpty(null)) MAP_NCRNA.out.unmapped.set { ncrna_reads } @@ -101,7 +101,7 @@ workflow CONTAMINANT_FILTER { ch_versions = ch_versions.mix(BLAT_PIRNA.out.versions) INDEX_PIRNA ( BLAT_PIRNA.out.filtered_set ) ch_versions = ch_versions.mix(INDEX_PIRNA.out.versions) - MAP_PIRNA (ncrna_reads, INDEX_PIRNA.out.bt_indices, 'piRNA' ) + MAP_PIRNA (ncrna_reads, INDEX_PIRNA.out.index, 'piRNA' ) ch_versions = ch_versions.mix(MAP_PIRNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_PIRNA.out.stats.ifEmpty(null)) MAP_PIRNA.out.unmapped.set { pirna_reads } @@ -114,7 +114,7 @@ workflow CONTAMINANT_FILTER { ch_versions = ch_versions.mix(BLAT_OTHER.out.versions) INDEX_OTHER ( BLAT_OTHER.out.filtered_set ) ch_versions = ch_versions.mix(INDEX_OTHER.out.versions) - MAP_OTHER (ncrna_reads, INDEX_OTHER.out.bt_indices, 'other' ) + MAP_OTHER (ncrna_reads, INDEX_OTHER.out.index, 'other' ) ch_versions = ch_versions.mix(MAP_OTHER.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_OTHER.out.stats.ifEmpty(null)) MAP_OTHER.out.unmapped.set { other_cont_reads } diff --git a/subworkflows/local/genome_quant.nf b/subworkflows/local/genome_quant.nf index 8cf812d7..033f59ad 100644 --- a/subworkflows/local/genome_quant.nf +++ b/subworkflows/local/genome_quant.nf @@ -9,24 +9,24 @@ include { BOWTIE_MAP_SEQ as BOWTIE_MAP_GENOME } from '../../modules/local/bowtie workflow GENOME_QUANT { take: fasta - bt_index - reads // channel: [ val(meta), [ reads ] ] + index + reads // channel: [ val(meta), [ reads ] ] main: ch_versions = Channel.empty() - if (!bt_index){ + if (!index){ INDEX_GENOME ( fasta ) - bowtie_indices = INDEX_GENOME.out.bowtie_indices + bowtie_index = INDEX_GENOME.out.index fasta_formatted = INDEX_GENOME.out.fasta ch_versions = ch_versions.mix(INDEX_GENOME.out.versions) } else { - bowtie_indices = Channel.fromPath("${bt_index}**ebwt", checkIfExists: true).ifEmpty { exit 1, "Bowtie1 index directory not found: ${bt_index}" } + bowtie_index = Channel.fromPath("${index}**ebwt", checkIfExists: true).ifEmpty { exit 1, "Bowtie1 index directory not found: ${index}" } fasta_formatted = fasta } - if (bowtie_indices){ - BOWTIE_MAP_GENOME ( reads, bowtie_indices.collect() ) + if (bowtie_index){ + BOWTIE_MAP_GENOME ( reads, bowtie_index.collect() ) ch_versions = ch_versions.mix(BOWTIE_MAP_GENOME.out.versions) BAM_SORT_SAMTOOLS ( BOWTIE_MAP_GENOME.out.bam, Channel.empty() ) @@ -35,7 +35,7 @@ workflow GENOME_QUANT { emit: fasta = fasta_formatted - indices = bowtie_indices + index = bowtie_index stats = BAM_SORT_SAMTOOLS.out.stats versions = ch_versions diff --git a/subworkflows/local/mirdeep2.nf b/subworkflows/local/mirdeep2.nf index a790189f..f8098ba5 100644 --- a/subworkflows/local/mirdeep2.nf +++ b/subworkflows/local/mirdeep2.nf @@ -10,7 +10,7 @@ workflow MIRDEEP2 { take: reads // channel: [ val(meta), [ reads ] ] fasta - indices + index hairpin mature @@ -20,7 +20,7 @@ workflow MIRDEEP2 { MIRDEEP2_PIGZ ( reads ) ch_versions = ch_versions.mix(MIRDEEP2_PIGZ.out.versions.first()) - MIRDEEP2_MAPPER ( MIRDEEP2_PIGZ.out.reads, indices ) + MIRDEEP2_MAPPER ( MIRDEEP2_PIGZ.out.reads, index ) ch_versions = ch_versions.mix(MIRDEEP2_MAPPER.out.versions.first()) MIRDEEP2_RUN ( fasta, MIRDEEP2_MAPPER.out.mirdeep2_inputs, hairpin, mature ) diff --git a/subworkflows/local/mirna_quant.nf b/subworkflows/local/mirna_quant.nf index 9a8259d4..dd8a8412 100644 --- a/subworkflows/local/mirna_quant.nf +++ b/subworkflows/local/mirna_quant.nf @@ -45,10 +45,10 @@ workflow MIRNA_QUANT { FORMAT_HAIRPIN ( hairpin_parsed ) ch_versions = ch_versions.mix(FORMAT_HAIRPIN.out.versions) - INDEX_MATURE ( FORMAT_MATURE.out.formatted_fasta ).bowtie_indices.set { mature_bowtie } + INDEX_MATURE ( FORMAT_MATURE.out.formatted_fasta ).index.set { mature_bowtie } ch_versions = ch_versions.mix(INDEX_MATURE.out.versions) - INDEX_HAIRPIN ( FORMAT_HAIRPIN.out.formatted_fasta ).bowtie_indices.set { hairpin_bowtie } + INDEX_HAIRPIN ( FORMAT_HAIRPIN.out.formatted_fasta ).index.set { hairpin_bowtie } ch_versions = ch_versions.mix(INDEX_HAIRPIN.out.versions) reads diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index f9675425..ca1b265e 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -20,19 +20,13 @@ for (param in checkPathParamList) { if (param) { file(param, checkIfExists: true if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input samplesheet not specified!' } // Check optional parameters -if (!params.mirtrace_species){ - exit 1, "Reference species for miRTrace is not defined." +if (!params.mirtrace_species) { + exit 1, "Reference species for miRTrace is not defined via the --mirtrace_species parameter." } -// Genome options -bt_index_from_species = params.genome ? params.genomes[ params.genome ].bowtie ?: false : false -bt_index = params.bowtie_indices ?: bt_index_from_species -mirtrace_species_from_species = params.genome ? params.genomes[ params.genome ].mirtrace_species ?: false : false -mirtrace_species = params.mirtrace_species ?: mirtrace_species_from_species -fasta_from_species = params.genome ? params.genomes[ params.genome ].fasta ?: false : false -fasta = params.fasta ?: fasta_from_species -mirna_gtf_from_species = params.mirtrace_species ? "https://mirbase.org/ftp/CURRENT/genomes/${params.mirtrace_species}.gff3" : false -mirna_gtf = params.mirna_gtf ? params.mirna_gtf : mirna_gtf_from_species +// Genome options +def mirna_gtf_from_species = params.mirtrace_species ? "https://mirbase.org/ftp/CURRENT/genomes/${params.mirtrace_species}.gff3" : false +def mirna_gtf = params.mirna_gtf ?: mirna_gtf_from_species /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -56,12 +50,10 @@ ch_multiqc_image = file("$projectDir/assets/smrnaseq_logo.png", checkIfE if (!params.mirgenedb) { if (params.mature) { reference_mature = file(params.mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mature}" } if (params.hairpin) { reference_hairpin = file(params.hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.hairpin}" } - params.filter_species = params.mirtrace_species } else { if (params.mirgenedb_mature) { reference_mature = file(params.mirgenedb_mature, checkIfExists: true) } else { exit 1, "Mature miRNA fasta file not found: ${params.mirgenedb_mature}" } if (params.mirgenedb_hairpin) { reference_hairpin = file(params.mirgenedb_hairpin, checkIfExists: true) } else { exit 1, "Hairpin miRNA fasta file not found: ${params.mirgenedb_hairpin}" } if (params.mirgenedb_gff) { mirna_gtf = file(params.mirgenedb_gff, checkIfExists: true) } else { exit 1, "MirGeneDB gff file not found: ${params.mirgenedb_gff}"} - params.filter_species = params.mirgenedb_species } include { INPUT_CHECK } from '../subworkflows/local/input_check' @@ -186,9 +178,9 @@ workflow SMRNASEQ { // GENOME // genome_stats = Channel.empty() - if (fasta){ - fasta_ch = file(fasta) - GENOME_QUANT ( fasta_ch, bt_index, MIRNA_QUANT.out.unmapped ) + if (params.fasta){ + ch_fasta = file(params.fasta) + GENOME_QUANT ( ch_fasta, params.bowtie_index, MIRNA_QUANT.out.unmapped ) GENOME_QUANT.out.stats .set { genome_stats } ch_versions = ch_versions.mix(GENOME_QUANT.out.versions) @@ -197,7 +189,7 @@ workflow SMRNASEQ { MIRDEEP2 ( FASTQC_TRIMGALORE.out.reads, GENOME_QUANT.out.fasta, - GENOME_QUANT.out.indices.collect(), + GENOME_QUANT.out.index.collect(), MIRNA_QUANT.out.fasta_hairpin, MIRNA_QUANT.out.fasta_mature ) From 1df1792130acd8eb4e448957e1c3926e8c081586 Mon Sep 17 00:00:00 2001 From: Robert Syme Date: Wed, 14 Sep 2022 02:38:58 +0000 Subject: [PATCH 078/145] Add basic tower.yml --- tower.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 tower.yml diff --git a/tower.yml b/tower.yml new file mode 100644 index 00000000..1a72c200 --- /dev/null +++ b/tower.yml @@ -0,0 +1,29 @@ +reports: + multiqc_report.html: + display: "MultiQC HTML report" + "**/edger/hairpin_counts.csv": + display: "EdgeR Read counts for known hairpins (raw counts)" + "**/edger/hairpin_logtpm.csv": + display: "EdgeR Read counts for known hairpins (log TPM)" + "**/edger/hairpin_normalized_CPM.txt": + display: "EdgeR Read counts for known hairpins (normalised)" + "**/edger/mature_counts.csv": + display: "EdgeR Read counts for known mature miRNAs (raw)" + "**/edger/mature_logtpm.csv": + display: "EdgeR Read counts for known mature miRNAs (log TPM)" + "**/edger/mature_normalized_CPM.txt": + display: "EdgeR Read counts for known mature miRNAs (normalised)" + "**/edger/hairpin_CPM_heatmap.pdf": + display: "EdgeR Heatmap of hairpin sequences for all samples" + "**/edger/hairpin_edgeR_MDS_plot.pdf": + display: "EdgeR Dimensionality reduction plot of samples from hairpin counts" + "**/edger/hairpin_log2CPM_sample_distances_dendrogram.pdf": + display: "EdgeR dendrogram from hairpin counts" + "**/edger/hairpin_log2CPM_sample_distances_heatmap.pdf": + display: "EdgeR distances from hairpin counts" + "**/mirtop/mirna.tsv": + display: "Mirtop miRNA summary table" + "**/mirtop/mirtop.tsv": + display: "Mirtop summary table" + "**/mirtrace/mirtrace-report.html": + display: "Mirtrace report" From 03142a903f22e767fffe41dd6cebae201ffd46a0 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 22 Sep 2022 14:00:19 +0000 Subject: [PATCH 079/145] Draft for fastp auto-detection and auto-setting of protocol params --- lib/WorkflowSmrnaseq.groovy | 36 +++++++++ modules.json | 4 + modules/local/trimgalore.nf | 73 ------------------ modules/nf-core/modules/fastp/main.nf | 98 ++++++++++++++++++++++++ modules/nf-core/modules/fastp/meta.yml | 69 +++++++++++++++++ nextflow.config | 4 +- nextflow_schema.json | 36 ++++++--- subworkflows/nf-core/fastp_fastqc.nf | 100 +++++++++++++++++++++++++ workflows/smrnaseq.nf | 24 +++--- 9 files changed, 348 insertions(+), 96 deletions(-) delete mode 100644 modules/local/trimgalore.nf create mode 100644 modules/nf-core/modules/fastp/main.nf create mode 100644 modules/nf-core/modules/fastp/meta.yml create mode 100644 subworkflows/nf-core/fastp_fastqc.nf diff --git a/lib/WorkflowSmrnaseq.groovy b/lib/WorkflowSmrnaseq.groovy index b0564d30..c1701ab5 100755 --- a/lib/WorkflowSmrnaseq.groovy +++ b/lib/WorkflowSmrnaseq.groovy @@ -54,4 +54,40 @@ class WorkflowSmrnaseq { System.exit(1) } } + + /* + * Format the protocol + * Given the protocol parameter (params.protocol), + * this function formats the protocol such that it is fit for the respective + * subworkflow + */ + static formatProtocol(protocol) { + Int clip_r1 = protocol + Int three_prime_clip_r1 = '' + String three_prime_adapter = '' + + switch(protocol){ + case 'illumina': + clip_r1 = 0 + three_prime_clip_r1 = 0 + three_prime_adapter = "TGGAATTCTCGGGTGCCAAGG" + case 'nextflex': + clip_r1 = 4 + three_prime_clip_r1 = 4 + three_prime_adapter = "TGGAATTCTCGGGTGCCAAGG" + case 'qiaseq': + clip_r1 = 0 + three_prime_clip_r1 = 0 + three_prime_adapter = "AACTGTAGGCACCATCAAT" + case 'cats': + clip_r1 = 3 + three_prime_clip_r1 = 0 + three_prime_adapter = "AAAAAAAA" + case 'custom': + clip_r1 = params.clip_r1 + three_prime_clip_r1 = params.three_prime_clip_r1 + three_prime_adapter = params.three_prime_adapter + } + return [clip_r1, three_prime_clip_r1, three_prime_adapter] + } } diff --git a/modules.json b/modules.json index d169ad11..7c426975 100644 --- a/modules.json +++ b/modules.json @@ -13,6 +13,10 @@ "branch": "master", "git_sha": "5e7b1ef9a5a2d9258635bcbf70fcf37dacd1b247" }, + "fastp": { + "branch": "master", + "git_sha": "2c70c1c1951aaf884d2e8d8d9c871db79f7b35aa" + }, "fastqc": { "branch": "master", "git_sha": "49b18b1639f4f7104187058866a8fab33332bdfe" diff --git a/modules/local/trimgalore.nf b/modules/local/trimgalore.nf deleted file mode 100644 index 3dec8f2e..00000000 --- a/modules/local/trimgalore.nf +++ /dev/null @@ -1,73 +0,0 @@ -process TRIMGALORE { - tag "$meta.id" - label 'process_high' - - conda (params.enable_conda ? 'bioconda::trim-galore=0.6.7' : null) - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/trim-galore:0.6.7--hdfd78af_0' : - 'quay.io/biocontainers/trim-galore:0.6.7--hdfd78af_0' }" - - input: - tuple val(meta), path(reads) - - output: - tuple val(meta), path("*.fq.gz") , emit: reads - tuple val(meta), path("*report.txt"), emit: log - path "versions.yml" , emit: versions - - tuple val(meta), path("*.html"), emit: html optional true - tuple val(meta), path("*.zip") , emit: zip optional true - - script: - def args = task.ext.args ?: '' - def cores = 1 - if (task.cpus) { - cores = (task.cpus as int) - 4 - if (meta.single_end) cores = (task.cpus as int) - 3 - if (cores < 1) cores = 1 - if (cores > 4) cores = 4 - } - def prefix = "${meta.id}" - // Define regular variables so that they can be overwritten - def clip_r1 = params.clip_r1 - def three_prime_clip_r1 = params.three_prime_clip_r1 - def three_prime_adapter = params.three_prime_adapter - def protocol = params.protocol - // Presets - if (params.protocol == "illumina"){ - clip_r1 = 0 - three_prime_clip_r1 = 0 - three_prime_adapter = "TGGAATTCTCGGGTGCCAAGG" - } else if (params.protocol == "nextflex"){ - clip_r1 = 4 - three_prime_clip_r1 = 4 - three_prime_adapter = "TGGAATTCTCGGGTGCCAAGG" - } else if (params.protocol == "qiaseq"){ - clip_r1 = 0 - three_prime_clip_r1 = 0 - three_prime_adapter = "AACTGTAGGCACCATCAAT" - } else if (params.protocol == "cats"){ - clip_r1 = 3 - three_prime_clip_r1 = 0 - // three_prime_adapter = "GATCGGAAGAGCACACGTCTG" - three_prime_adapter = "AAAAAAAA" - } else if (params.protocol == "custom"){ - //custom protocol - clip_r1 = params.clip_r1 - three_prime_clip_r1 = params.three_prime_clip_r1 - three_prime_adapter = params.three_prime_adapter - } - def tg_length = "--length ${params.min_length}" - def c_r1 = clip_r1 > 0 ? "--clip_r1 ${clip_r1}" : '' - def tpc_r1 = three_prime_clip_r1 > 0 ? "--three_prime_clip_r1 ${three_prime_clip_r1}" : '' - """ - [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz - trim_galore $args --cores $cores --adapter ${three_prime_adapter} $tg_length $c_r1 $tpc_r1 --max_length ${params.trim_galore_max_length} --gzip ${prefix}.fastq.gz - - cat <<-END_VERSIONS > versions.yml - ${task.process}": - trimgalore: \$(echo \$(trim_galore --version 2>&1) | sed 's/^.*version //; s/Last.*\$//') - cutadapt: \$(cutadapt --version) - END_VERSIONS - """ -} diff --git a/modules/nf-core/modules/fastp/main.nf b/modules/nf-core/modules/fastp/main.nf new file mode 100644 index 00000000..11ea4db3 --- /dev/null +++ b/modules/nf-core/modules/fastp/main.nf @@ -0,0 +1,98 @@ +process FASTP { + tag "$meta.id" + label 'process_medium' + + conda (params.enable_conda ? 'bioconda::fastp=0.23.2' : null) + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/fastp:0.23.2--h79da9fb_0' : + 'quay.io/biocontainers/fastp:0.23.2--h79da9fb_0' }" + + input: + tuple val(meta), path(reads) + val save_trimmed_fail + val save_merged + + output: + tuple val(meta), path('*.fastp.fastq.gz') , optional:true, emit: reads + tuple val(meta), path('*.json') , emit: json + tuple val(meta), path('*.html') , emit: html + tuple val(meta), path('*.log') , emit: log + path "versions.yml" , emit: versions + tuple val(meta), path('*.fail.fastq.gz') , optional:true, emit: reads_fail + tuple val(meta), path('*.merged.fastq.gz'), optional:true, emit: reads_merged + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def fail_fastq = save_trimmed_fail && meta.single_end ? "--failed_out ${prefix}.fail.fastq.gz" : save_trimmed_fail && !meta.single_end ? "--unpaired1 ${prefix}_1.fail.fastq.gz --unpaired2 ${prefix}_2.fail.fastq.gz" : '' + // Added soft-links to original fastqs for consistent naming in MultiQC + // Use single ended for interleaved. Add --interleaved_in in config. + if ( task.ext.args?.contains('--interleaved_in') ) { + """ + [ ! -f ${prefix}.fastq.gz ] && ln -sf $reads ${prefix}.fastq.gz + + fastp \\ + --stdout \\ + --in1 ${prefix}.fastq.gz \\ + --thread $task.cpus \\ + --json ${prefix}.fastp.json \\ + --html ${prefix}.fastp.html \\ + $fail_fastq \\ + $args \\ + 2> ${prefix}.fastp.log \\ + | gzip -c > ${prefix}.fastp.fastq.gz + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + fastp: \$(fastp --version 2>&1 | sed -e "s/fastp //g") + END_VERSIONS + """ + } else if (meta.single_end) { + """ + [ ! -f ${prefix}.fastq.gz ] && ln -sf $reads ${prefix}.fastq.gz + + fastp \\ + --stdout \\ + --in1 ${prefix}.fastq.gz \\ + --out1 ${prefix}.fastp.fastq.gz \\ + --thread $task.cpus \\ + --json ${prefix}.fastp.json \\ + --html ${prefix}.fastp.html \\ + $fail_fastq \\ + $args \\ + 2> ${prefix}.fastp.log + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + fastp: \$(fastp --version 2>&1 | sed -e "s/fastp //g") + END_VERSIONS + """ + } else { + def merge_fastq = save_merged ? "-m --merged_out ${prefix}.merged.fastq.gz" : '' + """ + [ ! -f ${prefix}_1.fastq.gz ] && ln -sf ${reads[0]} ${prefix}_1.fastq.gz + [ ! -f ${prefix}_2.fastq.gz ] && ln -sf ${reads[1]} ${prefix}_2.fastq.gz + fastp \\ + --in1 ${prefix}_1.fastq.gz \\ + --in2 ${prefix}_2.fastq.gz \\ + --out1 ${prefix}_1.fastp.fastq.gz \\ + --out2 ${prefix}_2.fastp.fastq.gz \\ + --json ${prefix}.fastp.json \\ + --html ${prefix}.fastp.html \\ + $fail_fastq \\ + $merge_fastq \\ + --thread $task.cpus \\ + --detect_adapter_for_pe \\ + $args \\ + 2> ${prefix}.fastp.log + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + fastp: \$(fastp --version 2>&1 | sed -e "s/fastp //g") + END_VERSIONS + """ + } +} diff --git a/modules/nf-core/modules/fastp/meta.yml b/modules/nf-core/modules/fastp/meta.yml new file mode 100644 index 00000000..2368fded --- /dev/null +++ b/modules/nf-core/modules/fastp/meta.yml @@ -0,0 +1,69 @@ +name: fastp +description: Perform adapter/quality trimming on sequencing reads +keywords: + - trimming + - quality control + - fastq +tools: + - fastp: + description: | + A tool designed to provide fast all-in-one preprocessing for FastQ files. This tool is developed in C++ with multithreading supported to afford high performance. + documentation: https://github.com/OpenGene/fastp + doi: https://doi.org/10.1093/bioinformatics/bty560 + licence: ["MIT"] +input: + - meta: + type: map + description: | + Groovy Map containing sample information. Use 'single_end: true' to specify single ended or interleaved FASTQs. Use 'single_end: false' for paired-end reads. + e.g. [ id:'test', single_end:false ] + - reads: + type: file + description: | + List of input FastQ files of size 1 and 2 for single-end and paired-end data, + respectively. If you wish to run interleaved paired-end data, supply as single-end data + but with `--interleaved_in` in your `modules.conf`'s `ext.args` for the module. + - save_trimmed_fail: + type: boolean + description: Specify true to save files that failed to pass trimming thresholds ending in `*.fail.fastq.gz` + - save_merged: + type: boolean + description: Specify true to save all merged reads to the a file ending in `*.merged.fastq.gz` + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. [ id:'test', single_end:false ] + - reads: + type: file + description: The trimmed/modified/unmerged fastq reads + pattern: "*fastp.fastq.gz" + - json: + type: file + description: Results in JSON format + pattern: "*.json" + - html: + type: file + description: Results in HTML format + pattern: "*.html" + - log: + type: file + description: fastq log file + pattern: "*.log" + - versions: + type: file + description: File containing software versions + pattern: "versions.yml" + - reads_fail: + type: file + description: Reads the failed the preprocessing + pattern: "*fail.fastq.gz" + - reads_merged: + type: file + description: Reads that were successfully merged + pattern: "*.{merged.fastq.gz}" +authors: + - "@drpatelh" + - "@kevinmenden" diff --git a/nextflow.config b/nextflow.config index 40795723..6ea448aa 100644 --- a/nextflow.config +++ b/nextflow.config @@ -31,13 +31,13 @@ params { // Trimming options clip_r1 = null three_prime_clip_r1 = null - three_prime_adapter = "TGGAATTCTCGGGTGCCAAGG" + three_prime_adapter = '' min_length = 17 skip_qc = false skip_fastqc = false skip_multiqc = false skip_mirdeep = false - skip_trimming = false + skip_fastp = false save_reference = false trim_galore_max_length = 40 diff --git a/nextflow_schema.json b/nextflow_schema.json index 7d89945c..82c2ee86 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -10,7 +10,10 @@ "type": "object", "fa_icon": "fas fa-terminal", "description": "Define where the pipeline should find input data and save output data.", - "required": ["input", "outdir"], + "required": [ + "input", + "outdir" + ], "properties": { "input": { "type": "string", @@ -28,7 +31,13 @@ "fa_icon": "fas fa-vial", "description": "Protocol for constructing smRNA-seq libraries.", "help_text": "Presets for trimming parameters and 3' adapter sequence with a specified protocol.\n\n| Protocol | Library Prep Kit | Trimming Parameter | 3' Adapter Sequence |\n| :------------ | :-------------------------------------- | :-------------------------------------- | :--------------------- |\n| illumina | Illumina TruSeq Small RNA | `clip_r1 = 0` `three_prime_clip_r1 = 0` | `TGGAATTCTCGGGTGCCAAGG` |\n| nextflex | BIOO SCIENTIFIC NEXTFLEX Small RNA-Seq | `clip_r1 = 4` `three_prime_clip_r1 = 4` | `TGGAATTCTCGGGTGCCAAGG` |\n| qiaseq | QIAGEN QIAseq miRNA | `clip_r1 = 0` `three_prime_clip_r1 = 0` | `AACTGTAGGCACCATCAAT` |\n| cats | Diagenode CATS Small RNA-seq | `clip_r1 = 3` `three_prime_clip_r1 = 0` | `AAAAAAAAAAA` + `GATCGGAAGAGCACACGTCTG` (only polyA is used for trimming) |\n| custom | user defined | user defined | user defined |\n\n> NB: When running `--protocol custom` the user ***must define the 3' Adapter Sequence***.\n> If trimming parameters aren't provided the pipeline will deafult to `clip_R1 = 0` and `three_prime_clip_R1 = 0` (i.e. no extra clipping).", - "enum": ["illumina", "nextflex", "qiaseq", "cats", "custom"] + "enum": [ + "illumina", + "nextflex", + "qiaseq", + "cats", + "custom" + ] }, "outdir": { "type": "string", @@ -257,12 +266,6 @@ "fa_icon": "fas fa-fast-forward", "description": "Skip MultiQC", "default": false - }, - "skip_trimming": { - "type": "boolean", - "fa_icon": "fas fa-fast-forward", - "description": "Skip all trimming steps.", - "default": false } } }, @@ -368,7 +371,14 @@ "description": "Method used to save pipeline results to output directory.", "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", "fa_icon": "fas fa-copy", - "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], + "enum": [ + "symlink", + "rellink", + "link", + "copy", + "copyNoFollow", + "move" + ], "hidden": true }, "email_on_fail": { @@ -460,5 +470,11 @@ { "$ref": "#/definitions/generic_options" } - ] + ], + "properties": { + "skip_fastp": { + "type": "string", + "default": "false" + } + } } diff --git a/subworkflows/nf-core/fastp_fastqc.nf b/subworkflows/nf-core/fastp_fastqc.nf new file mode 100644 index 00000000..87edb94f --- /dev/null +++ b/subworkflows/nf-core/fastp_fastqc.nf @@ -0,0 +1,100 @@ +// +// Read QC and trimming +// + +include { FASTQC as FASTQC_RAW } from '../../modules/nf-core/modules/fastqc/main' +include { FASTQC as FASTQC_TRIM } from '../../modules/nf-core/modules/fastqc/main' +include { FASTP } from '../../modules/nf-core/modules/fastp/main' + +// +// Function that parses fastp json output file to get total number of reads after trimming +// +import groovy.json.JsonSlurper + +def getFastpReadsAfterFiltering(json_file) { + def Map json = (Map) new JsonSlurper().parseText(json_file.text).get('summary') + return json['after_filtering']['total_reads'].toInteger() +} + +workflow FASTQC_FASTP { + take: + reads // channel: [ val(meta), [ reads ] ] + save_trimmed_fail // value: boolean + save_merged // value: boolean + + + main: + + ch_versions = Channel.empty() + + fastqc_raw_html = Channel.empty() + fastqc_raw_zip = Channel.empty() + if (!params.skip_fastqc) { + FASTQC_RAW ( + reads + ) + fastqc_raw_html = FASTQC_RAW.out.html + fastqc_raw_zip = FASTQC_RAW.out.zip + ch_versions = ch_versions.mix(FASTQC_RAW.out.versions.first()) + } + + trim_reads = reads + trim_json = Channel.empty() + trim_html = Channel.empty() + trim_log = Channel.empty() + trim_reads_fail = Channel.empty() + trim_reads_merged = Channel.empty() + fastqc_trim_html = Channel.empty() + fastqc_trim_zip = Channel.empty() + if (!params.skip_fastp) { + FASTP ( + reads, + save_trimmed_fail, + save_merged + ) + trim_reads = FASTP.out.reads + trim_json = FASTP.out.json + trim_html = FASTP.out.html + trim_log = FASTP.out.log + trim_reads_fail = FASTP.out.reads_fail + trim_reads_merged = FASTP.out.reads_merged + ch_versions = ch_versions.mix(FASTP.out.versions.first()) + + // + // Filter empty FastQ files after adapter trimming so FastQC doesn't fail + // + trim_reads + .join(trim_json) + .map { + meta, reads, json -> + if (getFastpReadsAfterFiltering(json) > 0) { + [ meta, reads ] + } + } + .set { trim_reads } + + if (!params.skip_fastqc) { + FASTQC_TRIM ( + trim_reads + ) + fastqc_trim_html = FASTQC_TRIM.out.html + fastqc_trim_zip = FASTQC_TRIM.out.zip + ch_versions = ch_versions.mix(FASTQC_TRIM.out.versions.first()) + } + } + + emit: + reads = trim_reads // channel: [ val(meta), [ reads ] ] + trim_json // channel: [ val(meta), [ json ] ] + trim_html // channel: [ val(meta), [ html ] ] + trim_log // channel: [ val(meta), [ log ] ] + trim_reads_fail // channel: [ val(meta), [ fastq.gz ] ] + trim_reads_merged // channel: [ val(meta), [ fastq.gz ] ] + + fastqc_raw_html // channel: [ val(meta), [ html ] ] + fastqc_raw_zip // channel: [ val(meta), [ zip ] ] + fastqc_trim_html // channel: [ val(meta), [ html ] ] + fastqc_trim_zip // channel: [ val(meta), [ zip ] ] + + versions = ch_versions.ifEmpty(null) // channel: [ versions.yml ] +} diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index ca1b265e..4c8a1d96 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -57,7 +57,7 @@ if (!params.mirgenedb) { } include { INPUT_CHECK } from '../subworkflows/local/input_check' -include { FASTQC_TRIMGALORE } from '../subworkflows/nf-core/fastqc_trimgalore' +include { FASTP_FASTQC } from '../subworkflows/nf-core/fastp_fastqc' include { CONTAMINANT_FILTER } from '../subworkflows/local/contaminant_filter' include { MIRNA_QUANT } from '../subworkflows/local/mirna_quant' include { GENOME_QUANT } from '../subworkflows/local/genome_quant' @@ -74,7 +74,6 @@ include { MIRDEEP2 } from '../subworkflows/local/mirdeep2' // MODULE: Installed directly from nf-core/modules // include { CAT_FASTQ } from '../modules/nf-core/modules/cat/fastq/main' -include { FASTQC } from '../modules/nf-core/modules/fastqc/main' include { MULTIQC } from '../modules/nf-core/modules/multiqc/main' include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/modules/custom/dumpsoftwareversions/main' @@ -133,16 +132,18 @@ workflow SMRNASEQ { ch_versions = ch_versions.mix(MIRTRACE.out.versions.ifEmpty(null)) // - // SUBWORKFLOW: Read QC, extract UMI and trim adapters + // SUBWORKFLOW: Read QC and trim adapters // - FASTQC_TRIMGALORE ( + + (clip_r1, three_prime_clip_r1, three_prime_adapter) = WorkflowSmrnaseq.formatProtocol(params.protocol) + + FASTP_FASTQC ( ch_cat_fastq, - params.skip_fastqc || params.skip_qc, - params.skip_trimming + false ) - ch_versions = ch_versions.mix(FASTQC_TRIMGALORE.out.versions) + ch_versions = ch_versions.mix(FASTP_FASTQC.out.versions) - reads_for_mirna = FASTQC_TRIMGALORE.out.reads + reads_for_mirna = FASTP_FASTQC.out.reads // // SUBWORKFLOW: remove contaminants from reads // @@ -156,7 +157,7 @@ workflow SMRNASEQ { params.ncrna, params.pirna, params.other_contamination, - FASTQC_TRIMGALORE.out.reads + FASTP_FASTQC.out.reads ) reads_for_mirna = CONTAMINANT_FILTER.out.filtered_reads @@ -187,7 +188,7 @@ workflow SMRNASEQ { if (!params.skip_mirdeep) { MIRDEEP2 ( - FASTQC_TRIMGALORE.out.reads, + FASTP_FASTQC.out.reads, GENOME_QUANT.out.fasta, GENOME_QUANT.out.index.collect(), MIRNA_QUANT.out.fasta_hairpin, @@ -217,7 +218,8 @@ workflow SMRNASEQ { ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.collect().ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(FASTQC_TRIMGALORE.out.fastqc_zip.collect{it[1]}.ifEmpty([])) + ch_multiqc_files = ch_multiqc_files.mix(FASTQC_FASTP.out.fastqc_raw_zip.collect{it[1]}.ifEmpty([])), + ch_multiqc_files = ch_multiqc_files.mix(FASTQC_FASTP.out.trim_json.collect{it[1]}.ifEmpty([])), ch_multiqc_files = ch_multiqc_files.mix(contamination_stats.collect().ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.mature_stats.collect({it[1]}).ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.hairpin_stats.collect({it[1]}).ifEmpty([])) From 4a15e26c82cdf05f26a53aaba9d2a75b34dfcf16 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 22 Sep 2022 14:37:19 +0000 Subject: [PATCH 080/145] Added fix for groovy replace --- conf/modules.config | 43 ++++++++++++++++++++++++++++++------- lib/WorkflowSmrnaseq.groovy | 31 +++++++++++--------------- nextflow.config | 5 +++-- workflows/smrnaseq.nf | 3 ++- 4 files changed, 53 insertions(+), 29 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 04d47b37..9fb5d7fc 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -89,29 +89,56 @@ process { if (!(params.skip_fastqc || params.skip_qc)) { process { - withName: '.*:FASTQC_TRIMGALORE:FASTQC' { + withName: '.*:FASTP_FASTQC:FASTQC' { ext.args = '--quiet' } } } -if (!params.skip_trimming) { +if (!params.skip_fastp) { process { - withName: '.*:FASTQC_TRIMGALORE:TRIMGALORE' { - ext.args = '--fastqc' + withName: 'FASTP' { + ext.args = [ "", + params.trim_fastq ?: "--disable_adapter_trimming", + params.clip_r1 > 0 ? "--trim_front1 ${params.clip_r1}" : "", // Remove bp from the 5' end of read 1. + params.three_prime_clip_r1 > 0 ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed. + params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}", + params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}", + params.three_prime_adapter ? "--adapter_sequence ${params.three_prime_adapter}" + ].join(" ").trim() publishDir = [ [ - path: { "${params.outdir}/trimmed/fastqc" }, + path: { "${params.outdir}/fastp" }, mode: params.publish_dir_mode, - pattern: "*.{html,zip}" + pattern: "*.{json,html}" ], [ - path: { "${params.outdir}/trimmed" }, + path: { "${params.outdir}/fastp/log" }, mode: params.publish_dir_mode, - pattern: "*.{txt,fastq.gz}" + pattern: "*.log" + ], + [ + path: { "${params.outdir}/fastp" }, + mode: params.publish_dir_mode, + pattern: "*.fail.fastq.gz", + enabled: params.save_trimmed_fail ] ] } + + } + + if (!params.skip_fastqc) { + process { + withName: '.*:.*:FASTQC_FASTP:FASTQC_TRIM' { + ext.args = '--quiet' + publishDir = [ + path: { "${params.outdir}/fastqc/trim" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + } } } diff --git a/lib/WorkflowSmrnaseq.groovy b/lib/WorkflowSmrnaseq.groovy index c1701ab5..f8ea2bfa 100755 --- a/lib/WorkflowSmrnaseq.groovy +++ b/lib/WorkflowSmrnaseq.groovy @@ -68,26 +68,21 @@ class WorkflowSmrnaseq { switch(protocol){ case 'illumina': - clip_r1 = 0 - three_prime_clip_r1 = 0 - three_prime_adapter = "TGGAATTCTCGGGTGCCAAGG" + params.replace("clip_r1", 0) + params.replace("three_prime_clip_r1",0) + params.replace("three_prime_adapter", "TGGAATTCTCGGGTGCCAAGG") case 'nextflex': - clip_r1 = 4 - three_prime_clip_r1 = 4 - three_prime_adapter = "TGGAATTCTCGGGTGCCAAGG" + params.replace("clip_r1", 4) + params.replace("three_prime_clip_r1", 4) + params.replace("three_prime_adapter", "TGGAATTCTCGGGTGCCAAGG") case 'qiaseq': - clip_r1 = 0 - three_prime_clip_r1 = 0 - three_prime_adapter = "AACTGTAGGCACCATCAAT" + params.replace("clip_r1",0) + params.replace("three_prime_clip_r1",0) + params.replace("three_prime_adapter","AACTGTAGGCACCATCAAT") case 'cats': - clip_r1 = 3 - three_prime_clip_r1 = 0 - three_prime_adapter = "AAAAAAAA" - case 'custom': - clip_r1 = params.clip_r1 - three_prime_clip_r1 = params.three_prime_clip_r1 - three_prime_adapter = params.three_prime_adapter - } - return [clip_r1, three_prime_clip_r1, three_prime_adapter] + params.replace("clip_r1",3) + params.replace("three_prime_clip_r1", 0) + params.replace("three_prime_adapter", "AAAAAAAA") + default: log.warn("No protocol specified, please ensure that you specified parameters for clipping/trimming or otherwise only auto-detection of adapters will be performed.") } } diff --git a/nextflow.config b/nextflow.config index 6ea448aa..28dbafbf 100644 --- a/nextflow.config +++ b/nextflow.config @@ -32,14 +32,15 @@ params { clip_r1 = null three_prime_clip_r1 = null three_prime_adapter = '' - min_length = 17 + trim_fastq = true + fastp_min_length = 17 skip_qc = false skip_fastqc = false skip_multiqc = false skip_mirdeep = false skip_fastp = false save_reference = false - trim_galore_max_length = 40 + fastp_max_length = 40 // Contamination filtering filter_contamination = false diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 4c8a1d96..28b870b0 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -135,12 +135,13 @@ workflow SMRNASEQ { // SUBWORKFLOW: Read QC and trim adapters // - (clip_r1, three_prime_clip_r1, three_prime_adapter) = WorkflowSmrnaseq.formatProtocol(params.protocol) + WorkflowSmrnaseq.formatProtocol(params.protocol) FASTP_FASTQC ( ch_cat_fastq, false ) + ch_versions = ch_versions.mix(FASTP_FASTQC.out.versions) reads_for_mirna = FASTP_FASTQC.out.reads From f8614894b9935d17128d9a83795339b0e856ab12 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 22 Sep 2022 14:46:24 +0000 Subject: [PATCH 081/145] Adjust configs --- conf/modules.config | 6 +++--- nextflow_schema.json | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 9fb5d7fc..fe7c83d4 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -102,9 +102,9 @@ if (!params.skip_fastp) { params.trim_fastq ?: "--disable_adapter_trimming", params.clip_r1 > 0 ? "--trim_front1 ${params.clip_r1}" : "", // Remove bp from the 5' end of read 1. params.three_prime_clip_r1 > 0 ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed. - params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}", - params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}", - params.three_prime_adapter ? "--adapter_sequence ${params.three_prime_adapter}" + params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}" : "", + params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}" : "", + params.three_prime_adapter ? "--adapter_sequence ${params.three_prime_adapter}" : "" ].join(" ").trim() publishDir = [ [ diff --git a/nextflow_schema.json b/nextflow_schema.json index 82c2ee86..b82d7d6c 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -164,12 +164,6 @@ "description": "Options for trimming reads and primers.", "fa_icon": "fas fa-cut", "properties": { - "min_length": { - "type": "integer", - "default": 17, - "description": "Discard reads that are shorter than this after quality / adapter trimming.", - "fa_icon": "fas fa-ruler" - }, "clip_r1": { "type": "integer", "fa_icon": "fas fa-cut", @@ -185,12 +179,6 @@ "default": "TGGAATTCTCGGGTGCCAAGG", "fa_icon": "fas fa-text-width", "description": "Sequencing adapter sequence to use for trimming." - }, - "trim_galore_max_length": { - "type": "integer", - "default": 40, - "description": "The max-length parameter used for trimming with TrimGalore!.", - "fa_icon": "fas fa-ruler" } } }, @@ -475,6 +463,18 @@ "skip_fastp": { "type": "string", "default": "false" + }, + "trim_fastq": { + "type": "string", + "default": "true" + }, + "fastp_min_length": { + "type": "integer", + "default": 17 + }, + "fastp_max_length": { + "type": "integer", + "default": 40 } } } From a8ec05584fdf85bb9b520c0bdd78c6da8c9eb476 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 22 Sep 2022 14:48:09 +0000 Subject: [PATCH 082/145] [automated] Fix linting with Prettier --- nextflow_schema.json | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index b82d7d6c..1ab04811 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -10,10 +10,7 @@ "type": "object", "fa_icon": "fas fa-terminal", "description": "Define where the pipeline should find input data and save output data.", - "required": [ - "input", - "outdir" - ], + "required": ["input", "outdir"], "properties": { "input": { "type": "string", @@ -31,13 +28,7 @@ "fa_icon": "fas fa-vial", "description": "Protocol for constructing smRNA-seq libraries.", "help_text": "Presets for trimming parameters and 3' adapter sequence with a specified protocol.\n\n| Protocol | Library Prep Kit | Trimming Parameter | 3' Adapter Sequence |\n| :------------ | :-------------------------------------- | :-------------------------------------- | :--------------------- |\n| illumina | Illumina TruSeq Small RNA | `clip_r1 = 0` `three_prime_clip_r1 = 0` | `TGGAATTCTCGGGTGCCAAGG` |\n| nextflex | BIOO SCIENTIFIC NEXTFLEX Small RNA-Seq | `clip_r1 = 4` `three_prime_clip_r1 = 4` | `TGGAATTCTCGGGTGCCAAGG` |\n| qiaseq | QIAGEN QIAseq miRNA | `clip_r1 = 0` `three_prime_clip_r1 = 0` | `AACTGTAGGCACCATCAAT` |\n| cats | Diagenode CATS Small RNA-seq | `clip_r1 = 3` `three_prime_clip_r1 = 0` | `AAAAAAAAAAA` + `GATCGGAAGAGCACACGTCTG` (only polyA is used for trimming) |\n| custom | user defined | user defined | user defined |\n\n> NB: When running `--protocol custom` the user ***must define the 3' Adapter Sequence***.\n> If trimming parameters aren't provided the pipeline will deafult to `clip_R1 = 0` and `three_prime_clip_R1 = 0` (i.e. no extra clipping).", - "enum": [ - "illumina", - "nextflex", - "qiaseq", - "cats", - "custom" - ] + "enum": ["illumina", "nextflex", "qiaseq", "cats", "custom"] }, "outdir": { "type": "string", @@ -359,14 +350,7 @@ "description": "Method used to save pipeline results to output directory.", "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", "fa_icon": "fas fa-copy", - "enum": [ - "symlink", - "rellink", - "link", - "copy", - "copyNoFollow", - "move" - ], + "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], "hidden": true }, "email_on_fail": { From 6c0150b09f47695c584e9efb3b897f416de42c8f Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 22 Sep 2022 14:54:31 +0000 Subject: [PATCH 083/145] Fix missing params --- nextflow.config | 3 +- nextflow_schema.json | 71 ++++++++++++++++++++++++++++---------------- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/nextflow.config b/nextflow.config index 28dbafbf..214d12bf 100644 --- a/nextflow.config +++ b/nextflow.config @@ -31,9 +31,10 @@ params { // Trimming options clip_r1 = null three_prime_clip_r1 = null - three_prime_adapter = '' + three_prime_adapter = null trim_fastq = true fastp_min_length = 17 + save_trimmed_fail = false skip_qc = false skip_fastqc = false skip_multiqc = false diff --git a/nextflow_schema.json b/nextflow_schema.json index 1ab04811..ce73b58a 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -10,7 +10,10 @@ "type": "object", "fa_icon": "fas fa-terminal", "description": "Define where the pipeline should find input data and save output data.", - "required": ["input", "outdir"], + "required": [ + "input", + "outdir" + ], "properties": { "input": { "type": "string", @@ -28,7 +31,13 @@ "fa_icon": "fas fa-vial", "description": "Protocol for constructing smRNA-seq libraries.", "help_text": "Presets for trimming parameters and 3' adapter sequence with a specified protocol.\n\n| Protocol | Library Prep Kit | Trimming Parameter | 3' Adapter Sequence |\n| :------------ | :-------------------------------------- | :-------------------------------------- | :--------------------- |\n| illumina | Illumina TruSeq Small RNA | `clip_r1 = 0` `three_prime_clip_r1 = 0` | `TGGAATTCTCGGGTGCCAAGG` |\n| nextflex | BIOO SCIENTIFIC NEXTFLEX Small RNA-Seq | `clip_r1 = 4` `three_prime_clip_r1 = 4` | `TGGAATTCTCGGGTGCCAAGG` |\n| qiaseq | QIAGEN QIAseq miRNA | `clip_r1 = 0` `three_prime_clip_r1 = 0` | `AACTGTAGGCACCATCAAT` |\n| cats | Diagenode CATS Small RNA-seq | `clip_r1 = 3` `three_prime_clip_r1 = 0` | `AAAAAAAAAAA` + `GATCGGAAGAGCACACGTCTG` (only polyA is used for trimming) |\n| custom | user defined | user defined | user defined |\n\n> NB: When running `--protocol custom` the user ***must define the 3' Adapter Sequence***.\n> If trimming parameters aren't provided the pipeline will deafult to `clip_R1 = 0` and `three_prime_clip_R1 = 0` (i.e. no extra clipping).", - "enum": ["illumina", "nextflex", "qiaseq", "cats", "custom"] + "enum": [ + "illumina", + "nextflex", + "qiaseq", + "cats", + "custom" + ] }, "outdir": { "type": "string", @@ -142,7 +151,6 @@ "igenomes_ignore": { "type": "boolean", "description": "Do not load the iGenomes reference config.", - "default": false, "fa_icon": "fas fa-ban", "help_text": "Do not load `igenomes.config` when running the pipeline. You may choose this option if you observe clashes between custom parameters and those supplied in `igenomes.config`.", "hidden": true @@ -170,6 +178,29 @@ "default": "TGGAATTCTCGGGTGCCAAGG", "fa_icon": "fas fa-text-width", "description": "Sequencing adapter sequence to use for trimming." + }, + "trim_fastq": { + "type": "string", + "default": "true", + "fa_icon": "fas fa-hand-scissors", + "description": "Trim FastQ files" + }, + "fastp_min_length": { + "type": "integer", + "default": 17, + "description": "Minimum filter length for raw reads.", + "fa_icon": "fas fa-ruler-horizontal" + }, + "fastp_max_length": { + "type": "integer", + "default": 40, + "description": "Maximum filter length for raw reads.", + "fa_icon": "fas fa-ruler-horizontal" + }, + "save_trimmed_fail": { + "type": "boolean", + "fa_icon": "fas fa-cloud-download-alt", + "description": "Save reads failing trimming" } } }, @@ -181,7 +212,6 @@ "properties": { "filter_contamination": { "type": "boolean", - "default": false, "description": "Enables the contamination filtering." }, "rrna": { @@ -225,26 +255,22 @@ "skip_qc": { "type": "boolean", "fa_icon": "fas fa-fast-forward", - "description": "Skip all QC steps", - "default": false + "description": "Skip all QC steps" }, "skip_fastqc": { "type": "boolean", "fa_icon": "fas fa-fast-forward", - "description": "Skip FastQC", - "default": false + "description": "Skip FastQC" }, "skip_mirdeep": { "type": "boolean", "fa_icon": "fas fa-fast-forward", - "description": "Skip miRDeep", - "default": false + "description": "Skip miRDeep" }, "skip_multiqc": { "type": "boolean", "fa_icon": "fas fa-fast-forward", - "description": "Skip MultiQC", - "default": false + "description": "Skip MultiQC" } } }, @@ -350,7 +376,14 @@ "description": "Method used to save pipeline results to output directory.", "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", "fa_icon": "fas fa-copy", - "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], + "enum": [ + "symlink", + "rellink", + "link", + "copy", + "copyNoFollow", + "move" + ], "hidden": true }, "email_on_fail": { @@ -447,18 +480,6 @@ "skip_fastp": { "type": "string", "default": "false" - }, - "trim_fastq": { - "type": "string", - "default": "true" - }, - "fastp_min_length": { - "type": "integer", - "default": 17 - }, - "fastp_max_length": { - "type": "integer", - "default": 40 } } } From b9e500aba94eda094b56efbb54405e4666c86f38 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 22 Sep 2022 15:02:19 +0000 Subject: [PATCH 084/145] Fix schema --- nextflow_schema.json | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index ce73b58a..81a320aa 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -180,8 +180,8 @@ "description": "Sequencing adapter sequence to use for trimming." }, "trim_fastq": { - "type": "string", - "default": "true", + "type": "boolean", + "default": true, "fa_icon": "fas fa-hand-scissors", "description": "Trim FastQ files" }, @@ -271,6 +271,11 @@ "type": "boolean", "fa_icon": "fas fa-fast-forward", "description": "Skip MultiQC" + }, + "skip_fastp": { + "type": "boolean", + "description": "Skip FastP", + "fa_icon": "fas fa-forward" } } }, @@ -475,11 +480,5 @@ { "$ref": "#/definitions/generic_options" } - ], - "properties": { - "skip_fastp": { - "type": "string", - "default": "false" - } - } + ] } From b59d3e7c734249bbae6f3c4491a35237c460a49d Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 22 Sep 2022 15:12:47 +0000 Subject: [PATCH 085/145] Adjust tests, add public statement --- conf/test.config | 1 + conf/test_full.config | 1 + lib/WorkflowSmrnaseq.groovy | 5 +---- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/conf/test.config b/conf/test.config index b4facb58..518ced5f 100644 --- a/conf/test.config +++ b/conf/test.config @@ -27,6 +27,7 @@ params { hairpin = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/hairpin.fa' mirna_gtf = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/reference/hsa.gff3' mirtrace_species = 'hsa' + protocol = 'illumina' skip_mirdeep = true } diff --git a/conf/test_full.config b/conf/test_full.config index cc5ecd92..964dc5b2 100644 --- a/conf/test_full.config +++ b/conf/test_full.config @@ -18,6 +18,7 @@ params { input = 'https://github.com/nf-core/test-datasets/raw/smrnaseq/samplesheet/v2.0/samplesheet-full.csv' genome = 'GRCh37' mirtrace_species = 'hsa' + protocol = 'illumina' } diff --git a/lib/WorkflowSmrnaseq.groovy b/lib/WorkflowSmrnaseq.groovy index f8ea2bfa..877fbda6 100755 --- a/lib/WorkflowSmrnaseq.groovy +++ b/lib/WorkflowSmrnaseq.groovy @@ -61,10 +61,7 @@ class WorkflowSmrnaseq { * this function formats the protocol such that it is fit for the respective * subworkflow */ - static formatProtocol(protocol) { - Int clip_r1 = protocol - Int three_prime_clip_r1 = '' - String three_prime_adapter = '' + public static void formatProtocol(protocol) { switch(protocol){ case 'illumina': From 1803adc7e6acfd688730ac96bb2a71f6af849bc2 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 22 Sep 2022 15:18:51 +0000 Subject: [PATCH 086/145] Adjust groovy functionality --- lib/WorkflowSmrnaseq.groovy | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/lib/WorkflowSmrnaseq.groovy b/lib/WorkflowSmrnaseq.groovy index 877fbda6..615b1cc9 100755 --- a/lib/WorkflowSmrnaseq.groovy +++ b/lib/WorkflowSmrnaseq.groovy @@ -64,22 +64,27 @@ class WorkflowSmrnaseq { public static void formatProtocol(protocol) { switch(protocol){ + log.info "Running with Protocol ${params.protocol} \n Clipping ${params.clip_r1} bases \n Clipping ${params.three_prime_clip_r1} 3' bases \n with Adapter ${params.three_prime_adapter}" case 'illumina': - params.replace("clip_r1", 0) - params.replace("three_prime_clip_r1",0) - params.replace("three_prime_adapter", "TGGAATTCTCGGGTGCCAAGG") + params.replace("clip_r1", 0); + params.replace("three_prime_clip_r1",0); + params.replace("three_prime_adapter", "TGGAATTCTCGGGTGCCAAGG"); + break case 'nextflex': - params.replace("clip_r1", 4) - params.replace("three_prime_clip_r1", 4) - params.replace("three_prime_adapter", "TGGAATTCTCGGGTGCCAAGG") + params.replace("clip_r1", 4); + params.replace("three_prime_clip_r1", 4); + params.replace("three_prime_adapter", "TGGAATTCTCGGGTGCCAAGG"); + break case 'qiaseq': - params.replace("clip_r1",0) - params.replace("three_prime_clip_r1",0) - params.replace("three_prime_adapter","AACTGTAGGCACCATCAAT") + params.replace("clip_r1",0); + params.replace("three_prime_clip_r1",0); + params.replace("three_prime_adapter","AACTGTAGGCACCATCAAT"); + break case 'cats': - params.replace("clip_r1",3) - params.replace("three_prime_clip_r1", 0) - params.replace("three_prime_adapter", "AAAAAAAA") + params.replace("clip_r1",3); + params.replace("three_prime_clip_r1", 0); + params.replace("three_prime_adapter", "AAAAAAAA"); + break default: log.warn("No protocol specified, please ensure that you specified parameters for clipping/trimming or otherwise only auto-detection of adapters will be performed.") } } From 3246932cf0d7f44749d728e4377e6272bc999ec1 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 22 Sep 2022 15:44:18 +0000 Subject: [PATCH 087/145] Conf not fully there --- conf/modules.config | 2 +- lib/WorkflowMain.groovy | 3 +++ lib/WorkflowSmrnaseq.groovy | 8 ++++---- .../nf-core/{fastp_fastqc.nf => fastqc_fastp.nf} | 0 workflows/smrnaseq.nf | 15 +++++++-------- 5 files changed, 15 insertions(+), 13 deletions(-) rename subworkflows/nf-core/{fastp_fastqc.nf => fastqc_fastp.nf} (100%) diff --git a/conf/modules.config b/conf/modules.config index fe7c83d4..30a4d558 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -89,7 +89,7 @@ process { if (!(params.skip_fastqc || params.skip_qc)) { process { - withName: '.*:FASTP_FASTQC:FASTQC' { + withName: '.*:FASTP_FASTQC:FASTQC_*' { ext.args = '--quiet' } } diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy index ce87a8d8..0e509e5d 100755 --- a/lib/WorkflowMain.groovy +++ b/lib/WorkflowMain.groovy @@ -77,6 +77,9 @@ class WorkflowMain { log.error "Please provide an input samplesheet to the pipeline e.g. '--input samplesheet.csv'" System.exit(1) } + + //Detect Protocol setting + WorkflowSmrnaseq.formatProtocol(params,log) } // // Get attribute from genome config file e.g. fasta diff --git a/lib/WorkflowSmrnaseq.groovy b/lib/WorkflowSmrnaseq.groovy index 615b1cc9..7551b441 100755 --- a/lib/WorkflowSmrnaseq.groovy +++ b/lib/WorkflowSmrnaseq.groovy @@ -61,10 +61,10 @@ class WorkflowSmrnaseq { * this function formats the protocol such that it is fit for the respective * subworkflow */ - public static void formatProtocol(protocol) { + public static String formatProtocol(params,log) { - switch(protocol){ - log.info "Running with Protocol ${params.protocol} \n Clipping ${params.clip_r1} bases \n Clipping ${params.three_prime_clip_r1} 3' bases \n with Adapter ${params.three_prime_adapter}" + switch(params.protocol){ + //log.warn("Running with Protocol ${params.protocol} \n Clipping ${params.clip_r1} bases \n Clipping ${params.three_prime_clip_r1} 3' bases \n with Adapter ${params.three_prime_adapter}") case 'illumina': params.replace("clip_r1", 0); params.replace("three_prime_clip_r1",0); @@ -85,6 +85,6 @@ class WorkflowSmrnaseq { params.replace("three_prime_clip_r1", 0); params.replace("three_prime_adapter", "AAAAAAAA"); break - default: log.warn("No protocol specified, please ensure that you specified parameters for clipping/trimming or otherwise only auto-detection of adapters will be performed.") + } } } diff --git a/subworkflows/nf-core/fastp_fastqc.nf b/subworkflows/nf-core/fastqc_fastp.nf similarity index 100% rename from subworkflows/nf-core/fastp_fastqc.nf rename to subworkflows/nf-core/fastqc_fastp.nf diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 28b870b0..816a1fe7 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -57,7 +57,7 @@ if (!params.mirgenedb) { } include { INPUT_CHECK } from '../subworkflows/local/input_check' -include { FASTP_FASTQC } from '../subworkflows/nf-core/fastp_fastqc' +include { FASTQC_FASTP } from '../subworkflows/nf-core/fastqc_fastp' include { CONTAMINANT_FILTER } from '../subworkflows/local/contaminant_filter' include { MIRNA_QUANT } from '../subworkflows/local/mirna_quant' include { GENOME_QUANT } from '../subworkflows/local/genome_quant' @@ -135,16 +135,15 @@ workflow SMRNASEQ { // SUBWORKFLOW: Read QC and trim adapters // - WorkflowSmrnaseq.formatProtocol(params.protocol) - - FASTP_FASTQC ( + FASTQC_FASTP ( ch_cat_fastq, + false, false ) - ch_versions = ch_versions.mix(FASTP_FASTQC.out.versions) + ch_versions = ch_versions.mix(FASTQC_FASTP.out.versions) - reads_for_mirna = FASTP_FASTQC.out.reads + reads_for_mirna = FASTQC_FASTP.out.reads // // SUBWORKFLOW: remove contaminants from reads // @@ -158,7 +157,7 @@ workflow SMRNASEQ { params.ncrna, params.pirna, params.other_contamination, - FASTP_FASTQC.out.reads + FASTQC_FASTP.out.reads ) reads_for_mirna = CONTAMINANT_FILTER.out.filtered_reads @@ -189,7 +188,7 @@ workflow SMRNASEQ { if (!params.skip_mirdeep) { MIRDEEP2 ( - FASTP_FASTQC.out.reads, + FASTQC_FASTP.out.reads, GENOME_QUANT.out.fasta, GENOME_QUANT.out.index.collect(), MIRNA_QUANT.out.fasta_hairpin, From 21b8aa8f50073dae5ba98405944a3a26d534e982 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 22 Sep 2022 15:52:56 +0000 Subject: [PATCH 088/145] Explicit conf --- conf/modules.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index 30a4d558..091b87a2 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -99,7 +99,7 @@ if (!params.skip_fastp) { process { withName: 'FASTP' { ext.args = [ "", - params.trim_fastq ?: "--disable_adapter_trimming", + params.trim_fastq ? "" : "--disable_adapter_trimming", params.clip_r1 > 0 ? "--trim_front1 ${params.clip_r1}" : "", // Remove bp from the 5' end of read 1. params.three_prime_clip_r1 > 0 ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed. params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}" : "", From fe231d8a044ffe14d2f8ebd4dc2fe01ad0455e04 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 22 Sep 2022 15:53:44 +0000 Subject: [PATCH 089/145] [automated] Fix linting with Prettier --- nextflow_schema.json | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index 81a320aa..7f95c921 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -10,10 +10,7 @@ "type": "object", "fa_icon": "fas fa-terminal", "description": "Define where the pipeline should find input data and save output data.", - "required": [ - "input", - "outdir" - ], + "required": ["input", "outdir"], "properties": { "input": { "type": "string", @@ -31,13 +28,7 @@ "fa_icon": "fas fa-vial", "description": "Protocol for constructing smRNA-seq libraries.", "help_text": "Presets for trimming parameters and 3' adapter sequence with a specified protocol.\n\n| Protocol | Library Prep Kit | Trimming Parameter | 3' Adapter Sequence |\n| :------------ | :-------------------------------------- | :-------------------------------------- | :--------------------- |\n| illumina | Illumina TruSeq Small RNA | `clip_r1 = 0` `three_prime_clip_r1 = 0` | `TGGAATTCTCGGGTGCCAAGG` |\n| nextflex | BIOO SCIENTIFIC NEXTFLEX Small RNA-Seq | `clip_r1 = 4` `three_prime_clip_r1 = 4` | `TGGAATTCTCGGGTGCCAAGG` |\n| qiaseq | QIAGEN QIAseq miRNA | `clip_r1 = 0` `three_prime_clip_r1 = 0` | `AACTGTAGGCACCATCAAT` |\n| cats | Diagenode CATS Small RNA-seq | `clip_r1 = 3` `three_prime_clip_r1 = 0` | `AAAAAAAAAAA` + `GATCGGAAGAGCACACGTCTG` (only polyA is used for trimming) |\n| custom | user defined | user defined | user defined |\n\n> NB: When running `--protocol custom` the user ***must define the 3' Adapter Sequence***.\n> If trimming parameters aren't provided the pipeline will deafult to `clip_R1 = 0` and `three_prime_clip_R1 = 0` (i.e. no extra clipping).", - "enum": [ - "illumina", - "nextflex", - "qiaseq", - "cats", - "custom" - ] + "enum": ["illumina", "nextflex", "qiaseq", "cats", "custom"] }, "outdir": { "type": "string", @@ -381,14 +372,7 @@ "description": "Method used to save pipeline results to output directory.", "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", "fa_icon": "fas fa-copy", - "enum": [ - "symlink", - "rellink", - "link", - "copy", - "copyNoFollow", - "move" - ], + "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], "hidden": true }, "email_on_fail": { From 853397e4cf69e15f77ebf96d132930439b0e4839 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 22 Sep 2022 16:00:07 +0000 Subject: [PATCH 090/145] Add a protocol to running the pipeline --- conf/test_no_genome.config | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/test_no_genome.config b/conf/test_no_genome.config index 8724f431..485870de 100644 --- a/conf/test_no_genome.config +++ b/conf/test_no_genome.config @@ -26,5 +26,6 @@ params { mirna_gtf = 'https://github.com/nf-core/test-datasets/raw/smrnaseq-better-input/reference/hsa.gff3' mirtrace_species = 'hsa' skip_mirdeep = true + protocol = 'illumina' } From 0224ede0b111d56904ec00a7b19203a975f37933 Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Thu, 22 Sep 2022 16:16:50 +0000 Subject: [PATCH 091/145] Process selector name fix --- conf/modules.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index 091b87a2..c47b5fea 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -89,7 +89,7 @@ process { if (!(params.skip_fastqc || params.skip_qc)) { process { - withName: '.*:FASTP_FASTQC:FASTQC_*' { + withName: '.*:FASTQC_FASTP:FASTQC_.*' { ext.args = '--quiet' } } From 6a295bbd1b55af283aa1241b6bab769a60cfbf3b Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Thu, 22 Sep 2022 23:00:28 +0000 Subject: [PATCH 092/145] Ensure that pigz works for fq and fastq extensions --- modules/local/mirdeep2_prepare.nf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/local/mirdeep2_prepare.nf b/modules/local/mirdeep2_prepare.nf index a62a5845..d49270ff 100644 --- a/modules/local/mirdeep2_prepare.nf +++ b/modules/local/mirdeep2_prepare.nf @@ -12,11 +12,10 @@ process MIRDEEP2_PIGZ { tuple val(meta), path(reads) output: - tuple val(meta), path("*.fq"), emit: reads - path "versions.yml" , emit: versions + tuple val(meta), path("*.{fastq,fq}"), emit: reads + path "versions.yml" , emit: versions script: - def unzip = reads.toString() - '.gz' """ pigz -f -d -p $task.cpus $reads From 56a67eebca604bc344feb37d9301aea972d9f00c Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 23 Sep 2022 11:39:04 +0000 Subject: [PATCH 093/145] Removed trimgalore --- modules.json | 4 - modules/nf-core/modules/trimgalore/main.nf | 86 --------------------- modules/nf-core/modules/trimgalore/meta.yml | 64 --------------- 3 files changed, 154 deletions(-) delete mode 100644 modules/nf-core/modules/trimgalore/main.nf delete mode 100644 modules/nf-core/modules/trimgalore/meta.yml diff --git a/modules.json b/modules.json index 7c426975..9ca284a9 100644 --- a/modules.json +++ b/modules.json @@ -44,10 +44,6 @@ "samtools/stats": { "branch": "master", "git_sha": "f4eab7945952dc4934224309701a49913ea05ae6" - }, - "trimgalore": { - "branch": "master", - "git_sha": "85ec13ff1fc2196c5a507ea497de468101baabed" } } } diff --git a/modules/nf-core/modules/trimgalore/main.nf b/modules/nf-core/modules/trimgalore/main.nf deleted file mode 100644 index 3a3fca90..00000000 --- a/modules/nf-core/modules/trimgalore/main.nf +++ /dev/null @@ -1,86 +0,0 @@ -process TRIMGALORE { - tag "$meta.id" - label 'process_high' - - conda (params.enable_conda ? 'bioconda::trim-galore=0.6.7' : null) - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/trim-galore:0.6.7--hdfd78af_0' : - 'quay.io/biocontainers/trim-galore:0.6.7--hdfd78af_0' }" - - input: - tuple val(meta), path(reads) - - output: - tuple val(meta), path("*{trimmed,val}*.fq.gz"), emit: reads - tuple val(meta), path("*report.txt") , emit: log - path "versions.yml" , emit: versions - - tuple val(meta), path("*unpaired*.fq.gz") , emit: unpaired, optional: true - tuple val(meta), path("*.html") , emit: html , optional: true - tuple val(meta), path("*.zip") , emit: zip , optional: true - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - // Calculate number of --cores for TrimGalore based on value of task.cpus - // See: https://github.com/FelixKrueger/TrimGalore/blob/master/Changelog.md#version-060-release-on-1-mar-2019 - // See: https://github.com/nf-core/atacseq/pull/65 - def cores = 1 - if (task.cpus) { - cores = (task.cpus as int) - 4 - if (meta.single_end) cores = (task.cpus as int) - 3 - if (cores < 1) cores = 1 - if (cores > 4) cores = 4 - } - - // Clipping presets have to be evaluated in the context of SE/PE - def c_r1 = params.clip_r1 > 0 ? "--clip_r1 ${params.clip_r1}" : '' - def c_r2 = params.clip_r2 > 0 ? "--clip_r2 ${params.clip_r2}" : '' - def tpc_r1 = params.three_prime_clip_r1 > 0 ? "--three_prime_clip_r1 ${params.three_prime_clip_r1}" : '' - def tpc_r2 = params.three_prime_clip_r2 > 0 ? "--three_prime_clip_r2 ${params.three_prime_clip_r2}" : '' - - // Added soft-links to original fastqs for consistent naming in MultiQC - def prefix = task.ext.prefix ?: "${meta.id}" - if (meta.single_end) { - """ - [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz - trim_galore \\ - $args \\ - --cores $cores \\ - --gzip \\ - $c_r1 \\ - $tpc_r1 \\ - ${prefix}.fastq.gz - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - trimgalore: \$(echo \$(trim_galore --version 2>&1) | sed 's/^.*version //; s/Last.*\$//') - cutadapt: \$(cutadapt --version) - END_VERSIONS - """ - } else { - """ - [ ! -f ${prefix}_1.fastq.gz ] && ln -s ${reads[0]} ${prefix}_1.fastq.gz - [ ! -f ${prefix}_2.fastq.gz ] && ln -s ${reads[1]} ${prefix}_2.fastq.gz - trim_galore \\ - $args \\ - --cores $cores \\ - --paired \\ - --gzip \\ - $c_r1 \\ - $c_r2 \\ - $tpc_r1 \\ - $tpc_r2 \\ - ${prefix}_1.fastq.gz \\ - ${prefix}_2.fastq.gz - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - trimgalore: \$(echo \$(trim_galore --version 2>&1) | sed 's/^.*version //; s/Last.*\$//') - cutadapt: \$(cutadapt --version) - END_VERSIONS - """ - } -} diff --git a/modules/nf-core/modules/trimgalore/meta.yml b/modules/nf-core/modules/trimgalore/meta.yml deleted file mode 100644 index 439f566d..00000000 --- a/modules/nf-core/modules/trimgalore/meta.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: trimgalore -description: Trim FastQ files using Trim Galore! -keywords: - - trimming - - adapters - - sequencing adapters - - fastq -tools: - - trimgalore: - description: | - A wrapper tool around Cutadapt and FastQC to consistently apply quality - and adapter trimming to FastQ files, with some extra functionality for - MspI-digested RRBS-type (Reduced Representation Bisufite-Seq) libraries. - homepage: https://www.bioinformatics.babraham.ac.uk/projects/trim_galore/ - documentation: https://github.com/FelixKrueger/TrimGalore/blob/master/Docs/Trim_Galore_User_Guide.md - licence: ["GPL-3.0-or-later"] -input: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - reads: - type: file - description: | - List of input FastQ files of size 1 and 2 for single-end and paired-end data, - respectively. -output: - - meta: - type: map - description: | - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] - - reads: - type: file - description: | - List of input adapter trimmed FastQ files of size 1 and 2 for - single-end and paired-end data, respectively. - pattern: "*.{fq.gz}" - - unpaired: - type: file - description: | - FastQ files containing unpaired reads from read 1 or read 2 - pattern: "*unpaired*.fq.gz" - - html: - type: file - description: FastQC report (optional) - pattern: "*_{fastqc.html}" - - zip: - type: file - description: FastQC report archive (optional) - pattern: "*_{fastqc.zip}" - - log: - type: file - description: Trim Galore! trimming report - pattern: "*_{report.txt}" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" -authors: - - "@drpatelh" - - "@ewels" - - "@FelixKrueger" From de95d049525510966c8a961a12e5e87cb768dc0c Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 23 Sep 2022 11:40:22 +0000 Subject: [PATCH 094/145] Adjust setting protocol early on in workflow --- lib/WorkflowMain.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy index 0e509e5d..f092760b 100755 --- a/lib/WorkflowMain.groovy +++ b/lib/WorkflowMain.groovy @@ -46,6 +46,10 @@ class WorkflowMain { // Validate parameters and print summary to screen // public static void initialise(workflow, params, log) { + + //Detect Protocol setting, set this early before help so help shows proper adapters etc pp + WorkflowSmrnaseq.formatProtocol(params,log) + // Print help to screen if required if (params.help) { log.info help(workflow, params, log) @@ -78,8 +82,6 @@ class WorkflowMain { System.exit(1) } - //Detect Protocol setting - WorkflowSmrnaseq.formatProtocol(params,log) } // // Get attribute from genome config file e.g. fasta From 5989773f2516e051d4e2d47442a6e29a50990e23 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 23 Sep 2022 13:39:10 +0000 Subject: [PATCH 095/145] More helpful error messages --- conf/modules.config | 2 +- lib/WorkflowSmrnaseq.groovy | 50 ++++++++++++++++++++----------------- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index c47b5fea..02237e06 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -104,7 +104,7 @@ if (!params.skip_fastp) { params.three_prime_clip_r1 > 0 ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed. params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}" : "", params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}" : "", - params.three_prime_adapter ? "--adapter_sequence ${params.three_prime_adapter}" : "" + params.three_prime_adapter != null ? "--adapter_sequence ${params.three_prime_adapter}" : "" ].join(" ").trim() publishDir = [ [ diff --git a/lib/WorkflowSmrnaseq.groovy b/lib/WorkflowSmrnaseq.groovy index 7551b441..6d530271 100755 --- a/lib/WorkflowSmrnaseq.groovy +++ b/lib/WorkflowSmrnaseq.groovy @@ -63,28 +63,32 @@ class WorkflowSmrnaseq { */ public static String formatProtocol(params,log) { - switch(params.protocol){ - //log.warn("Running with Protocol ${params.protocol} \n Clipping ${params.clip_r1} bases \n Clipping ${params.three_prime_clip_r1} 3' bases \n with Adapter ${params.three_prime_adapter}") - case 'illumina': - params.replace("clip_r1", 0); - params.replace("three_prime_clip_r1",0); - params.replace("three_prime_adapter", "TGGAATTCTCGGGTGCCAAGG"); - break - case 'nextflex': - params.replace("clip_r1", 4); - params.replace("three_prime_clip_r1", 4); - params.replace("three_prime_adapter", "TGGAATTCTCGGGTGCCAAGG"); - break - case 'qiaseq': - params.replace("clip_r1",0); - params.replace("three_prime_clip_r1",0); - params.replace("three_prime_adapter","AACTGTAGGCACCATCAAT"); - break - case 'cats': - params.replace("clip_r1",3); - params.replace("three_prime_clip_r1", 0); - params.replace("three_prime_adapter", "AAAAAAAA"); - break + switch(params.protocol){ + case 'illumina': + params.replace("clip_r1", 0); + params.replace("three_prime_clip_r1",0); + params.replace("three_prime_adapter", "TGGAATTCTCGGGTGCCAAGG"); + break + case 'nextflex': + params.replace("clip_r1", 4); + params.replace("three_prime_clip_r1", 4); + params.replace("three_prime_adapter", "TGGAATTCTCGGGTGCCAAGG"); + break + case 'qiaseq': + params.replace("clip_r1",0); + params.replace("three_prime_clip_r1",0); + params.replace("three_prime_adapter","AACTGTAGGCACCATCAAT"); + break + case 'cats': + params.replace("clip_r1",3); + params.replace("three_prime_clip_r1", 0); + params.replace("three_prime_adapter", "AAAAAAAA"); + break } - } + + log.warn "Running with Protocol ${params.protocol}" + log.warn "Therefore using Adapter: ${params.three_prime_adapter}" + log.warn "Clipping ${params.clip_r1} bases from R1" + log.warn "And clipping ${params.three_prime_clip_r1} bases from 3' end" + } } From 23f1ea25e3c8b7846e3dc7679f08dcb1322f183d Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 23 Sep 2022 13:48:00 +0000 Subject: [PATCH 096/145] Working default --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 214d12bf..09417e42 100644 --- a/nextflow.config +++ b/nextflow.config @@ -31,7 +31,7 @@ params { // Trimming options clip_r1 = null three_prime_clip_r1 = null - three_prime_adapter = null + three_prime_adapter = "TGGAATTCTCGGGTGCCAAGG" trim_fastq = true fastp_min_length = 17 save_trimmed_fail = false From f6ac76100334319c4f4ce6a34f64a00f964ec58c Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 23 Sep 2022 13:49:58 +0000 Subject: [PATCH 097/145] Proper default settings, should allow overriding with "" --- conf/modules.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index 02237e06..189e3011 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -104,7 +104,7 @@ if (!params.skip_fastp) { params.three_prime_clip_r1 > 0 ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed. params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}" : "", params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}" : "", - params.three_prime_adapter != null ? "--adapter_sequence ${params.three_prime_adapter}" : "" + params.three_prime_adapter.isEmpty() ? "" : "--adapter_sequence ${params.three_prime_adapter}" ].join(" ").trim() publishDir = [ [ From 8cb032d7189d13ee51b06fd9ccbd82f651f2dbec Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 23 Sep 2022 13:54:31 +0000 Subject: [PATCH 098/145] Better for custom profiles --- lib/WorkflowSmrnaseq.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/WorkflowSmrnaseq.groovy b/lib/WorkflowSmrnaseq.groovy index 6d530271..d2d1d66a 100755 --- a/lib/WorkflowSmrnaseq.groovy +++ b/lib/WorkflowSmrnaseq.groovy @@ -84,6 +84,8 @@ class WorkflowSmrnaseq { params.replace("three_prime_clip_r1", 0); params.replace("three_prime_adapter", "AAAAAAAA"); break + default: + log.warn "Please make sure to specify all required clipping and trimming parameters, otherwise only adapter detection will be performed." } log.warn "Running with Protocol ${params.protocol}" From 08ead6c74d6da537cebd0134cdcab3b37e3f9021 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 23 Sep 2022 14:44:11 +0000 Subject: [PATCH 099/145] Add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08ee6520..73b0b8ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[#161](https://github.com/nf-core/smrnaseq/issues/161)] - Trimmed output was not as documented and not correctly published - [[#168](https://github.com/nf-core/smrnaseq/issues/168)] - Removed `mirtrace_protocol` as the parameter was redundant and `params.protocol` is entirely sufficient - Updated pipeline template to [nf-core/tools 2.5.1](https://github.com/nf-core/tools/releases/tag/2.5.1) +- [[#188](https://github.com/nf-core/smrnaseq/pull/188)] - Dropped TrimGalore in favor of fastp QC and adapter trimming, improved handling of adapters and trimming parameters ### Parameters From 75e5e40917a3e3c2cbe1d57f94e53a990fb66c6d Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 23 Sep 2022 14:54:37 +0000 Subject: [PATCH 100/145] Add some details on how to use custom properly --- docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index ebcd6c15..bbaeeca6 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -16,7 +16,7 @@ This option indicates the experimental protocol used for the sample preparation. - 'cats': adapter (`GATCGGAAGAGCACACGTCTG), clip_r1(`3) - 'custom' (where the user can indicate the `three_prime_adapter`, `clip_r1` and `three_prime_clip_r1` manually) -:warning: At least the `custom` protocol has to be specified, otherwise the pipeline won't run. +:warning: At least the `custom` protocol has to be specified, otherwise the pipeline won't run. In case you specify the `custom` protocol, ensure that the parameters above are set accordingly or the defaults will be applied. If you want to auto-detect the adapters using `fastp`, please set `--three_prime_adapter` to `""`. ### `mirtrace_species` or `mirgenedb_species` From 47a8ebd90922ca11e0d3116641cea765a725f055 Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Sun, 2 Oct 2022 04:01:21 +0000 Subject: [PATCH 101/145] Ensure users can override params set by protocol --- lib/WorkflowSmrnaseq.groovy | 24 ++++++++++++------------ modules/local/mirtrace.nf | 18 ++---------------- 2 files changed, 14 insertions(+), 28 deletions(-) diff --git a/lib/WorkflowSmrnaseq.groovy b/lib/WorkflowSmrnaseq.groovy index d2d1d66a..cfcb9abd 100755 --- a/lib/WorkflowSmrnaseq.groovy +++ b/lib/WorkflowSmrnaseq.groovy @@ -65,24 +65,24 @@ class WorkflowSmrnaseq { switch(params.protocol){ case 'illumina': - params.replace("clip_r1", 0); - params.replace("three_prime_clip_r1",0); - params.replace("three_prime_adapter", "TGGAATTCTCGGGTGCCAAGG"); + params.putIfAbsent("clip_r1", 1); + params.putIfAbsent("three_prime_clip_r1",2); + params.putIfAbsent("three_prime_adapter", "TGGAATTCTCGGGTGCCAAGG"); break case 'nextflex': - params.replace("clip_r1", 4); - params.replace("three_prime_clip_r1", 4); - params.replace("three_prime_adapter", "TGGAATTCTCGGGTGCCAAGG"); + params.putIfAbsent("clip_r1", 4); + params.putIfAbsent("three_prime_clip_r1", 4); + params.putIfAbsent("three_prime_adapter", "TGGAATTCTCGGGTGCCAAGG"); break case 'qiaseq': - params.replace("clip_r1",0); - params.replace("three_prime_clip_r1",0); - params.replace("three_prime_adapter","AACTGTAGGCACCATCAAT"); + params.putIfAbsent("clip_r1",0); + params.putIfAbsent("three_prime_clip_r1",0); + params.putIfAbsent("three_prime_adapter","AACTGTAGGCACCATCAAT"); break case 'cats': - params.replace("clip_r1",3); - params.replace("three_prime_clip_r1", 0); - params.replace("three_prime_adapter", "AAAAAAAA"); + params.putIfAbsent("clip_r1",3); + params.putIfAbsent("three_prime_clip_r1", 0); + params.putIfAbsent("three_prime_adapter", "AAAAAAAA"); break default: log.warn "Please make sure to specify all required clipping and trimming parameters, otherwise only adapter detection will be performed." diff --git a/modules/local/mirtrace.nf b/modules/local/mirtrace.nf index e893c2e9..fee8e071 100644 --- a/modules/local/mirtrace.nf +++ b/modules/local/mirtrace.nf @@ -14,23 +14,9 @@ process MIRTRACE_RUN { path "versions.yml", emit: versions script: - def three_prime_adapter = params.three_prime_adapter - // Presets - if (params.protocol == "illumina"){ - three_prime_adapter = "TGGAATTCTCGGGTGCCAAGG" - } else if (params.protocol == "nextflex"){ - three_prime_adapter = "TGGAATTCTCGGGTGCCAAGG" - } else if (params.protocol == "qiaseq"){ - three_prime_adapter = "AACTGTAGGCACCATCAAT" - } else if (params.protocol == "cats"){ - three_prime_adapter = "AAAAAAAA" - } else if (params.protocol == "custom"){ - three_prime_adapter = params.three_prime_adapter - } - // mirtrace protocol defaults to 'params.protocol' if not set - def primer = (params.protocol=="cats") ? " " : " --adapter $three_prime_adapter " - def protocol = (params.protocol=="custom") ? " " : "--protocol $params.protocol" + def primer = params.protocol == 'cats' ? '' : "--adapter ${params.three_prime_adapter}" + def protocol = params.protocol == 'custom' ? '' : "--protocol $params.protocol" def java_mem = '' if(task.memory){ tmem = task.memory.toBytes() From 7c2e4b5444ba5f03de122584025561ce3d37c8fe Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Tue, 4 Oct 2022 22:05:52 +0000 Subject: [PATCH 102/145] Template update for nf-core/tools version 2.6 --- .github/workflows/awsfulltest.yml | 4 ++ .github/workflows/awstest.yml | 4 ++ .prettierignore | 1 + CITATION.cff | 8 +-- LICENSE | 2 +- README.md | 2 +- assets/adaptivecard.json | 67 +++++++++++++++++++ assets/methods_description_template.yml | 25 +++++++ assets/multiqc_config.yml | 6 +- docs/usage.md | 8 +++ lib/NfcoreTemplate.groovy | 55 +++++++++++++++ lib/Utils.groovy | 21 ++++-- lib/WorkflowSmrnaseq.groovy | 19 ++++++ main.nf | 3 +- modules.json | 27 ++++---- .../custom/dumpsoftwareversions/main.nf | 8 +-- .../custom/dumpsoftwareversions/meta.yml | 0 .../templates/dumpsoftwareversions.py | 0 modules/nf-core/{modules => }/fastqc/main.nf | 12 ++++ modules/nf-core/{modules => }/fastqc/meta.yml | 0 modules/nf-core/modules/multiqc/main.nf | 31 --------- modules/nf-core/multiqc/main.nf | 53 +++++++++++++++ .../nf-core/{modules => }/multiqc/meta.yml | 15 +++++ nextflow.config | 7 +- nextflow_schema.json | 18 +++++ workflows/smrnaseq.nf | 26 ++++--- 26 files changed, 348 insertions(+), 74 deletions(-) create mode 100644 assets/adaptivecard.json create mode 100644 assets/methods_description_template.yml mode change 100755 => 100644 lib/Utils.groovy rename modules/nf-core/{modules => }/custom/dumpsoftwareversions/main.nf (79%) rename modules/nf-core/{modules => }/custom/dumpsoftwareversions/meta.yml (100%) rename modules/nf-core/{modules => }/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py (100%) rename modules/nf-core/{modules => }/fastqc/main.nf (85%) rename modules/nf-core/{modules => }/fastqc/meta.yml (100%) delete mode 100644 modules/nf-core/modules/multiqc/main.nf create mode 100644 modules/nf-core/multiqc/main.nf rename modules/nf-core/{modules => }/multiqc/meta.yml (73%) diff --git a/.github/workflows/awsfulltest.yml b/.github/workflows/awsfulltest.yml index 94527ff0..76c16e75 100644 --- a/.github/workflows/awsfulltest.yml +++ b/.github/workflows/awsfulltest.yml @@ -28,3 +28,7 @@ jobs: "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/smrnaseq/results-${{ github.sha }}" } profiles: test_full,aws_tower + - uses: actions/upload-artifact@v3 + with: + name: Tower debug log file + path: tower_action_*.log diff --git a/.github/workflows/awstest.yml b/.github/workflows/awstest.yml index ea00f80e..03a442a1 100644 --- a/.github/workflows/awstest.yml +++ b/.github/workflows/awstest.yml @@ -23,3 +23,7 @@ jobs: "outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/smrnaseq/results-test-${{ github.sha }}" } profiles: test,aws_tower + - uses: actions/upload-artifact@v3 + with: + name: Tower debug log file + path: tower_action_*.log diff --git a/.prettierignore b/.prettierignore index d0e7ae58..eb74a574 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,4 +1,5 @@ email_template.html +adaptivecard.json .nextflow* work/ data/ diff --git a/CITATION.cff b/CITATION.cff index 4533e2f2..017666c0 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -13,8 +13,8 @@ authors: given-names: Johannes - family-names: Wilm given-names: Andreas - - family-names: Ulysse Garcia - given-names: Maxime + - family-names: Garcia + given-names: Maxime Ulysse - family-names: Di Tommaso given-names: Paolo - family-names: Nahnsen @@ -39,8 +39,8 @@ prefered-citation: given-names: Johannes - family-names: Wilm given-names: Andreas - - family-names: Ulysse Garcia - given-names: Maxime + - family-names: Garcia + given-names: Maxime Ulysse - family-names: Di Tommaso given-names: Paolo - family-names: Nahnsen diff --git a/LICENSE b/LICENSE index ad773211..499134b1 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) P. Ewels , C. Wang , R. Hammarén , L. Pantano +Copyright (c) P. Ewels, C. Wang, R. Hammarén, L. Pantano Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 166d0429..9caac333 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ The nf-core/smrnaseq pipeline comes with documentation about the pipeline [usage ## Credits -nf-core/smrnaseq was originally written by P. Ewels , C. Wang , R. Hammarén , L. Pantano . +nf-core/smrnaseq was originally written by P. Ewels, C. Wang, R. Hammarén, L. Pantano. We thank the following people for their extensive assistance in the development of this pipeline: diff --git a/assets/adaptivecard.json b/assets/adaptivecard.json new file mode 100644 index 00000000..0b804f3d --- /dev/null +++ b/assets/adaptivecard.json @@ -0,0 +1,67 @@ +{ + "type": "message", + "attachments": [ + { + "contentType": "application/vnd.microsoft.card.adaptive", + "contentUrl": null, + "content": { + "\$schema": "http://adaptivecards.io/schemas/adaptive-card.json", + "msteams": { + "width": "Full" + }, + "type": "AdaptiveCard", + "version": "1.2", + "body": [ + { + "type": "TextBlock", + "size": "Large", + "weight": "Bolder", + "color": "<% if (success) { %>Good<% } else { %>Attention<%} %>", + "text": "nf-core/smrnaseq v${version} - ${runName}", + "wrap": true + }, + { + "type": "TextBlock", + "spacing": "None", + "text": "Completed at ${dateComplete} (duration: ${duration})", + "isSubtle": true, + "wrap": true + }, + { + "type": "TextBlock", + "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors. The full error message was: ${errorReport}.<% } %>", + "wrap": true + }, + { + "type": "TextBlock", + "text": "The command used to launch the workflow was as follows:", + "wrap": true + }, + { + "type": "TextBlock", + "text": "${commandLine}", + "isSubtle": true, + "wrap": true + } + ], + "actions": [ + { + "type": "Action.ShowCard", + "title": "Pipeline Configuration", + "card": { + "type": "AdaptiveCard", + "\$schema": "http://adaptivecards.io/schemas/adaptive-card.json", + "body": [ + { + "type": "FactSet", + "facts": [<% out << summary.collect{ k,v -> "{\"title\": \"$k\", \"value\" : \"$v\"}"}.join(",\n") %> + ] + } + ] + } + } + ] + } + } + ] +} diff --git a/assets/methods_description_template.yml b/assets/methods_description_template.yml new file mode 100644 index 00000000..247628aa --- /dev/null +++ b/assets/methods_description_template.yml @@ -0,0 +1,25 @@ +id: "nf-core-smrnaseq-methods-description" +description: "Suggested text and references to use when describing pipeline usage within the methods section of a publication." +section_name: "nf-core/smrnaseq Methods Description" +section_href: "https://github.com/nf-core/smrnaseq" +plot_type: "html" +## TODO nf-core: Update the HTML below to your prefered methods description, e.g. add publication citation for this pipeline +## You inject any metadata in the Nextflow '${workflow}' object +data: | +

Methods

+

Data was processed using nf-core/smrnaseq v${workflow.manifest.version} ${doi_text} of the nf-core collection of workflows (Ewels et al., 2020).

+

The pipeline was executed with Nextflow v${workflow.nextflow.version} (Di Tommaso et al., 2017) with the following command:

+ 
${workflow.commandLine}
+

References

+
    +
  • Di Tommaso, P., Chatzou, M., Floden, E. W., Barja, P. P., Palumbo, E., & Notredame, C. (2017). Nextflow enables reproducible computational workflows. Nature Biotechnology, 35(4), 316-319. https://doi.org/10.1038/nbt.3820
    • +
  • Ewels, P. A., Peltzer, A., Fillinger, S., Patel, H., Alneberg, J., Wilm, A., Garcia, M. U., Di Tommaso, P., & Nahnsen, S. (2020). The nf-core framework for community-curated bioinformatics pipelines. Nature Biotechnology, 38(3), 276-278. https://doi.org/10.1038/s41587-020-0439-x
    • +
+
+
Notes:
+
    + ${nodoi_text} +
  • The command above does not include parameters contained in any configs or profiles that may have been used. Ensure the config file is also uploaded with your publication!
    • +
  • You should also cite all software used within this run. Check the "Software Versions" of this report to get version information.
    • +
+
diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index f7715e34..df5ec37a 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -3,9 +3,11 @@ report_comment: > analysis pipeline. For information about how to interpret these results, please see the documentation. report_section_order: - software_versions: + "nf-core-smrnaseq-methods-description": order: -1000 - "nf-core-smrnaseq-summary": + software_versions: order: -1001 + "nf-core-smrnaseq-summary": + order: -1002 export_plots: true diff --git a/docs/usage.md b/docs/usage.md index b75cbc81..3d01eda4 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -237,6 +237,14 @@ See the main [Nextflow documentation](https://www.nextflow.io/docs/latest/config If you have any questions or issues please send us a message on [Slack](https://nf-co.re/join/slack) on the [`#configs` channel](https://nfcore.slack.com/channels/configs). +## Azure Resource Requests + +To be used with the `azurebatch` profile by specifying the `-profile azurebatch`. +We recommend providing a compute `params.vm_type` of `Standard_D16_v3` VMs by default but these options can be changed if required. + +Note that the choice of VM size depends on your quota and the overall workload during the analysis. +For a thorough list, please refer the [Azure Sizes for virtual machines in Azure](https://docs.microsoft.com/en-us/azure/virtual-machines/sizes). + ## Running in the background Nextflow handles job submissions and supervises the running jobs. The Nextflow process must run until the pipeline is finished. diff --git a/lib/NfcoreTemplate.groovy b/lib/NfcoreTemplate.groovy index 2fc0a9b9..27feb009 100755 --- a/lib/NfcoreTemplate.groovy +++ b/lib/NfcoreTemplate.groovy @@ -145,6 +145,61 @@ class NfcoreTemplate { output_tf.withWriter { w -> w << email_txt } } + // + // Construct and send adaptive card + // https://adaptivecards.io + // + public static void adaptivecard(workflow, params, summary_params, projectDir, log) { + def hook_url = params.hook_url + + def summary = [:] + for (group in summary_params.keySet()) { + summary << summary_params[group] + } + + def misc_fields = [:] + misc_fields['start'] = workflow.start + misc_fields['complete'] = workflow.complete + misc_fields['scriptfile'] = workflow.scriptFile + misc_fields['scriptid'] = workflow.scriptId + if (workflow.repository) misc_fields['repository'] = workflow.repository + if (workflow.commitId) misc_fields['commitid'] = workflow.commitId + if (workflow.revision) misc_fields['revision'] = workflow.revision + misc_fields['nxf_version'] = workflow.nextflow.version + misc_fields['nxf_build'] = workflow.nextflow.build + misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp + + def msg_fields = [:] + msg_fields['version'] = workflow.manifest.version + msg_fields['runName'] = workflow.runName + msg_fields['success'] = workflow.success + msg_fields['dateComplete'] = workflow.complete + msg_fields['duration'] = workflow.duration + msg_fields['exitStatus'] = workflow.exitStatus + msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') + msg_fields['errorReport'] = (workflow.errorReport ?: 'None') + msg_fields['commandLine'] = workflow.commandLine + msg_fields['projectDir'] = workflow.projectDir + msg_fields['summary'] = summary << misc_fields + + // Render the JSON template + def engine = new groovy.text.GStringTemplateEngine() + def hf = new File("$projectDir/assets/adaptivecard.json") + def json_template = engine.createTemplate(hf).make(msg_fields) + def json_message = json_template.toString() + + // POST + def post = new URL(hook_url).openConnection(); + post.setRequestMethod("POST") + post.setDoOutput(true) + post.setRequestProperty("Content-Type", "application/json") + post.getOutputStream().write(json_message.getBytes("UTF-8")); + def postRC = post.getResponseCode(); + if (! postRC.equals(200)) { + log.warn(post.getErrorStream().getText()); + } + } + // // Print pipeline summary on completion // diff --git a/lib/Utils.groovy b/lib/Utils.groovy old mode 100755 new mode 100644 index 28567bd7..8d030f4e --- a/lib/Utils.groovy +++ b/lib/Utils.groovy @@ -21,19 +21,26 @@ class Utils { } // Check that all channels are present - def required_channels = ['conda-forge', 'bioconda', 'defaults'] - def conda_check_failed = !required_channels.every { ch -> ch in channels } + // This channel list is ordered by required channel priority. + def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults'] + def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean // Check that they are in the right order - conda_check_failed |= !(channels.indexOf('conda-forge') < channels.indexOf('bioconda')) - conda_check_failed |= !(channels.indexOf('bioconda') < channels.indexOf('defaults')) + def channel_priority_violation = false + def n = required_channels_in_order.size() + for (int i = 0; i < n - 1; i++) { + channel_priority_violation |= !(channels.indexOf(required_channels_in_order[i]) < channels.indexOf(required_channels_in_order[i+1])) + } - if (conda_check_failed) { + if (channels_missing | channel_priority_violation) { log.warn "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + " There is a problem with your Conda configuration!\n\n" + " You will need to set-up the conda-forge and bioconda channels correctly.\n" + - " Please refer to https://bioconda.github.io/user/install.html#set-up-channels\n" + - " NB: The order of the channels matters!\n" + + " Please refer to https://bioconda.github.io/\n" + + " The observed channel order is \n" + + " ${channels}\n" + + " but the following channel order is required:\n" + + " ${required_channels_in_order}\n" + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" } } diff --git a/lib/WorkflowSmrnaseq.groovy b/lib/WorkflowSmrnaseq.groovy index 5bc00333..3190e322 100755 --- a/lib/WorkflowSmrnaseq.groovy +++ b/lib/WorkflowSmrnaseq.groovy @@ -2,6 +2,8 @@ // This file holds several functions specific to the workflow/smrnaseq.nf in the nf-core/smrnaseq pipeline // +import groovy.text.SimpleTemplateEngine + class WorkflowSmrnaseq { // @@ -42,6 +44,23 @@ class WorkflowSmrnaseq { yaml_file_text += "data: |\n" yaml_file_text += "${summary_section}" return yaml_file_text + } + + public static String methodsDescriptionText(run_workflow, mqc_methods_yaml) { + // Convert to a named map so can be used as with familar NXF ${workflow} variable syntax in the MultiQC YML file + def meta = [:] + meta.workflow = run_workflow.toMap() + meta["manifest_map"] = run_workflow.manifest.toMap() + + meta["doi_text"] = meta.manifest_map.doi ? "(doi: ${meta.manifest_map.doi})" : "" + meta["nodoi_text"] = meta.manifest_map.doi ? "": "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " + + def methods_text = mqc_methods_yaml.text + + def engine = new SimpleTemplateEngine() + def description_html = engine.createTemplate(methods_text).make(meta) + + return description_html }// // Exit pipeline if incorrect --genome key provided // diff --git a/main.nf b/main.nf index a7e6c457..adbfad48 100644 --- a/main.nf +++ b/main.nf @@ -4,7 +4,8 @@ nf-core/smrnaseq ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Github : https://github.com/nf-core/smrnaseq -Website: https://nf-co.re/smrnaseq + + Website: https://nf-co.re/smrnaseq Slack : https://nfcore.slack.com/channels/smrnaseq ---------------------------------------------------------------------------------------- */ diff --git a/modules.json b/modules.json index 81f0ce77..70345a21 100644 --- a/modules.json +++ b/modules.json @@ -2,20 +2,21 @@ "name": "nf-core/smrnaseq", "homePage": "https://github.com/nf-core/smrnaseq", "repos": { - "nf-core/modules": { - "git_url": "https://github.com/nf-core/modules.git", + "https://github.com/nf-core/modules.git": { "modules": { - "custom/dumpsoftwareversions": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", - "branch": "master" - }, - "fastqc": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", - "branch": "master" - }, - "multiqc": { - "git_sha": "e745e167c1020928ef20ea1397b6b4d230681b4d", - "branch": "master" + "nf-core": { + "custom/dumpsoftwareversions": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "fastqc": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "multiqc": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + } } } } diff --git a/modules/nf-core/modules/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf similarity index 79% rename from modules/nf-core/modules/custom/dumpsoftwareversions/main.nf rename to modules/nf-core/custom/dumpsoftwareversions/main.nf index 327d5100..cebb6e05 100644 --- a/modules/nf-core/modules/custom/dumpsoftwareversions/main.nf +++ b/modules/nf-core/custom/dumpsoftwareversions/main.nf @@ -1,11 +1,11 @@ process CUSTOM_DUMPSOFTWAREVERSIONS { - label 'process_low' + label 'process_single' // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda (params.enable_conda ? "bioconda::multiqc=1.11" : null) + conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.11--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.11--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : + 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" input: path versions diff --git a/modules/nf-core/modules/custom/dumpsoftwareversions/meta.yml b/modules/nf-core/custom/dumpsoftwareversions/meta.yml similarity index 100% rename from modules/nf-core/modules/custom/dumpsoftwareversions/meta.yml rename to modules/nf-core/custom/dumpsoftwareversions/meta.yml diff --git a/modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py similarity index 100% rename from modules/nf-core/modules/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py rename to modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py diff --git a/modules/nf-core/modules/fastqc/main.nf b/modules/nf-core/fastqc/main.nf similarity index 85% rename from modules/nf-core/modules/fastqc/main.nf rename to modules/nf-core/fastqc/main.nf index ed6b8c50..05730368 100644 --- a/modules/nf-core/modules/fastqc/main.nf +++ b/modules/nf-core/fastqc/main.nf @@ -44,4 +44,16 @@ process FASTQC { END_VERSIONS """ } + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.html + touch ${prefix}.zip + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + fastqc: \$( fastqc --version | sed -e "s/FastQC v//g" ) + END_VERSIONS + """ } diff --git a/modules/nf-core/modules/fastqc/meta.yml b/modules/nf-core/fastqc/meta.yml similarity index 100% rename from modules/nf-core/modules/fastqc/meta.yml rename to modules/nf-core/fastqc/meta.yml diff --git a/modules/nf-core/modules/multiqc/main.nf b/modules/nf-core/modules/multiqc/main.nf deleted file mode 100644 index 1264aac1..00000000 --- a/modules/nf-core/modules/multiqc/main.nf +++ /dev/null @@ -1,31 +0,0 @@ -process MULTIQC { - label 'process_medium' - - conda (params.enable_conda ? 'bioconda::multiqc=1.12' : null) - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.12--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.12--pyhdfd78af_0' }" - - input: - path multiqc_files - - output: - path "*multiqc_report.html", emit: report - path "*_data" , emit: data - path "*_plots" , optional:true, emit: plots - path "versions.yml" , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - """ - multiqc -f $args . - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - multiqc: \$( multiqc --version | sed -e "s/multiqc, version //g" ) - END_VERSIONS - """ -} diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf new file mode 100644 index 00000000..a8159a57 --- /dev/null +++ b/modules/nf-core/multiqc/main.nf @@ -0,0 +1,53 @@ +process MULTIQC { + label 'process_single' + + conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : + 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" + + input: + path multiqc_files, stageAs: "?/*" + path(multiqc_config) + path(extra_multiqc_config) + path(multiqc_logo) + + output: + path "*multiqc_report.html", emit: report + path "*_data" , emit: data + path "*_plots" , optional:true, emit: plots + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + def config = multiqc_config ? "--config $multiqc_config" : '' + def extra_config = extra_multiqc_config ? "--config $extra_multiqc_config" : '' + """ + multiqc \\ + --force \\ + $args \\ + $config \\ + $extra_config \\ + . + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + multiqc: \$( multiqc --version | sed -e "s/multiqc, version //g" ) + END_VERSIONS + """ + + stub: + """ + touch multiqc_data + touch multiqc_plots + touch multiqc_report.html + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + multiqc: \$( multiqc --version | sed -e "s/multiqc, version //g" ) + END_VERSIONS + """ +} diff --git a/modules/nf-core/modules/multiqc/meta.yml b/modules/nf-core/multiqc/meta.yml similarity index 73% rename from modules/nf-core/modules/multiqc/meta.yml rename to modules/nf-core/multiqc/meta.yml index 6fa891ef..ebc29b27 100644 --- a/modules/nf-core/modules/multiqc/meta.yml +++ b/modules/nf-core/multiqc/meta.yml @@ -12,11 +12,25 @@ tools: homepage: https://multiqc.info/ documentation: https://multiqc.info/docs/ licence: ["GPL-3.0-or-later"] + input: - multiqc_files: type: file description: | List of reports / files recognised by MultiQC, for example the html and zip output of FastQC + - multiqc_config: + type: file + description: Optional config yml for MultiQC + pattern: "*.{yml,yaml}" + - extra_multiqc_config: + type: file + description: Second optional config yml for MultiQC. Will override common sections in multiqc_config. + pattern: "*.{yml,yaml}" + - multiqc_logo: + type: file + description: Optional logo file for MultiQC + pattern: "*.{png}" + output: - report: type: file @@ -38,3 +52,4 @@ authors: - "@abhi18av" - "@bunop" - "@drpatelh" + - "@jfy133" diff --git a/nextflow.config b/nextflow.config index e4a50caa..c9ccb710 100644 --- a/nextflow.config +++ b/nextflow.config @@ -21,7 +21,9 @@ params { // MultiQC options multiqc_config = null multiqc_title = null + multiqc_logo = null max_multiqc_email_size = '25.MB' + multiqc_methods_description = null // Boilerplate options outdir = null @@ -31,6 +33,7 @@ params { email_on_fail = null plaintext_email = false monochrome_logs = false + hook_url = null help = false validate_params = true show_hidden_params = false @@ -74,7 +77,6 @@ try { // } - profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { @@ -183,12 +185,13 @@ dag { manifest { name = 'nf-core/smrnaseq' - author = 'P. Ewels , C. Wang , R. Hammarén , L. Pantano ' + author = 'P. Ewels, C. Wang, R. Hammarén, L. Pantano' homePage = 'https://github.com/nf-core/smrnaseq' description = 'Small RNA-Seq Best Practice Analysis Pipeline.' mainScript = 'main.nf' nextflowVersion = '!>=21.10.3' version = '2.1.0dev' + doi = '' } // Load modules.config for DSL2 module specific options diff --git a/nextflow_schema.json b/nextflow_schema.json index 84b4768e..8aaa69c5 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -213,12 +213,30 @@ "fa_icon": "fas fa-palette", "hidden": true }, + "hook_url": { + "type": "string", + "description": "Incoming hook URL for messaging service", + "fa_icon": "fas fa-people-group", + "help_text": "Incoming hook URL for messaging service. Currently, only MS Teams is supported.", + "hidden": true + }, "multiqc_config": { "type": "string", "description": "Custom config file to supply to MultiQC.", "fa_icon": "fas fa-cog", "hidden": true }, + "multiqc_logo": { + "type": "string", + "description": "Custom logo file to supply to MultiQC. File name must also be set in the MultiQC config file", + "fa_icon": "fas fa-image", + "hidden": true + }, + "multiqc_methods_description": { + "type": "string", + "description": "Custom MultiQC yaml file containing HTML including a methods description.", + "fa_icon": "fas fa-cog" + }, "tracedir": { "type": "string", "description": "Directory to keep pipeline Nextflow logs and reports.", diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index b439836c..1d5a682d 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -23,8 +23,10 @@ if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input sample ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -ch_multiqc_config = file("$projectDir/assets/multiqc_config.yml", checkIfExists: true) -ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config) : Channel.empty() +ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) +ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath( params.multiqc_config, checkIfExists: true ) : Channel.empty() +ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath( params.multiqc_logo, checkIfExists: true ) : Channel.empty() +ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -46,9 +48,9 @@ include { INPUT_CHECK } from '../subworkflows/local/input_check' // // MODULE: Installed directly from nf-core/modules // -include { FASTQC } from '../modules/nf-core/modules/fastqc/main' -include { MULTIQC } from '../modules/nf-core/modules/multiqc/main' -include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/modules/custom/dumpsoftwareversions/main' +include { FASTQC } from '../modules/nf-core/fastqc/main' +include { MULTIQC } from '../modules/nf-core/multiqc/main' +include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoftwareversions/main' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -89,15 +91,20 @@ workflow SMRNASEQ { workflow_summary = WorkflowSmrnaseq.paramsSummaryMultiqc(workflow, summary_params) ch_workflow_summary = Channel.value(workflow_summary) + methods_description = WorkflowSmrnaseq.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description) + ch_methods_description = Channel.value(methods_description) + ch_multiqc_files = Channel.empty() - ch_multiqc_files = ch_multiqc_files.mix(Channel.from(ch_multiqc_config)) - ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.collect().ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect()) ch_multiqc_files = ch_multiqc_files.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) MULTIQC ( - ch_multiqc_files.collect() + ch_multiqc_files.collect(), + ch_multiqc_config.collect().ifEmpty([]), + ch_multiqc_custom_config.collect().ifEmpty([]), + ch_multiqc_logo.collect().ifEmpty([]) ) multiqc_report = MULTIQC.out.report.toList() ch_versions = ch_versions.mix(MULTIQC.out.versions) @@ -114,6 +121,9 @@ workflow.onComplete { NfcoreTemplate.email(workflow, params, summary_params, projectDir, log, multiqc_report) } NfcoreTemplate.summary(workflow, params, log) + if (params.hook_url) { + NfcoreTemplate.adaptivecard(workflow, params, summary_params, projectDir, log) + } } /* From b694ee171e1dcdcc01abb1443bd4c52a1f6971a8 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Fri, 7 Oct 2022 12:55:34 +0000 Subject: [PATCH 103/145] Tiny changes --- modules.json | 80 +++++++-------- .../nf-core/{modules => }/cat/fastq/main.nf | 10 +- .../nf-core/{modules => }/cat/fastq/meta.yml | 0 .../templates/dumpsoftwareversions.py | 98 +++++++++++-------- modules/nf-core/modules/multiqc/main.nf | 50 ---------- .../{modules => }/samtools/flagstat/main.nf | 0 .../{modules => }/samtools/flagstat/meta.yml | 0 .../{modules => }/samtools/idxstats/main.nf | 1 + .../{modules => }/samtools/idxstats/meta.yml | 0 .../{modules => }/samtools/index/main.nf | 0 .../{modules => }/samtools/index/meta.yml | 0 .../{modules => }/samtools/sort/main.nf | 1 + .../{modules => }/samtools/sort/meta.yml | 4 + .../{modules => }/samtools/stats/main.nf | 0 .../{modules => }/samtools/stats/meta.yml | 0 .../nf-core/{modules => }/trimgalore/main.nf | 2 +- .../nf-core/{modules => }/trimgalore/meta.yml | 0 subworkflows/nf-core/bam_sort_samtools.nf | 4 +- subworkflows/nf-core/bam_stats_samtools.nf | 6 +- subworkflows/nf-core/fastqc_trimgalore.nf | 2 +- workflows/smrnaseq.nf | 4 + 21 files changed, 118 insertions(+), 144 deletions(-) rename modules/nf-core/{modules => }/cat/fastq/main.nf (88%) rename modules/nf-core/{modules => }/cat/fastq/meta.yml (100%) mode change 100644 => 100755 modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py delete mode 100644 modules/nf-core/modules/multiqc/main.nf rename modules/nf-core/{modules => }/samtools/flagstat/main.nf (100%) rename modules/nf-core/{modules => }/samtools/flagstat/meta.yml (100%) rename modules/nf-core/{modules => }/samtools/idxstats/main.nf (96%) rename modules/nf-core/{modules => }/samtools/idxstats/meta.yml (100%) rename modules/nf-core/{modules => }/samtools/index/main.nf (100%) rename modules/nf-core/{modules => }/samtools/index/meta.yml (100%) rename modules/nf-core/{modules => }/samtools/sort/main.nf (95%) rename modules/nf-core/{modules => }/samtools/sort/meta.yml (92%) rename modules/nf-core/{modules => }/samtools/stats/main.nf (100%) rename modules/nf-core/{modules => }/samtools/stats/meta.yml (100%) rename modules/nf-core/{modules => }/trimgalore/main.nf (99%) rename modules/nf-core/{modules => }/trimgalore/meta.yml (100%) diff --git a/modules.json b/modules.json index 3c50ceaf..96095adf 100644 --- a/modules.json +++ b/modules.json @@ -4,45 +4,47 @@ "repos": { "https://github.com/nf-core/modules.git": { "modules": { - "cat/fastq": { - "branch": "master", - "git_sha": "b034029b59b1198075da8019074bc02051a6100e" - }, - "custom/dumpsoftwareversions": { - "branch": "master", - "git_sha": "5e7b1ef9a5a2d9258635bcbf70fcf37dacd1b247" - }, - "fastqc": { - "branch": "master", - "git_sha": "49b18b1639f4f7104187058866a8fab33332bdfe" - }, - "multiqc": { - "branch": "master", - "git_sha": "5587389874dac9c9953a2ab6f01d49af81969492" - }, - "samtools/flagstat": { - "branch": "master", - "git_sha": "bbb99cb8d679555cc01c98766de7869f83283545" - }, - "samtools/idxstats": { - "branch": "master", - "git_sha": "ecece498f10b47b7c9d06f53a310cea5811b4c5f" - }, - "samtools/index": { - "branch": "master", - "git_sha": "897c33d5da084b61109500ee44c01da2d3e4e773" - }, - "samtools/sort": { - "branch": "master", - "git_sha": "897c33d5da084b61109500ee44c01da2d3e4e773" - }, - "samtools/stats": { - "branch": "master", - "git_sha": "f4eab7945952dc4934224309701a49913ea05ae6" - }, - "trimgalore": { - "branch": "master", - "git_sha": "85ec13ff1fc2196c5a507ea497de468101baabed" + "nf-core": { + "cat/fastq": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "custom/dumpsoftwareversions": { + "branch": "master", + "git_sha": "8022c68e7403eecbd8ba9c49496f69f8c49d50f0" + }, + "fastqc": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "multiqc": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "samtools/flagstat": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "samtools/idxstats": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "samtools/index": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "samtools/sort": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "samtools/stats": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "trimgalore": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + } } } } diff --git a/modules/nf-core/modules/cat/fastq/main.nf b/modules/nf-core/cat/fastq/main.nf similarity index 88% rename from modules/nf-core/modules/cat/fastq/main.nf rename to modules/nf-core/cat/fastq/main.nf index d275f19c..4fa365d3 100644 --- a/modules/nf-core/modules/cat/fastq/main.nf +++ b/modules/nf-core/cat/fastq/main.nf @@ -1,6 +1,6 @@ process CAT_FASTQ { tag "$meta.id" - label 'process_low' + label 'process_single' conda (params.enable_conda ? "conda-forge::sed=4.7" : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? @@ -20,9 +20,9 @@ process CAT_FASTQ { script: def args = task.ext.args ?: '' def prefix = task.ext.prefix ?: "${meta.id}" - def readList = reads.collect{ it.toString() } + def readList = reads instanceof List ? reads.collect{ it.toString() } : [reads.toString()] if (meta.single_end) { - if (readList.size > 1) { + if (readList.size >= 1) { """ cat ${readList.join(' ')} > ${prefix}.merged.fastq.gz @@ -33,7 +33,7 @@ process CAT_FASTQ { """ } } else { - if (readList.size > 2) { + if (readList.size >= 2) { def read1 = [] def read2 = [] readList.eachWithIndex{ v, ix -> ( ix & 1 ? read2 : read1 ) << v } @@ -51,7 +51,7 @@ process CAT_FASTQ { stub: def prefix = task.ext.prefix ?: "${meta.id}" - def readList = reads.collect{ it.toString() } + def readList = reads instanceof List ? reads.collect{ it.toString() } : [reads.toString()] if (meta.single_end) { if (readList.size > 1) { """ diff --git a/modules/nf-core/modules/cat/fastq/meta.yml b/modules/nf-core/cat/fastq/meta.yml similarity index 100% rename from modules/nf-core/modules/cat/fastq/meta.yml rename to modules/nf-core/cat/fastq/meta.yml diff --git a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py old mode 100644 new mode 100755 index d1390392..da033408 --- a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py @@ -1,11 +1,16 @@ #!/usr/bin/env python + +"""Provide functions to merge multiple versions.yml files.""" + + import yaml import platform from textwrap import dedent def _make_versions_html(versions): + """Generate a tabular HTML output of all versions for MultiQC.""" html = [ dedent( """\\ @@ -44,46 +49,53 @@ def _make_versions_html(versions): return "\\n".join(html) -versions_this_module = {} -versions_this_module["${task.process}"] = { - "python": platform.python_version(), - "yaml": yaml.__version__, -} - -with open("$versions") as f: - versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module - -# aggregate versions by the module name (derived from fully-qualified process name) -versions_by_module = {} -for process, process_versions in versions_by_process.items(): - module = process.split(":")[-1] - try: - assert versions_by_module[module] == process_versions, ( - "We assume that software versions are the same between all modules. " - "If you see this error-message it means you discovered an edge-case " - "and should open an issue in nf-core/tools. " - ) - except KeyError: - versions_by_module[module] = process_versions - -versions_by_module["Workflow"] = { - "Nextflow": "$workflow.nextflow.version", - "$workflow.manifest.name": "$workflow.manifest.version", -} - -versions_mqc = { - "id": "software_versions", - "section_name": "${workflow.manifest.name} Software Versions", - "section_href": "https://github.com/${workflow.manifest.name}", - "plot_type": "html", - "description": "are collected at run time from the software output.", - "data": _make_versions_html(versions_by_module), -} - -with open("software_versions.yml", "w") as f: - yaml.dump(versions_by_module, f, default_flow_style=False) -with open("software_versions_mqc.yml", "w") as f: - yaml.dump(versions_mqc, f, default_flow_style=False) - -with open("versions.yml", "w") as f: - yaml.dump(versions_this_module, f, default_flow_style=False) +def main(): + """Load all version files and generate merged output.""" + versions_this_module = {} + versions_this_module["${task.process}"] = { + "python": platform.python_version(), + "yaml": yaml.__version__, + } + + with open("$versions") as f: + versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module + + # aggregate versions by the module name (derived from fully-qualified process name) + versions_by_module = {} + for process, process_versions in versions_by_process.items(): + module = process.split(":")[-1] + try: + if versions_by_module[module] != process_versions: + raise AssertionError( + "We assume that software versions are the same between all modules. " + "If you see this error-message it means you discovered an edge-case " + "and should open an issue in nf-core/tools. " + ) + except KeyError: + versions_by_module[module] = process_versions + + versions_by_module["Workflow"] = { + "Nextflow": "$workflow.nextflow.version", + "$workflow.manifest.name": "$workflow.manifest.version", + } + + versions_mqc = { + "id": "software_versions", + "section_name": "${workflow.manifest.name} Software Versions", + "section_href": "https://github.com/${workflow.manifest.name}", + "plot_type": "html", + "description": "are collected at run time from the software output.", + "data": _make_versions_html(versions_by_module), + } + + with open("software_versions.yml", "w") as f: + yaml.dump(versions_by_module, f, default_flow_style=False) + with open("software_versions_mqc.yml", "w") as f: + yaml.dump(versions_mqc, f, default_flow_style=False) + + with open("versions.yml", "w") as f: + yaml.dump(versions_this_module, f, default_flow_style=False) + + +if __name__ == "__main__": + main() diff --git a/modules/nf-core/modules/multiqc/main.nf b/modules/nf-core/modules/multiqc/main.nf deleted file mode 100644 index d10dae69..00000000 --- a/modules/nf-core/modules/multiqc/main.nf +++ /dev/null @@ -1,50 +0,0 @@ -process MULTIQC { - label 'process_medium' - - conda (params.enable_conda ? 'bioconda::multiqc=1.13' : null) - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.13--pyhdfd78af_0' : - 'quay.io/biocontainers/multiqc:1.13--pyhdfd78af_0' }" - - input: - path multiqc_files, stageAs: "?/*" - path(multiqc_config) - path(multiqc_logo) - - output: - path "*multiqc_report.html", emit: report - path "*_data" , emit: data - path "*_plots" , optional:true, emit: plots - path "versions.yml" , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - def config = multiqc_config ? "--config $multiqc_config" : '' - """ - multiqc \\ - --force \\ - $config \\ - $args \\ - . - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - multiqc: \$( multiqc --version | sed -e "s/multiqc, version //g" ) - END_VERSIONS - """ - - stub: - """ - touch multiqc_data - touch multiqc_plots - touch multiqc_report.html - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - multiqc: \$( multiqc --version | sed -e "s/multiqc, version //g" ) - END_VERSIONS - """ -} diff --git a/modules/nf-core/modules/samtools/flagstat/main.nf b/modules/nf-core/samtools/flagstat/main.nf similarity index 100% rename from modules/nf-core/modules/samtools/flagstat/main.nf rename to modules/nf-core/samtools/flagstat/main.nf diff --git a/modules/nf-core/modules/samtools/flagstat/meta.yml b/modules/nf-core/samtools/flagstat/meta.yml similarity index 100% rename from modules/nf-core/modules/samtools/flagstat/meta.yml rename to modules/nf-core/samtools/flagstat/meta.yml diff --git a/modules/nf-core/modules/samtools/idxstats/main.nf b/modules/nf-core/samtools/idxstats/main.nf similarity index 96% rename from modules/nf-core/modules/samtools/idxstats/main.nf rename to modules/nf-core/samtools/idxstats/main.nf index 4b245419..87618e5f 100644 --- a/modules/nf-core/modules/samtools/idxstats/main.nf +++ b/modules/nf-core/samtools/idxstats/main.nf @@ -24,6 +24,7 @@ process SAMTOOLS_IDXSTATS { """ samtools \\ idxstats \\ + --threads ${task.cpus-1} \\ $bam \\ > ${prefix}.idxstats diff --git a/modules/nf-core/modules/samtools/idxstats/meta.yml b/modules/nf-core/samtools/idxstats/meta.yml similarity index 100% rename from modules/nf-core/modules/samtools/idxstats/meta.yml rename to modules/nf-core/samtools/idxstats/meta.yml diff --git a/modules/nf-core/modules/samtools/index/main.nf b/modules/nf-core/samtools/index/main.nf similarity index 100% rename from modules/nf-core/modules/samtools/index/main.nf rename to modules/nf-core/samtools/index/main.nf diff --git a/modules/nf-core/modules/samtools/index/meta.yml b/modules/nf-core/samtools/index/meta.yml similarity index 100% rename from modules/nf-core/modules/samtools/index/meta.yml rename to modules/nf-core/samtools/index/meta.yml diff --git a/modules/nf-core/modules/samtools/sort/main.nf b/modules/nf-core/samtools/sort/main.nf similarity index 95% rename from modules/nf-core/modules/samtools/sort/main.nf rename to modules/nf-core/samtools/sort/main.nf index b4fc1cbe..ab7f1cca 100644 --- a/modules/nf-core/modules/samtools/sort/main.nf +++ b/modules/nf-core/samtools/sort/main.nf @@ -12,6 +12,7 @@ process SAMTOOLS_SORT { output: tuple val(meta), path("*.bam"), emit: bam + tuple val(meta), path("*.csi"), emit: csi, optional: true path "versions.yml" , emit: versions when: diff --git a/modules/nf-core/modules/samtools/sort/meta.yml b/modules/nf-core/samtools/sort/meta.yml similarity index 92% rename from modules/nf-core/modules/samtools/sort/meta.yml rename to modules/nf-core/samtools/sort/meta.yml index a820c55a..09289751 100644 --- a/modules/nf-core/modules/samtools/sort/meta.yml +++ b/modules/nf-core/samtools/sort/meta.yml @@ -39,6 +39,10 @@ output: type: file description: File containing software versions pattern: "versions.yml" + - csi: + type: file + description: BAM index file (optional) + pattern: "*.csi" authors: - "@drpatelh" - "@ewels" diff --git a/modules/nf-core/modules/samtools/stats/main.nf b/modules/nf-core/samtools/stats/main.nf similarity index 100% rename from modules/nf-core/modules/samtools/stats/main.nf rename to modules/nf-core/samtools/stats/main.nf diff --git a/modules/nf-core/modules/samtools/stats/meta.yml b/modules/nf-core/samtools/stats/meta.yml similarity index 100% rename from modules/nf-core/modules/samtools/stats/meta.yml rename to modules/nf-core/samtools/stats/meta.yml diff --git a/modules/nf-core/modules/trimgalore/main.nf b/modules/nf-core/trimgalore/main.nf similarity index 99% rename from modules/nf-core/modules/trimgalore/main.nf rename to modules/nf-core/trimgalore/main.nf index 3a3fca90..a69e3de6 100644 --- a/modules/nf-core/modules/trimgalore/main.nf +++ b/modules/nf-core/trimgalore/main.nf @@ -32,7 +32,7 @@ process TRIMGALORE { cores = (task.cpus as int) - 4 if (meta.single_end) cores = (task.cpus as int) - 3 if (cores < 1) cores = 1 - if (cores > 4) cores = 4 + if (cores > 8) cores = 8 } // Clipping presets have to be evaluated in the context of SE/PE diff --git a/modules/nf-core/modules/trimgalore/meta.yml b/modules/nf-core/trimgalore/meta.yml similarity index 100% rename from modules/nf-core/modules/trimgalore/meta.yml rename to modules/nf-core/trimgalore/meta.yml diff --git a/subworkflows/nf-core/bam_sort_samtools.nf b/subworkflows/nf-core/bam_sort_samtools.nf index a3f2741a..ded07b32 100644 --- a/subworkflows/nf-core/bam_sort_samtools.nf +++ b/subworkflows/nf-core/bam_sort_samtools.nf @@ -2,8 +2,8 @@ // Sort, index BAM file and run samtools stats, flagstat and idxstats // -include { SAMTOOLS_SORT } from '../../modules/nf-core/modules/samtools/sort/main' -include { SAMTOOLS_INDEX } from '../../modules/nf-core/modules/samtools/index/main' +include { SAMTOOLS_SORT } from '../../modules/nf-core/samtools/sort/main' +include { SAMTOOLS_INDEX } from '../../modules/nf-core/samtools/index/main' include { BAM_STATS_SAMTOOLS } from './bam_stats_samtools' workflow BAM_SORT_SAMTOOLS { diff --git a/subworkflows/nf-core/bam_stats_samtools.nf b/subworkflows/nf-core/bam_stats_samtools.nf index 857ae2c1..38d2569b 100644 --- a/subworkflows/nf-core/bam_stats_samtools.nf +++ b/subworkflows/nf-core/bam_stats_samtools.nf @@ -2,9 +2,9 @@ // Run SAMtools stats, flagstat and idxstats // -include { SAMTOOLS_STATS } from '../../modules/nf-core/modules/samtools/stats/main' -include { SAMTOOLS_IDXSTATS } from '../../modules/nf-core/modules/samtools/idxstats/main' -include { SAMTOOLS_FLAGSTAT } from '../../modules/nf-core/modules/samtools/flagstat/main' +include { SAMTOOLS_STATS } from '../../modules/nf-core/samtools/stats/main' +include { SAMTOOLS_IDXSTATS } from '../../modules/nf-core/samtools/idxstats/main' +include { SAMTOOLS_FLAGSTAT } from '../../modules/nf-core/samtools/flagstat/main' workflow BAM_STATS_SAMTOOLS { take: diff --git a/subworkflows/nf-core/fastqc_trimgalore.nf b/subworkflows/nf-core/fastqc_trimgalore.nf index ba2deecd..3eaee69f 100644 --- a/subworkflows/nf-core/fastqc_trimgalore.nf +++ b/subworkflows/nf-core/fastqc_trimgalore.nf @@ -2,7 +2,7 @@ // Read QC, UMI extraction and trimming // -include { FASTQC } from '../../modules/nf-core/modules/fastqc/main' +include { FASTQC } from '../../modules/nf-core/fastqc/main' include { TRIMGALORE } from '../../modules/local/trimgalore' workflow FASTQC_TRIMGALORE { diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index af0def5b..7fd887d9 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -37,6 +37,7 @@ def mirna_gtf = params.mirna_gtf ?: mirna_gtf_from_species ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath( params.multiqc_config, checkIfExists: true ) : Channel.empty() ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath( params.multiqc_logo, checkIfExists: true ) : Channel.empty() +ch_multiqc_image = file("$projectDir/assets/smrnaseq_logo.png", checkIfExists: true) ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) /* @@ -211,12 +212,15 @@ workflow SMRNASEQ { if (!params.skip_multiqc) { workflow_summary = WorkflowSmrnaseq.paramsSummaryMultiqc(workflow, summary_params) ch_workflow_summary = Channel.value(workflow_summary) + methods_description = WorkflowSmrnaseq.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description) + ch_methods_description = Channel.value(methods_description) ch_multiqc_files = Channel.empty() ch_multiqc_files = ch_multiqc_files.mix(Channel.from(ch_multiqc_config)) ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect()) ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.collect().ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(FASTQC_TRIMGALORE.out.fastqc_zip.collect{it[1]}.ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(contamination_stats.collect().ifEmpty([])) From 56b99c43ea29a733baf89dea70819004811cb7b1 Mon Sep 17 00:00:00 2001 From: Jose Espinosa-Carrasco Date: Fri, 7 Oct 2022 15:52:55 +0200 Subject: [PATCH 104/145] Update workflows/smrnaseq.nf --- workflows/smrnaseq.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 7fd887d9..d9a7d9c7 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -236,7 +236,7 @@ workflow SMRNASEQ { ch_multiqc_custom_config.collect().ifEmpty([]), ch_multiqc_logo.collect().ifEmpty([]) ) - multiqc_report = MULTIQC.out.report.toList() + multiqc_report = MULTIQC.out.report.toList(), ch_versions = ch_versions.mix(MULTIQC.out.versions) } From 1aa9e9f6db63e5bb38059ec60684efab551bbad2 Mon Sep 17 00:00:00 2001 From: Jose Espinosa-Carrasco Date: Fri, 7 Oct 2022 16:00:18 +0200 Subject: [PATCH 105/145] Update workflows/smrnaseq.nf --- workflows/smrnaseq.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index d9a7d9c7..7fd887d9 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -236,7 +236,7 @@ workflow SMRNASEQ { ch_multiqc_custom_config.collect().ifEmpty([]), ch_multiqc_logo.collect().ifEmpty([]) ) - multiqc_report = MULTIQC.out.report.toList(), + multiqc_report = MULTIQC.out.report.toList() ch_versions = ch_versions.mix(MULTIQC.out.versions) } From 68b0403c3647ccb3fac874d928ec74191198d79c Mon Sep 17 00:00:00 2001 From: Jose Espinosa-Carrasco Date: Fri, 7 Oct 2022 16:05:34 +0200 Subject: [PATCH 106/145] Update workflows/smrnaseq.nf --- workflows/smrnaseq.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 7fd887d9..b5d75ca0 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -238,6 +238,7 @@ workflow SMRNASEQ { ) multiqc_report = MULTIQC.out.report.toList() ch_versions = ch_versions.mix(MULTIQC.out.versions) + } } /* From 8a19dab58bd42cba80919220d54dd082d221cbd3 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Fri, 7 Oct 2022 16:11:35 +0200 Subject: [PATCH 107/145] Fix modules include --- workflows/smrnaseq.nf | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index b5d75ca0..6a70bb7d 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -75,10 +75,10 @@ include { MIRDEEP2 } from '../subworkflows/local/mirdeep2' // // MODULE: Installed directly from nf-core/modules // -include { CAT_FASTQ } from '../modules/nf-core/modules/cat/fastq/main' -include { FASTQC } from '../modules/nf-core/modules/fastqc/main' -include { MULTIQC } from '../modules/nf-core/modules/multiqc/main' -include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/modules/custom/dumpsoftwareversions/main' +include { CAT_FASTQ } from '../modules/nf-core/cat/fastq/main' +include { FASTQC } from '../modules/nf-core/fastqc/main' +include { MULTIQC } from '../modules/nf-core/multiqc/main' +include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoftwareversions/main' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -230,14 +230,14 @@ workflow SMRNASEQ { ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.mirtop_logs.collect().ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(MIRTRACE.out.results.collect().ifEmpty([])) - MULTIQC ( - ch_multiqc_files.collect(), - ch_multiqc_config.collect().ifEmpty([]), - ch_multiqc_custom_config.collect().ifEmpty([]), - ch_multiqc_logo.collect().ifEmpty([]) - ) - multiqc_report = MULTIQC.out.report.toList() - ch_versions = ch_versions.mix(MULTIQC.out.versions) + MULTIQC ( + ch_multiqc_files.collect(), + ch_multiqc_config.collect().ifEmpty([]), + ch_multiqc_custom_config.collect().ifEmpty([]), + ch_multiqc_logo.collect().ifEmpty([]) + ) + multiqc_report = MULTIQC.out.report.toList() + ch_versions = ch_versions.mix(MULTIQC.out.versions) } } From d6d5d61eff7de5ac4ec295bd9764c0a6688a756d Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Sun, 9 Oct 2022 11:18:02 +0200 Subject: [PATCH 108/145] Use toList() for input channels of multiqc --- workflows/smrnaseq.nf | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 6a70bb7d..bef00935 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -218,23 +218,23 @@ workflow SMRNASEQ { ch_multiqc_files = Channel.empty() ch_multiqc_files = ch_multiqc_files.mix(Channel.from(ch_multiqc_config)) ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect()) - ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.collect().ifEmpty([])) + ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.toList() ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(FASTQC_TRIMGALORE.out.fastqc_zip.collect{it[1]}.ifEmpty([])) - ch_multiqc_files = ch_multiqc_files.mix(contamination_stats.collect().ifEmpty([])) + ch_multiqc_files = ch_multiqc_files.mix(contamination_stats.toList()) ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.mature_stats.collect({it[1]}).ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.hairpin_stats.collect({it[1]}).ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(genome_stats.collect({it[1]}).ifEmpty([])) - ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.mirtop_logs.collect().ifEmpty([])) - ch_multiqc_files = ch_multiqc_files.mix(MIRTRACE.out.results.collect().ifEmpty([])) + ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.mirtop_logs.toList() + ch_multiqc_files = ch_multiqc_files.mix(MIRTRACE.out.results.toList() MULTIQC ( ch_multiqc_files.collect(), - ch_multiqc_config.collect().ifEmpty([]), - ch_multiqc_custom_config.collect().ifEmpty([]), - ch_multiqc_logo.collect().ifEmpty([]) + ch_multiqc_config.toList(), + ch_multiqc_custom_config.toList(), + ch_multiqc_logo.toList() ) multiqc_report = MULTIQC.out.report.toList() ch_versions = ch_versions.mix(MULTIQC.out.versions) From c4e15c10ebb49c5a7a21b409807e74e9c7f60dd1 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Sun, 9 Oct 2022 11:19:28 +0200 Subject: [PATCH 109/145] Typo fix --- workflows/smrnaseq.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index bef00935..a579685a 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -218,7 +218,7 @@ workflow SMRNASEQ { ch_multiqc_files = Channel.empty() ch_multiqc_files = ch_multiqc_files.mix(Channel.from(ch_multiqc_config)) ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect()) - ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.toList() + ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.toList()) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) From f34f0d0625f1c39ff121522f5be6209b07942c09 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Sun, 9 Oct 2022 11:20:40 +0200 Subject: [PATCH 110/145] Its typo sunday --- workflows/smrnaseq.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index a579685a..38a8bf03 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -227,8 +227,8 @@ workflow SMRNASEQ { ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.mature_stats.collect({it[1]}).ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.hairpin_stats.collect({it[1]}).ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(genome_stats.collect({it[1]}).ifEmpty([])) - ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.mirtop_logs.toList() - ch_multiqc_files = ch_multiqc_files.mix(MIRTRACE.out.results.toList() + ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.mirtop_logs.toList()) + ch_multiqc_files = ch_multiqc_files.mix(MIRTRACE.out.results.toList()) MULTIQC ( ch_multiqc_files.collect(), From e41c7d670492d20ab1fb8d0e22bf3df5d417c469 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Sun, 9 Oct 2022 14:11:37 +0200 Subject: [PATCH 111/145] Cleaned up MQC config stuff --- workflows/smrnaseq.nf | 3 --- 1 file changed, 3 deletions(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 38a8bf03..9ef916b9 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -37,7 +37,6 @@ def mirna_gtf = params.mirna_gtf ?: mirna_gtf_from_species ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath( params.multiqc_config, checkIfExists: true ) : Channel.empty() ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath( params.multiqc_logo, checkIfExists: true ) : Channel.empty() -ch_multiqc_image = file("$projectDir/assets/smrnaseq_logo.png", checkIfExists: true) ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) /* @@ -216,9 +215,7 @@ workflow SMRNASEQ { ch_methods_description = Channel.value(methods_description) ch_multiqc_files = Channel.empty() - ch_multiqc_files = ch_multiqc_files.mix(Channel.from(ch_multiqc_config)) ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect()) - ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.toList()) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) From 56e57fe37c32209a95cb6a10f05f6a597695fefd Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 10 Oct 2022 07:56:55 +0000 Subject: [PATCH 112/145] Fix everything, back to old normal --- workflows/smrnaseq.nf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 9ef916b9..f39fb7bd 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -218,14 +218,13 @@ workflow SMRNASEQ { ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect()) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(FASTQC_TRIMGALORE.out.fastqc_zip.collect{it[1]}.ifEmpty([])) - ch_multiqc_files = ch_multiqc_files.mix(contamination_stats.toList()) + ch_multiqc_files = ch_multiqc_files.mix(contamination_stats.collect().ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.mature_stats.collect({it[1]}).ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.hairpin_stats.collect({it[1]}).ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(genome_stats.collect({it[1]}).ifEmpty([])) - ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.mirtop_logs.toList()) - ch_multiqc_files = ch_multiqc_files.mix(MIRTRACE.out.results.toList()) + ch_multiqc_files = ch_multiqc_files.mix(MIRNA_QUANT.out.mirtop_logs.collect().ifEmpty([])) + ch_multiqc_files = ch_multiqc_files.mix(MIRTRACE.out.results.collect().ifEmpty([])) MULTIQC ( ch_multiqc_files.collect(), From 706ed69a2a9fc61f5d4389e87e8bc10849d2c3e8 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 10 Oct 2022 08:25:46 +0000 Subject: [PATCH 113/145] Added back fastp, not sure what went wrong --- modules.json | 80 ++++++++++---------- modules/nf-core/{modules => }/fastp/main.nf | 0 modules/nf-core/{modules => }/fastp/meta.yml | 0 subworkflows/nf-core/fastqc_fastp.nf | 6 +- 4 files changed, 44 insertions(+), 42 deletions(-) rename modules/nf-core/{modules => }/fastp/main.nf (100%) rename modules/nf-core/{modules => }/fastp/meta.yml (100%) diff --git a/modules.json b/modules.json index 4a74b7f4..173e2bf0 100644 --- a/modules.json +++ b/modules.json @@ -4,45 +4,47 @@ "repos": { "https://github.com/nf-core/modules.git": { "modules": { - "cat/fastq": { - "branch": "master", - "git_sha": "b034029b59b1198075da8019074bc02051a6100e" - }, - "custom/dumpsoftwareversions": { - "branch": "master", - "git_sha": "5e7b1ef9a5a2d9258635bcbf70fcf37dacd1b247" - }, - "fastp": { - "branch": "master", - "git_sha": "2c70c1c1951aaf884d2e8d8d9c871db79f7b35aa" - }, - "fastqc": { - "branch": "master", - "git_sha": "49b18b1639f4f7104187058866a8fab33332bdfe" - }, - "multiqc": { - "branch": "master", - "git_sha": "5587389874dac9c9953a2ab6f01d49af81969492" - }, - "samtools/flagstat": { - "branch": "master", - "git_sha": "bbb99cb8d679555cc01c98766de7869f83283545" - }, - "samtools/idxstats": { - "branch": "master", - "git_sha": "ecece498f10b47b7c9d06f53a310cea5811b4c5f" - }, - "samtools/index": { - "branch": "master", - "git_sha": "897c33d5da084b61109500ee44c01da2d3e4e773" - }, - "samtools/sort": { - "branch": "master", - "git_sha": "897c33d5da084b61109500ee44c01da2d3e4e773" - }, - "samtools/stats": { - "branch": "master", - "git_sha": "f4eab7945952dc4934224309701a49913ea05ae6" + "nf-core": { + "cat/fastq": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "custom/dumpsoftwareversions": { + "branch": "master", + "git_sha": "8022c68e7403eecbd8ba9c49496f69f8c49d50f0" + }, + "fastp": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "fastqc": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "multiqc": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "samtools/flagstat": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "samtools/idxstats": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "samtools/index": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "samtools/sort": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + }, + "samtools/stats": { + "branch": "master", + "git_sha": "5e34754d42cd2d5d248ca8673c0a53cdf5624905" + } } } } diff --git a/modules/nf-core/modules/fastp/main.nf b/modules/nf-core/fastp/main.nf similarity index 100% rename from modules/nf-core/modules/fastp/main.nf rename to modules/nf-core/fastp/main.nf diff --git a/modules/nf-core/modules/fastp/meta.yml b/modules/nf-core/fastp/meta.yml similarity index 100% rename from modules/nf-core/modules/fastp/meta.yml rename to modules/nf-core/fastp/meta.yml diff --git a/subworkflows/nf-core/fastqc_fastp.nf b/subworkflows/nf-core/fastqc_fastp.nf index 87edb94f..27fdce53 100644 --- a/subworkflows/nf-core/fastqc_fastp.nf +++ b/subworkflows/nf-core/fastqc_fastp.nf @@ -2,9 +2,9 @@ // Read QC and trimming // -include { FASTQC as FASTQC_RAW } from '../../modules/nf-core/modules/fastqc/main' -include { FASTQC as FASTQC_TRIM } from '../../modules/nf-core/modules/fastqc/main' -include { FASTP } from '../../modules/nf-core/modules/fastp/main' +include { FASTQC as FASTQC_RAW } from '../../modules/nf-core/fastqc/main' +include { FASTQC as FASTQC_TRIM } from '../../modules/nf-core/fastqc/main' +include { FASTP } from '../../modules/nf-core/fastp/main' // // Function that parses fastp json output file to get total number of reads after trimming From 4524c6245d464f3bc13a9d42ee47edc14bc83482 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 10 Oct 2022 08:32:15 +0000 Subject: [PATCH 114/145] Added when statements to most local modules --- modules/local/blat_mirna.nf | 4 +++- modules/local/bowtie_contaminants.nf | 3 +++ modules/local/bowtie_genome.nf | 3 +++ modules/local/bowtie_map_contaminants.nf | 5 ++++- modules/local/bowtie_map_mirna.nf | 3 +++ modules/local/bowtie_mirna.nf | 3 +++ modules/local/datatable_merge.nf | 3 +++ modules/local/edger_qc.nf | 3 +++ modules/local/filter_stats.nf | 3 +++ modules/local/format_fasta_mirna.nf | 3 +++ modules/local/mirdeep2_mapper.nf | 2 ++ modules/local/mirdeep2_prepare.nf | 3 +++ modules/local/mirdeep2_run.nf | 3 +++ modules/local/mirtop_quant.nf | 3 +++ modules/local/mirtrace.nf | 3 +++ modules/local/parse_fasta_mirna.nf | 3 +++ modules/local/samplesheet_check.nf | 3 +++ modules/local/seqcluster_collapse.nf | 3 +++ 18 files changed, 54 insertions(+), 2 deletions(-) diff --git a/modules/local/blat_mirna.nf b/modules/local/blat_mirna.nf index b0037565..da7f6449 100644 --- a/modules/local/blat_mirna.nf +++ b/modules/local/blat_mirna.nf @@ -17,8 +17,10 @@ process BLAT_MIRNA { path 'filtered.fa' , emit: filtered_set path "versions.yml" , emit: versions - script: + when: + task.ext.when == null || task.ext.when + script: if ( db_type == "cdna" ) """ echo $db_type diff --git a/modules/local/bowtie_contaminants.nf b/modules/local/bowtie_contaminants.nf index 4fd04643..bd8324b0 100644 --- a/modules/local/bowtie_contaminants.nf +++ b/modules/local/bowtie_contaminants.nf @@ -13,6 +13,9 @@ process INDEX_CONTAMINANTS { path 'fasta_bidx*' , emit: index path "versions.yml" , emit: versions + when: + task.ext.when == null || task.ext.when + script: """ bowtie2-build ${fasta} fasta_bidx --threads ${task.cpus} diff --git a/modules/local/bowtie_genome.nf b/modules/local/bowtie_genome.nf index cdf572f7..08379866 100644 --- a/modules/local/bowtie_genome.nf +++ b/modules/local/bowtie_genome.nf @@ -15,6 +15,9 @@ process INDEX_GENOME { path 'genome.edited.fa', emit: fasta path "versions.yml" , emit: versions + when: + task.ext.when == null || task.ext.when + script: """ # Remove any special base characters from reference genome FASTA file diff --git a/modules/local/bowtie_map_contaminants.nf b/modules/local/bowtie_map_contaminants.nf index 397aae82..07ce3a28 100644 --- a/modules/local/bowtie_map_contaminants.nf +++ b/modules/local/bowtie_map_contaminants.nf @@ -15,7 +15,10 @@ process BOWTIE_MAP_CONTAMINANTS { tuple val(meta), path("*sam") , emit: bam tuple val(meta), path('*.filter.unmapped.contaminant.fastq'), emit: unmapped path "versions.yml" , emit: versions - path "filtered.*.stats" , emit: stats + path "filtered.*.stats" + , emit: stats + when: + task.ext.when == null || task.ext.when script: """ diff --git a/modules/local/bowtie_map_mirna.nf b/modules/local/bowtie_map_mirna.nf index 316f94af..c35f9a68 100644 --- a/modules/local/bowtie_map_mirna.nf +++ b/modules/local/bowtie_map_mirna.nf @@ -16,6 +16,9 @@ process BOWTIE_MAP_SEQ { tuple val(meta), path('unmapped/*fq.gz'), emit: unmapped path "versions.yml" , emit: versions + when: + task.ext.when == null || task.ext.when + script: """ INDEX=`find -L ./ -name "*.3.ebwt" | sed 's/.3.ebwt//'` diff --git a/modules/local/bowtie_mirna.nf b/modules/local/bowtie_mirna.nf index 07050538..0da7349c 100644 --- a/modules/local/bowtie_mirna.nf +++ b/modules/local/bowtie_mirna.nf @@ -13,6 +13,9 @@ process INDEX_MIRNA { path 'fasta_bidx*' , emit: index path "versions.yml", emit: versions + when: + task.ext.when == null || task.ext.when + script: """ bowtie-build ${fasta} fasta_bidx --threads ${task.cpus} diff --git a/modules/local/datatable_merge.nf b/modules/local/datatable_merge.nf index 3fec291b..8a1ed3b5 100644 --- a/modules/local/datatable_merge.nf +++ b/modules/local/datatable_merge.nf @@ -13,6 +13,9 @@ process TABLE_MERGE { path "mirna.tsv" , emit: mirna_tsv path "versions.yml", emit: versions + when: + task.ext.when == null || task.ext.when + script: """ collapse_mirtop.r ${mirtop} diff --git a/modules/local/edger_qc.nf b/modules/local/edger_qc.nf index d8a4c520..023f70be 100644 --- a/modules/local/edger_qc.nf +++ b/modules/local/edger_qc.nf @@ -13,6 +13,9 @@ process EDGER_QC { path '*.{txt,pdf,csv}', emit: edger_files path "versions.yml" , emit: versions + when: + task.ext.when == null || task.ext.when + script: """ edgeR_miRBase.r $input_files diff --git a/modules/local/filter_stats.nf b/modules/local/filter_stats.nf index f0819e99..1db0d49b 100644 --- a/modules/local/filter_stats.nf +++ b/modules/local/filter_stats.nf @@ -14,6 +14,9 @@ process FILTER_STATS { path "*_mqc.yaml" , emit: stats tuple val(meta), path('*.filtered.fastq.gz') , emit: reads + when: + task.ext.when == null || task.ext.when + script: """ readnumber=\$(wc -l ${reads} | awk '{ print \$1/4 }') diff --git a/modules/local/format_fasta_mirna.nf b/modules/local/format_fasta_mirna.nf index ae8a3fda..d19901fd 100644 --- a/modules/local/format_fasta_mirna.nf +++ b/modules/local/format_fasta_mirna.nf @@ -16,6 +16,9 @@ process FORMAT_FASTA_MIRNA { path '*_idx.fa' , emit: formatted_fasta path "versions.yml", emit: versions + when: + task.ext.when == null || task.ext.when + script: """ fasta_formatter -w 0 -i $fasta -o ${fasta}_idx.fa diff --git a/modules/local/mirdeep2_mapper.nf b/modules/local/mirdeep2_mapper.nf index 7368f4b0..7d90712b 100644 --- a/modules/local/mirdeep2_mapper.nf +++ b/modules/local/mirdeep2_mapper.nf @@ -17,6 +17,8 @@ process MIRDEEP2_MAPPER { tuple path('*_collapsed.fa'), path('*reads_vs_refdb.arf'), emit: mirdeep2_inputs path "versions.yml" , emit: versions + when: + task.ext.when == null || task.ext.when script: def index_base = index.toString().tokenize(' ')[0].tokenize('.')[0] diff --git a/modules/local/mirdeep2_prepare.nf b/modules/local/mirdeep2_prepare.nf index d49270ff..9fcc2ed5 100644 --- a/modules/local/mirdeep2_prepare.nf +++ b/modules/local/mirdeep2_prepare.nf @@ -15,6 +15,9 @@ process MIRDEEP2_PIGZ { tuple val(meta), path("*.{fastq,fq}"), emit: reads path "versions.yml" , emit: versions + when: + task.ext.when == null || task.ext.when + script: """ pigz -f -d -p $task.cpus $reads diff --git a/modules/local/mirdeep2_run.nf b/modules/local/mirdeep2_run.nf index 8f72cbe2..ee4cfc0d 100644 --- a/modules/local/mirdeep2_run.nf +++ b/modules/local/mirdeep2_run.nf @@ -19,6 +19,9 @@ process MIRDEEP2_RUN { path 'result*.{bed,csv,html}', emit: result path "versions.yml" , emit: versions + when: + task.ext.when == null || task.ext.when + script: """ miRDeep2.pl \\ diff --git a/modules/local/mirtop_quant.nf b/modules/local/mirtop_quant.nf index 2ce3e1c6..b6f72a89 100644 --- a/modules/local/mirtop_quant.nf +++ b/modules/local/mirtop_quant.nf @@ -18,6 +18,9 @@ process MIRTOP_QUANT { path "mirtop/stats/*" , emit: logs path "versions.yml" , emit: versions + when: + task.ext.when == null || task.ext.when + script: def filter_species = params.mirgenedb ? params.mirgenedb_species : params.mirtrace_species """ diff --git a/modules/local/mirtrace.nf b/modules/local/mirtrace.nf index fee8e071..cd44adb8 100644 --- a/modules/local/mirtrace.nf +++ b/modules/local/mirtrace.nf @@ -13,6 +13,9 @@ process MIRTRACE_RUN { path "mirtrace/*" , emit: mirtrace path "versions.yml", emit: versions + when: + task.ext.when == null || task.ext.when + script: // mirtrace protocol defaults to 'params.protocol' if not set def primer = params.protocol == 'cats' ? '' : "--adapter ${params.three_prime_adapter}" diff --git a/modules/local/parse_fasta_mirna.nf b/modules/local/parse_fasta_mirna.nf index 88626bec..5a92b202 100644 --- a/modules/local/parse_fasta_mirna.nf +++ b/modules/local/parse_fasta_mirna.nf @@ -13,6 +13,9 @@ process PARSE_FASTA_MIRNA { path '*_igenome.fa', emit: parsed_fasta path "versions.yml", emit: versions + when: + task.ext.when == null || task.ext.when + script: def filter_species = params.mirgenedb ? params.mirgenedb_species : params.mirtrace_species """ diff --git a/modules/local/samplesheet_check.nf b/modules/local/samplesheet_check.nf index c0265a72..74f5a8c9 100644 --- a/modules/local/samplesheet_check.nf +++ b/modules/local/samplesheet_check.nf @@ -14,6 +14,9 @@ process SAMPLESHEET_CHECK { path '*.csv' , emit: csv path "versions.yml", emit: versions + when: + task.ext.when == null || task.ext.when + script: // This script is bundled with the pipeline, in nf-core/smrnaseq/bin/ """ check_samplesheet.py \\ diff --git a/modules/local/seqcluster_collapse.nf b/modules/local/seqcluster_collapse.nf index f430e65f..46239775 100644 --- a/modules/local/seqcluster_collapse.nf +++ b/modules/local/seqcluster_collapse.nf @@ -14,6 +14,9 @@ process SEQCLUSTER_SEQUENCES { tuple val(meta), path("final/*.fastq.gz"), emit: collapsed path "versions.yml" , emit: versions + when: + task.ext.when == null || task.ext.when + script: """ seqcluster collapse -f $reads -m 1 --min_size 15 -o collapsed From ccfd277101649e902e0b2b20688561ae52c6fb36 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 10 Oct 2022 08:38:31 +0000 Subject: [PATCH 115/145] Tiny module update --- modules/local/bowtie_genome.nf | 8 ++++---- modules/local/bowtie_mirna.nf | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/local/bowtie_genome.nf b/modules/local/bowtie_genome.nf index 08379866..1fb38551 100644 --- a/modules/local/bowtie_genome.nf +++ b/modules/local/bowtie_genome.nf @@ -2,10 +2,10 @@ process INDEX_GENOME { tag "$fasta" label 'process_medium' - conda (params.enable_conda ? 'bioconda::bowtie=1.3.0-2' : null) + conda (params.enable_conda ? 'bioconda::bowtie=1.3.1-4' : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/bowtie:1.3.0--py38hcf49a77_2' : - 'quay.io/biocontainers/bowtie:1.3.0--py38hcf49a77_2' }" + 'https://depot.galaxyproject.org/singularity/bowtie%3A1.3.1--py39hd400a0c_2' : + 'quay.io/biocontainers/bowtie:1.3.1--py310h4070885_4' }" input: path fasta @@ -17,7 +17,7 @@ process INDEX_GENOME { when: task.ext.when == null || task.ext.when - + script: """ # Remove any special base characters from reference genome FASTA file diff --git a/modules/local/bowtie_mirna.nf b/modules/local/bowtie_mirna.nf index 0da7349c..0be5888b 100644 --- a/modules/local/bowtie_mirna.nf +++ b/modules/local/bowtie_mirna.nf @@ -3,8 +3,8 @@ process INDEX_MIRNA { conda (params.enable_conda ? 'bioconda::bowtie=1.3.0-2' : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/bowtie:1.3.0--py38hcf49a77_2' : - 'quay.io/biocontainers/bowtie:1.3.0--py38hcf49a77_2' }" + 'https://depot.galaxyproject.org/singularity/bowtie%3A1.3.1--py39hd400a0c_2' : + 'quay.io/biocontainers/bowtie:1.3.1--py310h4070885_4' }" input: path fasta From df2ee0e13d5ef267bbb34b561055150cd97a3734 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 10 Oct 2022 08:41:43 +0000 Subject: [PATCH 116/145] Hopefully fixing editorconfig --- modules/local/bowtie_contaminants.nf | 4 ++-- modules/local/bowtie_map_mirna.nf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/local/bowtie_contaminants.nf b/modules/local/bowtie_contaminants.nf index bd8324b0..1145b56f 100644 --- a/modules/local/bowtie_contaminants.nf +++ b/modules/local/bowtie_contaminants.nf @@ -4,7 +4,7 @@ process INDEX_CONTAMINANTS { conda (params.enable_conda ? 'bowtie2=2.4.5' : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/bowtie2:2.4.5--py39hd2f7db1_2' : - 'quay.io/biocontainers/bowtie2:2.4.5--py36hfca12d5_2' }" + 'quay.io/biocontainers/bowtie2:2.4.5--py36hfca12d5_2'}" input: path fasta @@ -15,7 +15,7 @@ process INDEX_CONTAMINANTS { when: task.ext.when == null || task.ext.when - + script: """ bowtie2-build ${fasta} fasta_bidx --threads ${task.cpus} diff --git a/modules/local/bowtie_map_mirna.nf b/modules/local/bowtie_map_mirna.nf index c35f9a68..82d7acf8 100644 --- a/modules/local/bowtie_map_mirna.nf +++ b/modules/local/bowtie_map_mirna.nf @@ -18,7 +18,7 @@ process BOWTIE_MAP_SEQ { when: task.ext.when == null || task.ext.when - + script: """ INDEX=`find -L ./ -name "*.3.ebwt" | sed 's/.3.ebwt//'` From 2afe4181f50069993340372e743ee502c00e05fe Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 10 Oct 2022 08:59:53 +0000 Subject: [PATCH 117/145] Add back when statements --- modules/local/bowtie_map_contaminants.nf | 5 +++-- workflows/smrnaseq.nf | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/local/bowtie_map_contaminants.nf b/modules/local/bowtie_map_contaminants.nf index 07ce3a28..c3daa9be 100644 --- a/modules/local/bowtie_map_contaminants.nf +++ b/modules/local/bowtie_map_contaminants.nf @@ -15,8 +15,8 @@ process BOWTIE_MAP_CONTAMINANTS { tuple val(meta), path("*sam") , emit: bam tuple val(meta), path('*.filter.unmapped.contaminant.fastq'), emit: unmapped path "versions.yml" , emit: versions - path "filtered.*.stats" - , emit: stats + path "filtered.*.stats" , emit: stats + when: task.ext.when == null || task.ext.when @@ -30,6 +30,7 @@ process BOWTIE_MAP_CONTAMINANTS { -x \$INDEX \\ --un ${meta.id}.${contaminant_type}.filter.unmapped.contaminant.fastq \\ ${reads} \\ + ${args} \\ -S ${meta.id}.filter.contaminant.sam > ${meta.id}.contaminant_bowtie.log 2>&1 # extracting number of reads from bowtie logs diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 104b4c67..80aab3d7 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -74,9 +74,9 @@ include { MIRDEEP2 } from '../subworkflows/local/mirdeep2' // // MODULE: Installed directly from nf-core/modules // -include { CAT_FASTQ } from '../modules/nf-core/modules/cat/fastq/main' -include { MULTIQC } from '../modules/nf-core/modules/multiqc/main' -include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/modules/custom/dumpsoftwareversions/main' +include { CAT_FASTQ } from '../modules/nf-core/cat/fastq/main' +include { MULTIQC } from '../modules/nf-core/multiqc/main' +include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoftwareversions/main' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 75d11a2bd5d9f79684fbe4e1366a610a62ad0685 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 10 Oct 2022 09:16:03 +0000 Subject: [PATCH 118/145] Tried adding more stuff --- conf/modules.config | 2 +- lib/WorkflowSmrnaseq.groovy | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 189e3011..439d30b9 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -104,7 +104,7 @@ if (!params.skip_fastp) { params.three_prime_clip_r1 > 0 ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed. params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}" : "", params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}" : "", - params.three_prime_adapter.isEmpty() ? "" : "--adapter_sequence ${params.three_prime_adapter}" + params.three_prime_adapter == "" ? "" : "--adapter_sequence ${params.three_prime_adapter}" ].join(" ").trim() publishDir = [ [ diff --git a/lib/WorkflowSmrnaseq.groovy b/lib/WorkflowSmrnaseq.groovy index 9075116a..cd8b1e1e 100755 --- a/lib/WorkflowSmrnaseq.groovy +++ b/lib/WorkflowSmrnaseq.groovy @@ -11,11 +11,6 @@ class WorkflowSmrnaseq { // public static void initialise(params, log) { genomeExistsError(params, log) - - // if (!params.fasta) { - // log.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." - // System.exit(1) - // } } // @@ -103,6 +98,9 @@ class WorkflowSmrnaseq { params.putIfAbsent("three_prime_clip_r1", 0); params.putIfAbsent("three_prime_adapter", "AAAAAAAA"); break + case 'custom': + params.putIfAbsent("clip_r1", params.clip_r1) + params.putIfAbsent("three_prime_clip_r1", params.three_prime_clip_r1) default: log.warn "Please make sure to specify all required clipping and trimming parameters, otherwise only adapter detection will be performed." } From d0e64f1c50babce1fea990c975a9003e846452a8 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 10 Oct 2022 09:22:26 +0000 Subject: [PATCH 119/145] Try going back to '' --- conf/modules.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index 439d30b9..32386bfd 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -104,7 +104,7 @@ if (!params.skip_fastp) { params.three_prime_clip_r1 > 0 ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed. params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}" : "", params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}" : "", - params.three_prime_adapter == "" ? "" : "--adapter_sequence ${params.three_prime_adapter}" + params.three_prime_adapter == '' ? "" : "--adapter_sequence ${params.three_prime_adapter}" ].join(" ").trim() publishDir = [ [ From 5ba44edf4a8c9da206b644ab52b16799f74bda5f Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 10 Oct 2022 09:30:51 +0000 Subject: [PATCH 120/145] Hopefully should do the trick --- conf/modules.config | 2 +- nextflow.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 32386bfd..c7b7e140 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -104,7 +104,7 @@ if (!params.skip_fastp) { params.three_prime_clip_r1 > 0 ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed. params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}" : "", params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}" : "", - params.three_prime_adapter == '' ? "" : "--adapter_sequence ${params.three_prime_adapter}" + params.three_prime_adapter == '' ? '' : "--adapter_sequence ${params.three_prime_adapter}" ].join(" ").trim() publishDir = [ [ diff --git a/nextflow.config b/nextflow.config index 4003031f..34ed3c9b 100644 --- a/nextflow.config +++ b/nextflow.config @@ -71,7 +71,7 @@ params { help = false validate_params = true show_hidden_params = false - schema_ignore_params = 'genomes' + schema_ignore_params = 'genomes,three_prime_adapter' enable_conda = false // Config options From c861998ddebccc9b54b8fb9746544c7c5dd110ab Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 10 Oct 2022 09:36:47 +0000 Subject: [PATCH 121/145] Fix things properly --- conf/modules.config | 2 +- nextflow.config | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index c7b7e140..cdd5a80f 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -104,7 +104,7 @@ if (!params.skip_fastp) { params.three_prime_clip_r1 > 0 ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed. params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}" : "", params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}" : "", - params.three_prime_adapter == '' ? '' : "--adapter_sequence ${params.three_prime_adapter}" + params.three_prime_adapter == null ? '' : "--adapter_sequence ${params.three_prime_adapter}" ].join(" ").trim() publishDir = [ [ diff --git a/nextflow.config b/nextflow.config index 34ed3c9b..afb1b01f 100644 --- a/nextflow.config +++ b/nextflow.config @@ -31,7 +31,7 @@ params { // Trimming options clip_r1 = null three_prime_clip_r1 = null - three_prime_adapter = "TGGAATTCTCGGGTGCCAAGG" + three_prime_adapter = null trim_fastq = true fastp_min_length = 17 save_trimmed_fail = false @@ -71,7 +71,7 @@ params { help = false validate_params = true show_hidden_params = false - schema_ignore_params = 'genomes,three_prime_adapter' + schema_ignore_params = 'genomes' enable_conda = false // Config options From 43bed1c38b08d5517eed6958141e30aee6ebbba0 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 10 Oct 2022 10:37:36 +0000 Subject: [PATCH 122/145] add function for adapter sequence output --- subworkflows/nf-core/fastqc_fastp.nf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/subworkflows/nf-core/fastqc_fastp.nf b/subworkflows/nf-core/fastqc_fastp.nf index 27fdce53..a4b5f3db 100644 --- a/subworkflows/nf-core/fastqc_fastp.nf +++ b/subworkflows/nf-core/fastqc_fastp.nf @@ -16,6 +16,11 @@ def getFastpReadsAfterFiltering(json_file) { return json['after_filtering']['total_reads'].toInteger() } +def getFastpAdapterSequence(json_file){ + def Map json = (Map) new JsonSlurper().parseText(json_file.text).get('adapter_cutting') + return json['read1_adapter_sequence'].toString() +} + workflow FASTQC_FASTP { take: reads // channel: [ val(meta), [ reads ] ] From 171288d60d3632a84de6c2fd86f4ac5f54c09865 Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Mon, 10 Oct 2022 14:16:41 +0000 Subject: [PATCH 123/145] Document that a second fastq file will be ignored. --- docs/usage.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 89d438ad..b44ac8c6 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -56,7 +56,9 @@ Contamination filtering of the sequencing reads is optional and can be invoked u ## Samplesheet input -You will need to create a samplesheet with information about the samples you would like to analyse before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row as shown in the examples below. +You will need to create a samplesheet with information about the samples you would like to analyse before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 2 columns ("sample" and "fastq_1"), and a header row as shown in the examples below. + +If a second fastq file is provided using another column, the extra data are ignored by this pipeline. The smRNA species should be sufficiently contained in the first read, and so the second read is superfluous data in this smRNA context. ```bash --input '[path to samplesheet file]' From 9c845d37ee95dc010c51fe7c4834d3cead221bcc Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Mon, 10 Oct 2022 15:30:19 +0000 Subject: [PATCH 124/145] Include adapter sequence in fastq_fastp --- subworkflows/nf-core/fastqc_fastp.nf | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/subworkflows/nf-core/fastqc_fastp.nf b/subworkflows/nf-core/fastqc_fastp.nf index a4b5f3db..b93672bf 100644 --- a/subworkflows/nf-core/fastqc_fastp.nf +++ b/subworkflows/nf-core/fastqc_fastp.nf @@ -12,13 +12,17 @@ include { FASTP } from '../../modules/nf-core/fastp/main' import groovy.json.JsonSlurper def getFastpReadsAfterFiltering(json_file) { - def Map json = (Map) new JsonSlurper().parseText(json_file.text).get('summary') - return json['after_filtering']['total_reads'].toInteger() + return new JsonSlurper().parseText(json_file.text) + ?.get('summary') + ?.get('after_filtering') + ?.get('total_reads') + ?.toInteger() } -def getFastpAdapterSequence(json_file){ - def Map json = (Map) new JsonSlurper().parseText(json_file.text).get('adapter_cutting') - return json['read1_adapter_sequence'].toString() +String getFastpAdapterSequence(json_file){ + return new JsonSlurper().parseText(json_file.text) + ?.get('adapter_cutting') + ?.get('read1_adapter_sequence') } workflow FASTQC_FASTP { @@ -78,6 +82,10 @@ workflow FASTQC_FASTP { } .set { trim_reads } + trim_json + .map { meta, json -> [meta, getFastpAdapterSequence(json)] } + .set { adapterseq } + if (!params.skip_fastqc) { FASTQC_TRIM ( trim_reads @@ -95,6 +103,7 @@ workflow FASTQC_FASTP { trim_log // channel: [ val(meta), [ log ] ] trim_reads_fail // channel: [ val(meta), [ fastq.gz ] ] trim_reads_merged // channel: [ val(meta), [ fastq.gz ] ] + adapterseq // channel: [ val(meat), [ adapterseq ] ] fastqc_raw_html // channel: [ val(meta), [ html ] ] fastqc_raw_zip // channel: [ val(meta), [ zip ] ] From d70b70cf09f4a7a20320065a215ddc68a8ee91b7 Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Mon, 10 Oct 2022 15:30:39 +0000 Subject: [PATCH 125/145] Thread adapter sequence through workflow --- conf/modules.config | 2 +- modules/local/mirtrace.nf | 4 ++-- subworkflows/local/mirtrace.nf | 7 +------ workflows/smrnaseq.nf | 17 +++++++++++------ 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index cdd5a80f..0c5365fb 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -80,7 +80,7 @@ process { process { withName: 'MIRTRACE_RUN' { publishDir = [ - path: { "${params.outdir}/mirtrace" }, + path: { "${params.outdir}/mirtrace/${meta.id}" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] diff --git a/modules/local/mirtrace.nf b/modules/local/mirtrace.nf index cd44adb8..8683d5ce 100644 --- a/modules/local/mirtrace.nf +++ b/modules/local/mirtrace.nf @@ -7,7 +7,7 @@ process MIRTRACE_RUN { 'quay.io/biocontainers/mirtrace:1.0.1--hdfd78af_1' }" input: - path reads + tuple val(meta), path(reads) output: path "mirtrace/*" , emit: mirtrace @@ -18,7 +18,7 @@ process MIRTRACE_RUN { script: // mirtrace protocol defaults to 'params.protocol' if not set - def primer = params.protocol == 'cats' ? '' : "--adapter ${params.three_prime_adapter}" + def primer = meta.adapter ? "--adapter ${meta.adapter}" : "" def protocol = params.protocol == 'custom' ? '' : "--protocol $params.protocol" def java_mem = '' if(task.memory){ diff --git a/subworkflows/local/mirtrace.nf b/subworkflows/local/mirtrace.nf index ea4fc3a7..38d51975 100644 --- a/subworkflows/local/mirtrace.nf +++ b/subworkflows/local/mirtrace.nf @@ -10,12 +10,7 @@ workflow MIRTRACE { main: reads - .map { it[1] } - .flatten() - .dump(tag:'mirtrace') - .set { all_reads } - - MIRTRACE_RUN ( all_reads.collect() ) + | MIRTRACE_RUN emit: results = MIRTRACE_RUN.out.mirtrace diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 80aab3d7..5e1fca50 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -126,12 +126,6 @@ workflow SMRNASEQ { .set { ch_cat_fastq } ch_versions = ch_versions.mix(CAT_FASTQ.out.versions.first().ifEmpty(null)) - // - // SUBWORKFLOW: mirtrace QC - // - MIRTRACE (ch_cat_fastq) - ch_versions = ch_versions.mix(MIRTRACE.out.versions.ifEmpty(null)) - // // SUBWORKFLOW: Read QC and trim adapters // @@ -145,6 +139,17 @@ workflow SMRNASEQ { ch_versions = ch_versions.mix(FASTQC_FASTP.out.versions) reads_for_mirna = FASTQC_FASTP.out.reads + + // + // SUBWORKFLOW: mirtrace QC + // + FASTQC_FASTP.out.adapterseq + | join( ch_cat_fastq ) + | map { meta, adapterseq, fastq -> [meta + [adapter:adapterseq], fastq] } + | MIRTRACE + + ch_versions = ch_versions.mix(MIRTRACE.out.versions.ifEmpty(null)) + // // SUBWORKFLOW: remove contaminants from reads // From f8f30fa24b9d740783ebe9a77c00234b4267652a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 11 Oct 2022 09:12:56 +0000 Subject: [PATCH 126/145] Update changelog for 2.1.0 release, add config changes and update module --- CHANGELOG.md | 2 +- modules/local/parse_fasta_mirna.nf | 6 +++--- nextflow.config | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73b0b8ba..afa672fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## v2.1.0dev - [date] +## v2.1.0 - (https://github.com/nf-core/smrnaseq/releases/tag/2.1.0) - 2022-10-11 Maroon Tin Dalmatian ### Enhancements & fixes diff --git a/modules/local/parse_fasta_mirna.nf b/modules/local/parse_fasta_mirna.nf index 5a92b202..619cb427 100644 --- a/modules/local/parse_fasta_mirna.nf +++ b/modules/local/parse_fasta_mirna.nf @@ -1,10 +1,10 @@ process PARSE_FASTA_MIRNA { label 'process_medium' - conda (params.enable_conda ? 'bioconda::seqkit=2.0.0' : null) + conda (params.enable_conda ? 'bioconda::seqkit=2.3.1' : null) container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/seqkit:2.0.0--h9ee0642_0' : - 'quay.io/biocontainers/seqkit:2.0.0--h9ee0642_0' }" + 'https://depot.galaxyproject.org/singularity/seqkit:2.3.1--h9ee0642_0' : + 'quay.io/biocontainers/seqkit:2.3.1--h9ee0642_0' }" input: path fasta diff --git a/nextflow.config b/nextflow.config index afb1b01f..c1469e06 100644 --- a/nextflow.config +++ b/nextflow.config @@ -221,7 +221,7 @@ manifest { description = 'Small RNA-Seq Best Practice Analysis Pipeline.' mainScript = 'main.nf' nextflowVersion = '!>=21.10.3' - version = '2.1.0dev' + version = '2.1.0' doi = '' } From 577cd851df80dbcc46e39bf53cab3ae8579ab944 Mon Sep 17 00:00:00 2001 From: Rob Syme Date: Tue, 11 Oct 2022 12:51:35 +0000 Subject: [PATCH 127/145] More flexible samplesheet handling In preparation for handling adapter sequences in the samplesheet, we now pass through all columns in the samplesheet as keys in the meta map. The commit also removes a bunch of redundant fastq_2 checking which we don't use. --- bin/check_samplesheet.py | 74 +++++++++++++------------------ subworkflows/local/input_check.nf | 11 ++--- workflows/smrnaseq.nf | 6 --- 3 files changed, 34 insertions(+), 57 deletions(-) diff --git a/bin/check_samplesheet.py b/bin/check_samplesheet.py index 3ca65774..24252b4b 100755 --- a/bin/check_samplesheet.py +++ b/bin/check_samplesheet.py @@ -55,55 +55,45 @@ def check_samplesheet(file_in, file_out): MIN_COLS = 2 HEADER = ["sample", "fastq_1"] header = [x.strip('"') for x in fin.readline().strip().split(",")] - if header[: len(HEADER)] != HEADER: - print("ERROR: Please check samplesheet header -> {} != {}".format(",".join(header), ",".join(HEADER))) + if any([item not in header for item in HEADER]): + missing = [item for item in HEADER if item not in header] + eprint("ERROR: Please check samplesheet header. Missing columns: '{}'".format(",".join(missing))) sys.exit(1) ## Check sample entries - for line in fin: + for line_number, line in enumerate(fin): lspl = [x.strip().strip('"') for x in line.strip().split(",")] + row = {k: v for k, v in zip(header, lspl)} # Check valid number of columns per row - if len(lspl) < len(HEADER): + if len(lspl) != len(header): print_error( - "Invalid number of columns (minimum = {})!".format(len(HEADER)), - "Line", - line, - ) - num_cols = len([x for x in lspl if x]) - if num_cols < MIN_COLS: - print_error( - "Invalid number of populated columns (minimum = {})!".format(MIN_COLS), - "Line", + "Invalid number of columns: found {} columns (header has {})".format( + line_number, len(lspl), len(header) + ), + f"Line #{line_number+2}", line, ) ## Check sample name entries - sample, fastq_1 = lspl[: len(HEADER)] - sample = sample.replace(" ", "_") + sample = row.get("sample", "").replace(" ", "_") if not sample: - print_error("Sample entry has not been specified!", "Line", line) + print_error("Sample entry has not been specified!", f"Line #{line_number+2}", line) ## Check FastQ file extension - for fastq in [fastq_1]: - if fastq: - if fastq.find(" ") != -1: - print_error("FastQ file contains spaces!", "Line", line) - if not fastq.endswith(".fastq.gz") and not fastq.endswith(".fq.gz"): - print_error( - "FastQ file does not have extension '.fastq.gz' or '.fq.gz'!", - "Line", - line, - ) - - ## Auto-detect paired-end/single-end - sample_info = [] ## [single_end, fastq_1, fastq_2] - if sample and fastq_1: ## Single-end short reads - sample_info = ["1", fastq_1] - else: - print_error("Invalid combination of columns provided!", "Line", line) - - ## Create sample mapping dictionary = { sample: [ single_end, fastq_1 ] } + fastq = row.get("fastq_1", None) + if fastq: + if fastq.find(" ") != -1: + print_error("FastQ file contains spaces!", f"Line #{line_number+2}", line) + if not fastq.endswith(".fastq.gz") and not fastq.endswith(".fq.gz"): + print_error( + "FastQ file does not have extension '.fastq.gz' or '.fq.gz'!", + "Line", + line, + ) + + ## Create sample mapping dictionary + sample_info = {"single_end": "1", "fastq_1": fastq} if sample not in sample_mapping_dict: sample_mapping_dict[sample] = [sample_info] else: @@ -113,19 +103,17 @@ def check_samplesheet(file_in, file_out): sample_mapping_dict[sample].append(sample_info) ## Write validated samplesheet with appropriate columns + output_cols = ["id", "intrasample_id", "single_end", "fastq_1"] if len(sample_mapping_dict) > 0: out_dir = os.path.dirname(file_out) make_dir(out_dir) with open(file_out, "w") as fout: - fout.write(",".join(["sample", "single_end", "fastq_1"]) + "\n") + fout.write(",".join(output_cols) + "\n") for sample in sorted(sample_mapping_dict.keys()): - - ## Check that multiple runs of the same sample are of the same datatype - if not all(x[0] == sample_mapping_dict[sample][0][0] for x in sample_mapping_dict[sample]): - print_error("Multiple runs of a sample must be of the same datatype!", "Sample: {}".format(sample)) - - for idx, val in enumerate(sample_mapping_dict[sample]): - fout.write(",".join(["{}_T{}".format(sample, idx + 1)] + val) + "\n") + for intrasample_id, val in enumerate(sample_mapping_dict[sample]): + sample_info = {**{"id": sample, "intrasample_id": str(intrasample_id)}, **val} + outrow = [sample_info.get(colname, None) for colname in output_cols] + fout.write(",".join(outrow) + "\n") else: print_error("No entries to process!", "Samplesheet: {}".format(file_in)) diff --git a/subworkflows/local/input_check.nf b/subworkflows/local/input_check.nf index 136e28be..910afd05 100644 --- a/subworkflows/local/input_check.nf +++ b/subworkflows/local/input_check.nf @@ -20,16 +20,11 @@ workflow INPUT_CHECK { versions = SAMPLESHEET_CHECK.out.versions // channel: [ versions.yml ] } -// Function to get list of [ meta, [ fastq_1, fastq_2 ] ] +// Function to get list of [ meta, [ fastq_1 ] ] def create_fastq_channel(LinkedHashMap row) { - // create meta map - def meta = [:] - meta.id = row.sample - meta.single_end = 1 - def array = [] + def meta = row.findAll {it.key != "fastq_1"} if (!file(row.fastq_1).exists()) { exit 1, "ERROR: Please check input samplesheet -> Read 1 FastQ file does not exist!\n${row.fastq_1}" } - array = [ meta, [ file(row.fastq_1) ] ] - return array + return [ meta, [ file(row.fastq_1) ] ] } diff --git a/workflows/smrnaseq.nf b/workflows/smrnaseq.nf index 5e1fca50..09f2c2a6 100644 --- a/workflows/smrnaseq.nf +++ b/workflows/smrnaseq.nf @@ -98,12 +98,6 @@ workflow SMRNASEQ { ch_input ) .reads - .map { - meta, fastq -> - meta.id = meta.id.split('_')[0..-2].join('_') - [ meta, fastq ] } - .dump(tag: 'map') - .groupTuple(by: [0]) .dump(tag: 'group') .branch { meta, fastq -> From 334804bf0431f89d3621b0c510408c71e51aaa7a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 11 Oct 2022 16:28:22 +0000 Subject: [PATCH 128/145] Add known adapters to workflow --- CHANGELOG.md | 2 +- assets/known_adapters.fa | 6 ++++++ conf/modules.config | 1 + nextflow.config | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 assets/known_adapters.fa diff --git a/CHANGELOG.md b/CHANGELOG.md index 73b0b8ba..49871022 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,7 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[#168](https://github.com/nf-core/smrnaseq/issues/168)] - Removed `mirtrace_protocol` as the parameter was redundant and `params.protocol` is entirely sufficient - Updated pipeline template to [nf-core/tools 2.5.1](https://github.com/nf-core/tools/releases/tag/2.5.1) - [[#188](https://github.com/nf-core/smrnaseq/pull/188)] - Dropped TrimGalore in favor of fastp QC and adapter trimming, improved handling of adapters and trimming parameters - +- [[#194](https://github.com/nf-core/smrnaseq/issues/194)] - Added default adapters file for FastP improved miRNA adapter trimming ### Parameters | Old parameter | New parameter | diff --git a/assets/known_adapters.fa b/assets/known_adapters.fa new file mode 100644 index 00000000..6b54be80 --- /dev/null +++ b/assets/known_adapters.fa @@ -0,0 +1,6 @@ +> QIAseq miRNA adapter +AACTGTAGGCACCATCAAT +> Illumina miRNA adapter +TGGAATTCTCGGGTGCCAAGG +> Nextflex miRNA adapter +TGGAATTCTCGGGTGCCAAGG diff --git a/conf/modules.config b/conf/modules.config index 0c5365fb..648c5d20 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -105,6 +105,7 @@ if (!params.skip_fastp) { params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}" : "", params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}" : "", params.three_prime_adapter == null ? '' : "--adapter_sequence ${params.three_prime_adapter}" + params.fastp_known_mirna_adapters == null ? '' : "--adapter_fasta ${params.fastp_known_mirna_adapters}" ].join(" ").trim() publishDir = [ [ diff --git a/nextflow.config b/nextflow.config index afb1b01f..7794ab08 100644 --- a/nextflow.config +++ b/nextflow.config @@ -34,6 +34,7 @@ params { three_prime_adapter = null trim_fastq = true fastp_min_length = 17 + fastp_known_mirna_adapters = "$projectDir/assets/known_adapters.fa" save_trimmed_fail = false skip_qc = false skip_fastqc = false From 0e7398f8cc948d5cf332c22dba93c24da1ebefce Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 11 Oct 2022 16:32:48 +0000 Subject: [PATCH 129/145] Add parameter to schema --- conf/modules.config | 2 +- nextflow_schema.json | 27 ++++++++++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/conf/modules.config b/conf/modules.config index 648c5d20..951a4719 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -104,7 +104,7 @@ if (!params.skip_fastp) { params.three_prime_clip_r1 > 0 ? "--trim_tail1 ${params.three_prime_clip_r1}" : "", // Remove bp from the 3' end of read 1 AFTER adapter/quality trimming has been performed. params.fastp_min_length > 0 ? "-l ${params.fastp_min_length}" : "", params.fastp_max_length > 0 ? "--max_len1 ${params.fastp_max_length}" : "", - params.three_prime_adapter == null ? '' : "--adapter_sequence ${params.three_prime_adapter}" + params.three_prime_adapter == null ? '' : "--adapter_sequence ${params.three_prime_adapter}", params.fastp_known_mirna_adapters == null ? '' : "--adapter_fasta ${params.fastp_known_mirna_adapters}" ].join(" ").trim() publishDir = [ diff --git a/nextflow_schema.json b/nextflow_schema.json index 17e971e8..7eb0a44d 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -10,7 +10,10 @@ "type": "object", "fa_icon": "fas fa-terminal", "description": "Define where the pipeline should find input data and save output data.", - "required": ["input", "outdir"], + "required": [ + "input", + "outdir" + ], "properties": { "input": { "type": "string", @@ -28,7 +31,13 @@ "fa_icon": "fas fa-vial", "description": "Protocol for constructing smRNA-seq libraries.", "help_text": "Presets for trimming parameters and 3' adapter sequence with a specified protocol.\n\n| Protocol | Library Prep Kit | Trimming Parameter | 3' Adapter Sequence |\n| :------------ | :-------------------------------------- | :-------------------------------------- | :--------------------- |\n| illumina | Illumina TruSeq Small RNA | `clip_r1 = 0` `three_prime_clip_r1 = 0` | `TGGAATTCTCGGGTGCCAAGG` |\n| nextflex | BIOO SCIENTIFIC NEXTFLEX Small RNA-Seq | `clip_r1 = 4` `three_prime_clip_r1 = 4` | `TGGAATTCTCGGGTGCCAAGG` |\n| qiaseq | QIAGEN QIAseq miRNA | `clip_r1 = 0` `three_prime_clip_r1 = 0` | `AACTGTAGGCACCATCAAT` |\n| cats | Diagenode CATS Small RNA-seq | `clip_r1 = 3` `three_prime_clip_r1 = 0` | `AAAAAAAAAAA` + `GATCGGAAGAGCACACGTCTG` (only polyA is used for trimming) |\n| custom | user defined | user defined | user defined |\n\n> NB: When running `--protocol custom` the user ***must define the 3' Adapter Sequence***.\n> If trimming parameters aren't provided the pipeline will deafult to `clip_R1 = 0` and `three_prime_clip_R1 = 0` (i.e. no extra clipping).", - "enum": ["illumina", "nextflex", "qiaseq", "cats", "custom"] + "enum": [ + "illumina", + "nextflex", + "qiaseq", + "cats", + "custom" + ] }, "outdir": { "type": "string", @@ -192,6 +201,11 @@ "type": "boolean", "fa_icon": "fas fa-cloud-download-alt", "description": "Save reads failing trimming" + }, + "fastp_known_mirna_adapters": { + "type": "string", + "default": "${projectDir}/assets/known_adapters.fa", + "fa_icon": "far fa-question-circle" } } }, @@ -372,7 +386,14 @@ "description": "Method used to save pipeline results to output directory.", "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", "fa_icon": "fas fa-copy", - "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], + "enum": [ + "symlink", + "rellink", + "link", + "copy", + "copyNoFollow", + "move" + ], "hidden": true }, "email_on_fail": { From 1a09686d56542a9d30f7bb01628009ab382f8601 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 11 Oct 2022 16:33:40 +0000 Subject: [PATCH 130/145] Prettier --- CHANGELOG.md | 1 + nextflow_schema.json | 23 ++++------------------- 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49871022..245d985e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated pipeline template to [nf-core/tools 2.5.1](https://github.com/nf-core/tools/releases/tag/2.5.1) - [[#188](https://github.com/nf-core/smrnaseq/pull/188)] - Dropped TrimGalore in favor of fastp QC and adapter trimming, improved handling of adapters and trimming parameters - [[#194](https://github.com/nf-core/smrnaseq/issues/194)] - Added default adapters file for FastP improved miRNA adapter trimming + ### Parameters | Old parameter | New parameter | diff --git a/nextflow_schema.json b/nextflow_schema.json index 7eb0a44d..b5c52a3e 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -10,10 +10,7 @@ "type": "object", "fa_icon": "fas fa-terminal", "description": "Define where the pipeline should find input data and save output data.", - "required": [ - "input", - "outdir" - ], + "required": ["input", "outdir"], "properties": { "input": { "type": "string", @@ -31,13 +28,7 @@ "fa_icon": "fas fa-vial", "description": "Protocol for constructing smRNA-seq libraries.", "help_text": "Presets for trimming parameters and 3' adapter sequence with a specified protocol.\n\n| Protocol | Library Prep Kit | Trimming Parameter | 3' Adapter Sequence |\n| :------------ | :-------------------------------------- | :-------------------------------------- | :--------------------- |\n| illumina | Illumina TruSeq Small RNA | `clip_r1 = 0` `three_prime_clip_r1 = 0` | `TGGAATTCTCGGGTGCCAAGG` |\n| nextflex | BIOO SCIENTIFIC NEXTFLEX Small RNA-Seq | `clip_r1 = 4` `three_prime_clip_r1 = 4` | `TGGAATTCTCGGGTGCCAAGG` |\n| qiaseq | QIAGEN QIAseq miRNA | `clip_r1 = 0` `three_prime_clip_r1 = 0` | `AACTGTAGGCACCATCAAT` |\n| cats | Diagenode CATS Small RNA-seq | `clip_r1 = 3` `three_prime_clip_r1 = 0` | `AAAAAAAAAAA` + `GATCGGAAGAGCACACGTCTG` (only polyA is used for trimming) |\n| custom | user defined | user defined | user defined |\n\n> NB: When running `--protocol custom` the user ***must define the 3' Adapter Sequence***.\n> If trimming parameters aren't provided the pipeline will deafult to `clip_R1 = 0` and `three_prime_clip_R1 = 0` (i.e. no extra clipping).", - "enum": [ - "illumina", - "nextflex", - "qiaseq", - "cats", - "custom" - ] + "enum": ["illumina", "nextflex", "qiaseq", "cats", "custom"] }, "outdir": { "type": "string", @@ -205,6 +196,7 @@ "fastp_known_mirna_adapters": { "type": "string", "default": "${projectDir}/assets/known_adapters.fa", + "description": "FastA with known miRNA adapter sequences for adapter trimming", "fa_icon": "far fa-question-circle" } } @@ -386,14 +378,7 @@ "description": "Method used to save pipeline results to output directory.", "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", "fa_icon": "fas fa-copy", - "enum": [ - "symlink", - "rellink", - "link", - "copy", - "copyNoFollow", - "move" - ], + "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], "hidden": true }, "email_on_fail": { From ce95631e96cc2f2c789b56ec13a1fe2ac93efe2f Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 11 Oct 2022 16:43:37 +0000 Subject: [PATCH 131/145] Docs ++, remove old trimgalore stuff --- docs/images/cutadapt_plot.png | Bin 98738 -> 0 bytes docs/output.md | 20 ++++++++------------ 2 files changed, 8 insertions(+), 12 deletions(-) delete mode 100644 docs/images/cutadapt_plot.png diff --git a/docs/images/cutadapt_plot.png b/docs/images/cutadapt_plot.png deleted file mode 100644 index 8b4c978abeef6d0948191868ada3df8cf79d23f9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 98738 zcmeFZWmJ`G)Hb?6LPWqsB}GM~m2MQ36p-$c5-DjU7MO^FsFbupw{)k{p)@SIVbQSY zIP<~%?st2}`E|zmbH@1mvA4prp1WpT*EQ$8eD5nrksYHthC-ppWTfvXqfi8yDAbXQ zM~UDk+`|$0DAYxi%$-}Ru90&CTRNvT4i1(V56*OG7Z0o?HCitg<+yaJTk6bT)YLeW zRuFQ@DTS0pUA151%vqP4$%%;d(EYV*Yau$lHh`XfZUlweeT^cm5KhU>RZG_` zyaW#fpJi3A+ud3b_pLwK_HN^X<7xx>@0g{z8YA(<<<8Q&ZDQ zI%UmA*49oT>+9==6jP-+4}Kn-avm&pSu~_z8rLgvRTA4WcRu z-&NcFX_HSJYe3M|5X>5d8Dom^IQ71};K^#diJ6(Ky?s_wIIkQfhwe3OS6ZN--*IM6 zPV*RY6ds)7aFoNrzGs-Id!EPIZz(#(SY5e;{XNr_@p}8cZJZ&6`1naf!olan6hTfU zR=rt<6f>oFyiN5iEmMwD@zfQT?c*}M>OV5dne433fBg9IlAWF1sS6ioRJ?J$<}q4> zK0C^?a&mhs(XG*m$`9nD0QaBJ)6 znEw8Hw=GR4pFE}qrcbnE{Xzskwc>KcF){+pE>jbeYx&2;T$gTY)Y|0B$;rK)5F^Pv zarEi2%&hF}IH>@-ko@XTpU#(;@}Y*=@!3#|FG4vDWK~sz7Dz{beYwqeN0 zDl&zwt*wvI(HxB-?18rBdm*SsDFV%>Qr>5&j%KFr%;yh9Vjrzd|3=R^z=oVUdv@ac z+QNX&R;p_X{n@jO8w&%gjhja^2j!n0&17L^od{GEvs>R8@u6>6nP`mr{P`yGLpYKw zuLE?LuuJ+XpAO7uNgV#ekWpHy_u#>UpwqrK14Z>U{?u@U?$!t6^{4CW>fZPCXx6v1 zEId1T(E8=JkIDAh442#5!!VI2_Mv6YQZJ3Y)pT@NSEpJ-m&a-w^M}x^u#j7{6nG&A zX}X2V4~&gXI}(*XhKD~&BEt)DN*BM_^y$X8M*VJ_a~cfi)Qf%dTEt`XexA+1EL(1F zuHN?c^rxEyn&I(+JG+}>0iv(M*xA`XhK4c;yDY3VS{^~|`=Cf~*6Ea^OYO4`z#Zgn z-MW?OV=Cj}QR2{F{fdlq{V^L!EegB8x6|O>(A-Qm>v}YEH^X+QtOZvHhA(FgJ4!9^ z__Ko^yE%Fb?7q(P2{PsizmUD$38pGaX`3>ZZ27IK*jTVev5{f|Zw-mzFAFz$VmvW1)Ve+Q$^! z(;hdgHnBQhwm%>qk>0qoHZ!IhLqbAQCLqa9iN+POU>cS`-#TmA(cLYlpzsc2 z%U$FZV8?VDdigd3_V1Xvx!YI7dN8i6;6>BFzh&8tN%od|mDi#%7|cXloE)jE&f-*)2zpl&f`}c`RV9!GA8T@STs~GJSLB z&YkJ)eTV>zmoHc6D0X{S>1^#RmOIU_uB-$JxhxLm7UYSNs(N>JbX>_cYv%G$R)@D; zxpGBwyJ~-LV@MbyAHmnWwGkOfvo)KN;ppjt?UGl|(4Xhsb$V!P8_H|hb>3#E%GbL3nypj^z_4OGz2d8sN*@gx%Ci7Oe?(c=xQe{cIN#`N<=2|#5dZ? zy}U9gay2szpHAT1AuxNF1hf!on(YKbn5t70--~6FlaW#C(~kNvFvkl%cbH02G&l#g zx3{qw6A)!(#U~~vxXhZ)SL1|D8_qo0UQ4Roc*Hm8LqXpVD-(EphgKY2!eLTZUw@<*p`pz3-`}L<0Ch38*Y*u%DmoausXhV?6zv0bU){=A=uZ4g+ zgM&nX2n;j!_F%prgM!9)y&&RPTU!@BHiXnkm!X|iWs~1nb90iBUWIL zYoqG6vA_2U>++!&l$MR6`Qw4D{E=Z;5;F2hwWz?sWweMvVQmw z$;}z_5KpnVokZ!Scf2<hoGP{TayvCkvOq~ zU5BY5TGPeB(xQD@gDSW6IcA9ZA!}=ng^n{HRn^q$b1l0~aI>8aoW7nr>o`|oNeKTQ z8v`y4pQf54n6$LCR$9dm>L4PRZmmqTWf+K8bBe0tr)HGzYb&d$FtW1BL1t&9pc8rD z->*A>g@1JBaBc)vfVEuUdH8V~7k`B8T)q$&>7v9Y=A&NWiGNWgc2|VjkBZ z5Cy}$dtm`Te*BnTr;jqq$4u1Hd#UQX_ZQe9(HFq9vc5hCjL&6Ub3Cr4;^`4GTRaiZ z?Hq@_LVb4F*rA!OMKJokbury7;fg-U+NUJ_E^tdi_@fBpw~byPDEx-k)s?1$V5yda z0AJq^NEXa8eQOhQ{@gjHgWc5t1T;U$E)`V@2nYxT%-S&9nc_4OnDF_RXGwx$Oo-PW z)}RFqmc$MbDhQ64CBMJVim$iSJs&|8TOw?_1^1D>M@q@|9{7oHC;*A-vH2JKgY;l_ z%umOB=+W1hnVIePcb`nnYasc0i_4ntK=kF(=JHrkS^3r^^?-Fk*<&&~x3y^o@%n@L@vSQE7!txPvI5tmQibUOeqSa}*Y%fOKOTX8a zf|~ChfOr!yFfNbH#{+hCfm5w9Sc$~|@qNt@4m}p&IiZ+*TR9`6F9=jXD8+3b@?gkU zdg9=2vYE6OE--^@1j0j&rS2Q!J>QL2Vh(m|3rheA(j~k-+V!32=v4&&HsWG?&9cRA zWM}Ns8WzshU0LYPFhI7@Et1|6GH}Cr5xrzI!Z~bEtR|#_M*R|ki%au1bD53=EUS#tITrf z6?H7V`3PRERpgkYh)9w4@-6N~n{FYfx4|D(BsvQnrnKoYjA_0}`nM4(fMqnKc|w?Qdi4-|x5#q_(q(X&=m2_l zaql+}naR1uc13Ys(Lps;)w;S5mz@}%mw(dx@afa1Yn=KeOGz1T-W;14>YRQQm6yjC z26;cbr74VCekUb4Sy`-=+q{LRL^dZaO&*YnW4@EywjJ|*ypZzlr#tJ50#}jQD#VDT zb-;-S!#r7WqI)L<%v)&U7625-ef?U6EqXk)EjZhq5i?@_4CsYy-NH;mk|C|7x8bD< z{1aLIoNw3HNR7;2XnULfgl!JKK_lc?m#GgC&;F;Jl~rnY*}&~rEn+W!;bs%?ju3t7 z;x58*2?_NXKO9FFH$oaG!@gWqIjR@)zB5gSKY;03Ium}h!V7;JGG3khOJAEpx$Q>p zR10w$lsW2?Jpfz-jbh$w3=EZRzTdGKJFFV%IXodP_7rcw zef?UJXGcclhMQ#!pcCavqvpU1x%}&fo50iJcSM@+PU~j-P{+o_&CIrbx>4;YcCGvL zd<=TKe#Mm*lif?OGgjl@-Z}K$*vDu9Ks%$AW~p1Qk*L>VK>za#*~@}Cd3h`pip%ES zjF&z}MyBeR^I0@JK^4Q?uU}kLloH~DZtG$UyY`0FN-cD}sv(tuK z?nr{V{p$GdmsR8o8e7G!GER7&5Oovk-UFzW^R!tL*|WZ4lyd_r7-> zNl_<}6t$pnBudDMO=++AOjRdk-~`Qq2S z_vXlDmxX?%zFnY~@lMr@729j=itOUN*${;f_J$6gl>BV*ET;h$ZpY=&J@z``W`$?T zH>rU20?}fkEPz9xu^4M>+}ZOBYj&Ah-1}g8ex;I_Uehf(s+`Hr#btd?Yv72PZosWU z8{TrzM%pHUL(pN170%o6P^bbrv!}_i7a>`{vaqjDd}(|B09*(skgp~q;~Td~n7(r{ zLS9@WS}pyHDj0?cnh$76a5s-TeWgGbsuE4tAm%?TUR8I91a%#62NF zgp~Df3Rl>d5B#1x_b8j`$&<0|j&w24?ekqEQ>aF#XWF)v{@8MBaD0oK(^Q5}B0LQ4 z4730je56^6TkVGgTW#hBr zp#vd|KQ(_#j(!1wCbl~xzd!FAUPyk*%j~SIpzT5Y0Ei@RWCVUSSnHuZFdG4^q@<)< zD(lw;%O-p_uv6U_w?~X3CrLcK8CTZd)W*hMsjjarl^0k*r`Q4juI;vuQf!{q^jaC0 zi7BgoO(0&evLSThx9tIwV)R;1!p_Ds!2Z>42I#H0Yy}8j?0TgZKX49n^E;n+#C}>y zY#P!)7`Tjt0ZMl5Sej3wdZU9E7G=FV+uANclvdW(W&!S|l3u~JBe4N7S^%r*`R1EA ze(LtN8^W1&iyUP(Sj)?guys3GW%nMpJvi2l!U8vMAnAZSbq!Kv{Lj65DWKl3_KFlo z7yK;+OeqZuqrE+Fof;v71{&bT2`Ah}-G07qHls5v1tPcMx<)7JRtNZHd>e-ishMQ$ z#x!|6d60q?10yZFo^}qRK8WUmXlIGyrKSDh5b-``R%q)BGv0gi z_HFiBU)c*H$UO~bY2JY%$qdO9;h3h+Y$KWv@VdfzErXmTPv^a|^j5on|JMlT4+tPF zQNrdu8GqpWgj3oY^NJ0z<|W&y_%LqscO%x4@9f-)Mca>U;4E?39A9IIo92>5@Y(R$ zY#}*rNCE9{SOK2xPWw%|n^=<^;&{Wn8DGS5iC=nt>5jJFwnNnOUkz!WFI7_fXTb=p z+1~dD(i=ucoO;IgG)VUVk_ZVjQ4%3LGHBpXN9tk>b8MI9c(? z)>eZq*cx}TQT_IFaE}QR0RUYgK(eP5ypMN;skY%qyj-<-UHOx#5qR# zP*$3aGG6Q2=Ab zrHU0wNJL($s;aSTp&=n=NqU+1Y#;K!yvEvfo3#(MO$iwce7^NuXfIqzNvUNYh>k^) zeC820uZ$xfDhM>e!)0j=*3%Yry_&cqZl`A#jcOhtphmdsB>HR{$Z)&+RA0^*f^NfH z{+ZpNtUo6`Sf*dkiwtmoL3j7L?{riE`30~DuU@m8c}kvnRX%*&%9v05ZUH2pm6q{SaiC0 z>JzHbCZBaE7a~XZ$Z{cC>ab0 z>NF4^Rjj&v>5_=pV879PU#`(vLqUX9%$S2RL_1p>j1>`3k6F3)RsZaDqbh=SH^e*D z!8;cM`ZF#?3ppjnIfL(YWf)}a3hNd;z6Fc)<8|MTQcFro65A_&>nKvOvKFiSAwQqH zm`=PYPvf6MymSeW?)}*(?%HV=HWstN%V4x~PyZ(m#sa~XesT&bCV^mzn>b#OExxXK zfdbcFL_sg6diC2nvhW_ACvN^&l}9B}SF}kobvfJfd#&u3NACuB=H=$jL{mLHLp44% z#p(n$Vb*7U5dsy7I!K5QHCy-I&Vr^{x=t&-e>^jU(_mrQB~59@2upbnZY zkN%YP%(njh1+ZO9^2JE?0mOyiyS33JSIT+in0s?0W?! zlYp_syrhIYHZ$95RJ?he=QPbr7t3~9jLR1&)NWw(_mKE@4vB9LlT9&fRbdV_m2H+jSk|?fmuYv#ux4Xy^Z*B&qL&~*TS4k>%ZgK`h$00K>5xrOxNAZgoZZ$Qc_@_FU4yIf0)w)Ww- z9n?(6w6%TpP`th~@(4cOAOJ~ZXk@vwtBXbaA#NVi?HC2oe&BSpsOREIC9k79bJ?x; z0vbWo{e{C?xZ|dTPLSv#OSyQk;dL;TdhX=O5uB_iclywGpvp#@Q5{3OAe{Y4q?Xp! zY6=Rp2%USS80pYCILPtj$rCPc8{gNj--Aq}M_W9QzI#f*r0&F_Ff0Os3oLVNS7K6# zToEtC{nyw$v+TRmF14Xoz8PdFMvKhQy#3|@MWlGoqKx7nKkiJde+7ywhensJs*7}= zR*PsBh=c->%IqB}DQs0CAnbY|VNM4&po-*=2n;CVTk}QuoxdRg<%Z)}b2Bp*jVbG% z%t}Yv*NIF9P_PVz*SV*1uB?+T#LGuS{xk^P-Qlh{by5(dxVE8BH;l)D7Sdt*%e@R{ zu~@QD!b`|6oo0hZrd&s=UV>0|pP}h%W-c23`c0IAa z0a{`GJ6hp;8TTol6i0>aMHC(U;(yd}ZNe4AM?}Hp^4xZ;9xF-k{59De5nKcX4z&<* zQ0@hZ3eZw3Ii=g~v;-^_r+$m;PE}KgcUb$RLpgx#c6xF$%hpmgMbzH$^Ve^#tgec_ z%g)PF2Rw8#QyO;<#c2nq_4HZzO2moU%nHGwzkPf0qIlNt788rhHj zAStvPdo?~WaY5g66kUG+I6}1SmD&QHdO3Q}1hn^%x4+AM$A0`!LbOZ(-AFYCk@+BI z%SlSULRef2D?z$Jxs5>CO+tWCUU3`i0?VLF%D|X57#z>W>z#5nV7N$G3(g2JwWZJn zc=p`}Kj2d&7Sf2g-cQ#rWqJAXN)xc@4g&nZ}{G z`!P6}5t)Ka;8mFn{ZfLPIJA`6trfCOkiwgMj*48g0p7X4v8U!Lx>|&>sIR> zO>iFAZA7|ph$!;d%0lG0{cSAg)^aU9^njS92bI{zlkm(dbP# z7JARM4>P;!P-bXg9F;U3u6Sxc)j}H+EphEqQHpebIYALp3aYh3YTXE-ke8Ql2vwH4 zQ6xJJ5VRrVu_PlGmonr*Ht{Wh9hOB~_5E_Xx{)27oif_m;WoWhFG-ya%F-JlTPF&W zaIpXq3@HmLb;HSSq-0xyM)^B;CK=V(t$hFfXKXtx73`jjKE_)8*-1m+lG5DHhZunO zuRcIrI(za1$Wg8Xd3y3uLY-@+R8jW`geq3lk=jAIwF-G!T3VJ_Gt~qGX>H^;>GdaY;$zXj{(n*j;yAL0Y?)d2XTSh%x?nSTt)pM z-{?Nrwn*M$S6T#=G`kaI`tPotZl?|g*Zq!}7bDc%*CmGDg*$`#Mxh@+K6r9qLuu&y z`plU#(`ZASY6+p{pzOte>H)4({UuLS;uWHjckcd5gtD46v>-q`gPW)f6s|sSX?TA_ zwBxMDBN9`gN(M(Dlz@~$6`e^aJBX5`-4-HQ>^3%mUYY_}mJCfX*71?tUL zR~@ZSQ&VF-SRxT>HV#2?1$q@`w`sUcUfVzMgo>Yiig$B-%-hPjE03Xi+Yrg}t}Q{4 zUMtJQSJB~%sONSE&$w&%5jLNljm5?nB&Qu$r3&}my+9Pta-30a7Yx9sg@X{iP~b52 z4kq0IwABPix&2HVd%SP z>BthV0T($W%X5<#8= zFN(h<4--J}`>|ukL|R%$mR2U@kbpXe$yWrtI1W6Q7Oi;cfb^0=qyPgHgy>?VUH7(~ z1l`sWM?9_#D&Paux{{y}^%07}(%{_*HM)9w?BGPfAQj61?tn71<2H%(m&EvZNznD! zXJ%&XXFI?jnU^u^{bxhibte1ih$!f+3iYh5)6P&*%0eJVO8#)-9_Z=k;w4X3B^w$V zA`ggQcx82Uvl)k3PH;h_(gO4{7zVC`!1O$e1LZXjDw*53h@?S@>-WZW3lNLp16Sdg zp)G|4!t2NP?=M1$yrnfSVJjl_p^Z&QTbr`r<6&U1Qit&=)rE!!*tsjLtc`8D&&yW2g^uPRZ4!{Q09=5zNsU7dvV|Vsb7HeB z6e{#g3X$&{Lpd9v;Lny%@zR*=ml|}6pcuYzYsg*{DQt0{njr~Cp|CaO;EB9YQj&}8I zxPq#nLRybl;6FE|FIF7$Vq|1&@r?j+*`zaB4TT_c!^k6)BQV4-eT3tOPQrJZSOgJtrjlJjg5ftj?QDnWypPSO4+x?%v{Xg!( ztT$lYq{PJcNPSFCPcO8@3;=FWXlNs-u*IxB$A|&eg(BS*78Y3^`M~eS zwGCkBad~-NW3Iw|3jk$K(bF$_*di(*^kT?Bkh$4s3Naisoyd7eNV9gHheLXAK)cvo zaNe&L;Xi6?<=otg0J=3+J|~X)A`fLXP38MXobLxxe)&_D|L z5Gy7BY1zqg!m9hedGis(l?Etl@AjpGWb*afH%1N)1#()UD?p{Ppp)@8kNn5LK>6y# zcX4eNgoWw}s5hRLZ#MyUfVSWVsskee zBQWb}xv|j=IY($<0nVVMViAB&5g!EkmI0Qjp$;Zuf! zlh0V#*m!i&Ai%)eppM%&;|!RtMW%zAit0Ef&7fSLpI=Q11DYF5mdj;9x0>(LC7|3e z5fO!V6o;=Kq#)3Y>gegY2Ay>&u|?x6;89ky=$BhbklpZU^~RqzdL$B1PE1Y;O}U4|Iv@s9hX8`n!vM-`1n`4kLz}@8W|(#mkW1vCoaeRM zyHg>_ket3+2Tke|&;w#WRzrrsCQy-((mAveZRJLd9m|G7pDdJg&zw6ad^-{J@F%$0 z;GsT{@q{yOLdRvu*RPifACEi(y^a1=zQAEx_ue=MDFG0qJ&|Vgo8OTN?|eJ;8j*3J zzVZsE2i*kH!1LOi*XMen=jil%XM`<7JYtNJKg~l~q-*K$%4UIjajCy|Sj}HJ}ZouEpSBF#UHh_Xsh) zUorF0$`=kmEJ{1~ z1q`j^96}dR9%s(+t6aLgBV%GB4#d|w(+48#iS<9fTbk`;HFrIGp;z%0#erDQZSbn2 z?g++4i6thxB@$?U_^|g(^a={)ohb$Q9X0$M*uxBdYE4azAq9lDi>ZgEy!rX_=lkaj z0cR1rBOf*b_6b*3hzkq7u=uI)5yHQI44nd9gqnneFDvON0i3O`R{kHPBKp8r>dnW|NS(x z$|$^Bx$Wy;)4tjMz!5PUk4LI!|N61&{reB8456!#wWk_R?x@b6S4Roq@-4ibW|ZMp1BB-j zd562jCggg(ou{IqMeOX{2r@-D#PYaH~yZWg|?rCH?K|qkH1#K zbJLa{q+9#`o|w&ytgUiUczx{WfoH>8N#98&aHx{bl>JztGme&Y1ef zH|*l?-($h$0|iQyb4FnBFBlK^h!j+e;7k?{oA@6;n^pInmq+9N`uO{Zo%~bkj91v+ z4|ypn?}tQ1o&M|NKpRqDs5OS91;G_6W-mV?U%u>9-y(lS~JFVu>i=RbJva0Qh!GG&mgUxy>CI(2wfgm2FsB)w1rK_)nmDLc5F z@#_A`zs8?z+L9+p0hQ!3z$oQ?K4VFbf8Q_kIXztgxdZ~NTUkS6N(B=ie-$oOAq5#9 z$6_EPa(C~(1Ykl~5(VfGDi2UlU7mp?I4!FZ|AW~`P3ia9LUif9EuDK!FaF&YEfc74 zfO5eFg*kwTh^TJAvu+(QPcM#U1DpaEkQf1kKrTk=o60&m)XMXSIKI5R+_J%k-gWKG zu?U0?Gon~MP}reW{b#QY2;Lu~qUOJop-`1etF6CfHP!UdQh-i_%l*$U8~?i`Hn?>L zK=TSzFyML=qqw*MIi1M#acdxPNJn3$wUblsy$C*egjXR~D`H|e5fKiq5rzpm#KY|x zwkv&SK`;X1_~Yl#dbqR%R~!`8Q#Ba?V!;hGi}Rhoo#7fAC5!=Au2}JlOO?A2Cp5Ew~6^ zgaIB2?`VON=dP%*uns^ouo!|;Jy6r2Q=kqYQe1L!!M{$Db~n;P85l4;6zS6C+&aX8#98fFOOuF!1p92$0Eb+GNw zba+>5vOoxFs()pjZ;iPMwg8iO-e&`KNSKOlv2$}~l<`6a$gPe=10VoIJ%#S<3`gtP z+1bcOXrZ;};Y6YIpgCB9B#DS@dnoPMWy#${HehBezreGR%U6vQaPHUO%`sm4Njj6C z+LJOfe`U@Cu}Aj1$kF0gAHZQjCMDa453gJ01F;?GSEaSq03Zh+f*KeZ=JjjDP z{Bc(xRfavafWxEm>4{RJj3sbT>@%9st_{Wcr5nITj>}UAiPF zSSTwU{C=sNi6TR{P+Kd{)Fq5G80m?i79Us}rz1{HqYS@Q8ga zyC+1BTb3Kcl}F5%OxLGv|A#;#J_KcY>b#R%L;upqn;L?y*HuW8V4r4m^DNrcO?%_z zbwt((z%|}otXNTk?%hO&meux}D}^Wwh+Jl5OU}q}Y2geG z3>;Iz04oB54Ly>{yP^$Zf}HhhBeb(}t|+>E(ZePS!Pk{%*mccDM`y`Cu>; z(Wv^=p_3dcMM!xQxx|QQJxF~N5&2+sNEzKAI|U<{emw#%x|V|(<>1gEJC2kLuOzk|z&-yCHT&Q{B(&EVP8X?*GIMb7c@}*Z z$MEv=!|gKP{vxNwrDJ?3)HBc|Ps&QGsjSu&HbDniBd`K3w_k`#k@^RMKdA7&C3~~8 zrze(;o@d(9R8a9Zz^*0ELx4o#m!4_QZ$BQ;utH41^8EUtbt9@iYWSA5b<`;Z1efVW zefmTq8-?w*B``V+oqzD%a|jZQkyL^!QhHHPzhrh=S>+@$%3X|m5BTixdjbCYFg1Te z&|{XXX3x@FphOxh#H3(C_}AOyLXW`%0M*i4lGT`Hqt@2;ZknFFD#z>FgC5T|1g8|itpL7jPW!7d z41W*O^ywjEBnxNxw-jT@`$E9fe!!)Aqntkp3*leSjjAQDRdzrcA;Thv6Q>U*axUNu z?77DwCU(7USaFf@6GS+!)Q!Cjac11Ykk_xDm*Gs08XWSBt9~|QJgbeIfkAF?>vFN` zaEtg4?cp?XQQ{h}>yno|7ApBxaSL0kTY7(j-r*t)M=PJpG6rq$@5J20a%g5sC%KQ_ zQ9t(%8NcLk{7(gdDM{iUJh=L)OD~t@ZV~{leU3j^;?N?m+bxeWG6scGl$F9Hcuc~Yt{q+RV@$-oU%AlarVc{{EBm-spwEqrr9GW->BGm_> zr9>ymYe5%%&JQ`G-NQ3F4>@10DzWl;`EuW6fH)sc7M~rO3i8zf@I4^i8AJE05+O9`W8mVno<+9NZS7%VSmqKP6ev-s z>yJmO7#aPLbGD3lPRt$52SjS_uUGW;3d)|x?JUaZV^)V0+bu7MF+|c_I!47o|L&N6qau4)Z zLnkNyHnYv84}br1q$*DKJmSw8)>PCHL3~Guj?exzImnGFRf#1f4k3ModA^6oC|dE~ zErOokWCA6P3`(6x{3dnx(LjRe-+VnNX*F_Ff>2c2DyVoTxHd@6uj{NtD z;RxF@m1l`?r48PJ-ZhESlaaCTmO^Z0|BzxH-`yEH5_~Z^DXCt$cyqH*7D?H}j*)Nx zi$@O6fgipLfd4|V8W`^K*33)z#gZRv%`b*?27BzxoTp4R_6$~X(ysPY)KlT9(ma`mcsH&X=U2K_?#7__;P(J2)Ju_qD_vxc~RCL=-&X5-q0XkHLEdcM77x&%QV|8pW9UZYbdmH7%M2`{MTqK5*@oru3%IGH4 z#LaUaJ`VR3zrMI%`z~rJsnilvSBHjsZKpmX4pl;r9JLrmIbZplp{R8V2?$r^VZ4^n zr)~nqXh~Q%@Z`N`aWL1rit_)@Q5kUV|02{>OFJNN*YqB`7k(y%;J!vi$Y#erT&Yp- z(@^av{eg%LgVcw+Q|P177}FZ^CPW7mx?%orQc}!#9RtN@4`ND%_uhA#;>PU6gh=cjEBpHECmG9GFFaJenH_Mw%PnepZh zvCtQYEGX}^BS@l4Z5T^(hrQ9YE=rL*^7wUafSlImnUg0=XKxf04`tq1U%=T*#KH3@ z9mEz+}@{$P()6jKdRvJWEI^HZk6>*DgRG zFf#Jpe{yn?Mi<~5s-o*D61mkUs_w-8M6up|nUVU`{q?(d#+B{WK6fptrhMp~zuVQ+ z*3$m`XytPgDpi)In4ijFpyz0;-fNTo#wM!^hQ(<3x-LQl6rcVya&Ks!w40PjV&?7EPQ-V{Vz9C{5{iD^Ym&nZ(gp0 zmQF_ep83_*a?82Hu)WS2p+^nR1wVTBQUQ~{sJp7U zwZ%ovDz=|1_PxLWzZ1v5htr7{`=E_|n zA(e)P#-8iFq5rpG(3hSOs;6-~&9%NP!W<6Mojzdrr5#|+9MxiWYnjPdb5oN*;vfGa z{9{8a8RY9NPT$3cy3G#F2yXsI6Dokz;q1RL7Es*dYHU14p8VhCa9Kw_FzWp2`;aPf z(YV}HQQz4q4CM@8(q;U#{hQQJ_}o-Xv{gxTv_7%gp?T#VMxwJy?0(-fXTz`h)|`{< zvF+wH*VDV|t@rGigx0XCmX>w5D;j-}xf~v$L!-y=c7mY4blpGt)@*lXZi%gQHbY}bK z+a#-!)Ksg#7nKylVQi!pw_iQkRiFHLeMh9Zut3>67n3*7N-1#RI8kv;V?%@4m|n@J z*}tb}x-CJe%1g!K`+9ox4$qFBZO7pU!=j>0jRJjY`HE}Vu3a-3gIPM?{NoiAhhFhP zsz{hE(T%qq+FUg{6M7eC@BaK5jGyZNHBqxC$8$%~IY3KYy|U&w35m$J5cM4*8(Gfq^(_1p2@;W575sCnxL1-tNBL-=B;KCuK{d3ZuVx>8o&{ z#oWfzCp%?*M8hnc`Z?K|^|iIF3pd=|+&uh86*C#sOcqaj-K1L6D5ycsarF{*h0@TH?@nSV-v&JVN_#8W;z^Lb8ayEJj+k1{JmgUy~Pn0_^rm01=ZNSbb z4rO`n%$cfZ@yRThgM%{VFJEeV3rtNX$S)urREDg#ymMvC%F3?1xTvUzblqwzc?U;D z$x9sVEB6xL+T8p^3-C0n$JW*L6FxqE9R?EtEgCW)i913k?#kL)EOZWi_<#nM!*nqjhtk&JeIy!=8WYPx*^;r*0P==+Q3gV^=YMpE48S3m_ zrAsYR8Rzh=u7aOY^5E9XoazcfIpVVHxVdcw|Fz~2+QRP#(TWf&9WoR~p8u zAIO>T-!p-B;twaDEew`gF7Lu!dDs!?EGD0E*<3P$Lja-jCz^qQ0oD(qY4MK#yLTT? zJS!lfBdKwvpQ)&**rGW(Ihhy;7_VIE z8BomVj^I=3o^hnyiHna9pO??=LW9r%bNTh_{#(!2ufgbGzR+ot*GGivQcUNIDwh!0 zZ#ZN4Qe?q=ibLT}agnaKS_Fh_1|($r`0MHG*VmAd=j{gZATFrsgK8vCA|tztnS??5v5Z_LoXSib1g(*xfN zXf-G5{-U}D`G(aVmJ^1(MsavuwSJP9bMx}T^>J_g{9?DZo^;h1UWYAt`V{Z&nC7vs zU40q3xwybv9nk9q=K?pVoi~a-clbycruzl4)S|XQC`(t@^Tgyd)zv|%FS5O7ZB{Rm zW-9ncn@kOnZxoc6(FmwJhvzOdCCd|PGK_woaDLgV5&_?WS@%G3+bNauidhbB0|LA{QLyDwfR6;{Z84V-5 zLCP%iB72pY%0OCM3z3)N>gsb}HM7aL_d zmnXz&q8%K}UFgq$?dPU!s4bHF6fOw*1nkgVF?ake8NA>gQIZ8@f?PnH0!smuxB4p3 z9>SG@CX0c~2A(Is_eC1~yUgalD?YWri@xx~c8&+U%|!kFmD z)^?^b0h;0>*8I6v5%ef0p9wv5&4#RQG&AlJ_)YU&n$HSWNCPCZiPk~8?e_MEfjbo0 zUvtQM4AI$6O??hG#v1%=Z@+bWKNw$J&l(i<*~M**ZHZ}wY&far-PU$?aBW>*ZDc$(H{%QZ5rU(Eg;o%%p~;PyZ0p};$S|Zb)gv=Y{${tKSDT!d**@T z3oFxuYQ~CV(vtLZFM#n`Tw4?0x@M-Ye-e%@_>eyS7eI25W7I7 zGx2e4!RJ3!>bBHWY23?#GcvZem$Wu8G!(1O-U-rG+~@xI7KAGi??t|e@^R`ImU|yM z+|y&{*ofF314dvHHMO>;bBa6s0K==YB?3k-#<#F&trG-w7y0-C6XpEW5)u-$9z9a= z_4S33g7robbQ~RpxuiYbd;9oIfeL0BSXSkJG+FSg?Lag@{8ussRN~-0Af!lx-bO%X z5DkEq1`l=C$S7rEeZ4}*vBzNUpk^cwSAZ1+{|ju2bWao5?+7wwB&v)R{#A-#3!eLC zW#4Bdz_3Ke3HC^UDF)77zn$^AO_K=CcI#l$DLH|ey_~e(!e@$62v-z${&cl|S{im+ zPYMVP-!}#9G1P_MWU?}-+>1&XIvYH_y!fwNdB4JZ=urBHBmJDb*kWTny%P{e;C!uH z zV3!WCF-_R7Xlv{Fm3z9?d>f}_WgLUCojLO%G$~s5*SE!;qrklzuf;;FY%z3=9Ns z?W*4NxE9q`o0Z*YQ%H9?MZwBGJ45*qvf=CN+bz4(@pulM{wk7C4227Y66rfdTO7M= zxBdM)I;;u_g6T@$AQ_Y0rOpM?o*cfo@9gt@oIHhMZOa8Y&HGSMa@6$@dKCh}F&FP6 zBWAtoe)GD$moSudpY^Oyv>CfRrXW8ucL+%gdtBTXFZu1Z#dO#$_@-37E9OtE&8fzf zZU3zkV!zVs`K;{s;Je=87w*%~b7q6yGBWYxFPNJhJ>`QqT{;Kmf4^}jvbHuW__u6f z{nkR<$?o)`-j8g1c&M_bLycH90;b84|~Zj^xOkl(olWTtuymCR}A`%&)V7Bn~(BHPtJcc zKt}P@3yILH)30Y=VOL`#bjQGtG#TNJ^r!>!8zxLUu$WFuy8u&qYi8|aO$dKsz1L){0FRJ#_w|>6w#PnLeF8mn(NGbCN@_mG(EVi4VKfpaX0I-48(2TnIk@vET&GHRfDcg7_*A2SM&nDD_m_9{B zw&~gT=X}?Tr$u0Xe6MUDccpGNhZs(*kR23<5q!YEzprT5C$)@J=$-%}+A~)`&m3}k zAGF-g%GlhU6SDUD!{ct9lE+@^r~O0E%E-K;SDu<8<>Kv6#N}_go2ZnSEJ*?@(~9)O z&hFWAE;7E-TxL$B*U(ESi9a?odr6U%l2Qg#>NFT&NM#Pc1kmP*n_IU6G zC2dui;&P909_jkNYPf6T{L~5qw!-pe~7x6;yXk`^qWAYzR<&azo z!aO>4pncwm!Bo`W&eC_y>7u)gU%O`8FCIwoF>$`(CJ;lf+|?zj({Cke!~bW&K#Zgw z8t{SlvewXb`z6V&EZrVghk4Kc_=} zld)y@iH`0c7vS*AWlus#Ha*=__2>&`>xl6xpu0@;307mxf;|lOd>&nFe@&~R+}oe6 z7lL$l0l{HE=b-k-y!Ul3rYz*^`Lh|d-{!~`5yNWg>9yn?V`UwA*N@o5r^=jS)zg!c zSL}aOxp;Vb{seu8UIn;Y#)AhxLm{_3{Q(`FINtk%QD>UvE}bb`nFLSCTX1SE3}lCe zZ8x)Wni)_Z!tb1C3AY!yZvOoG>ixQrK-_A*Od*l8fqaQS&!yIkc;4P#xW^Ut#_j7Y zvhby8aFvc60X8te^okI?LDTupp2t*4C0pCeNaS7SER=beuG**Nq;=1383osu+?6qA zyL4I%_84aV^!B)vR`#}7m?Y=2B1q>QQFEp4cA9;ERY9NIO&OXgu_TYvYFBq-x>YU&19M{Kh^5l|W2`4S;G`p)TrmJ!scL6rj(4d#O3@|1J?;~*yF6@JW{t&BOI)6gE zcpDj~tlsHt!M6qzEWyK~toorUgnjllu>dXh2G{OAa3qE_0vf21#SMi{zlB);>C%Xl2|JE2d?@4_OUU%yxb9^!#wiGyQ%9 z8{=h1%h^Ic;eS)g&bF#I-d8z@N&Thx%Iddv%B7eQ26e;~kQ{N(35FHahCGOBMkWmq zWA{|q;fM)hY!1IHuc#m;Ujsxfx&1*s`y>Kz8bPjJ$Rr=S=Q_`BOa>Q*;@YC8JG{G^ zkBv>n>IDc=FkSfQ98uQI&mCs|uAVcz@bAMb?BpJfO_e1XVspsFsg4KWE7gMbn{>9` z-tiEW!bPM|C}b@lb8Y2xaum{gc6Dm98wIfYu8-(XOb!IfMK9x4pG3eKoN87?=Z4SJ~Xxxt!5OQN?w-iH&!^r5^h5BTKp6H zzq15Qzl%#J>%H9Z9x&UG%9WLS!!?M&Hk*Yk^#zHsn(`P z#y@oAdn_yaMbMA7nMqMFA9(eW-?I1G__OJB-?eV)6vF~->x3JDfn_nlX(uvru)af& zf&vVJqm}zoBmkzjh2)FZ!y`79YCF*x$B7_K8 z_DTV9u(p4!M(`THE-ac%yVq`+uH|}Mq;isy~ z1+3hNyQxl%E2c|5h#$F_rFD$Ge+)l;FQQ?x$V{SANsC_7PW=*`Nc%g5I@q-l1*wX} z@bIMnU93qu<&2Jpw{Gn+EO!Y4m9y+~IRECE9BlY2#B5oG9dw}hP)khTutT>`4^QS6 zBbjuals{KITp4T6$lx}+$+N1l7cF}zQd%0{K7$$iJ7qJ-8}M;ipBZ~pyM|Tny*yZ4 zOKi=^ad~+kAIgj;k+Pcczn_P_vaoo)y}f;JfBp!lLr$4nP`M0ZhT|&bvqVp_o4AJ4 zwgPT#j0I*Cnn_;QBkC5mkc}!bCVbfsySNke)=ru@Upa2o^Jrsx37d;oy`-s$+F%kR z3afs>$t8L6+=p-f&f(7)&Fr#qYndkwPrWmMufUlGHwtkfAZ*XQyfopC8&Wj1V6TbF6u~wfN&^q?+ZfdWlF|xmZpTMUVS5^Rs2ZXqy zoR!XBaAHj5+;Fwew2(q{Cdyx1XYl5kGYwNe(?l+j*Z&&8G3>_rmfIPUK@)nFi6J;d zxx08@k2Hd&Igho;$q3@nhF8EJu(oT}j;}#cp!~*>dS9mJ1B%f(_Va|*F5TfmvYM)% zpV>|-A`2cu=jqccV97E%%60~jQS+dNeQazqP@q!M)6pM#UvhTrmcB4mH{Nxmg+-63 z4!4g-hcA|iQmcUJ$8wlm6p~8hR229-q6;+B{e(yO-mGrwL)fxE;=I{;%+$MRoNoaE;B2- z+P^UE$;hu|XXL&iYj0?X#I>ZOnti2u|9*t4KP5pnCLY{>qc+dLc=#8_0Es4^a>9n6Jqwb66u|NxJbH9W zRb9QJ{1oi_Id!jBf=%QLMmRdMiOUoR+A`yw z|M~N$pzPV|%=3`hM9Q!t<4E90@3xNMJB_re7z%r@{!JLX*X+ys{I_q_;@?`31O+~3Ug}^-Ge}9|U zyXRZ~Y*tl8e{|`_2szHbBJ1z5;|6LmoVh;H<`=d@A^bb=?ioAZwv)`AcE5}-yz`8L z4Cx0sUm}O3s-j{=BDl}LD0Dy~pK5!s%a^ZS{qXv5jwJl&)x4}@8&)%SUI|n$Zu8X& zT-@XS%zda}d=~krRS{$i(3c%xYP~LqpTnF?&qfF~B@why z{wo!_|AAYx=0?g0WGEwe8j_8wB=7*@Z_fJ>^^gz0efQ4PIjywxi&@1-M$rJ-Srca| z!-RYC@^_XfGi#+(3s`|fH-q6#lQgUl5(-rl|HmHxUiZ$))vCDd6AsvI&;pJ7CpZNCVXD1;;UZitgQU)vC+(iON>|>)T-wU z+u5@}1=MiE$L^*;nCAJ{tZCXc1Q1$iBm7lzj0)eR8#sWt43qRwPyA^)xg+;kjQ-M* z$@AY|rBH4iVBLQDdPP+y0BbPF#Kgp2cNrr8Wh!N|ylWS_;l&!T z3z}#X@HKU=DXsRE%iSBv8{V-pp`SwncBm;y#*>u?2{Gx`SrHyYwTi$yG~rTISEguy z?k)OJ_B$yAhF4U_R9ouur}olPAy*cba1=96QmUdmZC}qL-z|G#G#|JCEBg!S959z) zJHcODwNoJ~_O0sjIOm%B`HN|{?N`cYXN7xRSx)_RZlWM5PwBCpuN`bz%dhuX{c~O^ zw><`)r|NeUcq3ooyMn~~os;*whfO-M%7c)&VD_P0U028&d+daojU}|RE`0A>R{b6c z=5g>t%eZUE?_m-%_DMuf87_khg5UzrK`PaKrkqGEZ{Xh%ivLoBJ#>Yr2wGtnAg5;4 z>-tu(;Mb)h@ky1~K2{ToPjS21z2z~I97OvcubgV$R2Pa1+}-*>f3wpD3MBzDtHpy!ja{j zGXrx4g$>@I;y{mhFVLJPbOIu9Y4MMR^&SCLZ|#2xjm#M^ z2Ok=IoLXA7grWWN9xm_n0qd0GYFPfUTnIeGRk5#Y&-(L6twUxRjeXYrYqHip*cp-8 z`Bd$jR>FJA-i`-?0cXyf$TgznV5_W2BpcU5L`7r?+za5|g4Qh)DGE1z`&R8&20+65 zXOuhm5E3p}Qw8@-$E7khB}TtU4D0sn#lrPzjI4wj#h34ZRAureQSZx~%pAh78B6<@t* zf$V*1adBwN5#!-I?EQwApAR12XF+#F^O@?!$gG1UIu7r0qlAU2 z1rG5bTTs!vysLY8SpNzIKTt3B`|odpM8BT@SP?fw4;eM1lBHim4o)#h`vn^4YD^G~wQu7;-~ zs5lW&@#Q!{`(Vn#0MGLn`UY zDMh#gY9T8=x!|14NW%QjNB(;4xJN#;Y&DP^M0^5pMB;GQ7=YKF2&u)a(#8WY8M*DV zn0*Rz&$%I;iCw`a3ORYUG>iYCj)XY&B*Df>WsvC5&|X0ZeQu)FP{cMBeHB6>mx8RA zmMO?vFoPJUHwkhw+)7~GqWbO^UXA8GpI}LX%&P788Um~CT=yt6S{Yg^;639dU4xN@ z044zdJRm;xhCz8lDBynhj7DngRkda_vkyuAdCGGB&t89e-`d((3#(;v3QZQ}RA1$i~u9 zXH5=w<`+mHH>M=Su5mi4DJm)VALXlE+%Bl*7v=Z zok|%E*V4iIO&s#uEg-QGe+QIGFpfd+)a1rUJ}6aNE`5XQ5iAUnWw@3BCVR`VGhbo7 zKImfkaQbz6RG_nGPs$bi3hTS6;{_R_j zrPqU~xZj*?ba!+iTa(9-ovUgNBva+we90v{|&hUzUrLU9IfeK#>3rxKm42JG+ z4M3aV+$j7^sdh(vRU_pgCDHUBOB(vlK8S#LvwbUORd00K+!c{LMH!;L$^-%pqaf|r#74y5!qssg-!TZ zGnPS@&o*lP^E`Vd5^)M8M8DHq(vxkzOy7K~zm|$Vfxsa( z;l`0hn~Oq1SG{V}OM5oZjdkNrzx;X7JmAq<`u-Godu;wKI$ArvjB)yknY}JU^SCAN zd<0B^Oz;>*0_39gdGoVAKmO0LMA%IJ_t$2W3rREphg5fcvL5ikPs%!-jX>@YapO^w zh*xlPFM^?pLi_5DGG~rn#}db3Cr>Wi^X=7sBQd$n&F@4D?=-iFmm&Llyk&DF9h@qG zQrRt(EfWp#>Ze;vxRAG#gB4(eXq0)D|FuG_yL6P5nSk>Y1$s07)2IDGGoc230zU&? zC@v|<-4)VAhgAFYM8R6+^f^tJARt7{9+)szlk!@XHY|M_OTmk8?&cgV=-cqxn(14S z)2XYjrn9sfG9!X}7+;n&frQnE56yF_>$M)bqwT`Q<=)HO-?9Z63b0mjdKEEwoUm0< z#8`ZfQEEGm?6)<`a)=;@oCx>AOMmucQz$2OS0#wv*DBP^9)19zfF^P#*&Z9uE6|*&u z5l2ki$cCm^bcl<)U|5zwrSX@kfVaD3bifA;o*U%C@5hqF^66|Fl4vnLWx0igA(YjwRwBe#3+fJ@ zvlpAOME7~wv)6`sy=cD18C`Vf=E=$9Xih1&ZM*Jqg4+u?Qs26F7iJt4yn1{EGnmyr z3d;r%Fe=x4ZEiT4uqio-`)BpV|0e+l{|>_B!d?eo-^AOuZv(v~gq>Z$FB<|)o$*Am zR}83QScugJ^N2k!D_}rc*#}npXa=k4&@kCFm?I@>PeO`tbHGR2<$C`%1Um zk&!8@s|%w!{ist~PQfi}=PK!W{Yux5%cA%kuS7jddCpgRqGU2M$JsAw1hKGge4ZOPK zaQa1Ps{V0T{g&pnUrP7x@t#e|%Hm($iZ=G00IbIudM=tf*R{>R*F(Q?zYYshzxuhL zY};xzs>`XPa{*(2Z;BJLR?M8e>-y8fIjMPhL#F>clLKtr7*w~RAZhfS3Wf0UurxzRUM z1J+4Cjjb&$EfTrl7cO1i(DaI6wdx*+>MO*8BMcD`pvyT@M2QBG9D4n@MIHE4m+PPQ zU?TZaQC#o}mE;UJ5{(bEfISA0ZzI%QtgK`Sj!HYMvB#c$iy7Ha)UpK9X3y@*N?m(K z-0p%3L5ELj@_)!)Zxcei>-S))Bqk^fZ(yeiL&u_LdKN-@kD1d!v1OrJ^uiFPg(AiU z`;>9_T+gFK{i3RN&nmIVVi_X?B&q!Ic}(z7L_}+WmI*lh`k@}rsnV%c7Z-xF+U$T z?Fl+-i^65ltpMFhzxR~o#59@H?%iI4`l!n!OqE+XZ`mW&f$&c#9-QnI3{2gr7aK zRemA1;AC++o${B(`qiLgAJVIu#Wg_P6(0(0KsWutX7Tq)aX-|H;3cXkZm*H`<*-?h0$bfX@p5#-Sqq#7L0ChQ3@h>4N%tCVf zr-|LMlK6F@gr?NVqfx*xE|VLWCQ+g5S5KMT=o&6xDLsD4wRn2^q}!msf9j0)ty^cL zczFE!kk1$R5rxql(7e!WVmNyoNx*H|ESJ{Hk_X01JyXMq+mBMFbeP&xPUfj}Y zm>cxF@qaEFWC-l+?2k=r7ALacnvZ>QylE(7XvT5cI=fR2%E5(nnX8Z)&d<-wYLiSGhs4LgC-=i! zK#22~jD1!Jn$!L*poZ!jkG>rA_8Mmq&=Oqq511}_wBFciT$s1T7Rol{kRbq>M#q%& z_^!>#p=9>#+2c06V-uzN4)4-2GUDE$7dxp^fGzy5gm)YF#XC*6qJ8l05hcv7iW4eiv?a6vDOjsU2!=; z=IH&T4i`-uTVu9$PVps5226t6;J^GI7OETi*TWv#6pyv3_$jHW9fDP>99W&mMaEZ{ zJ!<#ckm~l6)nvy>*nyKMq(((`(eN8I`~o(gzvd8DH)jmXK%I~?M%k^AjEyjcV2}by zyU~iu%9qhBzP^Dy&r^5cKETVx0-cMeE}ta2riZPBOn*a=cc_?z%*y$$bOScY#u44) z^ADDKd}l8Tx=BM?+}heIx}vV40vr>hyQElEFG@Ko~s!iZ>vCDe{d1 z*aS5(Y0IS&k4x?fqr})Tb%*-OGKBK&7P-;!LLU0qPM~Cf8(O&aFDhw=qYw2QST+!d z8@U?8^neZo<((|8OMF@kbib%YAt$S>qa;7gl}xGZ>gGw-?LE5%DpM)&vmG7SLQiNCWwj=Qgp;MI%tuV1zfJ@YxLH|?riBlH8HGYQ z$N=JZ%GaC*$E`x%hlHpod$BhFq|mk}m^9Imv%BoR2PxDii-MA( zATA}vOc0OcS+>!Ug#~<1s$~YD2HFznJRwxDG0kfqNS1U^|O+l7d>Y?(0 z0=i63y{=zH#E>%g@3(H4?dYqlO3v)Y^!DmOrxsW&cmXGjor6QS6I)mq39YsiZ|^U6 zW2V&~r8CLhoEX9>tPM8ieAG|9N{^DNtm`?u^}FtD$mq-vr6NnU$JtM89BOre-SLTu zH8x7B1hf3XP)@zeHVg)L^z|jtEGJIH&UmjF?16wJUna|ry81nLhV0MumA-qv=%`AB zwAz_~PkRO;j#d!|8Y9QC6Ug#a?>hjIHm3uH7b1nhd8*w9p!qsF#S>|Vt`r{-TC+4i^nFdTXuPyk z6O#_La6x=?TeKdC%{va|LagxlbD^E=K*gGWf?wx?>3qrh3wbd=MUFQ*-l0s#K42XG zj>^K(OrLmXWgkss2eePd6qsVyyYCPpWTX_C2!2Ctf8)$>|F(TqNbfx+D{{2m8 zk$;t57gQ}Jjyz(H;H)$YlivuICaH{RcAA4H-+b1PWcLA#QAxcM)YDb$!=xo~>3k@i zOIQ9L&&kTp{%Te+hIV6M2-LGbg1W1*r@{r8zR5Q zPBPzLJdUmaO2xI4ElG(57j)6Bn{RYw;HqSL4QwmNzj}2ClL#*qwha9%i79gm!EynR zkO5gdRH7v7zkdCRFpHtpmdi}fRV5Xb&!AU=&2|P|A)Ui$BP(sB&)Fw*C)u%PvV$Uq z4#yHO;Lv^Rmd^{W8XPP>l)HhT$f8+YHX*aXyhT;)AR67f(9Kjzm7Q#*_eOAny2rI! zXVSdYOSN1FvhyB_xJIwYEo`5FBMJNzD|JRBeJ=vGnN} z^mV@|Cj6WEs{Wes+Am*TPsRIo@051RFNwJ`xrpn!hK_kPDG(|VG9wi6LS~7SPsEfQ z9ow1mf3fCk1ooPIb0Gjxy0jvW=6NkU3G69eGVE^ZhWKRg1`p?T^_6NHVi&YtF%#X` zpqLj@!nouLSA$b-VAy*eeF}`_;f#F*C60?JWp7&stv#P>lRRX$Uao}CRv}Pq=!AaA zEaYOsK;TWKQP(*83Kf4;SO1Fphd~~w+`0kt(ZCw?f0UW|oc)HYt@N84hS8kUemz~@fos=zMi zJN4Ol*F}aKHXGJ50El$St6h|}Zzg8y8QO1z#38rE-~IhjCb5P-!d5s7c7pTBzky!t5}COdA&p=I2293VDsRiDXqmEq(GY$uq%ylF|q=+c%wGNa*glMdjZK-GyMJH=$^0&n8n*;o*+#)YJ~EtugGW^fW_PlpyXo^wZCO ze)!a>udNv*)d(nMJ~{uoWl&+XHXG}zn(*!GexIMf5BcHmIfSt{=x5T2E?KJ){THuR zz>5!bxcZeAf)&&3BYVlT?#B=`L`zXELXgJ=?8DZ6-I7I4ZX1yn5sDVE@=9IlT3_ww z(OJ>1+4hbthCOvb#=y!hYqoS6IyfY#12^T{s4w-;FZnNVd9K#iLPL)k^#VEONAm~0 zLVWYucwd#zL|h(1b^FzdV83=wT!n&m$&?4tcYwf^PDd^j*5zQ=)+N11O4Mx=Nrg_w z8tS+-=4)$r=QPxrqw9SFWFg9K>L3_Y{DYu^7(sID+Jo zdwbActTm{d5qz1==0Pj_(7UNqA5+h z6=F=DyBr{|P+BX(;mS!Dx_`Dk7Ww(>*J+TyV5aryejQhRV}t+Un;y+yF0TMn%dcF^ zITQk28e;kqxubh}zYY5SC>t@36PujoTL&Hu~oP z@4T-(YTWt#+c%K}wBczOXqYYuE%Tw2-EuR~F8=MfBHSLh8G?A)#>6nn9+Js!7enOw zx(CqNMghDjDkAa#h-|=gfM^S{BH3O?7nhGIDJlP=h9cLhtJA2eHA+{C_JQ_T2^MMcYLg&RnA+~GMgsklpVoWRH;W#8|!FHp5JlPqf zRUYk6#5m5SO-XRcy0(IH#avum4K@&@k^Zmkg>I2I;T;VT85!05{QS#|xPO2eG?SZI zg8~KQa~c{E&4?6kadq_q4-XGgU!*vQoScHzhmgO(%Q+GlW02i|##LH6o-lL+^m3jN zIVB~e5y~!qbLdP`5a5=IZ(*g*sE?Z>nkQY6aV&A}48$ESl^cgSD|F>7+z9~wVCKw4 zh$f$Zh1Xt&UYN3m(N=Qh^p{eMY7|`@0eICiWpnC>crmAYO2MgUVR5S5Ycc!mBqo4eD zLwsf!G@EFTp;BM`fhHv{bGCbW1mn0X$f`<`%{G8{+Dfh&RaEaxWV=P=$PxQtYDczM zD#P07tk3(F7tz8}DievW>IkEt-z&DIH?gc$)X>(eLi6>3gbpAz z7fMK#e(z2Z;f$nXm+)~^V`1T^y&p;mwof1JGTYXhX z^~+z?0fq)urdeaEL<@VDxTBSy=grMETgMk0Lv}MMYpaebvp|0C>KvjvUR+(KC8lhw zmo9AIIqW<6!?YWQ);OmRh}B#Lm)#-nZPQ53wqh`c2`8m$#xX;WH)z8;BT(T8Juh<# z()q{UP20)@Bm&1nerIqTVK|tOZ~I#YM!~}c_OtdX$e(A{={1~Rot4s4O-)TrcP>G@0tsKKKJYy>xV+bmEn3bR zEUq3mv*jrQs?~iK#U5`}jex#l-9EMzeleqOkq+PlJU>bl;(^9wwIV1G!?0uT4Ukjj zeW^MQxRLbI0CE@hq(k=l`tYCG+1UbLWzOua%}sp*30Po-%)dx|^R2YDAs?cG6M?da zHo6`cn0a61q^9hBm3Lt;Me(Rl>8BalZ+?9H_U+3QXnpzll|O5iG6y~D-WR>C1xyBW z^DLa~^MdR>gUOY~BvI0eH@qD*cm29-ud|nuo7jPuH+AzKEue(N#6Io4^&j*C4JXl6 zk4UIHm#0Xp))V$AE?SRv6ZHbi^B3x^2TStuUVy&*iDRFe8?C~zSMTyrHsHUYt{>Peaq*hc#LjLMN~&LDx7>SuB4mCn_sZ7 z5ELQjcZw>-8hmJ#3Zvv`DEr`^${VS!A+O50W#ky5%dTy``bX7`*s)T~8yT|vd4m2yYckp1nK7*1Gt**Wrsv3?8*9%VciJy1UEs*_ zGI{4P+-hQZ?p*45KkAKbcWFPPh9Me{-^|l^GS|t%`NKUA>Z(YAkxxRhmm1~SD~z?2 zuQvWUYbhF5L7@I=e86t5)4iuLiSvKd=X)A%vpC+_ZF@wP?}cde+b2()h^W5g0rVcmIC($muiKA{cX$;iq1wU3QQJcg$(9hwif2#*YrND~Vu=}?s~dS$EbMhPWo zOpAnl314y{2FUahgSVV!qh7Fqr{I*`+-7k&Uk8WDilvY^1!G(%S7 z7)GRZp7yIIHO$#j%L=wI*v6-+igf8tP0F-ge4g1_$ef4DwX%Ee=)rj$I^O$LJ=CZ3 zGdB^tuX7YRZO%uLcawSd?}itd%4Cv0e7FUh1cv1E7^vx&+dmj2Cnv+-f_Azh$jQ+Dt=<0aE@*ue!&a6o`e^nH`5sq`=hQXBd1br7 zO%96ZqO}`M`Be!!n!U;HWF1g(lzy4>=OM36}^Eo)Q5zaxi(#yTw zXT8;SBOpKAYiPn=G7-)tHSX5#>KBbR$hib>0HuBZ@?|d=00eg|?45~v=G>;1{Gjw> zPFB{&pR=Gc;A;7LGWrM~TwPdC$)~9wT+?Zrwi6tKTR)QhCL4$)xUD1}B_plMpdfFq z9C#8{s&&j=?YWAH_Og1%j~|>mZscwvI~$XLFh76R3}v`dB1t9p1ddOxZ6zarzSuq8 z=b`JNmPLi%=KJyxOJEn_;A=P{(1k5MIr+ z<>cfb*CQzCLuI^vZaPpZBt8C~=8sZxn}eq=kv$wA_so3+&#RsL#NvJ!=tyA?4P<0w z9QqmPhBVDDT5-+1nNm{x$#*09wQD+Pu~!w=WZmba?T3nk2IW>&QyN5(?PwepI8 zwj!aS>Dc}zm-CV2U(M@G>L&CI^X6BJOWMLmq;<0wW2(!$GS8!1#tYI(W)V+tmHThw zn&0eoo-3ZbcY$MSq(@?vV1sE2A6V!}x7YM$oR$APigsCBtG2k7f1HHT3voo6aQ%|I zXX2AwW=NIhV&Rs4YT#gE;t%abn(~+r97qNS9o{`L{|sIitnH}DbZGL#+#*G+wEONQMoUYnyaU~Y>YA~Dp)CEV|Kc<-%5s~N9OIR8!nhSL7 zr3k-ENOK~Lbp*)t6pV4OcMQy3xKP+)ju<}vWF=L3d9)biTT{r$$Wd0D_uhTJbwiXu zTLGh-uJ=IX$HiwLE6YV|4FSlIl~@i_j8%!>8FVuF?zK&4qVu> za|PcLGyCwWCS%4#Zqx$-T;;{6ORGcB<@H-@lCbW>dyR*;bRxB^D>dAMB|i;{W0pnX z?5%`V(B=nQ6DssD2?>TqLCBy@%I@rDqBCKuNDG>%$WyI!=Jmnv4J-_EdHApqZuSa! z7jQ`|*1|R4JeY_f&!r}rdwajEU!trnRh-2WuIInjwkmA3WbMX8O@g(Z;cjWkWddWx z5s(r|So`lkLYT7{I`lS^JY&wZ+3=LTYKkt%8aJW`#VeI=f&C42eCL2=VICt_Mn!3R zoJV>j_>Ubr005HqOx&e7;Ss8X?C(M+@-XJyU{BAo!dmy#G9-xi^6MJ(xY~vv%Nq8C z=H)rCx52EqCX_Z03g~J7wEmB2%v%-XLTYLw4 zrbAu5OSE8qIe)HOac+HM(dEQURmSx6eQ<{}X*IJex_R5pW?;-0l}^E!=$g5X0%bhs zV~dsZ2`+7!E3^03G5y>MZcLW&>YVJKitEjYh_$L{nekqxy?!LSoy0+qi zo;6GG##%?#DxlShKChU9*J}0*<%fq<%-o26kY?4GeIzuNyJ)&MiQr+IVo)-N^ZK0b zY5kE|!2ZY)?>~l4PFbUm*0xR{<~{XSPON4MT6=eKqZx?wnrX?M?&5G}pW%l07L38#cwb~qm`PxupS7IOBqL$2i_BJS?VH<-Bx8i#=de zbe}Hs0%WX{Wpn2U8AZGz>&)v#-A|~I{Bi$2iCwFqy81M|VgJ|ZAe+d@#%)Wc+K`lp z`lUhlrJ#&I6DL1qPAA2?#xuF=4-8&0Uuhay#(97K{JCJwXX(Q5^J^&&KH~~}S3}dY zg*`;ySaj*R(cX`ofjc&RJi*D!=h1*A`w}#RBAnbIFF$*|vt!!QD>1Iw1_y;t1trqB zs=i46SeuhUJ>c>rB)WZ})JzA>dCM!k+RD;QTwYyN7z5Jm*bNSCjK%#+Eest6BbT~n z24cEz5Bt&1RYYrK=|AW+Du2;+)8a^!>Q8}(xcFSF{Ep=r5}BJRs!ZKW7tkt|+)So(f zbgt{)jzLO@?3jT4HFc9LBdMb@oF$7~H0Tb_{;2K6x{Nnt@`&*?m%E&6Q>0?}Gn6!k z(3c}e+TS;QKdPmh-BtNW%9tm|^xHis;e45Oc!Rq}zt4BnNs$r>5Ki)2TN&ru;yHZ!eg=8E+@-3&8(+3GhA?$AB2B z@1qi%7YQCdu$Vwow|rA7xN(j}y_16q9kvleSQ^1%Q|M`Om zEo)Rp`R|0L50e`D-2ffY2c?F-(Y%*Tl zlH@A|m%uM8`(cpWo*GYI!u&}Lao%Kb-ZS1C%}%Su7>kLTQIAIN)OYdpD{zveWF!|A_45oy9nuLy?sMMGeLfZ7 z6Fjvc9yG(rb(Q70Xc#qC$c+|5)q=q>3LcjEv@t6#%Xrkb81lDjn}1$O#2#(NHSMQu zomonf9i00%Xw>aAR5aC~&=sZGM50M-NxlYi532(2(=voSc|6H{3v^D~$~2sNT1ojTBWqwv?@(@5)-ChOG1i4o3o%OEQRBa&&m$-r z6-}%^ES0sq@iTTuBD*`;^+NW+&Bkn((j}cuU)-v4$Q;KhcZd95i zli=JpVe4`*Ce7S&*zOI#CaKf0#+<#MgZCdCka(l0dwk}GO)DaLZ|1L>qyN$>r>UAc z%?LcsIN<77NNcAR4f(17T?ILtWH7N}WL)#?Y@b%13YPY|ZVsEtrC}oa$w!@36ch|y zeKA+U5l6pLT>ag~F@4yn@N4-?sg}XHAJ^Hc79B;ob_~(Nz3Wv@^Q0jT^qdmJw;E^z z32$v*;Q)>U?lsfg*wbF?n%Wv&RMSnkt-13J^X{RoxHx1uZ9x{t9FX>@VV-!EkN+Da0v?gd7ORt-37wnr;-oNPT?sP59-=!z zElX}fgxcIMPM0sUq_z@i_a*v|h_3#I1V4FMuWNMaMC6}C6P+InE(CT+O6>f z9t?K5m(_W^N>|w6Z?s3s3PM*R`(Y}``3K?5@JTEeMTlbBi0S6~+}8i+oV*?wemR=p zz%L+rSNx%$R5PT9$54Q)wowsIR?v82gR)V=d%`ynR%W{5Ah3Dz1hN~0dzK?yLoYMJ z_tq3OZ8tEG>c4KhPxIh`zUxC18z5>5OFBs2UagQQ^{f>B$wL#a*+FY=JK--cd;L2Q0|#n&U{DJ z&dXklCi{X`LAOs@=5mW284CwGqtvrgzS4CyQCN*vplk&}Gq;Ib-{@$iOchyN+?Ovo z5#fwJgQ7BEk9aSl#;Qa`@PjOy@b zJ0!t{E3eAELVG=>c+_F%Xs33#@sc)r9cwat+IzBY*IVgu(#7;+&o+uB^R#eR{Hsjz zTu0G&zWV_|sh0a-KCm8fMaVuI?}gXZdW{`+ThfAaYw(~3oLl*>@2YdlNoplty}M{< zpXiiPV=))zz>3+9=nJW>L^X*Lntvx{TGPlJz8u%V3p@%WvYj$~Z}b9Y|BAY*@dSGAMYV0iHHQE`LOA`dcJZCS4^~BCZN*zp4jTySln-H6?-`!(AIzQht_+m&&$8 zC(8>`gWp#x@H4xL+WtyvE59k7`EKiN1DH=l%IK#$Cx)QJ~pT$X?V{e5Lj~yD)snuuRo-8ALm&6oca96k}*!0)tX1l z5iu^e7!ccJj<(1#-NO5BrUh5bg8@sDA;gfm*Oq*yfK>T327JylJDa6O*m$`&boC z#PCkO;g>Vd!^U%h@iS0g7M(cnF+c zJBfH#TwD0}3@Daxc!45;T_$ZK=%FGwMC|J@Dfp;qX?20TRjH4-F46IqsQ=rkTS-@; zEJq)NkfjWhquREL}he7~!F=u5Kf3_V;F)PM(x! zs#S5eZbEsKgBuW%o|+A_{B;*#*~l5tm&GQA78H63F0@)M|bua2o+q|N6)Mm|4U8?z|$HC;guq_aT9 zfrhl^pp|h)PFjj8&Qc^}F1@lyJKrG7(wChADtX~zlL<~6^u%qedat7UYfh6^>sFJN zH*EB^0xxJAnV(e_kUE}pGE3r#z%#XG^AZCWPOVpfQc}Q2G8F4`^~FnGnRr&P&o=Wo zyslMEZtge-EU)khuG*dM2fe%8HiqOB8)w@h9QS;VyF9CCfv*s&8}?5Sa>K zY7Bo}mEOs6?w67sHUOL%0^n@Px#vV1pXgZX$c?~l!4Jb!&FuKJ5l6kQl9k0st&AP$ zU$lQQVh;o{2LiIdqW76M&QjSnPx4oY($UB<-Ks;?@_b zB-Qw67c|7-qRbN?atXQDYb8nPHX%DH{o*@hj^-oQR%bwU)6j1cY8+ZS0 zWBdQt-c-1CKw}Vi>s1yS_Lpm4(~nYR-}>=h&@R_yczdS_01BLO zgcC=O;46~%pVqc+4|7-%Ms1|M z6jIl>OlZdr4e>+8QAL^kW&tToKWwjtrW*LKy^oBQfGtYa`TO{!2`65^dy|NW>4XJF z3}R_hnZT?4`>*<*M4Zav5tAfF+5tMw0sL%~yQfOq`WV^HKn4*#ZMOXvGMSbp7>uUn z9rBbS7!p}r)6&iA_1xq3Ao=@25=m@~kGtk~DS6ru9-tKPV*vwdNb&d9Q6$dUE}^PY zp|98g3Ju!Bn*`qvk(01C8y8PP&ZFb@sU}RdKt!%j)C}qb2EUW8*L|B-Y zy&Ek8Q(~H0&&GJfoJA^MmkRYpJyo>f>2rn`!JgQY$U9MOE+|L9mJ8NXBaUBrw-kHT zpB`-9q3ANw}jJfgC?50@Z z>giHn@rJ>0hk(l|EiDw(tKfHe*L4CmQVSg&H2ZE!9;;fW43>}@^aF$F&x8kROi~z+gLQ* zhTcc0IJ>xn=wSUUSL{D#HZK_0NY)H*R^cV;z+WejdJNd+uZMGhHY#ucT?7SK<2K&zuwd|Gb0&F z0T`t3wMfrAd&cx|Go;T#!!!K$Nv*PfiOLormgr!MQj3gXS6Lj>-xI3hfJl%Eeq-ZW z$YOS?U*@lMwzy{YHGUtbTfHI8{@Yyii<#QqhC$Z3%TEOdn)3KvHGCu6x?9wP5RBJe zY_Pav#SAhs`m=FkRUW!jvsDj*ERD))%?yp)eIm^CLMHiaD*`L9L}2+%Jy@V~ikHu# zgD)<3a~*0!(-c>4ru*;i%4VDQ#w!otW3=a8wD%ibiJXk(k&O&&MWH=soP_u7*Iwyy zOsNy-ghr9T0&Uv5c2xpEGpVS(=EJg%rb2t9gF7|CtrGI~6pnd3hg`B4*itWPyJ%y3W5jA6AOWJwD#3J)Cgd$(TM7BUyQN9JP_R8^ zTfhLW;5|>AuOz&v%VuTx=c>Wp^yQ>lgCZ_AHnUPN`DGA!b ztH(*jh>@%fSq)!@R`K5{i%sk(WT(!8u?OxKuvb ze;x7N^o^PK_SqbDQQwP@szmHH_>AMKz7vc2aTm^T!pI9JzN4PnN>x$dr)RtAMnz}p z53Pz(C(c>XBfjOv!j~^y8f;*RYMPicO1(IxTRQ+8m5SFl&m420|1Anp!X@eNopMe(` zCxc(rY2;-XgdHo5F7O>0sqhP-0`?iR{lgtRGZf8eF}Ho`&z^`}E4bP7YkOYPFA*mN z>(Q}zK*9QUSDn6E{~>+q@3|T$yriNyGA@^b{3WQ@xTPAjo9Rk`I55OM%SiJ|365ps0+JP-YlOh~ zB4qSwbqrI=AT>qhX`fhO087cQQ;S_NLCdPNXD1}6Hi`Rs0f6mtTcR`-6xhAvL59VR z(%e%8-cj9Em_B&S)Mp(9~&hi&$kW;;mozC4aD7qcAUXiGU~&n zJ|qR3*TIFJ@C#HO7xNYx#lkqa<_p$`)`EcE!#6j2(KY9_A_H#^)7+8-rj!&{WiHmh zT&jn4eZBk%1t|vytw^CEo@BX=R<@Tp<4^_|Gzv)Xo8ros9b7aTvqQGOK;( ztd`=^zboq{N2(HzkHDdBwe5Sz5)&7P*b4|ea>PgjyWVG>JM$QfJLy~?=Nx-I)y z11tFa!UB0Qs<6sdcY5GNj%b(PbR(cGDJS{OXI=`j-`?E*yzXZ zim@1)21RH9mXL4VvW^=ZY3eji##{Q8=g+C0{=vsZ`gP0deOsn{3@Gn{8Qqrn#AkNG z_7=4BpzArc^-ul5K`nh_=KJ^0R@PIDigW6dRk32OoXJcjQMUTofxf{}lAq=cSyDG_ zbkCpv@K#_0Iy-A{O8IjuXes~ z^tGDIU@P9YW46Urd3h5~%0#+TS4YmJxaLlWAs+U?Hep$}rec)+(ZYnhqI@Ex*X)8_ zt2isnBkGmByanzl7uWP+qA)wc_mYtTt+!=Ot(ac~%Jb!?n`Q&yxXr`=FeEns4WM}< zA*rsGavi`h??*15UCUpdE4l%**8x(zEIs4h6!xMvyP?f=tlo|vxPA4uhC`Y8d-tND zoF-0Hy1#N)Mt0InJw$oC)~ifzrp5^hETCzZ8)LE;w1CI2RpL?)VgZAgUhUlm0K7p_ z?!z(`?)1dN3o9T{xGTMSM2Bf?&z86nVKF%9fcL3-4Ho8qQ^mBwM^?0dkCtzIBa(fK7J?``4k#l2gh-WEr$xYVO`$4-Ro zr@L0{9CbJ=6C~5iL;VFlXbG<+A@}1U&#zhHO7ipD7ZXF7@vh_iUupKbsk1nkzISL z_n;sXRS#kfZf*FrX5oE-)>DA5u*|n40$3u63W4nypR-*;eSa)=9AvGePxBR6~6~NKmK@}Z_AH;DH z%r2X0>i03fOf{k>o7=lCI~A z;W?1VB&I*K1N*w!2U9iMk|94mspt1@?oE?65oZ0R&FR_Z>*-J($4+uvvu1ES51CP> z+tl{{ypFE#^75(CFy7_<3VjJi2hh(5V!AMul{+UYM6bXj;*r`V*k`CLHnV@|PC3SZ zk8A#}^jczfn2#E6pww~MUeW61;xDkBue*%kLD zfA#leE9%lbm!BN_wF!^KzK#?(HVbtGLpl$s2qoy+j zUEP-r4W@ zd2VCl3w=5f4E}ceY>k?~c=lVY=%W4WHcyDJ%^mRC{I+rupt6I~cn?*It-zC$Gm47n zD7yOIxR+ki9*}dh#@_{T$b5=eABHO_pcEdhbtNgqA&&1@b6eXdcP%6^cao%avtfs-6V7X@RU4p6b{_ot^(~$Rz=+G1bU(BVL0<~3q?PEq3w6|-NuWdRj4o7TJ|rm@^fsMNEXP9X z?cY^KNYr@~xW){HV!KZ3M-KGoUY!uDb9n5^-1YiF;P(yWhBu@gD)$1V)$WBQqD!Er zXnm(vPdtN$MX?cvp%>8q4tZ^fhkf}Us80?+!d~aP+z(&9*WANCS^h59;aOS^ zu-o847_c-8x?HO7*O*)(41u&osGa$Xt7rb=Ps$!OmXoxn(Y4@W1@lH-1oOFin^y7G zxof!+e{zJ`mZ{;AzJbUmt>P)T;dqbrVgKCg4M-<6NKy>k+z{jS(X889QMojGtBIk4 z($YI1H8}AsTCZ%4Jc>}yHO1MV9EGhOhmOnsxV(M6KS!wXre(#CS82b4L8&ujanSsM z;wU?bcGV*FL!%NQdXaZyeUWF@UHoUAk(L0c4HZ7I5W*g()F+gOmWrrU&^Zr#uV|95 zO}c+WmSJpT(_i|Z)a>82Lr|D@d<3N&^Ih8tz2cEYF5L<2>NrXF9^fzl)D^$&s{D~? z*NcIlGjHB)NV@qsq<^eoC?b3qa_xcyKdDkwm)ZR6Fz|(8w9h?5K4nz^#`@qIpJ7?Y`ObK>?ZN;ow+ zFhqprzf}KIIMQXwEhzmZ0s3R#mQxL>(B1&gR%`U!9?ge&w%lK|*;^IR29&CVw?nIa zJRA&+;luuC+`Ev_P)p3s+HB|rjze0bjYj`0#kAjR4mNYT;wFo};m~cd?5Q~31f?$1 z)<;046;*(g%}(hh2p5~V%C3AnSAWI}yzlUfuM^TCJd6qc!U_nS_+nIP_y&0QeW^nR zER9QB>qXad@IhQnwXWR}m{5q{myuf!8mJXbD<+(9Bu#53yX@q&hJ|*ew<_88rtZef zt{!V?F^Wl9Y}a2qPym!XN&!_xMJr6P&qla~bELzU;8oGEgiK{kOFIo+aZu(6heq3q zM~F;SJKeO~2c70bxA#6X$ILbGL>T9=y>h^HF8?Oc2Xg2Wl;Vea?Q*510m~`lkB8*P zMF1(KwwWSy;iD&!HH?LS!<%7AW2B`sPPp4aO(!p~)%&Dks%5pQ68W391|fo=)YZ2Z z{`ZDhD)7v`P5e!ftm=fbG^H>jqua^BYqYbG9=t+!DjGI0ZFJ%Op6HETABFkcHe0!Z z$Ig+KTK#uF#J6_;(pYfswkp)S7(|F{u&{C;abj4STf@b47e z52uo11Zn?xI4;E|78rRK)DmZEYrSYN$poe&1(MW8>g-#QJ}cAwEgBSkhvB9a+usIP zEA>4NY)04T=G3L+0QXa|XX>xGi=27nzEIP7DEmL-Isi=B{i1{A7q9yI@ z!oowz2u$J%XNc8Wmg5w2iO>mbc$0y%;Ps9d4gIO&BCfr-q!u8W^@eCuP3nVn&)uQ> zt3DM!meMggdCkvG*Lu`DR(C->?>c?>Bs#=Ad3Rmv^!zIl;>Etd9vRM~Ps9;ouHbj_j84NXXlKDo zaVVYwDo+%BRZR{cwmz~U=(dxVd}hayesX-sE4tpaHf*oGzu2YI01T}54x(58ZdkG{ zqe05_b=aTC0J1t`0P$$&j71#CZVGzKCL}TjKS@Oir~eUq)!5W!+H39HS5WW^vuSP{ z1rk8doruIlOu-tl%J|Q| z&mv64AP%iXo3Xrr>Yi=zPvCKo94~cQ#?ATFaR;0r-t*n(FuGNniQKniv9NP2@u*xS z{ItKIC=$PRa9(@RYQu}hCA}Z~nmv6sD;PawCjy0ilS7phEK%%X!K!QTqwszrZ$V>V znD?#>Rw^LC=i8fD`(6;Ai#b9UZVTV6C2`8`gACP$&tS|U+z_lRPp!LyaAW43#x zyv{6{5B#%YkjNNxkBf_27uo+WmdE_fk-Fmb&Ubr0JKBYx2PQ>XSnkDUFXPIca-)1gcjrm*+lj!Sho1tB1AO*MPUSbkE&=|KQ9Yva=s$1NX)) zwSCGga!&o<%H<1*!xa3;?~$p6zfcY0uZ3Rq=!zp8GBIO5clUPrS2IS89;2(m>5VW7 zzQSdU>GGHyOn$L2Buw@VHNiHaxXZXNe{y(kuQn>TeYIA|tReMZY%>*3S3u8KMvtqa zu5dND-;&bv#q;&VAE|es3&0f1U+=8@e#R80{3k`6s&m3fO^xaK4iQ%s2OMzbl3zDv za$jEDU_Xl(OMHBpu{7?9iGT@)(3QhS9jaIYwT3-KrLZ9_&a4KTp8#!xB@tYY4-^Xb zr_0+|3^c;Ml2I4E@or;QyMDoMRz&vd+q&21ZNC?pqqL}QoY3SFh)o1V#MDS+-$Y#< z9Qy+j^fE4w-C4enlo@3O2n(SfO*;<+ehIUq*vdgO+$z;nox9P;C7`Ir_Now7t85Hz zuX3Mhm^4)M-{s~L(1Bt+#>AgVOQWy?dIP82U;utK7Z3SA=J+Ncp;x?);>QiL7OTHE18`hV|Pi~xBN z3`d6t;(n3qW@Lbg9IG|1p5H&R#ZtCesbF0sGAGVUl59C3u&$$Dzq0S$Qc1oR`N#{r z(>Gp-GZH-|q+Q<~aT!{4wwgb(>;aS#t`i991@&(mY=`;i`9?p95gw<8lr-&rGh6EPk+ZIv+|eFZ!}p74wXOS|p3*7+$Z=8uoD58p8T6`%7(UWy zj#`Mcr25HZy8c}#h&ROiJZ!=m-xR;X|BPrvhZ~siPBp@p5nfX9+qK&F16jgd0O*;W z0QX{7M}L;+z=+Q|OjMRf3C;kfMaPU5AX<=-WeRQxKD#j^h`>bV%14Pf>j0_xO~%YU zX0k}dtF4E)4+==sdHfO9k?9MV;33)A_kIl zSrS%UUq8s>6!vDOv^ou+1`g`a>r4bDs~E`$tM#ihl^V;OfvwWZ(X;_aS66i_lBj|A zX|>6ls?fs^_lqUA_UUc-*4(?)xZw}T7i45I;)_xfE+fZ>Bg*meG&LLpTPeuzMe<;ekChiI?b0=iIlQIzaMZFL6YDge2RZ(mt;;weHs}-1Cyn&TW*>V}0_3eJRCs zjcwKKgmS0w4Z*bUM-w5osRaOR!CE`p%iF!_k@xb3)q}465j?w2uK`G-?5fEvKx?OLhDGQQ+u zwE0lhMS+YJ$|slzML~W zg9dKwKJ)$r`)bs;cb81@uegJetd%!|%rvE}Dr*_KwIuG6yi4Z_sXS4-&zgm=~ zrFYtOHXU80uf4^uxK01=TqF5ejxh{_-NKTG#O+H<5A(fEyC;g%mz6lwNal+{R_>Zd zF@#ICI2^&Sm|uh}TiTlaD2WKJH-4L*)AalT_$Y~VK%YNjJ%fNn!WK33hm#%_Q@GpL z8+EEloZ4X2t2IR{V$9Z|G2jWf zBy+3__BSqI3g+wVP;^ag?k{~)kd)d-6ZYzDZ#PfA?H83ERs@0|tIFIB#sTj%=>s*T zm-}xFC>>M904$l%O%dN3HewlDK+*_m(j7Buz-!%i+uki+ulHLQ@LH=KHt{F+3sr57 z-r9gEmFc+_B!lEW1qm|Se|DxN{vGP+i<@wF^czG9U+~7Yl9eNx7 z*70yz`2PFiGt$llEl|hi*OxL_7jbV;dHF__8KL5+@YSCZ?^mE`wJ9^)_)gyVzAGMM zbkn}_kn!hV+V(LQFf@5r@m6n^y^plYN4_bG&Ak=W(AyJk@RnXJ^kN^BGPk&&B`XhX zCQ?{Z0fj{!^MDYUGL~k%)|B-%4zk)8l|7%{22lo$sWF-@?mDZ^6JvicsfRf~0bfe} zGl=!?vaZc*EQNTJNJ|UkRcur|<`}svcjFK3nd^O4Q#)C}KP54=B0_1DnFX>@6ZlLJ zDLr1W5fo%XaGEc}6faSupPYUSnn)h5Rl3X#hXUNL%SxXmWwLB9ywfJqJpZNn*=`XYhJgHH`Qu#uiAT6HNitI8>_v6+jYt~H}iqWp}yg&c&`uAlK%Hl zjSAH0j!ZD}qsv@8#XELBy*FdzgtS036Bu(kxpfFgkTG431jaH97|5H2Ha+-81RYcd zCzYx+E?II3-Ei`9plzej-U~i0Vzl1~`1mn<^#ly99m-UYrT-}%r0r^o_r8tVWY1bAdb1v(Rau?ecGYhUNa=4;HKiZbS;NXo$e9ah!ZxD@sz_2f5Xcev^0bcolQ z-K%T75#uhwKvMuE@>hH?bCWF&1`D^EuNRwdag+4@@9n$f*oM5XF&Y277RsTH5L0nt zD@-V&vvpLU-J+D&GBV$cNLZ^gPGS2?_&glaN0^%qpwLsv)f@yIXwWd78jl!%3&gWun3*a1 z$Fsg)Z1z0Km`s#z7acIB@7W}b7MQ^e3$-z*2X8|AbVCOWNXH`zKm?_VZVg!pS`GPu zQ{lAeTLhh_eqm-Rp!-=OwPZ`O9pNcy?fMWuNvt<^%l{Ai z(qB7P?FK!nrpv~#mlwK7fgaV;???u^h`~Yi2#faib~Xqk(QSPSbmsRL8Pu@?QI+A9 z2pnT+zq7r)z3bM3{M}0*fE+`E_rAKG9(6(wJ-u^nzw_bYq3gz<=W;bIL8SgWp68mg zVF?G+uRLpSwbz(bB!o-RS<=oFH|k;bABZXUaJC^D+`$satsN1Uz^iQydyBSu;CD(- zIPG)4uuorpzvV|*xyavQO7Qt5DO2>CalRndK&s&H%^IVE*EIzngd99E`)&tkw4W2{ z1(Kes2IJ(pSd)Tjc5KG*WB*7?Gm?-d>y^i3WLixrmbe2y>@hlGc9v{Usq*ciOdhy2 z%@Kymy=4R;MzO-krFspg@Lx?$h39pWMn{~qd%HORdT-L!h11d-HWBV^YxMlN=#BQ zXx+8!HZDTybY=g6QLk&+1ES;vz3xcy3BLHV%cL@W%_N5>pYhaIxJ1%610_u#Q(ez) zdp2^V_8Xpo3mvevs1$^roV8EOF~?a>*M=4JWMjb)b|W3+D{u*1=p+)(0{O?8c2UmZ zfR)zA^ytw>h@2{b4Y#)E3WHCoW}kBq!uq#U94|zT{s#ckbGYZ=#QeQ0xVC0g`N|m# zp}zj{SzoG@mA|2$o&u0^@bRffbuRCE-MD=l2t)1<{{H=CzRw`&;BM? z_G{KxK=$Ix7vrIa*;!dM3^hP2LR_5c+`;!3Cl^PrI|vJk^j6$TvNm$&!mh4QRJ7xG zA-HM^{%_rW=DVSIclec-0$W5Ca2; zSu2nh_s~8@AHiMZY*IhK0eiY+J3e`|{}MF(Tl3bZ0LBt9>{=OoRu7Nr$!#EN22_0X zN5k~2Yd(Iw0Imifzd==K>i6%Dp5Ivlb%)3KBP+4aV01GO^biylu8YYpDq;q@KTJ$W z`-kWL%&*%NmxdqruPV0JB%Ws=#zDR7!S^2fML2aI#~~l!nImy&{1zuW%JrP}{f7IA z&A74c0qlv`0+>C}YV!fF4^k3^oGtOo;&MMJo&BHcBiGf0VzKQM2Z7tBi(ZJcYg0J@D@y8L56v3;c#`aCViR9Xxn4- z7o{g@B;NfK%{ilIVAm<4nrofruYe5)QX{V3r$F=rd?X7C3(w3cFahHwSP?6$>`CW; z3NsazmE5&c2L~eH`=395E+DZ6QGS&^AkvF8j}dIKlb$fYPDo69lH0>3!2!_M?E37TYBHo|yFhq0wj*)Jm9% zX$0rAQmB{m?z;%)z-bqnCL{Z=V|{0h$0xh4x0k!A%}KF`(wMY(HSJ6Glb-Jqgr(#+ zEq3c7M);`d`K$?oynWb;ba~8jZWqa6+Fvf!<_X z468k?Py$W9V+eUg#dvlfkZ(pKGQ~wR6WwSV{6=hoKM3b79!7|rb0MDonoFab^XZk2tDWS~7!E03?}1 zxOITv0|!MZ6m!`PH0=ZZ8(VAZ4Ue$l0$g2(2VSS$WS#pPgz;QrNoZLw4#4+ zEwbW<`EP2R0$n)f-Gg+dpuQLO3HV`psXE+j!WX* zerl(<;@=@RB7o+Wm|H%6g#^E0?=3j(xaB=?LSB>**1f;eeIG9{LzOr2*EHNFHeq&= ze*OBD6+E2gd{k@Wi1x^x;IJ^JO!Z~wh7NOQ{q4QAH&y$qYimTsFA)(BARa}biEi?5 zY8P4nLQS0Kn);oAxC*#W1~tw|vzoCnC|FW32~on3L9pu?m!dH^9xdkvv8l_ZaP(U) zdIK1U?xj}6J0NxU?`uW1u86BE*=G9gAz%AXv}c=jES};CFBQ5jc{5YCuQCkHU=NzU zLvX*{|8|QiIO{Pi=WTJ;#gDFdk`yk2*|H=BJp^;oanz2-QTUv6%FfW6c$|tNMH15H z@%|k?-8BZEJ`v^1R3|*Mll}eRjsE#bF7qu&oestp$mu<4-~9Fau=mSB@}#~D-Bc%j zkth=8XNefkX=s?{9l>=c@S6*5hX)T>yE3MMj>PGic7rFjKmPy8phdKMpl`-C^G%^o z2CzHhsf})=Q!)~?S6ifvw? zgH0ke)$d4oOX#0s8k%;u0e2{+r=?uoP?Q5VidU23_FNUwM$Ov4GM1N4ALs`fm*6o3 zg)LXnCLonWyhq;N({ZUuH$1i$I%ML#@q2{~n-+)~BQxZ*W^+Is2NxGC^x3w#!~e>u z&8GW5mYW&eSwlnb+POj}Zj0QeIlSkt+g|1KqPJJ>zH%+6`pob!r+fh_S@v23wJ+N; zA_>oXV0FN?=VNNheZUnMwSTypAnh)ZQFi;(4ADfm4^#zOOm6}5)EmzVuqJcOv~I%?&>OVea!m$`ixv_Y8dsTr9V&dNpFW#@9*w< z)?2bYU8J+P&WUXu45!p?l&cjO-CP6Id<-aypQ00C1&I71d4%;57YV&ZT6w7A-S%Xs z249iJJKJLxUMWpeN!@eT3*v6@y*ZG&laV#!EN@68S?`wD`8?h)Vp%n;7-I6X3#z(L zE^c8zeV1n4+Ui#;)o8ED_WR*kZoQz~;0j_r?aP-dv4qryZNCuelZgYu-qy51OvS!| z!Dyf_tT5+4M(8DYmcf zq+Fna-r&=v@6=^f2THOBhlfAGJD2Py#LayhjQxJZ&8-sEr!4zPnX(2p`|I+?_Vy>> zzhwL!tgWws|NjHdr7>b$I(1;#^$!Mf6~DK35w3#SsbBwC_~$WS$=E1OCKFj#GUY^_ z^}JZ;ZIy;L5swYkmG)d{(-CXdZrCZ+AY}EOVO^irm@eG~b**=-0#^$May6_BvSw!= zso=dQ10VW9IVse*aynYj1o2CmglhQ)b;CzXmiwa?+uuNeh|;>K5tI7-S>BD0TTVSy zax02C*sN^6-rMP(5<~hI-v`?XynbCCU#i-z16^n=Dtg|*c89nkTUpeUR>kY?hRMEL zGnQM|lC#Fdo_pLdH)i{7?mF1F42u>J7XFl)O0VDGHL`3422{f-?2MK$%ywEJU{%>k zC@tkBuE#wVl`ktWt{oR)$C*n~Qc^kj`S*N_s;fOlpWxO%rmtV*D_OOevxa6(#Q+#jh0y zpHDg{HewMG1bBV@&vq*=agNdBC3T%0S~#}iG*|KOh$aHPaj=7pUZwN7pZ2M z8de+|$9`SB@8`>183EyYR)D10{xNQ;(98|YymR(n`30mO|A46qX}w<}BO{65MK9$Sv+5EX8>^uQje{S#2byH1RVF4S!F&7Q ze~#FlSblYPDJVXPW5#lavp?Lcm_T|SqSgrAS^$GX+gf{Gqr;RldJk&xL4Y6$00mema$rkqkD=YtOp zX{EQ1F`jIfGV=cT5zGYIuh_*7sx!@pNz(Kb4oPnM&t+ekQe&?h0i5j!x_}>oK!j39 zUGn*zOOF{)pR-~tyfhTXGOMkwAVMBZ(ejn(KXY{zU}R=~AtQ4c&^n(132PcLw_C)k zuspP!`UNPx`~)%XMG)-=gofS%o$MbM7R*K7{W+H`;&cJzFN`PpK(m^7&%pT5A7BZ= z5QS`|96MCe2jrZ1mDl6cc@YQp3tR#M&Jq`LehwbYiyo#I?Lb!vFN+3)}N$O7ad?lG5?m*{Sx&k-Ut;ob4^UgSJ!3 zj^Z>8ETV_^1{c;EZh(1_VW(?`2M=ANy^I1FwrJ`N_G*5Ai0GqE=00qEc!3{3+sJ;W zr>|SIxP2*jA<0Q@po7{HeYIdoES!A=J=G_rE>~ zpMyIEqY3^?6kDR<^N%19>OCAOtB?HC}TW0bDEzBai}uFf&@pANs#{UJViXZV245BxH<)+?E{|LzxQTBPs(=3`FZ ze>-!0b#P}@_w(;zh~J7h_b9~UDuBP?hlF$P(wXY>x4mcV9zW8$9N%?gU~0})H71Uo zO}UaJJp8UxRx9nk=*XTY2M{!sE159i^>N$~xJZonUj zp9e?fy-N0{-d+_}9-g3;7272oE_I~(E}wwFC2%ADQ|~F>n0X=DUdHL_>iP_TZ1qC+ zH&19xg zr+trvL@fO8zO;+QilB)^v>L-rV0CqW1CH+U2=R$=9uTsWBV4UN;JND#>GY!7IVFX4 zx~4K;V|#Qltw+RoL@EB|ACjTBfdic$>>VTK)VQAKfOt_C7ayV zWOpWd+($Hob5Ydj8%m|7<9{Dh+SpK4Gmrm#bnO-GC%jqwz*Hdlx+&Y$A1lTJaL9?tT_~d4=LR=Us;!Ix6Vs@HygxOm0#h_+d!7A*=Kq zL570%#ltBa8cRMjX`BDjYv__-UCc9!L}+%QTu~ZKK{=KM(*1Mmov4O{+PCBE3Xh!I ziC%r{)A(<3cBpqO;~>dTLq#MXs9nVlR;jzrkOu0Mb-z^k_^J56!?n6yvKK4S1NDPp z!3te5mre9G%wy&<1|Xi|tlyuCxvy=#v>Tb4O8e(Tt{CdfvQFU;F%y?b3gC*2jHA@?LD(*tu-^f zZ*;Cz>)F)QZU0f*J?w>;XZ-uO@`qLT%S+Dr)MZ6z*~^L zl?fnWt#*HknTY#t(pLo`e%gYGie&L-qNQ5`SueyG!g*}2Xx0Gb6OR(l!htehTUlVB#7g?SMPgy0f-Y% zTh#NrQj|D?+&E;OAk6d1|CuRCjAPMxVv4R=8WWRN<7b~4flDYDR?bD4Q%6&?`Ps%M zgXX@Zu5Ng*5xhYVQrTj)@=hC{CsxMd9}l@R)dZ=?13!<-Zyypo-fggHTcrFt7jc3b zR_l#=_ok!Y^T$ty`-53H^H$=I-epyuuZc%0y#Qk5TaabCtNOH{(*l`L!v|x1v>?d2 z!90L37H(dn4W3EjqeZxjd?7)`_hRtWHjAz_KXE^eTEuKi9NCCo$)IZ)d<{gY8P^03 zYPoZVk59&*mJTC6S&iFT1+kfpjB0#VQeyS2TpcHIsj|Ha&UHfJZlcNx*PfbQw{it> za@|xs|IeY;UvFlSygN?K(zk2k{X+vACMm>i-UH$XF3&AHqDyGtd`z+#c0kRaY!8={ zdZ{8XK4wl@CTh?M{ek%Jd{{5y80*iU3sjCEuFo5%*h9)=DYJrEyq@F}J6Wfd=1yLx zYl84}(E5-r&0XwjSkr8hQh{($qW>0@AFhewoDtIkL)IclxZ>P`ZFB2>?vP^qc z>DlY4!wl}+T!BwLj=eGU<_vOTl5M=@_cI3MO&?S4T|~7spUFchRev#$z`VWwc|&?} za9^Kkt*+NKp@O==x*Tq#n1uXj5OAQG+uHgA)pB=l?;BWDj7vfSQ?3d_jbUR6*Og&G zT$i)@{PS0%=$E7$)?KHQwhwB;0i@Xfd$}cdzGIPOZzT)K9##s)XbN}UqhaMrELTlZ zkYr$vi}HqTK7ATPG>a$6+<~l%SfbF(uj+nRq+9<7zg%_vM)^`^;rmayFQ)~A4K-`A zer5tYO)hEUX3vo&sS@&K-9!&q(9PX{?H+!_($u?TJrk-O`46e%Pb)@U7ffOv0|Oe?k;kK&E~==7|)k>gI( zd29ekO!Ze^4%F;pIh9Bs2r!=GsH0Mt`=^p^zXeu@aFwPDP!p*B~0%IYXDbQqRmudLEhpT zZ$0&iCS12N1h{HD{w7#$+h{XfpPp$TcwHZceEmuai{J$flJV)07M`=)&DUwEIyYFu zoP3V;`;fb%L>?bG+&V8dH{0S6$aUKoe^FF)16kf+Bzl*m*QoaHeV8#kB+N@UEc{)i z)^OtIhA}rVP&c6Dy?bF}lyjQL7T4kr+NAc!-}vQ8y{z?VJ@dF@JY%w(TfYozq1{RY zaN^r&$^x%$)&K-O@#wB9GPkohn(2v=#rVWMbR|Rw^JABp z(C;nCSaNW13{zbLKc3B($;owT$};V-G|REOy1F_Sy{03!$HAo9wN*d)&?uvRxel;n zuw&K@#d^7kXr_D-@`Gww=9#t<$@BA$P+ra-)PUrueo@WK=GCGMufG0BLZth$g`t6; z@uZvni1Yz5u;Rm7OQBV)a^RiN3JPBJt>~P#OQFC4Qz=|ndtAzgEGw*h_n zxF2?%d2xw|zJcKd=2liAO-;~X2YJZtEo6h!&c}3K7wq45tBvLuR?AJp3YxrL3+zJl z;uWUr3n5|$OL;%L+V{uI_`V$bF?~W6e!Y&N`pxbCiPbHFP?HiozMn${t1R^+)ldiMTiV|7f>@m(9jfeI8Vsb}tgI*Z!4qIdPztdwreJ zZ(Y4FW?^Ifwe9i@MyTd{+-%LgG!d?#Pe~ByTxxIT=j5ObZzQeGbPQLR1<0lKf|H9U zaP7)|gS(tpJS&WFHhH^n>@e4aC640hRT+?&s0wBF>5soy9-1f}d^A`a{mI=O(!5SH z$&vsM?;Aj9qznuYU_}xcYuK?-YeTl3 zxOmae=kfb1Eq!O_yRvyDQ*Up=<5iT5@kfmsQwkAL(ZJ4ZZyg~r&NL(-8GuTd@T2!F ztbxJ0{pO!`xt(gK9o^mieN}Z9p$3?4XzZtz)zPdhmW*Xfv8+*Bj=_9zv;$v{HmbpU z;i_>zM8_^G&(869=$0le3)-;ranwN0dQjTExjoB2e%x`mb=mT)r8T!ZQBc2v1R+(u zxlYVN5(Lhszi$@h1wUE+p3A+#8#W#PG5HpuLJxbw)6=pgKY!CBnhXT2#V`(3@Oi9; z;mp(Vw-0i+$+@~{B9%`ZQn#FjWecd9(rJ!#YM}$8fOhutxIWj4u22}5U8ty5)05x& zPQ4uo#3LXG@wy@I0^4;zmOIoLbgR09Im*n4d9_O(oeqROJw4q_?PZw51K{ol_@clW z7LT(Ver~@R-@5nkX_GpA>0QLXVQYXhz;JDc?yPF6_fK;kxRCDg`jHF#IDZa6pri{an(`aFBja`7EYZWGb z-=3@l;+f+_#9UC>^Jowu*ZTW&wVEmAERQQXRby!F27Ae3T6zxMjySE0(>>qC} zPHzNJaF}!h0C=>&Ifih5yrpe%M?M&$5hihzN0)BZ} zfSH{g5kC@o$dnE6$lZVmFpX$f_+`RJNktVnygeU@l$x5lJ}T&wF*3d?_@-7fV*(vc zm_@3|9xp&1sStcU<*{3e#}7{eXqhmtK#c4-vxktA<%bUHV^mY41erQEh+zSd4c3)p7G(@_9$~-+fcn&H{&o-04Q}hoP z31UDqg<9;h?ZbKLj!G|P9)|=MwOp$!6$Z}Lm!~NWI_}%BKd|2;B9!!@rBTv`p|`dY zP7gI859giqN9r>2`JZpQaX`Y0*M*m4NgGB*cbwew24ld$x(_TVX-Zg837a0;4bhB5 z51;3qkty&h4Y8kz1mQ;icV)!8-y4_ANhirC(W@B^wvbEB8!ukOCvG`Lq!Zg%exB}B zuIyFCeNAKp*K3BGB!W%ymtkWaw4BW$ds>3VfuD1D0*YRCc3hCdGXkp&Gyb%IUV&iW zBatU^1Ng0q13vw$eI$KOCjKXNbr`G3=TESI?L;tm3wap-c;P zoIBYOw5j0QJLkE$7q@aFbqA=iRFO zZouJjxUf~f+1}mK;;mk1**bYA==-Q*;Q_hsNPTiZ9etm3C#vn((%{64~KhaW@j*WMZkFHS<=SLTbDTv|*T!WN|SQE{)E9x5cd7`DrlpN3NAg?g4Vx zZ*n~n74ObbaY0LKF~-&|3kfxOUa)+8Pwx99j9O;`UApOg=c$xMO4!=kk-eV6>FQtE zDW1Pp^=q|pp6RN&O2r0G_YAj9*&{(t!llzfuC9#Q&8|+mM3r zJUl?jMse$eXYk!LO~05yl5ura)I2TR2@W-;grCT&P}^Rkpal{9mnC5B&~ZPUH76w2 zQaX6cJKO5_TEmDxi5F}&#*0~TER6ZG_Pl-bb%7s^+V_%&(*U5&&j!88)LKXrSq9!k zuNh;EjW9jTPROd>W>L@SzU*&Lm@O&R>fh zMP~4#vW5l}aOhOot>$!Rsl$jY;E`t{V zG}nLic|i8BJRzQIa^Wk}s)ti8Q%BO(T>;6Y+9ZP)hlrtKhtq@dy`PdMu~0)$wzIdM z!LMOK&;2_!LcEoawn;*uO#3f7u*f{x8vi_{PMC;G#N;2%`*)PPF-*Kxeq7Iz6keG~ zSewJZTm%@>8~;!_Ha@1dPh9tEmAY!`EQOOR^vr8Zd7K!d7xo`LK4Dg~59n_kqs@Sy<9B62`@;p}~_o{E{i~AW&hHLHHV6ft>P+7+YjZH~Qtf zK>qo*Oy*W5a29BzYa%a5I0GmVAT<8VnMSpw<oV?{p#Lwk?m5sHqzq>4oR&q+jOBNNlj>G6m)(GX^9L9XULq41Y>ty1|ud(;F*I3 z;z>}DNWRx(+na{*XY|NfYp2SQjNxJKe@98Hs-3-$@b0(DKLXU6LZk*&;x(=C?va%v z&Xg+!=sSw3bm(^Er@txMzW%U4!Cp%;Ai@NEG8Y$$>(O~yl0ltS5(&TWBj(?~e{q%= z;?;4n!=V^;TQ)Df=Bo@4RKAQ0;8e5YO>IQgpuqRU{&pfOnvNLxG5zZ_H4QQc@-;&A z%(aW^vlaLYFBbIZ-ooPit^EG>w*X3-W_4Rf+H_5pHQcEfNfc+Tz;!6>*M3oqwE35( zd%mVl5@KRSbr#cjB9Z%zS6Z1}^j3|iO#9)zCYTgxP;~NI^L=Bl_DE2$d6Vd5i;wrnCN2)%lEdZ5 z5$4w~$Burp$jOrr^oJ=E=di+#_`x>2T+d(bx^iPn@5^%^Fau60|HID)TCCNbN{1f_ z9OXmmQ@(~?^`hCP>aKYpmkqp6+G-M<)c#DMiNqDliole445=3yM4lY=9C<)(a*oV5b*)iD$LshlMR4GoabJ~b#seo>M*MEwqgx3^`Y z3rw3Emo@ataj$R9%whiWB!Bt|D}o(`*E2q`n1+5d48Ad~F*M*P!N*-0U5n+Zu*a~v z1C)f=HjgzmPC6V|qHEm9tZX&wfG{35SAPPB@)c@)5BGv7FSi+F4)(Y}pYX2IdTOY=G!2w?$gltJ1-ME)rT_0ktc#__#2U)-l-A}`K`h37 zo4vkhES_y>lTs9*mzylg*1SvEl?pz9aNy^1ar)==Oxu*x?z8AKlp{IRg0|;Z=#Yi4 z1K{Lb)dQwxH^4jK*WjJLqpRsXoN7s>y@z_Uw$|xKTDALga+jx=+YMQKp3v)C-WB>4 zT3l2;ZV;N$%fpo$EZt++Ww3?tABi_#xrvUo=S}QDkY=HfV}ct<<$-qrD(y_!Jk7!_ z7bj<-UeM#eDp9mW-obFR5 zf3Jxna6zH+qDem@TE%0-Th7Qi{gb+Sr;=EKEC78RvZryt3R%H(!{_SI=bAHK`sm2iB@a zFlmAUN=hvw31VfTu9rqI7k6D?jX1i`{iTL85!knAw{S443AqX?sFaRUG9G>AYl2Zitd=RrjT;qmh3j$f(l9wQ%EAHO0hrN zcD?V+rgQZyIW|E7uAtM2)?L2JfSH{ zsu0%3xJGv1%^Q0TX`~PmKOwHuX4gB~GFQ(ob~1Bf#1mp72WS^$?lgW-nyunsIb|JG z+pUtpBKf?#yZe@{YL#7F>_OhOJ0_NdS^4sG<03JdA=EO1Z>yc?u=80lExKQ_?)H zZ5dk;x)^ghMx!q5dnrtN?_IE(zK`c zWxLkDCM|^Zy9*p{AA7wwBA5T7LP6OKI1U2$%)pTS@PPJ|(96kd2kmsOGWX+g(f;Tx zY@MRf%trYP_BU^cx`lvLdAwsK+Ba}KSiRQjIaurG)Lx+u)3(#q0T$1{D)vBXt=Rdw zZkq`$j8J!4D*_V@UuRF{^jtV>7+plgD`CSfH2f7yOa~K=#nv##_YZW+u<3WY=|R?$ zpGpGkk@Wn-7CD%hn4xVifVxgPAcZYPU5VBn(bit3GnifYr7K2W07)kJn${7kh@iMN zX-=Xh;`s3^>IsXW+XR@6Qd3$tX`{NN?s42lS~H!ch2qw!7J7a!?5{s`h`j&xMe8Of z?WU{CuwYA0y6hM&wV#`>tTvz4bRj+CX$-}TqOIeH8PhtLh`4stYB>Tu6T91LK4OyS zt#R4z5vWW$YO;aZbs|$tzcCUb2(X=-5~+*}(cWPd!*JI>gsn(ooK3qL*?cK2qLy9W zBb9jr3BhJsLuOoqhwFCHSxYgo#HGm;G=jm;CRtxFSzh@ECz23@*#fn-R?EqVzUfu} z(e=X+-y{NM1OafeNPnrIq%fG8<5&Aft#^#bSsS`wBlktnBkpVLj&o=GEFsHMaQ-B! z6&B6HYI9{%7&Gu+;;fn6u!%%tG@;ks7}iF5m6CrLE~l@gt=hi*G3+9X3pU0ERRng2 zL^oPn2gIbl&47546`<4B9v*6OB`i;yKO~Mlfn>pXO__LWUv%a^-JMpxeq#g8?7aP} zh?AXROMOEhGy2;$%er;)oMf8a3iD!GZG=6oY5&A1(1$SIU0gHKO?$Vg}U0Pu88dUVNf7@N6TBF zfIhdr7KlhV{_CY`!qIFqa2+dEJgN1Ig`{$eJpdQr5aASfR#n%CV*vWU1DL_q0Y!H@ ztJhrE_QQ(7Y%G1c%v)#Gmy_}t{vIMS%`hU#W~mVs506TU#C!S%&*ITKeZERJihZ$- z&t?r21dZeDW=8+OP0p3NidnV{-bN!xw&ssdNC@YAKA5oy0Ck*XZRs@rpHZeDASDZ{ zsIV`!`JjwswZnqB8iB)FJvy7)US;Zaa8aw@gR24*-Iemc!9Zy?T!})E&0E4|26j49 zh(pLrJEN*ue2s_3z;ss6Zd!#WqT`Fe*U%?Kh$dG`;Rat%$nysr;DbdtY?jvD)w93V zbBMhLbFHH{2mISsMsNRmCgaIY4QwdpVE^^P(+F?N0vT9`0BB$_cFX}WvAJ@DEC~zT z_4mP(<9VHEI`Yy#5k9Q*ylNn<2C^Uhk;)wQ18PtSk!ZP78J_;7f9%(a>}Z(}q^NHj zR5p`GBFt?yGB;AKi#$^W z3`UX|9Qt(OlifyY<$n%jNO>Iu0u<{{M*X|QgPh0Ow;Q7wxtE>xS2p#n#7j^}e2igA zX+8Q|+wn6TKm0fRxJE)l>+rz+gao;>Gm41sm8`7-yrO!mCq$WgwdZ)>q+#^cL45}( zmOCsnS}$?~A*$v)Kp7vArj9#NGgj)*z}{;UM<(!^;%`;#0^W<(5C|#}M~73x|MfdC z&6fXQlhk8d2eG0;U^;D2`x&V?k{XvwQtv?;*MA@Qu^Jma;-WvJI{ujWN+J{Q%y8^q zmbpH5S7E|bQHU)B%rGE*QCr{L*u!giWrC=He$fBJooKOb=&mN}KAIc;gYEm%#DcT< z;@K5{&Z@H}=3OOOO#5e_Nn7V`U0vOr+9kXJ7OK3)DUHDdP*EL`BG<^L7^&5#;2MRD zdFR!Q@L5WUdV6N;dxLKNXy2%<1iiTMbZ0{zWjLRd zB|YI3(!!ckAY2!=zPXKnx2S`%C1d|W9RV2&3!F9QEDl0a_{VYxsV3al>kT6)%BC}* z(hcMkI{uej6%-aMPj_XTuQzpKw^RB7mxZC8YTt;k0n+_sl7(dnQ$EIS7>Jy(T(FE1 zsY={dtFb0Fa=1*8Cw0Y4TKzjadvMotzU|3Qa#S9*V(kl06m(K9Pr&Jk{D`?kK+ONX z7UAkD*+q{oS{^`8fdp0<1PT`t5StnV%%L!6HwyT)d4{u!W(?wKvnodKz)e__$(_Tj zQl|%DG*;M&H&vQS8L;~z21QW0gI5Y{q>-@`hNUhR_j~rw{QhqDIOgT4)+2m1&oa?{Z$&e(zn8LiT&2%jv6j#KsDvFRq5H-R^yA@OdD9#j`*;-`v6i4kx0^tE<5A-L|?XSQeyi&Z6F1zmI@M%dV5C z|8abBf}^7a_}#dSEK+TMW(GILNFLMA>Kjf&qC^UChSmSwvd-&xpxtPYNlBk`AFh&D z-Kjhpuk7eAI#$DBH$!kg9dgtzWso~jFSKB%)sPl;y?HqC7310$k81FP@K=&1P90Q} z1_|DHtCR|6xV|Tj;UQUYwzKK-i4xy$Zk*}icI4Knnc3%*qGjS=R6Ib{qO~6Hx>mpP zU+j2feYQ&ZIS@Z%98g5{kEWb0bh*lMaK$4{{uQ!-;vDj1dQpGto@lMc;eFY{FC5hu z2h^E6;y@&?k&fSavmH0uxQC7CC=VxvU9m;I_T+`geVogrvZ`HW5HIV!Btky(i1OOB0KM6;) z_R>Z$LXARSf<0M-P85H|oO|rrGhteod%}V>;UWr_+T%Q}P*T!oq$c-F^qFssul7yT zTkL=F`Pt3!anLhKE|0>{;(u>D>eSJ5N4^1x+dA=!ANqF_1QPrNf@p}0R-ASZgeKacJ>LME#XlMsH-l1z$%FJ*mXIL5tgkoYj*ak1cF)_qCssW0LV{fbH> z)CIG&oFT^@dlOy0j0v4C=Z=`kB6TbyvIUsE#1K%>uRYq>L&~+<3UTcM?+-t@Z0_*P z+C-{?RM#0Hzk=SU3lon!1UfEub~zm!FWZOQv(?g-!H;b*HUYa2r-r+~doYCR5NH5UQ{73xM^e>532Ja_-1zt34KFO4Gzs z2S_vt3xk6MK$4{5;S%~IxtxvkB-^7ViIG8Ltwb^H0VizxAMmTj30X(>9L56|7uJ0v znZ_`TD~e%1eaXC(8@{w%lf1X0w<~SD@XG)Gop%{m=gPs!*zn!iMo~f{?|aWfZkNMW zGAZ!0<*fIOYg5$K>##Bo{sNF1{#Aio+$)#(+#?Yr%`P)B{cRHF{VitERO8i$rHr&6-?(f ziQ?MAUboNC$r3m(cP;XiVTa_fjjujHHR(KVw96rc__m>z^e^nPhR&7lo$AqTi9@7oi-Vg%Ey$YlC){zQK<{H~9ROQ2EzH9IT-t$jXWi zHy=_g$273@rk4#3UbVa3o1O7IW;hj$Aa6(%y=*|(IHN%`(8wd3MFSsxWbgBo)eEwKXxa>=5Bn>N>@{ubY_u{q_Oo`1 zC&yHL+&i}pB|edo0854})ykh|*u<#6uP+dg36K9;g)xtM8pr-DpTo$J@Ov`h0goc7 z0HEjmE~GQ$ybv_&!F>dd!=79Vf*PT0(u7kXf9x`JXI*I-nJywWzbpUh>RwTIjG%Op zgmqj0;k=22VASV~c@4{ap@#n|vxrku4o;bWZXXy1j@Ear9VdfQ}hYisN9DvMziuap*sA`NFgQ{y?vqWqzukYtc7>GmAwLhN6TlE#T=VLfSYLMy`1c)?m5Apa>W0l z8K*_To_wfRV1XnEXGRD`jqMiV%$_g78@yzk_Ay)bVDb6g9L72Ha?Ha63=M%YD7xr?E*R<)w%4u@Og-0%lL*RaVnN~YYWa3DMFvz)6HL|u+~ zPf1C~;G`R5RguT@g|$~oI__Kl{-AkgM0jM=kN5BQ^)n=RzdE-ZJnYAvaoKe51s0L` zevJ-RUs9ZmJ3wcU{)TD9uh|ajUT~^~(Ses5RD^ESo7~9WK5ludxj@Q6FQ$gwi)ofi z(7vjJ6_$IAAt}{KqU+{aO(@_PYk{^BZ8rM7WcrO$)jS&35kSxtHt(X@5z`SbC5W2fQUQ}A+8&PW6S zI-qG{b^4X^02)ps-A%N`fHd@AwKZIKEYzGzQ83cgZq0mpKtWJC8I{gYCe2@e_==@- zbM_@zLvV6s&*`2fi4^EYCD=|eD!)0Ym z5beg?L3!A{ZCL$KKRIK+VP13Z{>XdSgc)wz^{RI>I0@zBuddZ{J|PrPNlLXaA`YK~ zOQtMX0pC%uYzU_zL@5_0qN)eQ3a8y_c*P`w{72!`GlmI8-JG+B@5`uv`9cxa1sWfc6gY(FqX30Kmat-+dG>_Z-2v==+ui~<77jtP z6rmI?u)@gyor`s%h>qm>xBLeRnu&|+KO;UL7!Nj*JtSL8C+;?{v-|vtZpLjoDmTx= z=e)$^JiZd=S{oS1icf4Lg(hmvjcf=bQq)YOtX;$xbt15L{*b%X6Uw$$V}Ui{=yq9B z)~n^^`5;2jJK6WS?U{kBJpZd!HJCfzaG5H=F3sRekv6i z)bNeKY_TsEqeC1eO3ITC83U6$rrz!2f5XWM;JTJyQ;GJQi)Yv<`iX{$i{$qm7lh1q zDA^o_UTp({jQ_)K>02xDK1;m{-QL||5@!4$&SEcDU0!=uz@e6UMQkV8NU3U0sp@Vd9==o(-xY6gVTEUCZD!OwvAxcOWxsrtx%vSdwLA}}0N7xaiJw~77<#al#2L)$%g;}ev;D%Yi#Mvr_ zk)Gz2cxb_^1iJptx{<4zXYxxfh#CP0rI^l8mk^mwvhnzfPhaAj{zU$6GCuqkV<#s{ zFdg1oITrc%9!V=9v4JMFY}(v6XP>m{JaZYVNRhedVgD1Z10qe~&chAwE7$+Q<-G&f zpypl0gY4qdn2YC@2CH#O+}Ld+^=Jg4GfYXcVQpe@9~Ut7!fLF!6O*@x@?of6GCaP7 zhT|~htQHA)T}%H=CdD2h5`qX++_-dE1rE>0^FgUt|EoKvg$N^{#^w0M;#UOP=`FlM7LvB#7)}@~M88~56l`b-X7wwo&yz6{`F-?e@5zqbZ?8NtGFl5e z;9Hao4RrgIOl>?M@}r6$0S$kNgS8j8vWz+r%I@TgJ?9SWgiCpgO5z!>*S+X6%z@Vz z5L|Sh(Iho~nF`x?>Ti6Wr^aLMI{637GsMy+O#>c@K3c((<&Cl=U z#ZOMKifkuVMb|0IR%C^ugI3TmusLuxSK zRTZwj^^sUiNR=7W%-7pFh_;VkD_kzho(<;P5m42G9{JpE)KeN3+Aj6&xGJSYu#+%prJu&c$ z^247`NS6pe;SZV#Q+0u>fA1%FerR!{l5FXXA^EV)u2gKAU5Q%}G@_bUyG@!()^Nk~ z^w?O;1UgP+3R@QhXlNS+lFG!d?Z}M!(A)vm$diObu+bG@xsDaDHi^ZnLP&Gm#OHy7 zd%^1*t4mW6R&SPqH5Zn~{w8-;VW;NB6(BtX#3bzTVld_hkIv2EKHit2+?Pv9N+t+# z2n|KravY=Ls+Of)w}Reei~BY0imfiWrM=S&a?8JHK_6b*S+w7b@_r4S_owhCybJlJ%%Mn9T$}Hq zY&szBe_7TWfnKtb*N$&2A}uN`TxdH#x!Rm;?NTtH8g_d%3vLV&^&c6RBLk=sHV z6_FcHoP}rfOzl)0FkJ-1WK0k{n05WJYl3p2Mnpj9e<`wkhOp+8Dk6{1X<7}dIN+L| z6jwAcHkP&09ZnQe7c~C7(W5FOi2Q$!m+&fI{%;81qQ@s4d|N(S!)yui$&)9My=PvYC-*AEQjVUOW>AWjU`{NTi%*qHM9E3V4hF zd$Bup zL|wu&O#S7c3W$MxJx!u>@Wg-|uxXT%RwDGWpsYYjbW>9lu(m?xocibX&dw!E!P0=M z`NHFQcbk&W%R)!Fx~bp1ZNAXBfND5}+^O(SK8urBfeG2OfPXXU3IR&cU<%vpTr!gt z)LtNpE;Vp81@dkWd>y;=DLgw2Muucj%RiI>&eU+I+@j zz}ulk2=0PvPDw>sK?-_X-_2=VXQ$lp2DQH%hJYc2m&qp8=eAH@+Uyv@6RrCjHmo5v zh<DB4-~&)&N_ja0*Z4x7)p$AJ`<0wR`nc`RBsE^!g{O9md;#PF8=cswTL zns8my%*e#YTNWbJGv0h98uW;-zV|2BR_{IC@wuLJQepkSzax20e$mR1EY%GHry}U1 z$@UZUaJ*;lT0?wvGpZgHgi0dShI&m0m93F9V+s1F>JVd;z5GYeP{9Vpq4ZQ14 zAyXj=+FrJ`Qqi2Lh0WW{1ikexL5gGI>B6|DTP#b1v!4t?p!S2UnX{d~&U3z~1avKY zRFp*q9Uv@C3hcUpbyC3^40`+J3Fbbl1i7!cnX%~SB>BSQETa03OB67mjN2tkiPrhU z&SqEJ@7}k@Zg{r?M`vKgc>7D=`0d#loRbuVj0JPfnN-sflVv(7^?zvTI|m198YlSk z(n^p90?byHUnk;FTQneK4)p+k@KBnY3ffnv5=IaFYTS}E;EhvP@;u(}Q^Y53VggqD zKFYG(L~1ZK)=ZDs0HTysqV^0y;!9^iMCgb}bpkXH5j*Q`p*6*hm@SPvk9!uc+A#uc zQ(qR`&@aodBdw-d#12$m(+@twy?(99)~8*@QRWcJVcLVN;Cz<`p!V zg(M_W{rf?NY~h{Lnpc~zaW@hF%}W1ErmLm?u&ef8*{JxJMfcg3FkEqf8$P_uwO2U z+fq~ZC$R0=3k1HX#ibKTlp+%JA|B-F4~0eDGeHz4=h1S@5lg;Rmz8<-_pt0Q|B)Q1 z-BG@=12fhS6}4p=iW>bf+XNYAYtF)p0WxJj&W7zT@3^>cNxLQQvsG2PhnMpv`4fOP zSUVECu&BS5F4+41dx4Av3dMZ}-9d()g;w zj6OaYTo)csp4aSMLIUA^Uh3Hi-g2Nn=Rar#h=yfeml3SF@GO0dN@f)c$w|Y()^B{` zrh3XU(pbt7oCPm!PiX7sM~f3&??6+E`7`L=sX9Ma&LdMK3B7EcXgqOY5HWA*^_2S~umz5Rjxl`{6dn1_9NKezVeeYoWJ zE+So*Jt{WueT$=l-3}r5I>~K*u*$Qf5uR(_9hpV|o z(O{vXRt*acr)?o7Ii5vNeS+71gx@xcEakKuSsPc)@~0rb8?}swd=NOI3Wqq%HLnN9 z_WR5V48n=;cm*Wcd*0wceIYu7+u1vDO5GOsQ1WWnq%&=W?s6kSu0WQQq1=OUjr%o9 zCrJv-=M4IMe*~e1cdrMmml^IMmaL0lsWLHI`*gv#s!j0T==mg?hQUI$%mQbb+>n0Y z6x~Lw9>zM2CaQIZMD&tI2uk_;45_fwAOoVlfmy4&PVz+DcJE+kh{Q|QvlbOg#p`4R z&>)vROR$zfD*Rd_c`iuf6WD#@UdDJ&Jp)0PE>F)FbQOfXU~<$2#aazJYkv?9ty4+I z);2pZ(76`~=0scdUBFGkMw19Kc#h@?GvGUY+KF-g|0gQE+C--pO`vrP&=*M949s?q z_Gk~O^-`s&SQVARnBRH{#C0ktq`Yu!L}nk#VVIcg8t#}Uwlw1d)`w6_kcAJ%Osy1D zl+((EjL{M2yfCNP^;oYYC0+pFMPMxj6A<2kW4?Yk|LL_Yrl+s(vzZwVbGk;Ym}la? z)t|@&qx4jD5TvUdkAL_|qgIeM5f-8A$|(CU^o%_4<1j-02o?PU-+w{gov`7+uA;9$ zmOcUV9*>yVK`9MgoV-s3cPtsG;{!Wls7kK}um)VOyARg?rl@Hd2B=^UBv}9xuP%@W z9d)%KN%-x*0?G+U&QmyJ)Kgw{DDEB}pDjGS{PKaq-Ue}qPoLAYB-DgmDU7k&UBwhx zrwBCf?7EmO!z?cHnF>^bD7Mj$d3nDhhJjyS<>V1xM9Poy-zlrI(X%%=4S)gc7SZn- z3C!LesQ3lXLBETXmSnx!I| zVuTQ3<{F4Lzdh4a=Us40N-VRBig08Kd-Zk(g4K2(SLqwke^PJ5cc&C_R9m++Ff$9$ ze*x(=_fKNW^*5V|3{k_QRX_E(RT)E%Nfz6zYEvy+TjHi;(DB4mbO9F9xc0Gk#W60` zT__9a3qql@SH%}RZeJ3?; zxshfHcMh**H_?M!gG?abncT~}a`D8yKyuz7(JkPr4ck?x&pXWiDHo?Q=U)1U8_!G} ztde2j!d>|{)#F*}hm2e`>iaH0oxDf&pGp^k1s(3hX#32lbF4b(N6nR>H&j%)Tu-$L zvwZ9Wa_d_jLhZr6 z;-H(0x(>&SlLj$rY%&-P22*FB?b_#t_G-;?1Syhq0oA~}Mc|()4ixNOWawz=i#4`(3YP9;WHrl0+R6F8yase|D3dxyyj3C( zG8#+0PRu52YreU)l@;4dF@Hlb&BujPgkrn++emi@+m+(+ zW>s0ak_B0t|09hK)p!J~u;%M$hN|*ujEszto3&}vfwG(Vt_*`9(Cl85N2jauJ#%M! zwb#52R`zQK8Q+>;`!n)29=sWoOGz@icD_wIN{zYe;KRPnR_XwAM8qD z+X3mAa$O@7Stl){gaqvu4x_g4?A*nUlQc zD!dVQc~?=SvXxTjWZIR8MJ~~}2-7mn{Q$F!yHfU+`kKl5qbt_nM_UjE7R+rwHo>_P za#K7s97iOWiEK-xmFqq`@SHVxY>)nmK7j9=M^o&)%8CXywbS1Ccc#9!1gr)nW)0iH zLu{nmVWDiF@~@+I`JTYn3ihz;aAa64z$H^cdDP``L&Y={a#^-zx*j@gQABOtB&O>P{qkdOR@a?^TXMqat zx^le%)-SC6&?vYhaGd($hM{y_YIlN7U`M)Q@5j%?=q1b)Hpr{`03BUFY1?2zpHtNr zZ%v91Dny8)+lczNDa~#s_^WxNJ}d=8x4vC98j3<#WUjaE~oWME2Sy+^9G@UzKpGx$@hM&z0 zGbNNN(R#j5IaSvSre#JTW9E-1m|_6}o<7VO5@Wy9sk#qdYy0SUaR>eP6lQzg?_*2o zKJ|NjB5NK}L}ZB{v+!leyR*exPYa>JYlIst1tAo#-k(#nGSTd^HWh-bfX&_~V-aK_ zdwY_TZL|&Z<^SyY?@sw)WQLHn|13AL+gc|Y?q{QFtq@s{ndl_gl)cJx>bk{(aP97u zZ~NrX(BHg~3{GPFP?%|TPs3_oMo6*fek>eP4rl0`Gl{Pu&q1azfawq6Ykd#K?d%E| zq@qg6Iw@b4^%aLEn8>_-&T}xe3dp=w7BQu7(~Q<ubzGik@_V27pVTX?E_k;=lR0i$K*fOnn?f(0 zckV*gLL;ubn73C^oz5dyTGPhGT-k$jhvfD`|0}(b2l?`~>UiH`mvq)d(MHd>E|yH} z=0*b>!*9*+=+1taGYutmiEqDhx3Udh3K?AW$n5eU@z1Vc`V_@qOE!c#xI(foHIN(K z{Om0b3SVvj0DwN|T-Tbk_2Huf(e@UWL>xFAAU^{4B53X558!idqn~8*1b~L)H2@0F zA&?m46;VlBNQQS=?CVz?@OC%iYl&5n%AWhvZ^UcCj|HEbr;|Vm`hFNx&-^QWJLaAjngtZvq4q8(Qc0 zV!)j8r-!`9m~-RcRC-YaeihULRFWW;ySs}x>y%96O}Z+wL`G(maK z6s(^P3u2|DH8JIsVem30ei;WIRHBO-oQYW+FwMJvc>V8b1Fdc^!SQw%C%iTBpy?}@ z!?U85cJJ=X;dgUoJu@AASvc&^NEcY3dX}{^eoD8Lbce>5+nVa38|xX_nd|txc=@>^@bEjV9wpJkk`kH4Wdr^-KOAcV8J+RoAskgEWGGq@W_w(%qn< zf{JvPbeA+pDgr9fB}g~YT_WAxAkrz_oUwi0_j^xV->>uMTxb3AlD+m`YtA{wxbJ(+ zIfmR@H6p(fgo=UPc%ZU?mH0sy))IM{hytu7km^~~=3Qvul!g%(T|$fsWS$ByrsT;V z6n+jVCMopAOEk&<>S$KX^T_EEa&-<{z&Ac8Ob*(5Q!_`31sYIfa|kN1JY-H(6X5>6`Ar>?rJ8pda2WRx83qV+nK z<5ZTEOc}KDgOj6=d>iEsn6DinHr+qRn&A(4mUKJ~bM;pVINve~7d+;&c!+$7zu@j; zOzLH?HTepMV6q^{YOeP+t&e4($mq(Ka1?tA^_=ok|G?3lu@}cZUnH~tgbZSRBh|?B z>xjZoPV~{YkIa1XZRj=qJjKw)N^ZZSUTn_J3?Vu=Ta)i24U9sWB^FKSz1G2GAINMiW02Fl~lG>tskwP#=jrYe`D6Xb$_f8V_^epe{AyWh&l#%dN66p z6j2oV((=1pUjYp?YIvZLL=S##SLZs=uw2ONGsz1e2v`wFTF(|yiS7xRiJjp(P! zJonED2~OpI7J}+6e$kRtJ7x!&1C} zQfl(Ee0r)Lkg-1hkY(Ky%ElxsdJoR;oKkCjPd_8vn_U{BVh%p(vxoe+rht??af2r4Pu7FnSj-AZo?l7fcJONV^};wF6snXP?X z&F|KXKJ0eMJvwE~ELZk+{FJ_$`RT^ZYef+QGH#_MB~6zXff+pCp==u-B&$D%JbZ&; z@OFKxhYXM8M#Q5DkLN-ExayBHhLVE2ldE)S^V$ekfO99G>7~P-1H%P}#q^(hBxGI* zg95O3&SqcUce#W`r{}+L>ogg4NS6GphRP@s5fQuY(UwOF{e%`YWZ zhb+ceBf(Rugp`Bgo!8_>m|)jFaBSIyQaQYpd0I-Xit@VOy*^88U)Yk7O1>bYB}Qfs z;oo0#@Lp!Iazl`T_^K&ew5=04VDjhs?T9sTe{ef%i2_dkHn~ZWq*6jD%rj4O&L>kC zm#v#_n?3TgtHoT!X9-R#@5j(3NLWu2E_oVfCqFZ4#Z-|+=_ zmaJ>1Xe$ORcUHD3rQcHWw3iZD&PoBFq}Q-*-K_e} zxUD}?@`qQ6C)F0KC|`vw`C@Yf_a2(}+qaVLCyRV+zPXd-3(5T)!ei{GukoBVH|2d% zmV2~R`x&6^T|5_8G5%(Ls^dUDP35Kr7XekjvYxk0i3Ii@ef~2+PZPnjQ!LT5KRA8? z0S0<{2aCqfL|P8Ry+?m+KHuc)4G+LCD|fzif0ZqiO2FsaLj`)44Hh=7s`q1K{BYcT zq0#p#?6&-xno7dM<7i@ODP?U<`}60|2V!Cb&lWt++&*m*1dpSI$oh74FiFd5{~*(; zxcs=rR3mtFD*X1Pwq+-KL*iQE`Dvs(Q8jy53(f&C?n*U?wUzcMN8`#->wUG7{#Aqo zH{Mm0+E_i&b~D}8U{vdAG`AF)p?G*SMEI&!CfIg?Bet{@@!zN?IvS*x=cR9uk2ag~ z>0uha!88yxdk>jsOKJt(%f@Vb{=<5;P}I3I&)@#hr&;m*T^hNly}rz>hQefm&ZPtw zua_FHO!`OY6+g`>dlcVr?km_H>$7h+Tg0b}%rk!49OU#Q|6%g;e8JqEIBWh~n|c$f z{)y7!jU{yV-F7mk&1Bg9*O}zJ$RVjF7b^8pwj_$9fqufzWsutn8a7nd3wvIlr9H<_ z^18gSyQ_JW&8gr#j!s*3FT<(ukaF2!Wo|BTP{;^MQ@~zkEPVWyR$8z5UEgoji#oJd z#2wXQ$A!GTAKLrsf=0Q~@*JFYLc}#iaLTSU=2L5};vL z{8(6WlKh2M^08*shrst}!b&32AysI36BxKRT^|_7PVD#9H4*%D8mvRqzlUs(wj+9}&f)s^X2o>+>0)_ z683JBZ$;uZ<3}Ffd_o>QDI`K5N+L@>MZplw5GIv&dOrYH#BJ-z%}@6pGzL9hAfK3B z&vUe&-$?h36ndia;;US%lnniMlE)e^6EkTnObfDWm9LY2`^XU1Y+!`l5xMtAPQ#zZ zgBjN2*Qu0w&uSUd$ZwA|LL5#udA6mJnNW#;@>wM~%NjKm)9^F}YhkL*x9l-)y|8YV z-d0EyZmGXK*ko$PHd>E0k0@*X{PV*H(!2Rz*i&pB%v~3ZO2!ltJ-H1R*L*dE-1}a? ze*H5!fx45bBnPKoy9vgLnBfM)5Ux2#7DuvsWy30F+U?4^0ZK_Zx$I_v61BQRW}=Rx zL*ZNaQkbylf?f5>a~8isT_ES}^oCh6o8cLS*;5bSS+>i!s7F`QJHe<}S;9d&x4pET zy$=@1Xr;pSXK~hldx@1#E`RH-YazhwX}-5~Uw)Bbez%P^@X*fr5TBa?LnRKz015{T zE|$MYWBd6r+4-TXHDUi6$&w7LnEtOWKJG>opQG|OT^ovHbQvGIkVUM~JFLaR>MLV5 z_(`|FF4UK9F8wVX<;I#MN!Is|-)e%nU8p`s!qLW|V*i(w9VA~^mcJ_RQL#=4?ypV4 zeLvrtVob1VCT~j>8fuH5*fqu;am#q3T2{%D7arCq3w}etfl8tIrf!b24f}rlLr-Y& zJ`{IG_cH4h950E>!DnWdh7z6SJI8=~laPZn>W(@N^w- z^7(9SIU9@}^sA20f8_gDyk(8;|4L&3$)oc zULi2%vJJIjM4GlpWSF4^r;UeeD>vi2!y5lQd5ncye=RBe#qmX-+Ij2I9na$n!n(_L zoFtDuo%6lo?VYR3%lnsyt)DXC^=rDbA@F)hC z_(Qb^vxsE*77qnC`_9k4)`s^hsCs_$UbUs#zx+?Hao+f&!CO9}{N=bMxJ`LV0&HW# zk^RpIf7Rku9b>m$S1iK=Gji#II=q6?1CzcQ@!&FL?6jq4TzwnG&&_V{lTxdfIGXp| za5A5#>WIQAG;y+ym<=FeWs73Zr!X`sV|bbH%rt zo%-IsSYtxxzZdkT%)6|d^6Kc;_utdb*t3*(oxIk!>t_?^eR-J!i9D~?C92IS)qWc* zP6(0f5?MZPe&0@Nc{e*5m2ZhEkVDJW{`%}`{W>WPB_(EvEUJ`Z_64?6o?5kssl3`N zdpE0-6T~y&H27Z{#GN+TS*~5DkLy&v z_{x;`e7|snZmv3JRKL7;)mX8Dm!fs&eNfvoyFCKfU_v)$vidxBmAA@ z-J4|3x4c@n89rF^4;`cO8LqTI3M$s?%%X)F7Vwmy@uUW|eomzFg{IAR8j4h$rbtC$ zp~lR3$|;-NMl)hSsoms-q=D+J&CA+$edo~ai=Od|5bNpF&Q1H;p;z>_ ztDK41G*u$Yaf5n(1emF(CGtCg;{OaxpWoEVZZ~tT1G&o21a_WFw;`_$&ljugkIM-B zwKXRjcQ>k^H+(17b3`>g!p^Od!A75Ax_K*SyYXiA z*-)(QpXtEZ5%j4zXH9|CPjBaM5cRu~e{D^*?b555dEKG0fHg#lft+a3$Q}HB(ftRT zUqS!m*5}Fax}zTQkI$E@jO8EA3tNr0_4AY-K9^B}w#+HttSfMf>dtp5g08U;V56%`SZ;OU&t7+ok>q)WjxV-5?!f4{xM`$GQ4zdniO#Qab=`P9mXN39 zbYKdGp3OM{;gg#wMJX;?4&Trf7?3tD^HNiZdeL-J_@9b#M*F2Q#rc zSaM5o^+q4;ie0AC;iNBpTxs_4hGLQ5t|#8lNZ!M5u+?ZM+urT+c)^QbY=rdws#5j2 z(2^s|lN^N+d?DITL@)Wn-u4)gPmw=v*&%cM_g;Vrp>I*4e}YSCMIlQnmha-8REKIC z92$%=E*DYtrrSQ3O7?h>9r-U!I?&g6*wy&NQdx+l8EQs-RcJU!V3eWj*r)S zcm9Zp+F=RlP8=6FXvk&wna-h?tk&m|3%=2Nn5donoc`YO1E}tU>2)^w@OZzvmneFx zbnENd#Kuicns?1DTTi9AqZXbjVak&!)Kea+l?A0@pwpD+?C6X88sGj_!Ix0ACa|!? zo}!zO~CL!1$jWv$DB4SynYPGIE<`BX=L;w4E<$K{wXMI&*fC5f+~>V)3b5 zUDsQbLOe~{fXtm?4Jp7nc&_OO(b@)Yhh_GrVrg*=gUR}H^Gercj9&CgU2RNd(?uR$ z_V%S;U`yrUF2&4PR!2!oOPBDj?=;o$1d{z^QB^6k92gNda=K`l8F}Kh_}($Uc>a0kY^0PWerPe*5y3YQ^-cs*#tbKvZBT@DWhuYZBF!l(_S5uh@J0nxx1s- zz;kF;UJpr4HT?Zs_-8O#1h9?Pm6hmezeJ`8Y$>Vc%mQn1ZEfUV3*m%Bw#b8(E~8d#WSo6rC*#jM>(KMl0Srw}*RPFm4{`0KuSp!)7p98Q{USds z>;^4hJMbJ_I_b&U9+zXCuW}t<9p4l^TS;PT&MoaA5Ig^%AbK*eTGlM)6|gZ^@ZfMX z%>0PtRLAA|VRvlrN&7vWTZ}8@Fdfg;)cjXXNB3kiUoi=L#uMnwqB<=fALl3#*=#Fl zCSW^F51ReLPSG&`Q=<^_*#FwkgM+%+k4C}QZ>%*bc_oBsd}v^{yfGAMB>d`*T$tD@ zZqxI5TB~Ve4;kDp>+ypm=o8F`w^IE)wZkq)MY*fV^!I(m{tRnp!Sb|5v{(3MHaV^b z5jZN8R%86x{x6r$QG17mxO`qQLzJ-ZO>Hnh695z3?Gr?XWi}PLO=2m5YduORE z4h?s8B0-*O+ht7|?;k1g)ah9kUfv!nu{5>Nt{cRqlO45b>(N}##y1PfGC$_oh|*fe z3gbU{eby5F+IX32qPrh=oulah-~R3d8*X8cQ&6KJ^@3-qI}w>H=f;fmt`I>Ug~`*F zD@yinbq}rDMuS9l*r_fSxz6_&Ti3CRMNeAAh)yS=2KAN(%bkngP9{oTt3p4Hz20q$ z>CEnpKSKlV&+oF!{qp6@7do>+P<$<3gmSfTR;>P<1ljqjA>#39qf(ax2~7yE?$361 zRUsJzHQ9(Nta8$x$^)On3mwytj%(x^oL{?lHL6uczWH*LiM*BR)~O2$3UUfYb_JZW zqv|r7cXSqq1D$m{nkdPb8%%LY9iFaz8cc8UUtV92xr@iQp)Na{+o)GD4<#v~=kV4p z*EmobYCAic?FOyMNx_VLw(Sd@Qo9V1w2}1#Km6rH0k2R$SApMGIQ|y3dzY6*>+`-B zIZubQ>og13;zMX=Lk3Pa0^XFp_}2;r%!dw%*-|uM5IT* z(AbKwr}BOL@h;p7Q;zwGaTfujz!~djd@uiEFL&gmZLt=yE33`YgW-Eu(qDR`GV68{ zCYKeb&d?9aQ3z@6(E|DmYPHqUqIBIa2!BT6k^>eq zk@KHl6d|Vw*MpO^$!y30>NjzD%Jnkq(4~}E_9I4n@H&`BY?MZg2PHg)GwzVjImi2^(Ir*V_09KvX=ba z?jlMJPiTek@|OX>SUuLQ+E*1K#v>gZsroH8%>e;X#~04*QAxF)mnSN(l!iBcuudwz z5)9@xv^Ek@+!^KU>%cBMwowIdsG@3}~p7W{uaw6*^;)=6b7JG70EY;>g z%h+H~p(FiR@yLD22JX@i&^WE)XbX$u>cwSE^*1T(>`;iIGwK z`Ex=)LPE(o=;6r7IxW_DRmOy2W6(q4Sz{uIBy&1pf45TOHSyKvVO?;RSBfYX=@GUZ z(P0X~d}(*k8zp~e4Oa0WI%KTh@JHy~xt}(IY>p(iQnFCs&-{m(DB3E;JKJBGkI!dC z*9r1P&(P}r)X(o$tj{r7%?oRr=EOU-&NAcneN)p2K9*Uwj~97)r$F1QoBqRQW#3}% z46(I}TqL{0Noz9uJHNVJL0r7Y9A&Q?hqTZJZScAVMTDHcsc@PgCtQEd-jovSFY|{q z67ph8fx3yS{w`Imr}f1bgZs^|7e+X6RtxQZ$rw z(~ejij8M%9sNw!SYe`*~C$z`kP zwzqQ#_)ES^Pk(J_XgGg|8ov-AXOW{FGC(yzhRrSft9`tM~f#$SOzEchAeh$8M;oEE7C2F)@Ua zw>UYkLkB;``}ZZFO^_cUqUn^CighbvS8zqO!@@LWTigQU3a;d{TCu7pXz4T;RjiFa zUZskvGnH7vPGiQb$JUPQ zahQUbkHVC7g>ac%SdM>j=I9p*>ytvyKc<5`^roxt9jFC5s@Jh+yA-uJUd>Ki%$Zep z_|7JG?+5Ts(pJTbuk413tJqwxY8vb#3zX8K!q=KNUBQ-DEnh^FjN7ypS!&#_*6|`W zp*@vUu^Dg~;~8!#3_iTOXCd}M%fC3 z&T6w-q?an;<7i0H>Ytgxr$cLMZr=3FQq|U$wXxywBP6GxnQ5WeVgc0SZZwfiYxQ!L zN=ok`Av9L(Q)J-?zwlbW{3Q^3MVU=;Ve9q#_-AuAUSuYpG2tZ>T1Fz~e1{*H^6PnS zreT>rRnZL{*D6QWl>r{>b2L|;r?tIbJH?95e9&<^WuKIZIVd!LEeHxoesJ?=sPMYY za67*0Z(^!!Iodm&yHrna<8g4ONve=69j;3p9djTI3vU#RdU~_-t0lJ))k9F2|F8vE zthk&HxJOg#qtND9E*V`=e~Bz1(Mm||fePB? zx2&w2ii(5P9_9(LN-=$x%3&GPogaqB{)dd; zmL3gpW4#xWEY!TIv{0eF(ts9>N$y7iO_caTmN@NC`lETDS+tA9afj0D-S1f4Ypcqi zr%m$w#qoh7)rN+sm-OS*S8>}gih_F%(XTdU;!8@z9>6>h5fh`*p~c2g*Uzb_s7PV? zKoMW9%<-)_u4l;#v!SlfZtg~yR#cNLR7DMFVhsd_eapNcTrobW{fBMX7(<}tMrl*sSP&?0Wf!hRpFmXB%Ye{XT?16JS5ZotnEnur3*<`Rh zW0@8wSnBZ=UR7%#!k;Fel73*lIYO6x8*gMjsq;KVp}JFe5XWGYc!fD7xqElIsw`=f zKLsp|wAa2??kj07+xvm$ZmroJK1Q3e zW37Uc+osd}ggThp<_bG!<7dYwV|eb%lmYBl_Jp)v2RN6Pe@2g6GiLMF)vrz#Gmg(Q z*mok1v1c(o19c=lfB3|SVs#3>D|~owj{1Vq6aR#-PVAWa@?f%lr=2hKHo?Zym(yRA zP$2FW-R1*HPQlz}Oaj$Ff=jCN+iYP%S2fz-<55xfHC)+CNkcmQuh9#I&FR$FuHSgA z5z?nrsZ{HiJIeHmz?6f-*!!pM_^W9Wb*`ULA68-qzUfH^FMF67mS=yQ%4`6RX3A*t zreC-xie=7joL-+P60q&HYmOt{e0h)G1r{g)IhlVv%9(hO{RilGtR5c4!xEF5qsay@ z!)9)x1}x!4zlhPD(+_r|0Z(ho#l=M^*#M1|P05TZHQ6jxm~Orv1Cg))o3#vN0(QR~ zscfcrn)9L&1cXZP*19U(GhXZ&P%>n?{yeio9Mr)z9wVnTS*^E{N(`Jgu02w9xa{n9 z?c^H14Q{&HBe?xq=dLgdj?HmDHacQzL`U9Cb;n-@&L&)mx{e)Wq9SBWq0Xc<6R z9Ifgy=DI72qV=_>z7ZTDL(Rf7D0{ESy0~f~qN^=aDb|!82K2M2FW=!ZaXvG8SlyNq z>$Ccd8kc=rLgkIC@uW*6JFZd@$AL)s;qyCTTzc-l@>;}ZL%kn!o=WV#+%LQ0e`6_l zUFD|%ArDp96OSzZ*%;rp5#yjYYW~D0{>LZ3>D6?D3sD{UJMdUzW5Z zq__JDW@iLCVoQ103Kotsb#}_@J^ZX|C3j|ZV_tcRgntn0HEvknH8|Gx3PAmChBo8! zuK7sFW@8>nekpXJ$==*L&sfB)b! znu+Il%hVzT+fk67ps@tsF6_c<5;jP1nC9d&@pyVUQWfTk+;=~>HTH1J@bb)Gh6iVF z%G=TIjcHui1_mS-IHGwu((Dd#MiIq zGW+qSsV`%hyP+98OmxgIoH6D&h|-WM<#Xf+t1HPz5*#nmpZ7_2#vy2%)p?_PINgM* ze&U8hU{dZi?^D}mSa=zi5+7h>`@{8$LA!ZB9Y6ij4ipJMoJILl>PaSA$)y|LOY=kp z(6zScOlV!NwykR$o#HNq_V0cIG~9e%)_rPy11Z=YC4J_)d2ikaF+RenQ*j*^>hzh< zT6OIu(;)tnbMAkAq3ENp#LG#-`Gd_u-)aEw%M)u!+a)6o$@NhBRHi^iblx% zFE+vq6wYn#jFL&H&0sw}YVsjLzDk|_<2J;8ywy;yq3kJqih4vfjX0K}6 zx1LQ3tzIp?jWfsFF}^#MF6f@Nd~?;j+LYwL)t6@jyj`r%(pr-}R z2OAi7L|oo|_^iU1Yta~dI-2P={@$3U;D!2V!276#Osh3n4|6eXN8FUOGrWmY*?tjj zpBcVza;mtWZNI5zdf4<`wL{S>vF#!#m_ioLDjRWEo{^e_kY?hMvaGWLc>ieL4Fc5gJ|vkI(sM)R9USJdQ=4W) z{Z5>#dM}p{mOJ+(E!+B-zWOS5NKD!r_yq z?;IA??^RttHJ7NBrFMy&vvpF`ZvSo<_OmmOV7Q$D(`w>Q#v^&u#h?>I@~-z^ppU(!*#|H#8|NlzpQY zeK+Du4m~GObx70>t^3Ime2N69m-?f;m5Yp?%_4w|Iq!apvGWCmn&?!D=MJ-WR>$)? zO>giLZ!#ZA?+)-xY4TUE>)41Sw~?~XZN20kSS-~6PCGtY3mezH>Y&vxtcC?d(O>hv5IydIqv}a>+;s1QO2mSN$(Wxoi zzqnm%%OLJYZ9#!Grn&6z@v`+0zZI#5Lxr=OlVU_Vm?iZta%aZgGD22%cGo4bpqC_c zuoJR~>Rbf$(AkM&chZwvBT+*8rh}5KU$*dK(DUb^!jDnpP3y#g%bOquK&juigeaSv zoA{F;-i*ZmP6!(5Rh+2^;(fcBbjVfaMPG++;a|M76uO%}YjA3PwE#Ddc(DQRiwczAdf(1pj3@X?be zlbcM}U8odX7U(Ci%Dg6e$1dL;3x2)mR-6z+FwWP-H;=Zaeap-F5PC(!=qQmr>#c2`uR|zl<7NB1^PMu}x!(D`|J?zRRWin@Px|ME#=k{~!Jo*Nt+bne zqizEntao*Dd%(qo`mKNEum~}z|B(#Kb0kZcNl8gLgoSa#+e;>u)QC}8=;;ZPUorI$ z4f*ZwJNWtgOJMn&o}TiA#l^%l`s0wnQ3`ZAw3^ylTW}dHtgM^jdGB~z^^3k177`{W zClgAJMc6O*B!gx+yl~(u0v*<&FKg<8q?ALSGUPHc zGC4GzhYE?0*rFEpR|olB_g7-KkWS4r{Qov4Y9CZ`f{9WsJb zQBE!v=H|?AeSOmoX$c9vtDU#=*D*gLHp&FGNl^*EA(X^|BTtQh8a~fy%#N1FXVTG4H?VJ&Y6eOlm^@sbbjK%)4?yk(9sF2f$hGiF=1h0odnj< z@(Elz=yr#jlbLyWGFDb>Z{NO!{jW3_CQ4cy=mTr!Lgf)#XQs^6AqjMrLNI$B(apOaPuZ{GNhS;fOS) z^VXEWeRn%$3{1=fn$e*lG+J6(M1$LtlatK+{GSKwqwt&-78{9pMUd<#atVSbEh#OP z_}2eOM{WeWkpA{<_<)vH!t_E~S{gj`>B0J>no-I#uDEVQGu#$^+p)+mU(o2#dXmJp z_s2FiEU@T4#>ZP69q;ULxO;dIk&|Q9YS&W_GfZ&i2Y5_QT#XkSCs-Mu-BW$`77qwGl!_q`%Saz%p17@XGwBKH4&8@EodwU~+Lj5RAgM}1o zio$r1kdawGbmR*Gv;Bl=cdr53b%7@|?uoapx)>`mAe6L`$2!$LI68V#Tc~{bIdgO! z@jUG8?2Hj|fd-a3mpR$lPT5t`8rc~cn37m9)qQ%DlVUC@^YeOd+}v){)1%zKf1gnD zYjySHYTbhe58$7bmE(FEj*dLw`JO(V+F~NMoRCGyg~F;3LSUH(S{4@j@Nw^cHM7VJscW(1)~%bQ>it4XF?ej6?K694~B1}Roo^{2iQeC z&Dq*;j*?pe;ijXdr6t=WKRglzZ1RIyrN4Cx33%lr1qJ7uy4_t}kHoA@xG4fR?J6}V zwpixwPCVhgu1hWENhOKp;N&DYZ55}o53{VH)Xm!bU(d1R0&{0qRhylSEr|u~upO(j zAD86q?(W|5YbZ4zr7DB^5*6nML~oAJTPG(cS9hdSYP$?M=XV!7VF2>-y7h$MiDHkB z-(byjgaDp$e5xV{gl+y%dFJQ~-#Nj#&~b1)Kjo~y4o`MnBdiIGSS~D&R2Q1_g4@Z=WQ4o#r*Gg-29U>j@d3vaH4Y&VTO($eE!0 z>k3fwZs@K$2(EUUTUd0hiq_WF4(u?e5WH(QY#Po{fyZ}nC>bO$DMQEA%c_LpA?c4br)+15-gPI&w8!urjNrt$M2UNB)drw8{C!HmWv z59vwHu$+$k&(_w2EtqIgX@>B}v9qiU3_lmKEy3~Uc>3_gbq|;unwXdfrJ#Q`^i;Wf@YTc1|2}1z*De}{FY^5SJTybMH(Aa>SA{aZ;9_iTZP|Le5nt_E zOLODKjRD4B=!hNYVX=2RpHz1|T{r06yYZeiE3ChM|91~z6QbKQ790l9&X}~6!t(X? zEzgOS*61+drsw4)mBhMp=T60j2MQ8YUIU|r*O4!`$GYLs#zKm-HC>HKhbAA-uQ$qt zu6uT`cjuBL0|HgG-Pxh#{Zwjla&O7xkD<)0tN>l2*may-tp34c2^AzMtVU-<>R3MQ z?T@Q14GathcvbAF?6Bw{_%S-8uuKLPM@Evu6NOjkRN5IKR=&|whH&nJUg!cp68~o6 zaDD??q|292G87%|?oNbvJ|fO*G2w0+9UWcOr?$BIea&Kzs!B0CJ3Dp3Lafg}FtA#< z-17QSU0vPZUpIF17#bcP7rDG7Ac3rpd|2tTU)s-~7ZNLJk^P;lwgjZkGX1a|ow>n* zfm?p!>gp4SMgK&Vg%pNF9Lopns$BV5PG%-h<74PG1M9c1uP-m3tv^)2?U2Qfkdck8 z%q<>Pl7y-0eFzirk#lpp;8DJO@q7Cg>2#tR52Ilg|}w z7wXs6JLB~H_2XIwairk4eiK4MLL_Tz>%YFI%%w!Ln@D<}j^APB;~+a7jL|Bh%efss z%iQ#G>fxbNpd}9vkGcG=e90)akW<&9Tx^*@KEy`qvhJQ9SR>7W1O{@y^{oNsoza(AvGHj(`td90;lWp6C34fD$gv9+!+loL@kog8Y002Ai0efF>gJwQPx^9&7vj z|9X$t#cHw3_rQR_;estUK-+l%4-^j$ZVEho1A`aOpL;`(au~N;TwKI}O2CEI)J!cF zmCdZ{>FFW9Swka^L|d!EMz8nM`)_EH8gosqU{X_4lMkb$p^2=zIoYu~nDKyzr?eR$ zmM0wQv3fSQZjngbAQ+Rii4&ZMhJjI~Ipu9qP~`DKM-4WvzlHugO3Qu*5Y$VRz0p;r za@~%$8Yd?;%&x#H@N2KXJIp@1XXsV$=jT_!lxDTrMJD>XxxmR^4d3#;b2Cfu)`(r>~cDEK}-ouQGow%RUOhh0=%3 zAU_gXS0|cL>-g@FDY)hB__*5s;bBU0GIByf0_+dc9$hP92)_mn2ne=)CDS}&YcFKs z|6YgO2&4SxD#<`tz&}@t2m)37>-q;a|8s+hf&r}3e_d`A2yge#H8J@2CH~&hpNNPU z|GNGk_=!(}W)>FG+S)YQ^`5npO39YGym6nRq5=W#JylS!Ueh!yrC^GXJKdf(5dwH2 zbf_sUy+4mIM~E(20FKyo>ZHBAuB=pyMyIMAaU`*(YTcZl^xTW;gmOilcoFUG?T#W* z>Uoi&!|MkcFTE5Dpn-^=_g`8{DcTFh)e|q^TU7L5wJ6U@XbVWEjIr@@s0b|)k%#Kv zar^Y}{a+r-XFW5unY|I4hfgOcD45z{Si1x#JE6d7afF<=04171}eJm|ai`b|rE&cnGut>we`av{sj-jD1ogQU^e(-Mz z2?-ym_O`Zgz!1Pyo?EMlZB1V>hXn0w=$Y4z`mWMBj**ma!TaL zeiLa!!#gObsB}@C&KoZ8f>xRv{?4@U=L}5MqwU#1LKN8eWK-EI?@tIw@a4y*87?8? zVq|4YOIGmQjEpv8Wt7OINZ_Tx@l{k*PjEg2d(W1a$MkrWq4 zij0b)F55=fm7D)Hy2FEWM|i8Czl+l)H3UNy!f_alNYlyQGCX@AIP}!iRQ`+41W4c^ zfEg=ofmR9t@BPQrg$IBJR8|W31qMQD@b<&dTTD#YFk}!&`T56I4S)SZ zpoX#(Rn^rcA3X{heB1@C2B|`aR~;%`_R>pw+r#hK9&V@s)udx&#DW+D;0B1@z=LfC ztF7FM*r4iy0)orSOZ+oT0M2PI^Y*qTG!(oKJQ3&>&?M7^vx!OO;Dz$c(b4wF`MEG0 zFN4G}7)6=^AUyO0`9J_;$ET-&41fcfx;=V?1<*(80-r`0iw;r1NX4$o~x)V?@t5=2hRgxhky%K^~lSd4hgaL~7*K^%(?5P>%i z8QUXAaM`mq-Up*PUI1L6`zQbOrRSuniHXg`2qH}9Q&hwlAg}+}1-`{^gM{I$r!@+0 zCNcQmD;iA|5M2Nvim9pT%L+XX^3;#_@7`^hC)bF+4#63aPCUOYN^g=_Yf}>nbdn{c zq5_pIU#oh_#=^;|Y><5q820Mw>Oj;*P!Lx20t;y{x8Dx%Rw$eQ(j|%Y5f%Xf0y03K zxS)Ul4A?XWaP4tp50I;=x`#a$+zo;NvS7UYcX#c;4d|Piwy%2TqvU@5dJ7s9V}pN( zFyaf2z|F0i1be#1g%}wbIVdDV#@d>F-&G`0#El`L97s~KXPM0kKzOLgAqk#$X=w?8 z!<-P~6bBXI8+=fKvinLVHvk;~=eV%H4Hm9Fz9!7e%OEl{I*JKH4J1@(AA8V*`=tlq zf9BH15Z(a3g4Xe7*jz1ywaa~PDP(Z7tc;kH6azk=nx39gUQPzy96}$cAB=v42nHcV z2xD#uks5{0Z-_V);?lnc8|YbqPJN`ScHYLp0}&BC;Nx&M%9VM%iDa?}+?UfNOgrca z^UZ;qG&-uPA5l`5dy|Cn6xG#ZJH-t}>~*iv*?@WzxL)OXQ8g59%*9Kg3v!FV6ua?R zC3c}7VTH{+3O7Z_^dU@!shhq&eb?*al5W7$y}iH=OoJxREPNmmkpXqzC@U>3eY`C4 zX6F`2s~crvF1ym$6*I=bqy!@(BL#(xB_&Z{ugdXG^59+qziks;JX&tE65n;*`w?=h z{3=rSVaqDO^m$RFZ4?1IG&6Ja@(oo=4TZL~HM8;ph{wW^)&h&&eYB2Oc%7J71<&3> zL>U7EV}qvL|2|lj{PIB&Uh?kVUa?zEcD5XLONn~k;Br@Yw?E;b6qe-(u`a|P0Y70R zo9!8qEV9wpuv*sE*6hsdhh23As`FD*@7$kq6|4CU$AIm}_qs@RMLbR&hSwc<4GXfe z9%I+f@Nse7re3z_d7x2H7*V^AFiSc#J3Bk7P-I@wch#0@Vsw6kXP7nSLJ(2edhG#m zXSM!PrZzWN$BRiys$V)FBQ4g#H^Cf zZ|NH+F!FOms86W@Z264fKf^qeq+n9|-41I^7L_IDDF$7xJ!@qXQYRNm?)&%H| zpVJJ$Hb!tW%2F^R)$^RH!i4Bv{iXb%Qcrw0u(T+c1zN+G56kut^S6B&tI)_#Ud+gx8DrSKeFf(T;R-96LFy!Rtym-a)d5dDX$t_BZJDmh?T8JPg#fE? zMl3%gqjiy|zSo3XF6Vw^M1%n1%Sm^v9hEJzSL zpGYx<$SzdnBNjc`e`W(|4rcWw<8O;n{yb%z2sxu8#JnIum8sBunai14H-sPniX^Cr zpp*qc_=8yXB#KRd%G^zx4h`lhUX%|tx0Xm3S=`#Xz`;UKFS&Uwh%~rGkAL>eYau(E zY1P(W(x;?^8(4zeB$Vxr*Un)sd$*{SS7mB0l{LJ}`vj{zhH zXib@0VmBH{O%wpWK1yO~{5d?7RZs}y#*8dxKLyMI@F3hZJ2fqhj*oBDiWWoF?Op>4 z9PvfT8r4!&B?rmpV_aPD`e?yz4h}-V4aJi>1*1|x0snb7ezrf0=s=LsjZQbksKTxb9Vj6l*Wx&Lah|I z81W)*r0{NlBPnYTPW#ng8wxsvW`Or0E=pmIGPe*}MryLPW+`S6UcatSrfeA?ltU;O z9vh2IOPl1_yNvKfu;~%*PSF~en0OEGKaTgQSe0dEdpk-wFBlZ77K#DSug@6?iHM+e zK@qFKtSo9S)XM%^Uhe#4O&oAZ;NMW$&(_}2@p%bj_vq*$==vbvrM1G>#^(n-Dj5bT z5z*QW&;QWHk%VX@xm_Gp*#}QDY(wW%&f;7=Un|> zdROI*YccVOL#^E=NlOr zna+Q2=;TD|;^G3cHMW^ukycej0jFih2kb$uefsQ~M9y&jI=^qDcTCJp2!9e8gO;WV z%KHB%z9ml})q^k?KR>iKL(VKf&-9=-VEgNr=TxMTV538>1_ZLQO#zT-Z-uu{;kS=e z*K>ggG%zdA7F9jPE+RtB8G{Sy8%oc7U~!OI?9cP)e$53pBIFJSH!y%#_cR_WC_Dt( znY~O+L4nW{H$ubP4IjzN%R(vYs)Bavf0hKeRwu*?hLBD~OB+^IB?w1NU?V%WmX;PH z3ya@VhGxhd=2kVeE6E#~Qx!-_G$yb=!>qp~vIDC9^k7(DpOH)n@= zfH~bhCnZ@Qy^)v4he?~MaVbu&vc~vpYzTX!j*O500Mr^XE#-$v2?>Zi7x*TSmHeBU zBw;MZH@QSbLqUK*j0-Ft0Ic}SH)&}fYH3m1+1q~tF%ywanM=>i!~!e?zuVT{E@5Sr zKVA7j_BKq;r^v_vFdN{10@PL}9OQ5g<{h|3L^=*ME8zIH%nfTB8`5w~^F`6H$6{Xx zOtrAvVb{Di@<^fuj4gcWBRM%qcr^G$L}s%=v;odAGqSU{UYs2veqd&1MrqyD0H;u0 zLn9$TuWY>x{n<7qBG4eFMv5Rr_BPzw-`A&mblico4tW5`ha;8^s1fi7$n+sY{AT6j zBXe|g9JmBDjTpe=o%!t1FEKIKVxBS_l*A?_MY&o+Qm=fg^MA52UjUyZycM0K-yu_W zOF-ZzWX)ebLPi1|>*d_g%naK>q1sJ-pr+qaQ_&#%53D3y^8KezM5aj)F--NhCnnT^ z+UPkrka#=Fr$K6N3RV=%HpqtJK=mOsY`U6*mw==c*0pQb5dZD$%qJ%&XLGcrRgRJi zf(sb8gR}FKFnCUwCRpzCkW6wqo2~cK!S+`^%!KS50xsP?ds=sy#2KpF*5Z_f%_86Jm?Hx(Z>r*Cap z1-2rsXFG5C-ID!A^zPj|r}+@enRL)I$ESrRFG{*cf15p;QSvGb&5N0imQZ@(AFIgaB`5C;&4y{czGY zpCN13>OU^*qb2>uz`)D$up)n!p0~7nx zVXSEptLlFu8<_Zn1P%@ktY-OU5*6h*lc^DAru+A=fvLWH{PZcp=R#s_`V#a3KrJnh zZ%ch#IgM9nVQ)}-C0y1zvwb7t^XIpa3Jt8vzt`~HpHop$5h*wPD<2P!-kUe%;F!Y0 z!;y0Bd1`g8{zqxo|J2kG!~w-90$K$@L>*f}5Q1n?K@t%J1XK!=geZbIe2GNFihvM- zFd(KAkQOw2DU(n{w235OBLqQ1stpuch$fSuqzcwXh!H{sg5+t^#Zmu(-n<`fZruPJ25d}$0&DZbTY z_HJoq{`T4WqCgEmlL4|E9Pe$3UfPnnk_Hvr4UpBXf9y;Y5ZztSXt-DqLoX?Lq)<3p z8PDz%!9EBelR2`{9O^pmJVtmnmkV_E+s(X^yQz(m52s@2_gm0tpC-CM36S19qi%w? z>6EACi?_4J#mgF9vJ`KHfA%LH5dRQgx@}ckjg#}5z1?1c+yEPn;w0+2WKUe8?4Y%w z>%wJerSRJfl~7AcN)i*ga#*AoubvGeddyY`XKBx5u#=-ZPyCfeF;Wb6X6`2Q&A=Cn zgH|U5$EnhY1-63at1OD^N56Z(8TfUAstm9fXsrIuVSzD|==_$nT8!5bRS z8b(qC%JkUi-bCrZ!OpvPuV^~LHm(4rIX_v0+5VZRBV6Z%QjA2VR;LW-f$F4Yd%BA( z9z>`!j<7~&EK{^Lf(vr;_D=4j6VuF|);o7x6H3Fw!#mn~itg=ils-gaVq?eWxGwZ7 z{mDib5xqGfKVPV+W-`}0tylsF?aV?<5sRTk!fwd0_|wKOMc}Xi9ow#-Z|%)DRmkNG z$DUhgn)InJv$l4PkBClt(Q|WIx89s6)wan7G-Ec+d$OtR#)%1D=~fEJQY2r!#lu6N zMs@$l1Iyo|iqF4kQC!H8bAU9QM-@Xd$_Q&rRSmS5?ANc22e)brnblep?op*5h6Fiu zDsat%Nu>ZjSM;fV(dl$vSS)%lQ-uft62~4pDrfX2X5{9kc9mQW0azv7ai6cRFK18# zkA7MmHP|XPjormmfG33C%(&g&exisQxu6N4nIT&Y%FAk5{n>6U*fhO ztE;mobKYBhTA^OS>0#jT)Pw4`5O~%f^&k~B(9GP`X~E@MMCW+sr!P>%3aBGVqVeEU z=P`hcvMv%xlgoq3ckI~lPsjYq8x8l~dsyyG;ve=YOQ3?8>f;gYkTaLr;K2q^?ui7(Q@At$A9vgeA>|_ z9#YyVP(RXb-+0JXHI-9*+&jI%>1lt!6y}?5du~%xh=XH%=((>SWt}@W?)&Op6OU(| zy7JQ1km3M;HjXpuJv@tdU$s6rj#>nv6tn`8B2D06=U@eWb!ebJE;Bp(XRtpmiwjYt z16DF}>qXX!moI}a1vi|G0V0$uvhwnH9Rt<0`N__tXyvACEU;jj~e z@<^>grz7|;&nQlASQo{{;n=v>CVBu$U8??IA7WePQb8;r92$*=HL`C67r Date: Thu, 20 Oct 2022 10:06:20 +0200 Subject: [PATCH 132/145] Adjust title of changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fb037eb..20c42871 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## v2.1.0 - (https://github.com/nf-core/smrnaseq/releases/tag/2.1.0) - 2022-10-11 Maroon Tin Dalmatian +## [v2.0.0](https://github.com/nf-core/smrnaseq/releases/tag/2.1.0) - 2022-10-20 Maroon Tin Dalmatian ### Enhancements & fixes From 8beb03e5208cece49f839e03d11fd0ca21b30947 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 20 Oct 2022 10:06:41 +0200 Subject: [PATCH 133/145] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20c42871..c20ab933 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[#159](https://github.com/nf-core/smrnaseq/issues/159)] - Index files were not collected when `bowtie_index` was used and thus mapping was failing - [[#161](https://github.com/nf-core/smrnaseq/issues/161)] - Trimmed output was not as documented and not correctly published - [[#168](https://github.com/nf-core/smrnaseq/issues/168)] - Removed `mirtrace_protocol` as the parameter was redundant and `params.protocol` is entirely sufficient -- Updated pipeline template to [nf-core/tools 2.5.1](https://github.com/nf-core/tools/releases/tag/2.5.1) +- Updated pipeline template to [nf-core/tools 2.5.1](https://github.com/nf-core/tools/releases/tag/2.6.0) - [[#188](https://github.com/nf-core/smrnaseq/pull/188)] - Dropped TrimGalore in favor of fastp QC and adapter trimming, improved handling of adapters and trimming parameters - [[#194](https://github.com/nf-core/smrnaseq/issues/194)] - Added default adapters file for FastP improved miRNA adapter trimming From af2fed5271e5c18ff49ea0fea1605e3e85d77c1c Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Thu, 20 Oct 2022 10:06:57 +0200 Subject: [PATCH 134/145] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c20ab933..fc29c0db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [[#159](https://github.com/nf-core/smrnaseq/issues/159)] - Index files were not collected when `bowtie_index` was used and thus mapping was failing - [[#161](https://github.com/nf-core/smrnaseq/issues/161)] - Trimmed output was not as documented and not correctly published - [[#168](https://github.com/nf-core/smrnaseq/issues/168)] - Removed `mirtrace_protocol` as the parameter was redundant and `params.protocol` is entirely sufficient -- Updated pipeline template to [nf-core/tools 2.5.1](https://github.com/nf-core/tools/releases/tag/2.6.0) +- Updated pipeline template to [nf-core/tools 2.6.0](https://github.com/nf-core/tools/releases/tag/2.6.0) - [[#188](https://github.com/nf-core/smrnaseq/pull/188)] - Dropped TrimGalore in favor of fastp QC and adapter trimming, improved handling of adapters and trimming parameters - [[#194](https://github.com/nf-core/smrnaseq/issues/194)] - Added default adapters file for FastP improved miRNA adapter trimming From b12a6036163921b507c38e780a9f4d8cb4631d18 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 24 Oct 2022 12:47:35 +0000 Subject: [PATCH 135/145] Add online video as per Joses suggestion :-) --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index e1054a4c..b0cecd05 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,12 @@ The pipeline is built using [Nextflow](https://www.nextflow.io), a workflow tool On release, automated continuous integration tests run the pipeline on a full-sized dataset on the AWS cloud infrastructure. This ensures that the pipeline runs on AWS, has sensible resource allocation defaults set to run on real-world datasets, and permits the persistent storage of results to benchmark between pipeline releases and other analysis sources. The results obtained from the full-sized test can be viewed on the [nf-core website](https://nf-co.re/smrnaseq/results). +## Online videos + +A short talk about the history, current status and functionality on offer in this pipeline was given by Lorena Pantano (@lpantano) on [9th November 2021](https://youtu.be/4YLQ2VwpCJE) as part of the nf-core/bytesize series. + +You can find numerous talks on the nf-core events page from various topics including writing pipelines/modules in Nextflow DSL2, using nf-core tooling, running nf-core pipelines as well as more generic content like contributing to Github. Please check them out! + ## Pipeline summary 1. Raw read QC ([`FastQC`](https://www.bioinformatics.babraham.ac.uk/projects/fastqc/)) From 051128f3ea013faf691b5903334c6d33ea0935f7 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 24 Oct 2022 14:47:51 +0200 Subject: [PATCH 136/145] Update docs/usage.md Co-authored-by: Jose Espinosa-Carrasco --- docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 3e3a3960..8a924d3b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -24,7 +24,7 @@ It should point to the 3-letter species name used by [miRBase](https://www.mirba ### miRNA related files -Different parameters can be set for the two supported datbases. By default `miRBase` will be used with the parameters below. +Different parameters can be set for the two supported databases. By default `miRBase` will be used with the parameters below. - `mirna_gtf`: If not supplied by the user, then `mirna_gtf` will point to the latest GFF3 file in miRbase: `https://mirbase.org/ftp/CURRENT/genomes/${params.mirtrace_species}.gff3` - `mature`: points to the FASTA file of mature miRNA sequences. `https://mirbase.org/ftp/CURRENT/mature.fa.gz` From b9afa72f4c0e436c1aa6f2b411e147d2da36b51a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 24 Oct 2022 14:48:01 +0200 Subject: [PATCH 137/145] Update docs/usage.md Co-authored-by: Jose Espinosa-Carrasco --- docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 8a924d3b..b45f449a 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -45,7 +45,7 @@ If MirGeneDB should be used instead it needs to be specified using `--mirgenedb` This step has, until now, only been tested for human data. Unexpected behaviour can occur when using it with a different species. -Contamination filtering of the sequencing reads is optional and can be invoked using `filter_contamination`. FASTA files with contamination sequences to use need to be supplied using the following commands. Otherwise the contamination filtering of the specific type will be omitted. +Contamination filtering of the sequencing reads is optional and can be invoked using the `filter_contamination` parameter. FASTA files with - `rrna`: Used to supply a FASTA file containing rRNA contamination sequence. - `trna`: Used to supply a FASTA file containing tRNA contamination sequence. e.g. `http://gtrnadb.ucsc.edu/genomes/eukaryota/Hsapi38/hg38-tRNAs.fa` From 6f85c3604c1a0dbf865fa422099aeed96bdab589 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 24 Oct 2022 14:51:39 +0200 Subject: [PATCH 138/145] Apply suggestions from code review Co-authored-by: Jose Espinosa-Carrasco --- modules/local/blat_mirna.nf | 3 ++- modules/local/filter_stats.nf | 4 ++-- subworkflows/local/contaminant_filter.nf | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/local/blat_mirna.nf b/modules/local/blat_mirna.nf index da7f6449..2b50d7db 100644 --- a/modules/local/blat_mirna.nf +++ b/modules/local/blat_mirna.nf @@ -52,7 +52,8 @@ process BLAT_MIRNA { echo $db_type blat -out=blast8 $mirna $contaminants /dev/stdout | awk 'BEGIN{FS="\t"}{if(\$11 < 1e-5)print \$1;}' | uniq > mirnahit.txt awk 'BEGIN { while((getline<"mirnahit.txt")>0) l[">"\$1]=1 } /^>/ {x = l[\$1]} {if(!x) print }' $contaminants > filtered.fa - cat <<-END_VERSIONS > versions.yml + +cat <<-END_VERSIONS > versions.yml "${task.process}": blat: \$(echo \$(blat) | grep Standalone | awk '{ if (match(\$0,/[0-9]*[0-9]/,m)) print m[0] }') END_VERSIONS diff --git a/modules/local/filter_stats.nf b/modules/local/filter_stats.nf index 1db0d49b..4d94d9b1 100644 --- a/modules/local/filter_stats.nf +++ b/modules/local/filter_stats.nf @@ -11,8 +11,8 @@ process FILTER_STATS { path stats_files output: - path "*_mqc.yaml" , emit: stats - tuple val(meta), path('*.filtered.fastq.gz') , emit: reads + path "*_mqc.yaml" , emit: stats + tuple val(meta), path('*.filtered.fastq.gz'), emit: reads when: task.ext.when == null || task.ext.when diff --git a/subworkflows/local/contaminant_filter.nf b/subworkflows/local/contaminant_filter.nf index 4ab180c7..19ce7bea 100644 --- a/subworkflows/local/contaminant_filter.nf +++ b/subworkflows/local/contaminant_filter.nf @@ -44,7 +44,6 @@ workflow CONTAMINANT_FILTER { reads.set { rrna_reads } - if (params.rrna) { // Index DB and filter $reads emit: $rrna_reads INDEX_RRNA ( rrna ) From 5aa27a312b2ec39c80e90455ae5b90f42e0d3875 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 24 Oct 2022 12:53:43 +0000 Subject: [PATCH 139/145] Added url for mirgenedb --- docs/usage.md | 2 +- nextflow_schema.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 3e3a3960..901084fc 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -20,7 +20,7 @@ This option indicates the experimental protocol used for the sample preparation. ### `mirtrace_species` or `mirgenedb_species` -It should point to the 3-letter species name used by [miRBase](https://www.mirbase.org/help/genome_summary.shtml) or MirGeneDB. Note the difference in case for the two databases. +It should point to the 3-letter species name used by [miRBase](https://www.mirbase.org/help/genome_summary.shtml) or [MirGeneDB](https://www.mirgenedb.org/browse). Note the difference in case for the two databases. ### miRNA related files diff --git a/nextflow_schema.json b/nextflow_schema.json index b5c52a3e..77b04352 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -76,7 +76,7 @@ "mirgenedb_species": { "type": "string", "description": "Species of MirGeneDB.", - "help_text": "This replaces the value of `--mirtrace_species` if `--mirgenedb` is used. \n Note the difference in case for species names used in MirGeneDB and miRBase." + "help_text": "This replaces the value of `--mirtrace_species` if `--mirgenedb` is used. \n Note the difference in case for species names used in MirGeneDB and miRBase. See https://www.mirgenedb.org/browse for more information." }, "fasta": { "type": "string", From 4a30e4aff4e567bc55e573fefc24bd0e3908ad3b Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 25 Oct 2022 00:10:39 +0200 Subject: [PATCH 140/145] Update CHANGELOG.md Co-authored-by: Edmund Miller --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc29c0db..ee8bd744 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [v2.0.0](https://github.com/nf-core/smrnaseq/releases/tag/2.1.0) - 2022-10-20 Maroon Tin Dalmatian +## [v2.1.0](https://github.com/nf-core/smrnaseq/releases/tag/2.1.0) - 2022-10-20 Maroon Tin Dalmatian ### Enhancements & fixes From 9fe7a4f8d04374cb2c4a70e58db60a3bc5f6bb0d Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 24 Oct 2022 22:15:35 +0000 Subject: [PATCH 141/145] Add note why no download possible --- docs/usage.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 1bfa8051..bc6c3a4b 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -32,10 +32,12 @@ Different parameters can be set for the two supported databases. By default `miR If MirGeneDB should be used instead it needs to be specified using `--mirgenedb` and use the parameters below . -- `mirgenedb_gff`: The data can not be downloaded automatically, thus the user needs to supply the gff file for either his species, or all species downloaded from `https://mirgenedb.org/download`. The total set will automatically be subsetted to the species specified with `--mirgenedb_species`. +- `mirgenedb_gff`: The data can not be downloaded automatically (URLs are created with short term tokens in it), thus the user needs to supply the gff file for either his species, or all species downloaded from `https://mirgenedb.org/download`. The total set will automatically be subsetted to the species specified with `--mirgenedb_species`. - `mirgenedb_mature`: points to the FASTA file of mature miRNA sequences. Download from `https://mirgenedb.org/download`. - `mirgenedb_hairpin`: points to the FASTA file of precursor miRNA sequences. Download from `https://mirgenedb.org/download`. Note that MirGeneDB does not have a dedicated `hairpin` file, but the `Precursor sequences` are to be used. + + ### Genome - `fasta`: the reference genome FASTA file From 4a6d9d94005ae0976b97933511db72636bd4da4e Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 25 Oct 2022 00:15:43 +0200 Subject: [PATCH 142/145] Update modules/local/bowtie_map_contaminants.nf Co-authored-by: Edmund Miller --- modules/local/bowtie_map_contaminants.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/local/bowtie_map_contaminants.nf b/modules/local/bowtie_map_contaminants.nf index c3daa9be..bbfb66fd 100644 --- a/modules/local/bowtie_map_contaminants.nf +++ b/modules/local/bowtie_map_contaminants.nf @@ -15,7 +15,7 @@ process BOWTIE_MAP_CONTAMINANTS { tuple val(meta), path("*sam") , emit: bam tuple val(meta), path('*.filter.unmapped.contaminant.fastq'), emit: unmapped path "versions.yml" , emit: versions - path "filtered.*.stats" , emit: stats + path "filtered.*.stats" , emit: stats when: task.ext.when == null || task.ext.when From 6363ae31c5d15aa397d24d6339e11e7b4b11743a Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Tue, 25 Oct 2022 00:19:56 +0200 Subject: [PATCH 143/145] Apply suggestions from code review Co-authored-by: Edmund Miller --- subworkflows/local/contaminant_filter.nf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/subworkflows/local/contaminant_filter.nf b/subworkflows/local/contaminant_filter.nf index 19ce7bea..383c85ad 100644 --- a/subworkflows/local/contaminant_filter.nf +++ b/subworkflows/local/contaminant_filter.nf @@ -74,7 +74,7 @@ workflow CONTAMINANT_FILTER { ch_versions = ch_versions.mix(BLAT_CDNA.out.versions) INDEX_CDNA ( BLAT_CDNA.out.filtered_set ) ch_versions = ch_versions.mix(INDEX_CDNA.out.versions) - MAP_CDNA ( trna_reads, INDEX_CDNA.out.index, 'cDNA' ) + MAP_CDNA ( trna_reads, INDEX_CDNA.out.index, 'cDNA' ) ch_versions = ch_versions.mix(MAP_CDNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_CDNA.out.stats.ifEmpty(null)) MAP_CDNA.out.unmapped.set { cdna_reads } @@ -100,7 +100,7 @@ workflow CONTAMINANT_FILTER { ch_versions = ch_versions.mix(BLAT_PIRNA.out.versions) INDEX_PIRNA ( BLAT_PIRNA.out.filtered_set ) ch_versions = ch_versions.mix(INDEX_PIRNA.out.versions) - MAP_PIRNA (ncrna_reads, INDEX_PIRNA.out.index, 'piRNA' ) + MAP_PIRNA ( ncrna_reads, INDEX_PIRNA.out.index, 'piRNA' ) ch_versions = ch_versions.mix(MAP_PIRNA.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_PIRNA.out.stats.ifEmpty(null)) MAP_PIRNA.out.unmapped.set { pirna_reads } @@ -113,7 +113,7 @@ workflow CONTAMINANT_FILTER { ch_versions = ch_versions.mix(BLAT_OTHER.out.versions) INDEX_OTHER ( BLAT_OTHER.out.filtered_set ) ch_versions = ch_versions.mix(INDEX_OTHER.out.versions) - MAP_OTHER (ncrna_reads, INDEX_OTHER.out.index, 'other' ) + MAP_OTHER ( ncrna_reads, INDEX_OTHER.out.index, 'other' ) ch_versions = ch_versions.mix(MAP_OTHER.out.versions) ch_filter_stats = ch_filter_stats.mix(MAP_OTHER.out.stats.ifEmpty(null)) MAP_OTHER.out.unmapped.set { other_cont_reads } From 2ff08d92da17f7ae78da85380f959dc3e74aa3e6 Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 24 Oct 2022 22:20:23 +0000 Subject: [PATCH 144/145] Adjust = to be same everywhere --- nextflow.config | 208 ++++++++++++++++++++++++------------------------ 1 file changed, 104 insertions(+), 104 deletions(-) diff --git a/nextflow.config b/nextflow.config index b1b24dbc..759cf40d 100644 --- a/nextflow.config +++ b/nextflow.config @@ -10,84 +10,84 @@ params { // Input options - input = null + input = null // Workflow flags - protocol = 'illumina' + protocol = 'illumina' // References - genome = null - igenomes_base = 's3://ngi-igenomes/igenomes' - igenomes_ignore = false - mirna_gtf = null - mature = "https://mirbase.org/ftp/CURRENT/mature.fa.gz" - hairpin = "https://mirbase.org/ftp/CURRENT/hairpin.fa.gz" - mirgenedb = false - mirgenedb_mature = null - mirgenedb_hairpin = null - mirgenedb_gff = null - mirgenedb_species = null + genome = null + igenomes_base = 's3://ngi-igenomes/igenomes' + igenomes_ignore = false + mirna_gtf = null + mature = "https://mirbase.org/ftp/CURRENT/mature.fa.gz" + hairpin = "https://mirbase.org/ftp/CURRENT/hairpin.fa.gz" + mirgenedb = false + mirgenedb_mature = null + mirgenedb_hairpin = null + mirgenedb_gff = null + mirgenedb_species = null // Trimming options - clip_r1 = null - three_prime_clip_r1 = null - three_prime_adapter = null - trim_fastq = true - fastp_min_length = 17 - fastp_known_mirna_adapters = "$projectDir/assets/known_adapters.fa" - save_trimmed_fail = false - skip_qc = false - skip_fastqc = false - skip_multiqc = false - skip_mirdeep = false - skip_fastp = false - save_reference = false - fastp_max_length = 40 + clip_r1 = null + three_prime_clip_r1 = null + three_prime_adapter = null + trim_fastq = true + fastp_min_length = 17 + fastp_known_mirna_adapters = "$projectDir/assets/known_adapters.fa" + save_trimmed_fail = false + skip_qc = false + skip_fastqc = false + skip_multiqc = false + skip_mirdeep = false + skip_fastp = false + save_reference = false + fastp_max_length = 40 // Contamination filtering - filter_contamination = false - rrna = null - trna = null - cdna = null - ncrna = null - pirna = null - other_contamination = null + filter_contamination = false + rrna = null + trna = null + cdna = null + ncrna = null + pirna = null + other_contamination = null // MultiQC options - multiqc_config = null - multiqc_title = null - multiqc_logo = null - max_multiqc_email_size = '25.MB' + multiqc_config = null + multiqc_title = null + multiqc_logo = null + max_multiqc_email_size = '25.MB' multiqc_methods_description = null // Boilerplate options - outdir = null - tracedir = "${params.outdir}/pipeline_info" - publish_dir_mode = 'copy' - email = null - email_on_fail = null - plaintext_email = false - monochrome_logs = false - hook_url = null - help = false - validate_params = true - show_hidden_params = false - schema_ignore_params = 'genomes' - enable_conda = false + outdir = null + tracedir = "${params.outdir}/pipeline_info" + publish_dir_mode = 'copy' + email = null + email_on_fail = null + plaintext_email = false + monochrome_logs = false + hook_url = null + help = false + validate_params = true + show_hidden_params = false + schema_ignore_params = 'genomes' + enable_conda = false // Config options - custom_config_version = 'master' - custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" - config_profile_description = null - config_profile_contact = null - config_profile_url = null - config_profile_name = null + custom_config_version = 'master' + custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" + config_profile_description = null + config_profile_contact = null + config_profile_url = null + config_profile_name = null // Max resource options // Defaults only, expecting to be overwritten - max_memory = '128.GB' - max_cpus = 16 - max_time = '240.h' + max_memory = '128.GB' + max_cpus = 16 + max_time = '240.h' } @@ -112,64 +112,64 @@ try { profiles { debug { process.beforeScript = 'echo $HOSTNAME' } conda { - params.enable_conda = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false + params.enable_conda = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false } mamba { - params.enable_conda = true - conda.useMamba = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false + params.enable_conda = true + conda.useMamba = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false } docker { - docker.enabled = true - docker.userEmulation = true - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - docker.runOptions = '-u $(id -u):$(id -g)' + docker.enabled = true + docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + docker.runOptions = '-u $(id -u):$(id -g)' } singularity { - singularity.enabled = true - singularity.autoMounts = true - docker.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false + singularity.enabled = true + singularity.autoMounts = true + docker.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false } podman { - podman.enabled = true - docker.enabled = false - singularity.enabled = false - shifter.enabled = false - charliecloud.enabled = false + podman.enabled = true + docker.enabled = false + singularity.enabled = false + shifter.enabled = false + charliecloud.enabled = false } shifter { - shifter.enabled = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - charliecloud.enabled = false + shifter.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + charliecloud.enabled = false } charliecloud { - charliecloud.enabled = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false + charliecloud.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false } gitpod { - executor.name = 'local' - executor.cpus = 16 - executor.memory = 60.GB + executor.name = 'local' + executor.cpus = 16 + executor.memory = 60.GB } test { includeConfig 'conf/test.config' } test_no_genome { includeConfig 'conf/test_no_genome.config' } From bf3ef92df3bd77a90a44bc18f500021067c5e6fd Mon Sep 17 00:00:00 2001 From: Alexander Peltzer Date: Mon, 24 Oct 2022 22:20:47 +0000 Subject: [PATCH 145/145] Fix prettier --- docs/usage.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index bc6c3a4b..28373e00 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -36,8 +36,6 @@ If MirGeneDB should be used instead it needs to be specified using `--mirgenedb` - `mirgenedb_mature`: points to the FASTA file of mature miRNA sequences. Download from `https://mirgenedb.org/download`. - `mirgenedb_hairpin`: points to the FASTA file of precursor miRNA sequences. Download from `https://mirgenedb.org/download`. Note that MirGeneDB does not have a dedicated `hairpin` file, but the `Precursor sequences` are to be used. - - ### Genome - `fasta`: the reference genome FASTA file