Merge branch 'dev' into release-checklist

nf-core · May 31, 2024 · 301e6fa · 301e6fa
2 parents 814dc85 + 26fb9e5
commit 301e6fa
Show file tree

Hide file tree

Showing 19 changed files with 1,616 additions and 1,540 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.4
+    rev: v0.4.6
     hooks:
       - id: ruff # linter
         args: [--fix, --exit-non-zero-on-fix] # sort imports and fix

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,8 @@
 
 ### Template
 
+- Change paths to test data ([#2985](https://github.com/nf-core/tools/pull/2985))
+
 ### Linting
 
 - Fix linting fail on nfcore_external_java_deps if nf_schema is used ([#2976](https://github.com/nf-core/tools/pull/2976))
@@ -12,18 +14,25 @@
 
 ### Components
 
+- The `modules_nfcore` tag in the `main.nf.test` file of modules/subworkflows now displays the organization name in custom modules repositories ([#3005](https://github.com/nf-core/tools/pull/3005))
+
 ### General
 
 - Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.4 ([#2974](https://github.com/nf-core/tools/pull/2974))
 - Update gitpod/workspace-base Docker digest to 92dd1bc ([#2982](https://github.com/nf-core/tools/pull/2982))
 - Update output of generation script for API docs to new structure ([#2988](https://github.com/nf-core/tools/pull/2988))
 - Remove `rich-codex.yml` action, images are now generated on the website repo ([#2989](https://github.com/nf-core/tools/pull/2989))
+- Add no clobber and put bash options on their own line ([#2991](https://github.com/nf-core/tools/pull/2991))
+- update minimal textual version and snapshots ([#2998](https://github.com/nf-core/tools/pull/2998))
+- return directory if base_dir is the root directory ([#3003](https://github.com/nf-core/tools/pull/3003))
+- Update pre-commit hook astral-sh/ruff-pre-commit to v0.4.6 ([#3006](https://github.com/nf-core/tools/pull/3006))
 
 ## [v2.14.1 - Tantalum Toad - Patch](https://github.com/nf-core/tools/releases/tag/2.14.1) - [2024-05-09]
 
 ### Template
 
 - Don't cache pip in `linting.yml` ([#2961](https://github.com/nf-core/tools/pull/2961))
+- Lint pipelines with the nf-core template version and post comment if it is outdated ([#2978](https://github.com/nf-core/tools/pull/2978))
 
 ### General
 

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.12-slim@sha256:2be8daddbb82756f7d1f2c7ece706aadcb284bf6ab6d769ea695cc3ed6016743
+FROM python:3.12-slim@sha256:afc139a0a640942491ec481ad8dda10f2c5b753f5c969393b12480155fe15a63
 LABEL authors="[email protected],[email protected]" \
     description="Docker image containing requirements for nf-core/tools"
 

diff --git a/nf_core/components/create.py b/nf_core/components/create.py
@@ -157,6 +157,10 @@ def create(self):
             if self.component_type == "modules":
                 self._get_module_structure_components()
 
+        # Add a valid organization name for nf-test tags
+        not_alphabet = re.compile(r"[^a-zA-Z]")
+        self.org_alphabet = not_alphabet.sub("", self.org)
+
         # Create component template with jinja2
         self._render_template()
         log.info(f"Created component template: '{self.component_name}'")

diff --git a/nf_core/module-template/tests/main.nf.test.j2 b/nf_core/module-template/tests/main.nf.test.j2
@@ -7,7 +7,7 @@ nextflow_process {
     process "{{ component_name_underscore|upper }}"
 
     tag "modules"
-    tag "modules_nfcore"
+    tag "modules_{{ org_alphabet }}"
     {%- if subtool %}
     tag "{{ component }}"
     {%- endif %}
@@ -28,10 +28,10 @@ nextflow_process {
                 {% if has_meta %}
                 input[0] = [
                     [ id:'test', single_end:false ], // meta map
-                    file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)
+                    file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.bam', checkIfExists: true)
                 ]
                 {%- else %}
-                input[0] = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true)
+                input[0] = file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.single_end.bam', checkIfExists: true)
                 {%- endif %}
                 """
             }
@@ -60,10 +60,10 @@ nextflow_process {
                 {% if has_meta %}
                 input[0] = [
                     [ id:'test', single_end:false ], // meta map
-                    file(params.test_data['sarscov2']['illumina']['test_paired_end_bam'], checkIfExists: true)
+                    file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.paired_end.bam', checkIfExists: true)
                     ]
                 {%- else %}
-                input[0] = file(params.test_data['sarscov2']['illumina']['test_single_end_bam'], checkIfExists: true)
+                input[0] = file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/bam/test.single_end.bam', checkIfExists: true)
                 {%- endif %}
                 """
             }

diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py
@@ -4,6 +4,7 @@
 
 import json
 import logging
+import re
 from pathlib import Path
 
 import yaml
@@ -137,7 +138,10 @@ def module_tests(_, module: NFCoreComponent):
                     )
             # Verify that tags are correct.
             main_nf_tags = module._get_main_nf_tags(module.nftest_main_nf)
-            required_tags = ["modules", "modules_nfcore", module.component_name]
+            not_alphabet = re.compile(r"[^a-zA-Z]")
+            org_alp = not_alphabet.sub("", module.org)
+            org_alphabet = org_alp if org_alp != "" else "nfcore"
+            required_tags = ["modules", f"modules_{org_alphabet}", module.component_name]
             if module.component_name.count("/") == 1:
                 required_tags.append(module.component_name.split("/")[0])
             chained_components_tags = module._get_included_components_in_chained_tests(module.nftest_main_nf)

diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml
@@ -41,10 +41,16 @@ jobs:
           python-version: "3.12"
           architecture: "x64"
 
+      - name: read .nf-core.yml
+        uses: pietrobolcato/[email protected]
+        id: read_yml
+        with:
+          config: ${{ github.workspace }}/.nf-core.yaml
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install nf-core
+          pip install nf-core==${{ steps.read_yml.outputs['nf_core_version'] }}
 
       - name: Run nf-core lint
         env:

diff --git a/nf_core/pipeline-template/.github/workflows/template_version_comment.yml b/nf_core/pipeline-template/.github/workflows/template_version_comment.yml
@@ -0,0 +1,42 @@
+name: nf-core template version comment
+# This workflow is triggered on PRs to check if the pipeline template version matches the latest nf-core version.
+# It posts a comment to the PR, even if it comes from a fork.
+
+on: pull_request_target
+
+jobs:
+  template_version:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+
+      - name: Read template version from .nf-core.yml
+        uses: pietrobolcato/[email protected]
+        id: read_yml
+        with:
+          config: ${{ github.workspace }}/.nf-core.yml
+
+      - name: Install nf-core
+        run: |
+          python -m pip install --upgrade pip
+          pip install nf-core==${{ steps.read_yml.outputs['nf_core_version'] }}
+
+      - name: Check nf-core outdated
+        id: nf_core_outdated
+        run: pip list --outdated | grep nf-core
+
+      - name: Post nf-core template version comment
+        uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2
+        if: |
+          ${{ steps.nf_core_outdated.outputs.stdout }} =~ 'nf-core'
+        with:
+          repo-token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }}
+          allow-repeats: false
+          message: |
+            ## :warning: Newer version of the nf-core template is available.
+
+            Your pipeline is using an old version of the nf-core template: ${{ steps.read_yml.outputs['nf_core_version'] }}.
+            Please update your pipeline to the latest version.
+
+            For more documentation on how to update your pipeline, please see the [nf-core documentation](https://github.com/nf-core/tools?tab=readme-ov-file#sync-a-pipeline-with-the-template) and [Synchronisation documentation](https://nf-co.re/docs/contributing/sync).
diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md
@@ -80,8 +80,7 @@ nextflow run {{ name }} \
 ```
 
 > [!WARNING]
-> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_;
-> see [docs](https://nf-co.re/usage/configuration#custom-configuration-files).
+> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; see [docs](https://nf-co.re/docs/usage/getting_started/configuration#custom-configuration-files).
 
 {% if is_nfcore -%}
 

diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config
@@ -187,13 +187,14 @@ profiles {
     test_full { includeConfig 'conf/test_full.config' }
 }
 
-// Set default registry for Apptainer, Docker, Podman and Singularity independent of -profile
-// Will not be used unless Apptainer / Docker / Podman / Singularity are enabled
+// Set default registry for Apptainer, Docker, Podman, Charliecloud and Singularity independent of -profile
+// Will not be used unless Apptainer / Docker / Podman / Charliecloud / Singularity are enabled
 // Set to your registry if you have a mirror of containers
-apptainer.registry   = 'quay.io'
-docker.registry      = 'quay.io'
-podman.registry      = 'quay.io'
-singularity.registry = 'quay.io'
+apptainer.registry    = 'quay.io'
+docker.registry       = 'quay.io'
+podman.registry       = 'quay.io'
+singularity.registry  = 'quay.io'
+charliecloud.registry = 'quay.io'
 
 // Nextflow plugins
 plugins {
@@ -220,8 +221,15 @@ env {
     JULIA_DEPOT_PATH = "/usr/local/share/julia"
 }
 
-// Capture exit codes from upstream processes when piping
-process.shell = ['/bin/bash', '-euo', 'pipefail']
+// Set bash options
+process.shell = """\
+bash
+
+set -e # Exit if a tool returns a non-zero status/exit code
+set -u # Treat unset variables and parameters as an error
+set -o pipefail # Returns the status of the last command to exit with a non-zero status or zero if all successfully execute
+set -C # No clobber - prevent output redirection from overwriting files.
+"""
 
 // Disable process selector warnings by default. Use debug profile to enable warnings.
 nextflow.enable.configProcessNamesValidation = false

diff --git a/nf_core/subworkflow-template/tests/main.nf.test.j2 b/nf_core/subworkflow-template/tests/main.nf.test.j2
@@ -7,7 +7,7 @@ nextflow_workflow {
     workflow "{{ component_name_underscore|upper }}"
 
     tag "subworkflows"
-    tag "subworkflows_nfcore"
+    tag "subworkflows_{{ org_alphabet }}"
     tag "subworkflows/{{ component_name }}"
     // TODO nf-core: Add tags for all modules used within this subworkflow. Example:
     tag "samtools"

diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py
@@ -4,6 +4,7 @@
 
 import json
 import logging
+import re
 from pathlib import Path
 
 import yaml
@@ -144,10 +145,13 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent):
                     )
             # Verify that tags are correct.
             main_nf_tags = subworkflow._get_main_nf_tags(subworkflow.nftest_main_nf)
+            not_alphabet = re.compile(r"[^a-zA-Z]")
+            org_alp = not_alphabet.sub("", subworkflow.org)
+            org_alphabet = org_alp if org_alp != "" else "nfcore"
             required_tags = [
                 "subworkflows",
                 f"subworkflows/{subworkflow.component_name}",
-                "subworkflows_nfcore",
+                f"subworkflows_{org_alphabet}",
             ]
             included_components = []
             if subworkflow.main_nf.is_file():

diff --git a/nf_core/utils.py b/nf_core/utils.py
@@ -1069,7 +1069,7 @@ def determine_base_dir(directory="."):
         config_fn = get_first_available_path(base_dir, CONFIG_PATHS)
         if config_fn:
             break
-    return directory if base_dir == start_dir else base_dir
+    return directory if (base_dir == start_dir or str(base_dir) == base_dir.root) else base_dir
 
 
 def get_first_available_path(directory, paths):

diff --git a/requirements.txt b/requirements.txt
@@ -21,6 +21,6 @@ requests_cache
 rich-click>=1.6.1
 rich>=13.3.1
 tabulate
-textual>=0.47.1
+textual>=0.63.1
 trogon
 pdiff