perf: datavzrd wrapper v3.12.1, offer-excel configurable, free disk space for CI, dynamic sleuth_init mem_mb, pure download rules as localrules

.github/workflows/main.yml

@@ -69,6 +69,22 @@ jobs:
       - formatting
     steps:
 
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/[email protected]
+      with:
+        # this might remove tools that are actually needed,
+        # if set to "true" but frees about 6 GB
+        tool-cache: false
+
+        # all of these default to true, but feel free to set to
+        # "false" if necessary for your workflow
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: true
+        docker-images: false
+        swap-storage: true
+
     - name: Checkout repository
       uses: actions/checkout@v3
       with:
@@ -88,6 +104,22 @@ jobs:
       - formatting
     steps:
 
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/[email protected]
+      with:
+        # this might remove tools that are actually needed,
+        # if set to "true" but frees about 6 GB
+        tool-cache: false
+
+        # all of these default to true, but feel free to set to
+        # "false" if necessary for your workflow
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: true
+        docker-images: false
+        swap-storage: true
+
     - name: Checkout repository
       uses: actions/checkout@v3
 

.test/config/config.yaml

@@ -61,7 +61,7 @@ diffexp:
     genelist: "resources/gene_list.tsv"
 
 diffsplice:
-  activate: true
+  activate: false
   # codingCutoff parameter of isoformSwitchAnalyzer, see
   # https://rdrr.io/bioc/IsoformSwitchAnalyzeR/man/analyzeCPAT.html
   coding_cutoff: 0.725
@@ -93,6 +93,12 @@ enrichment:
     # the species specified by resources -> ref -> species above
     pathway_database: "panther"
 
+report:
+  # make this `true`, to get excel files for download in the snakemake
+  # report, BUT: this can drastically increase the runtime of datavzrd report
+  # generation, especially on larger cohorts
+  offer_excel: true
+
 bootstrap_plots:
   # desired false discovery rate for bootstrap plots, i.e. a lower FDR will result in fewer boxplots generated
   FDR: 0.01
@@ -133,4 +139,4 @@ params:
   #     of expected adapter matches by chance
   cutadapt-pe: 
     adapters: "-a ACGGATCGATCGATCGATCGAT -g GGATCGATCGATCGATCGAT -A ACGGATCGATCGATCGATCGAT -G GGATCGATCGATCGATCGAT"
-    extra: "--minimum-length 33 -e 0.005 --overlap 7"
+    extra: "--minimum-length 33 -e 0.005 --overlap 7"

.test/three_prime/config/config.yaml

@@ -18,7 +18,7 @@ resources:
     # ensembl species name
     species: homo_sapiens
     # ensembl release version
-    release: "111"
+    release: "112"
     # genome build
     build: GRCh38
     # pfam release to use for annotation of domains in differential splicing analysis
@@ -63,7 +63,7 @@ diffexp:
     genelist: "resources/gene_list.tsv"
 
 diffsplice:
-  activate: true
+  activate: false
   # codingCutoff parameter of isoformSwitchAnalyzer, see
   # https://rdrr.io/bioc/IsoformSwitchAnalyzeR/man/analyzeCPAT.html
   coding_cutoff: 0.725

config/config.yaml

@@ -18,7 +18,7 @@ resources:
     # ensembl species name
     species: homo_sapiens
     # ensembl release version
-    release: "104"
+    release: "112"
     # genome build
     build: GRCh38
     # pfam release to use for annotation of domains in differential splicing analysis
@@ -71,7 +71,7 @@ diffexp:
     genelist: "resources/gene_list.tsv"
 
 diffsplice:
-  activate: true
+  activate: false
   # codingCutoff parameter of isoformSwitchAnalyzer, see
   # https://rdrr.io/bioc/IsoformSwitchAnalyzeR/man/analyzeCPAT.html
   coding_cutoff: 0.725
@@ -117,6 +117,12 @@ plot_vars:
   # significance level used for plot_vars() plots
   sig_level: 0.1
 
+report:
+  # make this `true`, to get excel files for download in the snakemake
+  # report, BUT: this can drastically increase the runtime of datavzrd report
+  # generation, especially on larger cohorts
+  offer_excel: false
+
 params:
   #For reads that are produced by 3’-end sequencing, the --single-overhang option does not discard
   #reads where the expected fragment size goes beyond the transcript start

workflow/Snakefile

@@ -1,6 +1,6 @@
 from snakemake.utils import min_version
 
-min_version("7.17.0")
+min_version("8.13.0")
 
 
 configfile: "config/config.yaml"

workflow/envs/isoform-switch-analyzer.yaml

@@ -3,7 +3,6 @@ channels:
   - bioconda
   - nodefaults
 dependencies:
-  - bioconductor-isoformswitchanalyzer =1.14
-  - bioconductor-rhdf5 =2.36
-  - r-tidyverse =1.3
-  - r-base =4.1
+  - bioconductor-isoformswitchanalyzer =2.2.0
+  - bioconductor-rhdf5 =2.46.1
+  - r-tidyverse =2.0

workflow/resources/datavzrd/diffexp-template.yaml

@@ -1,8 +1,10 @@
+__use_yte__: true
+
 name: ?f"Differential expression analysis for model {wildcards.model}"
 datasets:
   genes_representative:
     path: ?input.genes_representative
-    offer-excel: true
+    offer-excel: ?params.offer_excel
     links:
       link to transcripts:
         column: ens_gene
@@ -19,19 +21,19 @@ datasets:
     separator: "\t"
   transcripts:
     path: ?input.transcripts
-    offer-excel: true
+    offer-excel: ?params.offer_excel
     links:
       link to genes representative:
         column: ens_gene
         table-row: genes_representative/ens_gene
     separator: "\t"  
   genes_aggregated:
     path: ?input.genes_aggregated
-    offer-excel: true
+    offer-excel: ?params.offer_excel
     separator: "\t"
   logcount_matrix:
     path: ?input.logcount_matrix
-    offer-excel: true
+    offer-excel: ?params.offer_excel
     links:  
       link to transcripts:
         column: transcript
@@ -292,4 +294,4 @@ views:
                 domain:
                   - -1
                   - 0
-                  - 1
+                  - 1

workflow/resources/datavzrd/go-enrichment-template.yaml

@@ -1,12 +1,14 @@
+__use_yte__: true
+
 name: ?f"Gene ontology (GO) term enrichment analysis performed by goatools {wildcards.model}"
 datasets:
   significant_terms:
     path: ?input.significant_terms
-    offer-excel: true
+    offer-excel: ?params.offer_excel
     separator: "\t"
   go_enrichment:
     path: ?input.enrichment
-    offer-excel: true
+    offer-excel: ?params.offer_excel
     separator: "\t"
 default-view: significant_terms
 views:

workflow/resources/datavzrd/spia-template.yaml

@@ -1,8 +1,10 @@
+__use_yte__: true
+
 name: ?f"spia pathway impact analysis for model {wildcards.model}"
 datasets:
   spia_table:
     path: ?input.spia_table
-    offer-excel: true
+    offer-excel: ?params.offer_excel
     separator: "\t"
 default-view: spia_table
 views:

workflow/rules/common.smk

@@ -80,10 +80,6 @@ def check_config():
 check_config()
 
 
-def is_activated(config_element):
-    return config_element["activate"] in {"true", "True"}
-
-
 def get_model(wildcards):
     if wildcards.model == "all":
         return {"full": None}
@@ -360,7 +356,7 @@ def all_input(wildcards):
         )
     )
 
-    if is_activated(config["diffsplice"]):
+    if config["diffsplice"]["activate"]:
         # diffsplice analysis
         wanted_input.extend(
             expand(

workflow/rules/datavzrd.smk

@@ -1,56 +1,6 @@
-rule render_datavzrd_config_spia:
-    input:
-        template=workflow.source_path("../resources/datavzrd/spia-template.yaml"),
-        spia_table="results/tables/pathways/{model}.pathways.tsv",
-    output:
-        "results/datavzrd/spia/{model}.yaml",
-    log:
-        "logs/yte/render-datavzrd-config-spia/{model}.log",
-    params:
-        pathway_db=config["enrichment"]["spia"]["pathway_database"],
-    template_engine:
-        "yte"
-
-
-rule render_datavzrd_config_diffexp:
-    input:
-        template=workflow.source_path("../resources/datavzrd/diffexp-template.yaml"),
-        logcount_matrix="results/tables/logcount-matrix/{model}.logcount-matrix.tsv",
-        transcripts="results/tables/diffexp/{model}.transcripts.diffexp.tsv",
-        genes_aggregated="results/tables/diffexp/{model}.genes-aggregated.diffexp.tsv",
-        genes_representative="results/tables/diffexp/{model}.genes-representative.diffexp.tsv",
-        volcano_plots="results/plots/interactive/volcano/{model}.vl.json",
-    output:
-        "results/datavzrd/diffexp/{model}.yaml",
-    params:
-        samples=get_model_samples,
-    log:
-        "logs/yte/render-datavzrd-config-diffexp/{model}.log",
-    template_engine:
-        "yte"
-
-
-rule render_datavzrd_config_go_enrichment:
-    input:
-        template=workflow.source_path(
-            "../resources/datavzrd/go-enrichment-template.yaml"
-        ),
-        enrichment="results/tables/go_terms/{model}.go_term_enrichment.gene_fdr_{gene_fdr}.go_term_fdr_{go_term_fdr}.tsv",
-        significant_terms="results/tables/go_terms/{model}.go_term_enrichment.gene_fdr_{gene_fdr}.go_term_fdr_{go_term_fdr}.sig_terms.tsv",
-        genes_representative="results/tables/diffexp/{model}.genes-representative.diffexp.tsv",
-    output:
-        "results/datavzrd/go_terms/{model}_{gene_fdr}.go_term_fdr_{go_term_fdr}.yaml",
-    params:
-        samples=get_model_samples,
-    log:
-        "logs/yte/render-datavzrd-config-go_terms/{model}_{gene_fdr}.go_term_fdr_{go_term_fdr}.log",
-    template_engine:
-        "yte"
-
-
 rule spia_datavzrd:
     input:
-        config="results/datavzrd/spia/{model}.yaml",
+        config=workflow.source_path("../resources/datavzrd/spia-template.yaml"),
         # files required for rendering the given configs
         spia_table="results/tables/pathways/{model}.pathways.tsv",
     output:
@@ -64,13 +14,17 @@ rule spia_datavzrd:
         ),
     log:
         "logs/datavzrd-report/spia-{model}/spia-{model}.log",
+    params:
+        offer_excel=lookup(within=config, dpath="report/offer_excel", default=False),
+        pathway_db=config["enrichment"]["spia"]["pathway_database"],
     wrapper:
-        "v3.5.2/utils/datavzrd"
+        "v3.12.1/utils/datavzrd"
 
 
 rule diffexp_datavzrd:
     input:
-        config="results/datavzrd/diffexp/{model}.yaml",
+        config=workflow.source_path("../resources/datavzrd/diffexp-template.yaml"),
+        # optional files required for rendering the given config
         logcount_matrix="results/tables/logcount-matrix/{model}.logcount-matrix.tsv",
         transcripts="results/tables/diffexp/{model}.transcripts.diffexp.tsv",
         genes_aggregated="results/tables/diffexp/{model}.genes-aggregated.diffexp.tsv",
@@ -85,19 +39,22 @@ rule diffexp_datavzrd:
             patterns=["index.html"],
             labels={"model": "{model}"},
         ),
-    params:
-        model=get_model,
     log:
         "logs/datavzrd-report/diffexp.{model}/diffexp.{model}.log",
+    params:
+        extra="",
+        model=get_model,
+        offer_excel=lookup(within=config, dpath="report/offer_excel", default=False),
+        samples=get_model_samples,
     wrapper:
-        "v3.5.2/utils/datavzrd"
+        "v3.12.1/utils/datavzrd"
 
 
 rule go_enrichment_datavzrd:
     input:
-        config="results/datavzrd/go_terms/{model}_{gene_fdr}.go_term_fdr_{go_term_fdr}.yaml",
+        config=workflow.source_path("../resources/datavzrd/go-enrichment-template.yaml"),
+        significant_terms="results/tables/go_terms/{model}.go_term_enrichment.gene_fdr_{gene_fdr}.go_term_fdr_{go_term_fdr}.sig_terms.tsv",
         enrichment="results/tables/go_terms/{model}.go_term_enrichment.gene_fdr_{gene_fdr}.go_term_fdr_{go_term_fdr}.tsv",
-        sig_go="results/tables/go_terms/{model}.go_term_enrichment.gene_fdr_{gene_fdr}.go_term_fdr_{go_term_fdr}.sig_terms.tsv",
     output:
         report(
             directory(
@@ -116,5 +73,8 @@ rule go_enrichment_datavzrd:
         ),
     log:
         "logs/datavzrd-report/go_enrichment-{model}/go_enrichment-{model}_{gene_fdr}.go_term_fdr_{go_term_fdr}.log",
+    params:
+        offer_excel=lookup(within=config, dpath="report/offer_excel", default=False),
+        samples=get_model_samples,
     wrapper:
-        "v3.5.2/utils/datavzrd"
+        "v3.12.1/utils/datavzrd"

workflow/rules/diffexp.smk

@@ -45,6 +45,9 @@ rule sleuth_init:
     group:
         "sleuth-init"
     threads: 6
+    resources:
+        # based on: https://github.com/pachterlab/sleuth/issues/139#issuecomment-331157007
+        mem_mb=lambda wc, threads: threads * 8000,
     script:
         "../scripts/sleuth-init.R"
 

workflow/rules/diffsplice.smk

@@ -23,6 +23,8 @@ rule init_isoform_switch:
         seq_dir=lambda _, output: os.path.dirname(output.seqs[0]),
         min_effect_size=config["diffsplice"]["min_effect_size"],
         fdr=config["diffsplice"]["fdr"],
+    resources:
+        mem_mb=lambda wc, input: 3 * input.size_mb,
     script:
         "../scripts/isoform-switch-analysis-init.R"