Fix logs

scripts/MPRA_Snakemake_Pipeline/config/config.yaml

@@ -0,0 +1,4 @@
+input_directory: "/your/input/directory"
+output_directory: "/your/output/directory"
+reference_data: "/directory/with/reference/data"
+filepaths_df: "/csv/with/your/filepaths.csv"

scripts/MPRA_Snakemake_Pipeline/config/filepaths.csv

@@ -0,0 +1,3 @@
+ID,Type
+SRR22253236down,o
+SRR22253239down,m

scripts/MPRA_Snakemake_Pipeline/config/slurm_scg/config.yaml

@@ -0,0 +1,35 @@
+# helpful to give path e.g. if you write to scratch/not in current dir
+cluster:
+  mkdir -p ../logs/{rule} &&
+  sbatch
+    --partition={resources.partition}
+    --account={resources.account}
+    --time={resources.time}
+    --job-name={rule}.{wildcards}
+    --output=../logs/{rule}/%j.{wildcards}.out
+    --error=../logs/{rule}/%j.{wildcards}.err
+    --mem-per-cpu={resources.mem}
+    --nodes={resources.nodes}
+    --cpus-per-task={resources.threads}
+    --parsable
+default-resources:
+  - partition=batch
+  - account=smontgom
+  - time="02:00:00"
+  - mem="64G"
+  - nodes=1
+  - threads=1
+latency-wait: 120
+# restart-times: 3
+jobs: 50
+keep-going: True
+rerun-incomplete: True
+printshellcmds: True
+scheduler: greedy
+use-conda: True
+# Singularity args (binds to oak)
+#use-singularity: True
+#singularity-args: "-B /oak:/oak"
+#cluster-status: "/oak/stanford/groups/smontgom/maurertm/MPRA/MPRA_snakemake_pipeline/config/slurm_scg/status-sacct.sh" 
+max-status-checks-per-second: 10
+#cluster-cancel: scancel

scripts/MPRA_Snakemake_Pipeline/config/slurm_scg/status-sacct.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Check status of slurm jobs
+jobid="$1"
+if [[ "$jobid" == Submitted ]]
+then
+  echo smk-simple-slurm: Invalid job ID: "$jobid" >&2
+  echo smk-simple-slurm: Did you remember to add the flag --parsable to your sbatch call? >&2
+  exit 1
+fi
+output=`sacct -j "$jobid" --format State --noheader | head -n 1 | awk '{print $1}'`
+if [[ $output =~ ^(COMPLETED).* ]]
+then
+  echo success
+elif [[ $output =~ ^(RUNNING|PENDING|COMPLETING|CONFIGURING|SUSPENDED).* ]]
+then
+  echo running
+else
+  echo failed
+fi

scripts/MPRA_Snakemake_Pipeline/config/slurm_sherlock/config.yaml

@@ -0,0 +1,36 @@
+cluster:
+  mkdir -p ../logs/{rule} &&
+  sbatch
+    --partition={resources.partition}
+    --time={resources.time}
+    --job-name={rule}.{wildcards}
+    --output=../logs/{rule}/%j.out
+    --error=../logs/{rule}/%j.err
+    --parsable
+    --mem={resources.mem}
+    --gpus-per-task={resources.gpus}
+    --nodes={resources.nodes}
+    --ntasks-per-node={resources.tasks}
+    --cpus-per-task={resources.threads}
+default-resources:
+  - partition=normal,owners
+  - time="00:10:00"
+  - mem=4000
+  - nodes=1
+  - threads=1
+  - tasks=1
+  - gpus=0
+latency-wait: 120
+# restart-times: 3
+jobs: 50
+keep-going: True
+rerun-incomplete: True
+printshellcmds: True
+scheduler: greedy
+use-conda: True
+# Singularity args (binds to oak)
+#use-singularity: True
+#singularity-args: "-B /oak:/oak"
+cluster-status: "config/slurm_sherlock/status-sacct.sh"
+max-status-checks-per-second: 10
+cluster-cancel: scancel

scripts/MPRA_Snakemake_Pipeline/config/slurm_sherlock/status-sacct.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Check status of slurm jobs
+jobid="$1"
+if [[ "$jobid" == Submitted ]]
+then
+  echo smk-simple-slurm: Invalid job ID: "$jobid" >&2
+  echo smk-simple-slurm: Did you remember to add the flag --parsable to your sbatch call? >&2
+  exit 1
+fi
+output=`sacct -j "$jobid" --format State --noheader | head -n 1 | awk '{print $1}'`
+if [[ $output =~ ^(COMPLETED).* ]]
+then
+  echo success
+elif [[ $output =~ ^(RUNNING|PENDING|COMPLETING|CONFIGURING|SUSPENDED).* ]]
+then
+  echo running
+else
+  echo failed
+fi

scripts/MPRA_Snakemake_Pipeline/workflow/Snakefile

@@ -0,0 +1,34 @@
+import os 
+import sys
+import pandas as pd
+
+###################################---Step 1: Define all Config Variables---######################################################
+input_directory=config["input_directory"]
+output_directory=config["output_directory"]
+reference_data=config["reference_data"]
+filepaths_df=config["filepaths_df"]
+
+#########################################---Step 1: Define Wildcard---############################################################
+filepaths_dataframe = pd.read_csv(filepaths_df)
+
+IDs = list(filepaths_dataframe['ID'])
+print('Running on samples:{}'.format(IDs))
+
+########################################---Step 1: Define Dictionary---###########################################################
+#this dictionary maps the sampleIDs in filepaths_dictionary to their corresponding barcode type
+#these values will be passed on to get_barcodes.smk and clean_umis.smk to select further parameters
+filepaths_dictionary = filepaths_dataframe.set_index("ID")['Type'].to_dict()
+barcode_types = [*filepaths_dictionary.values()]
+
+###########################################---Step 1: Define Rules--##############################################################
+include: 'rules/cellranger.smk'
+include: 'rules/get_barcodes.smk'
+include: 'rules/clean_umis.smk'
+
+rule all:
+    input:
+        expand(config["input_directory"] + "/" + "{ID}" + "/" + "{ID}" + "_S1_L001_R1_001.fastq", ID = IDs),
+        expand(config["output_directory"] + "/" + "{ID}" + "/outs/possorted_genome_bam.bam", ID = IDs),
+        expand(config["output_directory"] + "/" + "{ID}" + "/outs/" + "{ID}" + "_get_bc_v3.txt", ID = IDs),
+        expand(config["output_directory"] + "/" + "{ID}" + "/outs/" + "{ID}" + "_get_bc_v3_no_G.txt", ID = IDs),
+        expand(config["output_directory"] + "/" + "{ID}" + "/outs/" + "{ID}" + "_get_bc_v3_no_G_cleaned_UMI.txt", ID = IDs)