hilbert initial options

.gitignore

@@ -7,4 +7,5 @@ config.yaml
 __pycache__/
 benchmarks/
 logs/
-report/
+report/
\ No newline at end of file
+clusterLogs/
\ No newline at end of file

clusterExecution.sh

@@ -22,5 +22,5 @@ mkdir -p clusterLogs
 
 type snakemake >/dev/null 2>&1 || { echo >&2 "I require snakemake but it's not installed or added to your path.  Aborting..."; exit 1; }
 
-snakemake --jobs $maxNrOfConcurrentJobs --use-singularity --use-conda --reason --jobscript cluster/jobscript.sh --cluster "qsub -e clusterLogs/{rule}.{wildcards}.{jobid}.errors -o clusterLogs/{rule}.{wildcards}.{jobid}.output -A ${projectID} -l select=1:ncpus={params.cpus}:ngpus={params.gpus}:mem={params.mem} -l walltime={params.walltime}"
+snakemake --jobs $maxNrOfConcurrentJobs --use-singularity --use-conda --reason --jobscript cluster/jobscript.sh --cluster "qsub -e clusterLogs/{rule}.{wildcards}.{jobid}.errors -o clusterLogs/{rule}.{wildcards}.{jobid}.output -A ${projectID} -l select=1:ncpus={params.cpus}:ngpus={params.gpus}:mem={params.mem} -l walltime={params.walltime}" --latency-wait 900 --keep-going
 

envs/biopythonworkbench.yaml

 name: BioPythonWorkbench
 
 channels:
-  - conda-forge
-  - bioconda
-  - defaults
+  - nodefaults
+  - http://conda.repo.test.hhu.de/conda-forge
+  - http://conda.repo.test.hhu.de/bioconda
+  - http://conda.repo.test.hhu.de/main
 
 dependencies:
   - biopython=1.71
   - python=3.7
   - regex
-  - openssl = 1.1.1h
-  - samtools = 1.11
-  - seaborn = 0.11.0
-  - scipy = 1.5.2
-  - pysam = 0.16.0.1
-  - mpmath = 1.1.0
+  - openssl=1.1.1h
+  - samtools=1.11
+  - seaborn=0.11.0
+  - scipy=1.5.2
+  - pysam=0.16.0.1
+  - mpmath=1.1.0
   - matplotlib-venn=0.11.5

envs/bwa.yaml

 name: bwa
 
 channels:
-  - defaults
-  - bioconda
+  - nodefaults
+  - http://conda.repo.test.hhu.de/bioconda
+  - http://conda.repo.test.hhu.de/main
 
 dependencies:
   - bwa=0.7.17

envs/fastp.yaml

 name: FastP
 
 channels:
-  - bioconda
-  - defaults
+  - nodefaults
+  - http://conda.repo.test.hhu.de/bioconda
+  - http://conda.repo.test.hhu.de/main
 
 dependencies:
   - fastp=0.20.0

rules/coverageBased.snk

@@ -12,7 +12,7 @@ rule estimateKmerCoverage:
         #cluster execution
         cpus = '1', #TODO: This could in theory be sped up significantly using a shared cache and multithreading
         gpus = '0',
-        mem = '64G',
+        mem = '10G',
         walltime = '00:45:00'
     log:
         'logs/' + config['input_folder'] + '/kmers/{kmer}/{id}/estimateKmerCoverage.log'

rules/kmerApproach.snk

@@ -82,7 +82,7 @@ rule bwa:
     params:
         #cluster execution
         cpus = '1',
-        mem = '32G',
+        mem = '4G',
         gpus = '0',
         walltime = '00:30:00'
     singularity:
@@ -343,7 +343,12 @@ rule createSpaTypeVennDiagram:
         venntopsix = 'data/output/'+config['input_folder']+'/kmers/{kmer}/{id}/spaTypesTopSixVennDia.svg',
         vennrandomsix = 'data/output/'+config['input_folder']+'/kmers/{kmer}/{id}/spaTypesRandomSixVennDia.svg'
     params:
-        gtt = lambda wildcards : getGroundTruthType(wildcards.id)
+        gtt = lambda wildcards : getGroundTruthType(wildcards.id),
+        #cluster execution
+        cpus = '1',
+        mem = '1G',
+        gpus = '0',
+        walltime = '00:05:00'
     log:
         'logs/'+config['input_folder']+'/probabilistic/kmers/{kmer}/{id}/spaTypeVennDia.log'
     conda:
@@ -375,7 +380,12 @@ rule calcKmerErrorRates:
     output:
         error = 'data/auxiliary/'+config['input_folder']+'/kmers/{kmer}/{id}/error_estimate.txt'
     params:
-        k = lambda wildcards: wildcards.kmer
+        k = lambda wildcards: wildcards.kmer,
+        #cluster execution
+        cpus = '1',
+        mem = '2G',
+        gpus = '0',
+        walltime = '00:30:00'
     log:
         'logs/'+config['input_folder']+'/kmers/{kmer}/{id}/calcKmerErrorRates.log'
     conda:

rules/probabilistic.snk

@@ -79,8 +79,8 @@ rule calcLikelihoods:
         deviationCutoff = (lambda wildcards,input : round(config['deviationCutoff']*extractCoverageEstimateFile(input.kmerCoverageEstimate,config))),
         itersetType = config['itersetType'],
         #cluster exectuion
-        cpus = '1',
-        mem = '4G',
+        cpus = '8',
+        mem = '10G',
         gpus = '0',
         walltime = '01:30:00'
     singularity:
@@ -155,7 +155,12 @@ rule calcKmerStats:
         stats = 'data/auxiliary/'+config['input_folder']+'/kmers/{kmer}/{id}/stats.tsv'
     params:
         id = lambda wildcards: wildcards.id,
-        k = lambda wildcards: wildcards.kmer
+        k = lambda wildcards: wildcards.kmer,
+        # cluster execution
+        cpus = '1',
+        mem = '5G',
+        gpus = '0',
+        walltime = '00:30:00'
     conda:
         '../envs/biopythonworkbench.yaml'
     log: