introducing attempts

clusterExecution.sh

@@ -4,6 +4,8 @@ module load Singularity/3.5.2
 module load Miniconda/3_snakemake
 module load Snakemake/5.10.0
 
+export SINGULARITY_CACHEDIR=/tmp/.singularity-$(id -u)
+
 while getopts "p:j:" opt
 do
    case $opt in
@@ -22,5 +24,5 @@ mkdir -p clusterLogs
 
 type snakemake >/dev/null 2>&1 || { echo >&2 "I require snakemake but it's not installed or added to your path.  Aborting..."; exit 1; }
 
-snakemake --jobs $maxNrOfConcurrentJobs --use-singularity --use-conda --reason --jobscript cluster/jobscript.sh --cluster "qsub -e clusterLogs/{rule}.{wildcards}.{jobid}.errors -o clusterLogs/{rule}.{wildcards}.{jobid}.output -A ${projectID} -l select=1:ncpus={params.cpus}:ngpus={params.gpus}:mem={params.mem} -l walltime={params.walltime}"
+snakemake --jobs $maxNrOfConcurrentJobs --use-singularity --use-conda --reason --jobscript cluster/jobscript.sh --cluster "qsub -e clusterLogs/{rule}.{wildcards}.{jobid}.errors -o clusterLogs/{rule}.{wildcards}.{jobid}.output -A ${projectID} -l select=1:ncpus={params.cpus}:ngpus={params.gpus}:mem={params.mem} -l walltime={params.walltime}" --restart-times 3 --envvars SINGULARITY_CACHEDIR
 

rules/probabilistic.smk

@@ -81,10 +81,10 @@ rule calcLikelihoods:
         deviationCutoff = (lambda wildcards,input : round(config['deviationCutoff']*extractCoverageEstimateFile(input.kmerCoverageEstimate,config))),
         itersetType = lambda wildcards: wildcards.iterset,
         #cluster exectuion
-        cpus = '8',
-        mem = '15G',
+        cpus = lambda wildcards, attempt: str(2*attempt),
+        mem = '10G',
         gpus = '0',
-        walltime = '00:30:00'
+        walltime = lambda wildcards, attempt: '0' + str(attempt) + ':00:00'
     singularity:
         'singularity_container/ckmertools_iterset.sif'
         #'docker://phspo/ckmertools:iterationset-tests'