workflow bugs

Snakefile

@@ -8,16 +8,21 @@ configfile: "config.yaml"
 validate(config, "schemas/config.schema.yaml")
 
 
-#Generate Input/Output Files from specified folder
+def get_input_ids():
+	inputIDs, = glob_wildcards('data/input/'+wildcards.dataset+'/{id}'+config["datasets"][wildcards.dataset]['input_read_1_ending'])
+	return inputIDs
 
+#Generate Input/Output Files from specified folder
 kmer_lengths = config['kmers']
 dataset_inputIDs = {}
+for d in config['datasets'].keys():
+	inputIDs = get_input_ids()
+	dataset_inputIDs[d] = [str(x) for x in inputIDs]
 
 #kmer_lengths = [24]
 def get_general_inputs(wildcards):
 	run_dir = wildcards.dataset
-	inputIDs, = glob_wildcards('data/input/'+wildcards.dataset+'/{id}'+config["datasets"][wildcards.dataset]['input_read_1_ending'])
-	dataset_inputIDs[wildcards.dataset] = inputIDs
+	inputIDs = get_input_ids()
 
 	possible_params = {
 	'assembly_model': expand('data/output/'+run_dir+'/{id}/exactMatches.tsv',id=inputIDs),
@@ -57,6 +62,12 @@ rule run_dataset:
 		summarys = expand('data/auxiliary/{dataset}/{iterset}_summary.md', iterset=use_itersets(), allow_missing=True)
 	output:
 		summary = 'data/output/{dataset}_summary.md'
+	params:
+		# cluster execution
+		cpus = '1',
+		mem = '1G',
+		gpus = '0',
+		walltime = '00:01:00'
 	#  TODO create summary
 	shell:
 		'touch {output.out}'
@@ -66,6 +77,12 @@ rule run_iterset:
 		get_iterset_inputs
 	output:
 		out = 'data/auxiliary/{dataset}/{iterset}_summary.md'
+	params:
+		# cluster execution
+		cpus = '1',
+		mem = '1G',
+		gpus = '0',
+		walltime = '00:01:00'
 	# TODO create summary
 	shell:
 		'touch {output.out}'

clusterExecution.sh

@@ -24,5 +24,5 @@ mkdir -p clusterLogs
 
 type snakemake >/dev/null 2>&1 || { echo >&2 "I require snakemake but it's not installed or added to your path.  Aborting..."; exit 1; }
 
-snakemake --jobs $maxNrOfConcurrentJobs --use-singularity --use-conda --reason --jobscript cluster/jobscript.sh --cluster "qsub -e clusterLogs/{rule}.{wildcards}.{jobid}.errors -o clusterLogs/{rule}.{wildcards}.{jobid}.output -A ${projectID} -l select=1:ncpus={params.cpus}:ngpus={params.gpus}:mem={params.mem} -l walltime={params.walltime}" --restart-times 3 --envvars SINGULARITY_CACHEDIR
+snakemake --jobs $maxNrOfConcurrentJobs --use-singularity --use-conda --reason --jobscript cluster/jobscript.sh --cluster "qsub -e clusterLogs/{rule}.{wildcards}.{jobid}.errors -o clusterLogs/{rule}.{wildcards}.{jobid}.output -A ${projectID} -l select=1:ncpus={params.cpus}:ngpus={params.gpus}:mem={params.mem} -l walltime={params.walltime}" --restart-times 3
 

rules/probabilistic.smk

@@ -77,14 +77,14 @@ rule calcLikelihoods:
     benchmark:
         'benchmarks/{dataset}/{iterset}/probabilistic/kmers/{kmer}/{id}/calcLikelihoodsCoverageBasedModel.txt'
     params:
-        e = (lambda wildcards,input : extractTsvValue(input.kmerError,0)),
-        deviationCutoff = (lambda wildcards,input : round(config['deviationCutoff']*extractCoverageEstimateFile(input.kmerCoverageEstimate,config))),
+        e = lambda wildcards,input : extractTsvValue(input.kmerError,0),
+        deviationCutoff = lambda wildcards,input : round(config['deviationCutoff']*extractCoverageEstimateFile(input.kmerCoverageEstimate,config)),
         itersetType = lambda wildcards: wildcards.iterset,
         #cluster exectuion
-        cpus = lambda wildcards, attempt: str(2*attempt),
+        cpus = '4',
         mem = '10G',
         gpus = '0',
-        walltime = lambda wildcards, attempt: '0' + str(attempt) + ':00:00'
+        walltime = lambda wildcards, attempt: '01:00:00'
     singularity:
         'singularity_container/ckmertools_iterset.sif'
         #'docker://phspo/ckmertools:iterationset-tests'