dump dst eval results

convlab2/dst/dstc9/eval_file.py

@@ -5,7 +5,7 @@
 import json
 import os
 
-from convlab2.dst.dstc9.utils import prepare_data, extract_gt, eval_states, get_subdir
+from convlab2.dst.dstc9.utils import prepare_data, extract_gt, eval_states, get_subdir, dump_result
 
 
 def evaluate(model_dir, subtask, gt):
@@ -18,7 +18,7 @@ def evaluate(model_dir, subtask, gt):
         results[i] = eval_states(gt, pred, subtask)
 
     print(json.dumps(results, indent=4, ensure_ascii=False))
-    json.dump(results, open(os.path.join(model_dir, 'file-results.json'), 'w'), indent=4, ensure_ascii=False)
+    dump_result(model_dir, 'file-results.json', results)
 
 
 # generate submission examples

convlab2/dst/dstc9/eval_model.py

@@ -7,7 +7,7 @@ import json
 import importlib
 
 from convlab2.dst import DST
-from convlab2.dst.dstc9.utils import prepare_data, eval_states
+from convlab2.dst.dstc9.utils import prepare_data, eval_states, dump_result
 
 
 def evaluate(model_dir, subtask, test_data, gt):
@@ -23,7 +23,7 @@ def evaluate(model_dir, subtask, test_data, gt):
         pred[dialog_id] = [model.update_turn(sys_utt, user_utt) for sys_utt, user_utt, gt_turn in turns]
     result = eval_states(gt, pred, subtask)
     print(json.dumps(result, indent=4))
-    json.dump(result, open(os.path.join(model_dir, 'model-result.json'), 'w'), indent=4, ensure_ascii=False)
+    dump_result(model_dir, 'model-result.json', result)
 
 
 if __name__ == '__main__':

convlab2/dst/dstc9/utils.py

@@ -144,3 +144,9 @@ def eval_states(gt, pred, subtask):
             'f1': f1,
         }
     }
+
+
+def dump_result(model_dir, filename, result):
+    output_dir = os.path.join('../results', model_dir)
+    os.makedirs(output_dir, exist_ok=True)
+    json.dump(result, open(os.path.join(output_dir, filename), 'w'), indent=4, ensure_ascii=False)

data/.gitignore

+*/dstc9*.json.zip