Skip to content
Snippets Groups Projects
Select Git revision
  • a26a42b2b361f5234fd43bc9a6fe0cca9951a74b
  • master default protected
  • emoUS
  • add_default_vectorizer_and_pretrained_loading
  • clean_code
  • readme
  • issue127
  • generalized_action_dicts
  • ppo_num_dialogues
  • crossowoz_ddpt
  • issue_114
  • robust_masking_feature
  • scgpt_exp
  • e2e-soloist
  • convlab_exp
  • change_system_act_in_env
  • pre-training
  • nlg-scgpt
  • remapping_actions
  • soloist
20 results

merge_predict_res.py

Blame
  • Code owners
    Assign users and groups as approvers for specific file changes. Learn more.
    merge_predict_res.py 2.05 KiB
    import json
    import os
    from convlab2.util import load_dataset, load_dst_data
    from convlab2.base_models.t5.dst.serialization import deserialize_dialogue_state
    
    
    def merge(dataset_name, speaker, save_dir, context_window_size, predict_result):
        assert os.path.exists(predict_result)
        dataset = load_dataset(dataset_name, args.dial_ids_order)
        data = load_dst_data(dataset, data_split='test', speaker=speaker, use_context=context_window_size>0, context_window_size=context_window_size)['test']
        
        if save_dir is None:
            save_dir = os.path.dirname(predict_result)
        else:
            os.makedirs(save_dir, exist_ok=True)
        predict_result = [deserialize_dialogue_state(json.loads(x)['predictions'].strip()) for x in open(predict_result)]
    
        for sample, prediction in zip(data, predict_result):
            sample['predictions'] = {'state': prediction}
    
        json.dump(data, open(os.path.join(save_dir, 'predictions.json'), 'w', encoding='utf-8'), indent=2, ensure_ascii=False)
    
    
    if __name__ == '__main__':
        from argparse import ArgumentParser
        parser = ArgumentParser(description="merge predict results with original data for unified NLU evaluation")
        parser.add_argument('--dataset', '-d', metavar='dataset_name', type=str, help='name of the unified dataset')
        parser.add_argument('--speaker', '-s', type=str, choices=['user', 'system', 'all'], help='speaker(s) of utterances')
        parser.add_argument('--save_dir', type=str, help='merged data will be saved as $save_dir/predictions.json. default: on the same directory as predict_result')
        parser.add_argument('--context_window_size', '-c', type=int, default=0, help='how many contextual utterances are considered')
        parser.add_argument('--predict_result', '-p', type=str, required=True, help='path to the output file generated_predictions.json')
        parser.add_argument('--dial_ids_order', '-o', type=int, default=None, help='which data order is used for experiments')
        args = parser.parse_args()
        print(args)
        merge(args.dataset, args.speaker, args.save_dir, args.context_window_size, args.predict_result)