From df0c81b008315ba2e3fc66e823f060c972b134f2 Mon Sep 17 00:00:00 2001 From: zqwerty <zhuq96@hotmail.com> Date: Fri, 22 Apr 2022 14:43:58 +0800 Subject: [PATCH] add t5nlu few-shot script, 1% multiwoz21: acc 54, f1 72.1; 10% multiwoz21: acc73.3, f1 84.6 --- convlab2/base_models/t5/create_data.py | 9 +- .../base_models/t5/nlu/merge_predict_res.py | 3 +- .../base_models/t5/nlu/run_nlu_fewshot.sh | 83 +++++++++++++++++++ 3 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 convlab2/base_models/t5/nlu/run_nlu_fewshot.sh diff --git a/convlab2/base_models/t5/create_data.py b/convlab2/base_models/t5/create_data.py index c2f3da96..b2091f52 100644 --- a/convlab2/base_models/t5/create_data.py +++ b/convlab2/base_models/t5/create_data.py @@ -138,14 +138,19 @@ if __name__ == '__main__': parser.add_argument('--speaker', '-s', type=str, choices=['user', 'system', 'all'], help='speaker(s)') parser.add_argument('--context_window_size', '-c', type=int, default=0, help='how many contextual utterances are considered') parser.add_argument('--len_tokenizer', '-l', type=str, default=None, help='name or path of tokenizer that used to get seq len') + parser.add_argument('--ratio', '-r', type=float, default=None, help='how many data is used for training and evaluation') + parser.add_argument('--dial_ids_order', '-o', type=int, default=None, help='which data order is used for experiments') args = parser.parse_args() print(args) if args.len_tokenizer: tokenizer = AutoTokenizer.from_pretrained(args.len_tokenizer) for dataset_name in tqdm(args.datasets, desc='datasets'): - dataset = load_dataset(dataset_name) + dataset = load_dataset(dataset_name, args.dial_ids_order) + if args.ratio: + dataset['train'] = dataset['train'][:round(len(dataset['train'])*args.ratio)] + dataset['validation'] = dataset['validation'][:round(len(dataset['validation'])*args.ratio)] for task_name in tqdm(args.tasks, desc='tasks', leave=False): - data_dir = os.path.join('data', task_name, dataset_name) + data_dir = os.path.join('data', task_name, (dataset_name if not args.ratio else f'{dataset_name}_{args.ratio}_order{args.dial_ids_order}')) data_by_split = eval(f"create_{task_name}_data")(dataset, data_dir, args) if args.len_tokenizer: get_max_len(data_by_split, tokenizer) diff --git a/convlab2/base_models/t5/nlu/merge_predict_res.py b/convlab2/base_models/t5/nlu/merge_predict_res.py index f3386b21..cc7c9913 100755 --- a/convlab2/base_models/t5/nlu/merge_predict_res.py +++ b/convlab2/base_models/t5/nlu/merge_predict_res.py @@ -6,7 +6,7 @@ from convlab2.base_models.t5.nlu.serialization import deserialize_dialogue_acts def merge(dataset_name, speaker, save_dir, context_window_size, predict_result): assert os.path.exists(predict_result) - dataset = load_dataset(dataset_name) + dataset = load_dataset(dataset_name, args.dial_ids_order) data = load_nlu_data(dataset, data_split='test', speaker=speaker, use_context=context_window_size>0, context_window_size=context_window_size)['test'] if save_dir is None: @@ -29,6 +29,7 @@ if __name__ == '__main__': parser.add_argument('--save_dir', type=str, help='merged data will be saved as $save_dir/predictions.json. default: on the same directory as predict_result') parser.add_argument('--context_window_size', '-c', type=int, default=0, help='how many contextual utterances are considered') parser.add_argument('--predict_result', '-p', type=str, required=True, help='path to the output file generated_predictions.json') + parser.add_argument('--dial_ids_order', '-o', type=int, default=None, help='which data order is used for experiments') args = parser.parse_args() print(args) merge(args.dataset, args.speaker, args.save_dir, args.context_window_size, args.predict_result) diff --git a/convlab2/base_models/t5/nlu/run_nlu_fewshot.sh b/convlab2/base_models/t5/nlu/run_nlu_fewshot.sh new file mode 100644 index 00000000..026e50aa --- /dev/null +++ b/convlab2/base_models/t5/nlu/run_nlu_fewshot.sh @@ -0,0 +1,83 @@ +n_gpus=1 +task_name="nlu" +dataset_name=$1 +speaker="user" +context_window_size=$2 +ratio=$3 +dial_ids_order=$4 +data_dir="data/${task_name}/${dataset_name}_${ratio}_order${dial_ids_order}/${speaker}/context_${context_window_size}" +output_dir="output/${task_name}/${dataset_name}_${ratio}_order${dial_ids_order}/${speaker}/context_${context_window_size}" +cache_dir="../cache" +logging_dir="${output_dir}/runs" +train_file="${data_dir}/train.json" +validation_file="${data_dir}/validation.json" +test_file="${data_dir}/test.json" +metric_name_or_path="nlu_metric.py" +metric_for_best_model="overall_f1" +source_column="context" +target_column="dialogue_acts_seq" +truncation_side="left" +max_source_length=512 +max_target_length=512 +model_name_or_path="t5-small" +per_device_train_batch_size=128 +per_device_eval_batch_size=64 +gradient_accumulation_steps=2 +lr=1e-3 +num_train_epochs=100 + +python ../create_data.py -t ${task_name} -d ${dataset_name} -s ${speaker} -c ${context_window_size} -r ${ratio} -o ${dial_ids_order} + +python ../run_seq2seq.py \ + --task_name ${task_name} \ + --train_file ${train_file} \ + --validation_file ${validation_file} \ + --source_column ${source_column} \ + --target_column ${target_column} \ + --max_source_length ${max_source_length} \ + --max_target_length ${max_target_length} \ + --truncation_side ${truncation_side} \ + --model_name_or_path ${model_name_or_path} \ + --do_train \ + --do_eval \ + --save_strategy epoch \ + --evaluation_strategy epoch \ + --save_total_limit 3 \ + --prediction_loss_only \ + --load_best_model_at_end \ + --cache_dir ${cache_dir} \ + --output_dir ${output_dir} \ + --logging_dir ${logging_dir} \ + --overwrite_output_dir \ + --preprocessing_num_workers 4 \ + --per_device_train_batch_size ${per_device_train_batch_size} \ + --per_device_eval_batch_size ${per_device_eval_batch_size} \ + --gradient_accumulation_steps ${gradient_accumulation_steps} \ + --learning_rate ${lr} \ + --num_train_epochs ${num_train_epochs} \ + --debug underflow_overflow \ + --adafactor \ + --gradient_checkpointing + +python ../run_seq2seq.py \ + --task_name ${task_name} \ + --test_file ${test_file} \ + --source_column ${source_column} \ + --target_column ${target_column} \ + --max_source_length ${max_source_length} \ + --max_target_length ${max_target_length} \ + --truncation_side ${truncation_side} \ + --model_name_or_path ${output_dir} \ + --do_predict \ + --predict_with_generate \ + --metric_name_or_path ${metric_name_or_path} \ + --cache_dir ${cache_dir} \ + --output_dir ${output_dir} \ + --logging_dir ${logging_dir} \ + --overwrite_output_dir \ + --preprocessing_num_workers 4 \ + --per_device_eval_batch_size ${per_device_eval_batch_size} + +python merge_predict_res.py -d ${dataset_name} -s ${speaker} -c ${context_window_size} -p ${output_dir}/generated_predictions.json -o ${dial_ids_order} + +python ../../../nlu/evaluate_unified_datasets.py -p ${output_dir}/predictions.json -- GitLab