Merge pull request #43 from ConvLab/nlu

t5 nlu benchmark

convlab2/base_models/t5/create_data.py

@@ -43,15 +43,18 @@ def create_nlu_data(dataset, data_dir, args):
         return ';'.join(da_seqs)
 
     def deserialize_dialogue_acts(das_seq):
-        pattern = re.compile(r'\[(.*?)\]')
-        da_seqs = das_seq.split('];[')
         dialogue_acts = {'binary': [], 'categorical': [], 'non-categorical': []}
+        if len(das_seq) == 0:
+            return dialogue_acts
+        da_seqs = das_seq.split('];[')
         for i, da_seq in enumerate(da_seqs):
-            if i > 0:
-                da_seq = '[' + da_seq
-            if i < len(da_seq) - 1:
-                da_seq = da_seq + ']'
-            da = pattern.findall(da_seq)
+            if i == 0:
+                assert da_seq[0] == '['
+                da_seq = da_seq[1:]
+            if i == len(da_seqs) - 1:
+                assert da_seq[-1] == ']'
+                da_seq = da_seq[:-1]
+            da = da_seq.split('][')
             if len(da) == 0:
                 continue
             da_type = da[0]

convlab2/base_models/t5/nlu/nlu_metric.py

@@ -71,15 +71,20 @@ class NLUMetrics(datasets.Metric):
         )
 
     def deserialize_dialogue_acts(self, das_seq):
-        pattern = re.compile(r'\[(.*?)\]')
-        da_seqs = das_seq.split('];[')
         dialogue_acts = {'binary': [], 'categorical': [], 'non-categorical': []}
+        if len(das_seq) == 0:
+            return dialogue_acts
+        da_seqs = das_seq.split('];[')
         for i, da_seq in enumerate(da_seqs):
-            if i > 0:
-                da_seq = '[' + da_seq
-            if i < len(da_seq) - 1:
-                da_seq = da_seq + ']'
-            da = pattern.findall(da_seq)
+            if len(da_seq) == 0:
+                continue
+            if i == 0:
+                if da_seq[0] == '[':
+                    da_seq = da_seq[1:]
+            if i == len(da_seqs) - 1:
+                if da_seq[-1] == ']':
+                    da_seq = da_seq[:-1]
+            da = da_seq.split('][')
             if len(da) == 0:
                 continue
             da_type = da[0]

convlab2/base_models/t5/nlu/run_nlu.sh → convlab2/base_models/t5/nlu/run_multiwoz21_user.sh

-n_gpus=2
+n_gpus=1
 task_name="nlu"
 dataset_name="multiwoz21"
 speaker="user"
@@ -17,8 +17,8 @@ source_column="context"
 target_column="dialogue_acts_seq"
 model_name_or_path="t5-small"
 per_device_train_batch_size=128
-per_device_eval_batch_size=256
-gradient_accumulation_steps=1
+per_device_eval_batch_size=64
+gradient_accumulation_steps=2
 lr=1e-3
 num_train_epochs=10
 
@@ -28,21 +28,13 @@ python -m torch.distributed.launch \
     --nproc_per_node ${n_gpus} ../run_seq2seq.py \
     --task_name ${task_name} \
     --train_file ${train_file} \
-    --validation_file ${validation_file} \
-    --test_file ${test_file} \
     --source_column ${source_column} \
     --target_column ${target_column} \
     --source_prefix ${source_prefix} \
     --model_name_or_path ${model_name_or_path} \
     --do_train \
-    --do_eval \
-    --do_predict \
     --save_strategy epoch \
-    --evaluation_strategy epoch \
-    --load_best_model_at_end \
-    --predict_with_generate \
-    --metric_name_or_path ${metric_name_or_path} \
-    --metric_for_best_model ${metric_for_best_model} \
+    --prediction_loss_only \
     --cache_dir ${cache_dir} \
     --output_dir ${output_dir} \
     --logging_dir ${logging_dir} \
@@ -56,3 +48,21 @@ python -m torch.distributed.launch \
     --debug underflow_overflow \
     --adafactor \
     --gradient_checkpointing
+
+python -m torch.distributed.launch \
+    --nproc_per_node ${n_gpus} ../run_seq2seq.py \
+    --task_name ${task_name} \
+    --test_file ${test_file} \
+    --source_column ${source_column} \
+    --target_column ${target_column} \
+    --source_prefix ${source_prefix} \
+    --model_name_or_path ${output_dir} \
+    --do_predict \
+    --predict_with_generate \
+    --metric_name_or_path ${metric_name_or_path} \
+    --cache_dir ${cache_dir} \
+    --output_dir ${output_dir} \
+    --logging_dir ${logging_dir} \
+    --overwrite_output_dir \
+    --preprocessing_num_workers 4 \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \

convlab2/base_models/t5/nlu/run_multiwoz21_user_context3.sh

+n_gpus=1
+task_name="nlu"
+dataset_name="multiwoz21"
+speaker="user"
+context_window_size=3
+data_dir="data/${task_name}/${dataset_name}/${speaker}/context_${context_window_size}"
+output_dir="output/${task_name}/${dataset_name}/${speaker}/context_${context_window_size}"
+cache_dir="../cache"
+logging_dir="${output_dir}/runs"
+train_file="${data_dir}/train.json"
+validation_file="${data_dir}/validation.json"
+test_file="${data_dir}/test.json"
+metric_name_or_path="nlu_metric.py"
+metric_for_best_model="overall_f1"
+source_prefix="${data_dir}/source_prefix.txt"
+source_column="context"
+target_column="dialogue_acts_seq"
+model_name_or_path="t5-small"
+per_device_train_batch_size=128
+per_device_eval_batch_size=64
+gradient_accumulation_steps=2
+lr=1e-3
+num_train_epochs=10
+
+python ../create_data.py --tasks ${task_name} --datasets ${dataset_name} --speaker ${speaker} --context_window_size ${context_window_size}
+
+python -m torch.distributed.launch \
+    --nproc_per_node ${n_gpus} ../run_seq2seq.py \
+    --task_name ${task_name} \
+    --train_file ${train_file} \
+    --source_column ${source_column} \
+    --target_column ${target_column} \
+    --source_prefix ${source_prefix} \
+    --model_name_or_path ${model_name_or_path} \
+    --do_train \
+    --save_strategy epoch \
+    --prediction_loss_only \
+    --cache_dir ${cache_dir} \
+    --output_dir ${output_dir} \
+    --logging_dir ${logging_dir} \
+    --overwrite_output_dir \
+    --preprocessing_num_workers 4 \
+    --per_device_train_batch_size ${per_device_train_batch_size} \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \
+    --gradient_accumulation_steps ${gradient_accumulation_steps} \
+    --learning_rate ${lr} \
+    --num_train_epochs ${num_train_epochs} \
+    --debug underflow_overflow \
+    --adafactor \
+    --gradient_checkpointing
+
+python -m torch.distributed.launch \
+    --nproc_per_node ${n_gpus} ../run_seq2seq.py \
+    --task_name ${task_name} \
+    --test_file ${test_file} \
+    --source_column ${source_column} \
+    --target_column ${target_column} \
+    --source_prefix ${source_prefix} \
+    --model_name_or_path ${output_dir} \
+    --do_predict \
+    --predict_with_generate \
+    --metric_name_or_path ${metric_name_or_path} \
+    --cache_dir ${cache_dir} \
+    --output_dir ${output_dir} \
+    --logging_dir ${logging_dir} \
+    --overwrite_output_dir \
+    --preprocessing_num_workers 4 \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \

convlab2/base_models/t5/nlu/run_tm1_user.sh

+n_gpus=1
+task_name="nlu"
+dataset_name="tm1"
+speaker="user"
+context_window_size=0
+data_dir="data/${task_name}/${dataset_name}/${speaker}/context_${context_window_size}"
+output_dir="output/${task_name}/${dataset_name}/${speaker}/context_${context_window_size}"
+cache_dir="../cache"
+logging_dir="${output_dir}/runs"
+train_file="${data_dir}/train.json"
+validation_file="${data_dir}/validation.json"
+test_file="${data_dir}/test.json"
+metric_name_or_path="nlu_metric.py"
+metric_for_best_model="overall_f1"
+source_prefix="${data_dir}/source_prefix.txt"
+source_column="context"
+target_column="dialogue_acts_seq"
+model_name_or_path="t5-small"
+per_device_train_batch_size=128
+per_device_eval_batch_size=64
+gradient_accumulation_steps=2
+lr=1e-3
+num_train_epochs=10
+
+python ../create_data.py --tasks ${task_name} --datasets ${dataset_name} --speaker ${speaker} --context_window_size ${context_window_size}
+
+python -m torch.distributed.launch \
+    --nproc_per_node ${n_gpus} ../run_seq2seq.py \
+    --task_name ${task_name} \
+    --train_file ${train_file} \
+    --source_column ${source_column} \
+    --target_column ${target_column} \
+    --source_prefix ${source_prefix} \
+    --model_name_or_path ${model_name_or_path} \
+    --do_train \
+    --save_strategy epoch \
+    --prediction_loss_only \
+    --cache_dir ${cache_dir} \
+    --output_dir ${output_dir} \
+    --logging_dir ${logging_dir} \
+    --overwrite_output_dir \
+    --preprocessing_num_workers 4 \
+    --per_device_train_batch_size ${per_device_train_batch_size} \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \
+    --gradient_accumulation_steps ${gradient_accumulation_steps} \
+    --learning_rate ${lr} \
+    --num_train_epochs ${num_train_epochs} \
+    --debug underflow_overflow \
+    --adafactor \
+    --gradient_checkpointing
+
+python -m torch.distributed.launch \
+    --nproc_per_node ${n_gpus} ../run_seq2seq.py \
+    --task_name ${task_name} \
+    --test_file ${test_file} \
+    --source_column ${source_column} \
+    --target_column ${target_column} \
+    --source_prefix ${source_prefix} \
+    --model_name_or_path ${output_dir} \
+    --do_predict \
+    --predict_with_generate \
+    --metric_name_or_path ${metric_name_or_path} \
+    --cache_dir ${cache_dir} \
+    --output_dir ${output_dir} \
+    --logging_dir ${logging_dir} \
+    --overwrite_output_dir \
+    --preprocessing_num_workers 4 \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \

convlab2/base_models/t5/nlu/run_tm1_user_context3.sh

+n_gpus=1
+task_name="nlu"
+dataset_name="tm1"
+speaker="user"
+context_window_size=3
+data_dir="data/${task_name}/${dataset_name}/${speaker}/context_${context_window_size}"
+output_dir="output/${task_name}/${dataset_name}/${speaker}/context_${context_window_size}"
+cache_dir="../cache"
+logging_dir="${output_dir}/runs"
+train_file="${data_dir}/train.json"
+validation_file="${data_dir}/validation.json"
+test_file="${data_dir}/test.json"
+metric_name_or_path="nlu_metric.py"
+metric_for_best_model="overall_f1"
+source_prefix="${data_dir}/source_prefix.txt"
+source_column="context"
+target_column="dialogue_acts_seq"
+model_name_or_path="t5-small"
+per_device_train_batch_size=64
+per_device_eval_batch_size=64
+gradient_accumulation_steps=4
+lr=1e-3
+num_train_epochs=10
+
+python ../create_data.py --tasks ${task_name} --datasets ${dataset_name} --speaker ${speaker} --context_window_size ${context_window_size}
+
+python ../run_seq2seq.py \
+    --task_name ${task_name} \
+    --train_file ${train_file} \
+    --source_column ${source_column} \
+    --target_column ${target_column} \
+    --source_prefix ${source_prefix} \
+    --model_name_or_path ${model_name_or_path} \
+    --do_train \
+    --save_strategy epoch \
+    --prediction_loss_only \
+    --cache_dir ${cache_dir} \
+    --output_dir ${output_dir} \
+    --logging_dir ${logging_dir} \
+    --overwrite_output_dir \
+    --preprocessing_num_workers 4 \
+    --per_device_train_batch_size ${per_device_train_batch_size} \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \
+    --gradient_accumulation_steps ${gradient_accumulation_steps} \
+    --learning_rate ${lr} \
+    --num_train_epochs ${num_train_epochs} \
+    --debug underflow_overflow \
+    --adafactor \
+    --gradient_checkpointing
+
+python ../run_seq2seq.py \
+    --task_name ${task_name} \
+    --test_file ${test_file} \
+    --source_column ${source_column} \
+    --target_column ${target_column} \
+    --source_prefix ${source_prefix} \
+    --model_name_or_path ${output_dir} \
+    --do_predict \
+    --predict_with_generate \
+    --metric_name_or_path ${metric_name_or_path} \
+    --cache_dir ${cache_dir} \
+    --output_dir ${output_dir} \
+    --logging_dir ${logging_dir} \
+    --overwrite_output_dir \
+    --preprocessing_num_workers 4 \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \

convlab2/base_models/t5/nlu/run_tm2_user.sh

+n_gpus=1
+task_name="nlu"
+dataset_name="tm2"
+speaker="user"
+context_window_size=0
+data_dir="data/${task_name}/${dataset_name}/${speaker}/context_${context_window_size}"
+output_dir="output/${task_name}/${dataset_name}/${speaker}/context_${context_window_size}"
+cache_dir="../cache"
+logging_dir="${output_dir}/runs"
+train_file="${data_dir}/train.json"
+validation_file="${data_dir}/validation.json"
+test_file="${data_dir}/test.json"
+metric_name_or_path="nlu_metric.py"
+metric_for_best_model="overall_f1"
+source_prefix="${data_dir}/source_prefix.txt"
+source_column="context"
+target_column="dialogue_acts_seq"
+model_name_or_path="t5-small"
+per_device_train_batch_size=128
+per_device_eval_batch_size=64
+gradient_accumulation_steps=2
+lr=1e-3
+num_train_epochs=10
+
+python ../create_data.py --tasks ${task_name} --datasets ${dataset_name} --speaker ${speaker} --context_window_size ${context_window_size}
+
+python -m torch.distributed.launch \
+    --nproc_per_node ${n_gpus} ../run_seq2seq.py \
+    --task_name ${task_name} \
+    --train_file ${train_file} \
+    --source_column ${source_column} \
+    --target_column ${target_column} \
+    --source_prefix ${source_prefix} \
+    --model_name_or_path ${model_name_or_path} \
+    --do_train \
+    --save_strategy epoch \
+    --prediction_loss_only \
+    --cache_dir ${cache_dir} \
+    --output_dir ${output_dir} \
+    --logging_dir ${logging_dir} \
+    --overwrite_output_dir \
+    --preprocessing_num_workers 4 \
+    --per_device_train_batch_size ${per_device_train_batch_size} \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \
+    --gradient_accumulation_steps ${gradient_accumulation_steps} \
+    --learning_rate ${lr} \
+    --num_train_epochs ${num_train_epochs} \
+    --debug underflow_overflow \
+    --adafactor \
+    --gradient_checkpointing
+
+python -m torch.distributed.launch \
+    --nproc_per_node ${n_gpus} ../run_seq2seq.py \
+    --task_name ${task_name} \
+    --test_file ${test_file} \
+    --source_column ${source_column} \
+    --target_column ${target_column} \
+    --source_prefix ${source_prefix} \
+    --model_name_or_path ${output_dir} \
+    --do_predict \
+    --predict_with_generate \
+    --metric_name_or_path ${metric_name_or_path} \
+    --cache_dir ${cache_dir} \
+    --output_dir ${output_dir} \
+    --logging_dir ${logging_dir} \
+    --overwrite_output_dir \
+    --preprocessing_num_workers 4 \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \

convlab2/base_models/t5/nlu/run_tm2_user_context3.sh

+n_gpus=1
+task_name="nlu"
+dataset_name="tm2"
+speaker="user"
+context_window_size=3
+data_dir="data/${task_name}/${dataset_name}/${speaker}/context_${context_window_size}"
+output_dir="output/${task_name}/${dataset_name}/${speaker}/context_${context_window_size}"
+cache_dir="../cache"
+logging_dir="${output_dir}/runs"
+train_file="${data_dir}/train.json"
+validation_file="${data_dir}/validation.json"
+test_file="${data_dir}/test.json"
+metric_name_or_path="nlu_metric.py"
+metric_for_best_model="overall_f1"
+source_prefix="${data_dir}/source_prefix.txt"
+source_column="context"
+target_column="dialogue_acts_seq"
+model_name_or_path="t5-small"
+per_device_train_batch_size=128
+per_device_eval_batch_size=64
+gradient_accumulation_steps=2
+lr=1e-3
+num_train_epochs=10
+
+python ../create_data.py --tasks ${task_name} --datasets ${dataset_name} --speaker ${speaker} --context_window_size ${context_window_size}
+
+python -m torch.distributed.launch \
+    --nproc_per_node ${n_gpus} ../run_seq2seq.py \
+    --task_name ${task_name} \
+    --train_file ${train_file} \
+    --source_column ${source_column} \
+    --target_column ${target_column} \
+    --source_prefix ${source_prefix} \
+    --model_name_or_path ${model_name_or_path} \
+    --do_train \
+    --save_strategy epoch \
+    --prediction_loss_only \
+    --cache_dir ${cache_dir} \
+    --output_dir ${output_dir} \
+    --logging_dir ${logging_dir} \
+    --overwrite_output_dir \
+    --preprocessing_num_workers 4 \
+    --per_device_train_batch_size ${per_device_train_batch_size} \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \
+    --gradient_accumulation_steps ${gradient_accumulation_steps} \
+    --learning_rate ${lr} \
+    --num_train_epochs ${num_train_epochs} \
+    --debug underflow_overflow \
+    --adafactor \
+    --gradient_checkpointing
+
+python -m torch.distributed.launch \
+    --nproc_per_node ${n_gpus} ../run_seq2seq.py \
+    --task_name ${task_name} \
+    --test_file ${test_file} \
+    --source_column ${source_column} \
+    --target_column ${target_column} \
+    --source_prefix ${source_prefix} \
+    --model_name_or_path ${output_dir} \
+    --do_predict \
+    --predict_with_generate \
+    --metric_name_or_path ${metric_name_or_path} \
+    --cache_dir ${cache_dir} \
+    --output_dir ${output_dir} \
+    --logging_dir ${logging_dir} \
+    --overwrite_output_dir \
+    --preprocessing_num_workers 4 \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \

convlab2/base_models/t5/nlu/run_tm3_user.sh

+n_gpus=1
+task_name="nlu"
+dataset_name="tm3"
+speaker="user"
+context_window_size=0
+data_dir="data/${task_name}/${dataset_name}/${speaker}/context_${context_window_size}"
+output_dir="output/${task_name}/${dataset_name}/${speaker}/context_${context_window_size}"
+cache_dir="../cache"
+logging_dir="${output_dir}/runs"
+train_file="${data_dir}/train.json"
+validation_file="${data_dir}/validation.json"
+test_file="${data_dir}/test.json"
+metric_name_or_path="nlu_metric.py"
+metric_for_best_model="overall_f1"
+source_prefix="${data_dir}/source_prefix.txt"
+source_column="context"
+target_column="dialogue_acts_seq"
+model_name_or_path="t5-small"
+per_device_train_batch_size=128
+per_device_eval_batch_size=64
+gradient_accumulation_steps=2
+lr=1e-3
+num_train_epochs=10
+
+python ../create_data.py --tasks ${task_name} --datasets ${dataset_name} --speaker ${speaker} --context_window_size ${context_window_size}
+
+python ../run_seq2seq.py \
+    --task_name ${task_name} \
+    --train_file ${train_file} \
+    --source_column ${source_column} \
+    --target_column ${target_column} \
+    --source_prefix ${source_prefix} \
+    --model_name_or_path ${model_name_or_path} \
+    --do_train \
+    --save_strategy epoch \
+    --prediction_loss_only \
+    --cache_dir ${cache_dir} \
+    --output_dir ${output_dir} \
+    --logging_dir ${logging_dir} \
+    --overwrite_output_dir \
+    --preprocessing_num_workers 4 \
+    --per_device_train_batch_size ${per_device_train_batch_size} \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \
+    --gradient_accumulation_steps ${gradient_accumulation_steps} \
+    --learning_rate ${lr} \
+    --num_train_epochs ${num_train_epochs} \
+    --debug underflow_overflow \
+    --adafactor \
+    --gradient_checkpointing
+
+python ../run_seq2seq.py \
+    --task_name ${task_name} \
+    --test_file ${test_file} \
+    --source_column ${source_column} \
+    --target_column ${target_column} \
+    --source_prefix ${source_prefix} \
+    --model_name_or_path ${output_dir} \
+    --do_predict \
+    --predict_with_generate \
+    --metric_name_or_path ${metric_name_or_path} \
+    --cache_dir ${cache_dir} \
+    --output_dir ${output_dir} \
+    --logging_dir ${logging_dir} \
+    --overwrite_output_dir \
+    --preprocessing_num_workers 4 \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \

convlab2/base_models/t5/nlu/run_tm3_user_context3.sh

+n_gpus=1
+task_name="nlu"
+dataset_name="tm3"
+speaker="user"
+context_window_size=3
+data_dir="data/${task_name}/${dataset_name}/${speaker}/context_${context_window_size}"
+output_dir="output/${task_name}/${dataset_name}/${speaker}/context_${context_window_size}"
+cache_dir="../cache"
+logging_dir="${output_dir}/runs"
+train_file="${data_dir}/train.json"
+validation_file="${data_dir}/validation.json"
+test_file="${data_dir}/test.json"
+metric_name_or_path="nlu_metric.py"
+metric_for_best_model="overall_f1"
+source_prefix="${data_dir}/source_prefix.txt"
+source_column="context"
+target_column="dialogue_acts_seq"
+model_name_or_path="t5-small"
+per_device_train_batch_size=128
+per_device_eval_batch_size=64
+gradient_accumulation_steps=2
+lr=1e-3
+num_train_epochs=10
+
+python ../create_data.py --tasks ${task_name} --datasets ${dataset_name} --speaker ${speaker} --context_window_size ${context_window_size}
+
+python ../run_seq2seq.py \
+    --task_name ${task_name} \
+    --train_file ${train_file} \
+    --source_column ${source_column} \
+    --target_column ${target_column} \
+    --source_prefix ${source_prefix} \
+    --model_name_or_path ${model_name_or_path} \
+    --do_train \
+    --save_strategy epoch \
+    --prediction_loss_only \
+    --cache_dir ${cache_dir} \
+    --output_dir ${output_dir} \
+    --logging_dir ${logging_dir} \
+    --overwrite_output_dir \
+    --preprocessing_num_workers 4 \
+    --per_device_train_batch_size ${per_device_train_batch_size} \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \
+    --gradient_accumulation_steps ${gradient_accumulation_steps} \
+    --learning_rate ${lr} \
+    --num_train_epochs ${num_train_epochs} \
+    --debug underflow_overflow \
+    --adafactor \
+    --gradient_checkpointing
+
+python ../run_seq2seq.py \
+    --task_name ${task_name} \
+    --test_file ${test_file} \
+    --source_column ${source_column} \
+    --target_column ${target_column} \
+    --source_prefix ${source_prefix} \
+    --model_name_or_path ${output_dir} \
+    --do_predict \
+    --predict_with_generate \
+    --metric_name_or_path ${metric_name_or_path} \
+    --cache_dir ${cache_dir} \
+    --output_dir ${output_dir} \
+    --logging_dir ${logging_dir} \
+    --overwrite_output_dir \
+    --preprocessing_num_workers 4 \
+    --per_device_eval_batch_size ${per_device_eval_batch_size} \

convlab2/util/unified_datasets_util.py

@@ -204,7 +204,7 @@ def create_delex_data(dataset, delex_format='[({domain})-({slot})]', ignore_valu
             substring = texts_placeholders[idx][0]
             searchObj = re.search(value_pattern, substring)
             assert searchObj
-            start, end = searchObj.span(2)
+            start, end = searchObj.span(1)
             texts_placeholders[idx:idx+1] = [(substring[0:start], False), (placeholder, True), (substring[end:], False)]
             return True
         return False
@@ -238,7 +238,7 @@ def create_delex_data(dataset, delex_format='[({domain})-({slot})]', ignore_valu
                     domain, slot, value = da['domain'], da['slot'], da['value']
                     if value.lower() not in ignore_values:
                         placeholder = delex_format.format(domain=domain, slot=slot, value=value)
-                        pattern = re.compile(r'(\W|^)'+f'({value})'+r'(\W|$)', flags=re.I)
+                        pattern = re.compile(r'\b'+f'({value})'+r'\b', flags=re.I)
                         if delex_inplace(delex_utt, pattern):
                             delex_vocab.add(placeholder)
 
@@ -252,7 +252,7 @@ def create_delex_data(dataset, delex_format='[({domain})-({slot})]', ignore_valu
                             for value in values.split('|'):
                                 if value.lower() not in ignore_values:
                                     placeholder = delex_format.format(domain=domain, slot=slot, value=value)
-                                    pattern = re.compile(r'(\W|^)'+f'({value})'+r'(\W|$)', flags=re.I)
+                                    pattern = re.compile(r'\b'+f'({value})'+r'\b', flags=re.I)
                                     if delex_inplace(delex_utt, pattern):
                                         delex_vocab.add(placeholder)
 
@@ -275,9 +275,9 @@ if __name__ == "__main__":
     pprint(data_by_split['test'][0])
 
     dataset, delex_vocab = create_delex_data(dataset)
-    json.dump(dataset['test'], open('delex_multiwoz21_test.json', 'w', encoding='utf-8'), indent=2, ensure_ascii=False)
-    json.dump(delex_vocab, open('delex_vocab.json', 'w', encoding='utf-8'), indent=2, ensure_ascii=False)
-    with open('delex_cmp.txt', 'w') as f:
+    json.dump(dataset['test'], open('new_delex_multiwoz21_test.json', 'w', encoding='utf-8'), indent=2, ensure_ascii=False)
+    json.dump(delex_vocab, open('new_delex_vocab.json', 'w', encoding='utf-8'), indent=2, ensure_ascii=False)
+    with open('new_delex_cmp.txt', 'w') as f:
         for dialog in dataset['test']:
             for turn in dialog['turns']:
                 f.write(turn['utterance']+'\n')