diff --git a/.gitignore b/.gitignore index b8268b152da605f2187579dec5ec4f664edceefd..cd356208b42f43779d592bd7dd043a4bcbf306b6 100644 --- a/.gitignore +++ b/.gitignore @@ -33,12 +33,21 @@ convlab2/nlg/sclstm/**/sclstm.log convlab2/nlg/sclstm/**/sclstm_usr.pt convlab2/nlg/sclstm/**/sclstm_usr.res convlab2/nlg/sclstm/**/sclstm_usr.log -convlab2/nlu/jointBERT/**/output/ convlab2/dst/sumbt/multiwoz/output/ convlab2/nlg/sclstm/**/generated_sens_sys.json convlab2/nlg/template/**/generated_sens_sys.json convlab2/nlu/jointBERT/crosswoz/**/data convlab2/nlu/jointBERT/multiwoz/**/data +convlab2/nlu/jointBERT/**/output/ +convlab2/nlu/jointBERT_new/crosswoz/**/data +convlab2/nlu/jointBERT_new/multiwoz/**/data +convlab2/nlu/jointBERT_new/crosswoz/**/log +convlab2/nlu/jointBERT_new/multiwoz/**/log +convlab2/nlu/jointBERT_new/**/output/ +convlab2/nlu/milu/09* +convlab2/nlu/jointBERT/multiwoz/configs/multiwoz_new_usr_context.json +convlab2/nlu/milu/multiwoz/configs/system_without_context.jsonnet +convlab2/nlu/milu/multiwoz/configs/user_without_context.jsonnet # test script *_test.py diff --git a/convlab2/nlu/jointBERT/dataloader.py b/convlab2/nlu/jointBERT/dataloader.py index fba4ebf125d71bc6a40d5f3a438faf63a7b0db3d..38fc24ea0fdc288410a65146716996432ebd896c 100755 --- a/convlab2/nlu/jointBERT/dataloader.py +++ b/convlab2/nlu/jointBERT/dataloader.py @@ -57,6 +57,7 @@ class Dataloader: new2ori = None d.append(new2ori) d.append(word_seq) + d.append(self.seq_tag2id(tag_seq)) d.append(self.seq_intent2id(d[2])) # d = (tokens, tags, intents, da2triples(turn["dialog_act"]), context(token id), new2ori, new_word_seq, tag2id_seq, intent2id_seq) @@ -95,7 +96,7 @@ class Dataloader: return split_tokens, new_tag_seq, new2ori def seq_tag2id(self, tags): - return [self.tag2id[x] for x in tags if x in self.tag2id] + return [self.tag2id[x] if x in self.tag2id else self.tag2id['O'] for x in tags] def seq_id2tag(self, ids): return [self.id2tag[x] for x in ids] diff --git a/convlab2/nlu/milu/dataset_reader.py b/convlab2/nlu/milu/dataset_reader.py index 3a8cf77818d2a8c8431337593eca532d55ec6cc8..5e00af04e7fe6c13ddbb21d60f22c51d5cbbc106 100755 --- a/convlab2/nlu/milu/dataset_reader.py +++ b/convlab2/nlu/milu/dataset_reader.py @@ -75,9 +75,11 @@ class MILUDatasetReader(DatasetReader): dialog = dialogs[dial_name]["log"] context_tokens_list = [] for i, turn in enumerate(dialog): - if self._agent and self._agent == "user" and i % 2 != 1: + if self._agent and self._agent == "user" and i % 2 == 1: + context_tokens_list.append(turn["text"].lower().split()+ ["SENT_END"]) continue - if self._agent and self._agent == "system" and i % 2 != 0: + if self._agent and self._agent == "system" and i % 2 == 0: + context_tokens_list.append(turn["text"].lower().split()+ ["SENT_END"]) continue tokens = turn["text"].split() diff --git a/convlab2/nlu/milu/multiwoz/nlu.py b/convlab2/nlu/milu/multiwoz/nlu.py index 5417c6d958954bf1005d399895bf7e2972379861..002a7dc86f6f0bcce81d9410193d069f442195ef 100755 --- a/convlab2/nlu/milu/multiwoz/nlu.py +++ b/convlab2/nlu/milu/multiwoz/nlu.py @@ -28,7 +28,7 @@ class MILU(NLU): def __init__(self, archive_file=DEFAULT_ARCHIVE_FILE, cuda_device=DEFAULT_CUDA_DEVICE, - model_file="https://convlab.blob.core.windows.net/convlab-2/milu_multiwoz_all_context.tar.gz", + model_file="https://convlab.blob.core.windows.net/convlab-2/new_milu(20200922)_multiwoz_all_context.tar.gz", context_size=3): """ Constructor for NLU class. """ diff --git a/convlab2/nlu/milu/train.py b/convlab2/nlu/milu/train.py index 99db49f91c3ae90f5c97287835f483c28fe1a832..9507a3a8ba920622ce84b26fa24513cdd2f2bb53 100755 --- a/convlab2/nlu/milu/train.py +++ b/convlab2/nlu/milu/train.py @@ -16,7 +16,8 @@ from allennlp.common.checks import check_for_gpu from allennlp.common.util import prepare_environment, prepare_global_logging, cleanup_global_logging, dump_metrics from allennlp.models.archival import archive_model, CONFIG_NAME from allennlp.models.model import Model, _DEFAULT_WEIGHTS -from allennlp.training.trainer import Trainer, TrainerPieces +from allennlp.training.trainer import Trainer +from allennlp.training.trainer_pieces import TrainerPieces from allennlp.training.trainer_base import TrainerBase from allennlp.training.util import create_serialization_dir, evaluate