Skip to content
Snippets Groups Projects
Commit 8fcf187d authored by zqwerty's avatar zqwerty
Browse files

add milu exp on unified datasets

parent 196e2ade
No related branches found
No related tags found
No related merge requests found
Showing
with 879 additions and 1 deletion
......@@ -43,6 +43,50 @@ Note that the config file is different from the above. You should set:
## Predict
See `nlu.py` under `multiwoz` and `unified_datasets` directories.
## Performance on unified format datasets
To illustrate that it is easy to use the model for any dataset that in our unified format, we report the performance on several datasets in our unified format. We follow `README.md` and config files in `unified_datasets/` to generate `predictions.json`, then evaluate it using `../evaluate_unified_datasets.py`. Note that we use almost the same hyper-parameters for different datasets, which may not be optimal.
<table>
<thead>
<tr>
<th></th>
<th colspan=2>MultiWOZ 2.1</th>
<th colspan=2>Taskmaster-1</th>
<th colspan=2>Taskmaster-2</th>
<th colspan=2>Taskmaster-3</th>
</tr>
</thead>
<thead>
<tr>
<th>Model</th>
<th>Acc</th><th>F1</th>
<th>Acc</th><th>F1</th>
<th>Acc</th><th>F1</th>
<th>Acc</th><th>F1</th>
</tr>
</thead>
<tbody>
<tr>
<td>MILU</td>
<td>72.9</td><td>85.2</td>
<td>72.9</td><td>49.2</td>
<td>79.1</td><td>68.7</td>
<td>85.4</td><td>80.3</td>
</tr>
<tr>
<td>MILU (context=3)</td>
<td>76.6</td><td>87.9</td>
<td>72.4</td><td>48.5</td>
<td>78.9</td><td>68.4</td>
<td>85.1</td><td>80.1</td>
</tr>
</tbody>
</table>
- Acc: whether all dialogue acts of an utterance are correctly predicted
- F1: F1 measure of the dialogue act predictions over the corpus.
## References
```
@inproceedings{lee2019convlab,
......
......@@ -76,6 +76,8 @@ class MILUDatasetReader(DatasetReader):
for sample in data:
utterance = sample['utterance']
if len(utterance) == 0:
continue
sentences = self._sent_tokenizer.tokenize(utterance)
sent_spans = self._sent_tokenizer.span_tokenize(utterance)
tokens = [token for sent in sentences for token in self._word_tokenizer.tokenize(sent)]
......
......@@ -98,7 +98,7 @@
"num_epochs": 40,
"grad_norm": 5.0,
"patience": 75,
"cuda_device": 4
"cuda_device": 0
},
"evaluate_on_test": true
}
{
"dataset_reader": {
"type": "milu",
"token_indexers": {
"tokens": {
"type": "single_id",
"lowercase_tokens": true
},
"token_characters": {
"type": "characters",
"min_padding_length": 3
},
},
"context_size": 0,
"agent": "user",
"use_unified_datasets": true,
"dataset_name": "sgd",
"random_context_size": false
},
"train_data_path": "train",
"validation_data_path": "validation",
"test_data_path": "test",
"model": {
"type": "milu",
"label_encoding": "BIO",
"use_unified_datasets": true,
"dropout": 0.3,
"include_start_end_transitions": false,
"text_field_embedder": {
"token_embedders": {
"tokens": {
"type": "embedding",
"embedding_dim": 50,
"pretrained_file": "https://s3-us-west-2.amazonaws.com/allennlp/datasets/glove/glove.6B.50d.txt.gz",
"trainable": true
},
"token_characters": {
"type": "character_encoding",
"embedding": {
"embedding_dim": 16
},
"encoder": {
"type": "cnn",
"embedding_dim": 16,
"num_filters": 128,
"ngram_filter_sizes": [3],
"conv_layer_activation": "relu"
}
}
}
},
"encoder": {
"type": "lstm",
"input_size": 178,
"hidden_size": 200,
"num_layers": 1,
"dropout": 0.5,
"bidirectional": true
},
"intent_encoder": {
"type": "lstm",
"input_size": 400,
"hidden_size": 200,
"num_layers": 1,
"dropout": 0.5,
"bidirectional": true
},
"attention": {
"type": "bilinear",
"vector_dim": 400,
"matrix_dim": 400
},
"context_for_intent": true,
"context_for_tag": false,
"attention_for_intent": false,
"attention_for_tag": false,
"regularizer": [
[
"scalar_parameters",
{
"type": "l2",
"alpha": 0.1
}
]
]
},
"iterator": {
"type": "basic",
"batch_size": 64
},
"trainer": {
"optimizer": {
"type": "adam",
"lr": 0.001
},
"validation_metric": "+f1-measure",
"num_serialized_models_to_keep": 3,
"num_epochs": 40,
"grad_norm": 5.0,
"patience": 75,
"cuda_device": 0
},
"evaluate_on_test": true
}
{
"dataset_reader": {
"type": "milu",
"token_indexers": {
"tokens": {
"type": "single_id",
"lowercase_tokens": true
},
"token_characters": {
"type": "characters",
"min_padding_length": 3
},
},
"context_size": 3,
"agent": "user",
"use_unified_datasets": true,
"dataset_name": "sgd",
"random_context_size": false
},
"train_data_path": "train",
"validation_data_path": "validation",
"test_data_path": "test",
"model": {
"type": "milu",
"label_encoding": "BIO",
"use_unified_datasets": true,
"dropout": 0.3,
"include_start_end_transitions": false,
"text_field_embedder": {
"token_embedders": {
"tokens": {
"type": "embedding",
"embedding_dim": 50,
"pretrained_file": "https://s3-us-west-2.amazonaws.com/allennlp/datasets/glove/glove.6B.50d.txt.gz",
"trainable": true
},
"token_characters": {
"type": "character_encoding",
"embedding": {
"embedding_dim": 16
},
"encoder": {
"type": "cnn",
"embedding_dim": 16,
"num_filters": 128,
"ngram_filter_sizes": [3],
"conv_layer_activation": "relu"
}
}
}
},
"encoder": {
"type": "lstm",
"input_size": 178,
"hidden_size": 200,
"num_layers": 1,
"dropout": 0.5,
"bidirectional": true
},
"intent_encoder": {
"type": "lstm",
"input_size": 400,
"hidden_size": 200,
"num_layers": 1,
"dropout": 0.5,
"bidirectional": true
},
"attention": {
"type": "bilinear",
"vector_dim": 400,
"matrix_dim": 400
},
"context_for_intent": true,
"context_for_tag": false,
"attention_for_intent": false,
"attention_for_tag": false,
"regularizer": [
[
"scalar_parameters",
{
"type": "l2",
"alpha": 0.1
}
]
]
},
"iterator": {
"type": "basic",
"batch_size": 64
},
"trainer": {
"optimizer": {
"type": "adam",
"lr": 0.001
},
"validation_metric": "+f1-measure",
"num_serialized_models_to_keep": 3,
"num_epochs": 40,
"grad_norm": 5.0,
"patience": 75,
"cuda_device": 0
},
"evaluate_on_test": true
}
{
"dataset_reader": {
"type": "milu",
"token_indexers": {
"tokens": {
"type": "single_id",
"lowercase_tokens": true
},
"token_characters": {
"type": "characters",
"min_padding_length": 3
},
},
"context_size": 0,
"agent": "user",
"use_unified_datasets": true,
"dataset_name": "tm1",
"random_context_size": false
},
"train_data_path": "train",
"validation_data_path": "validation",
"test_data_path": "test",
"model": {
"type": "milu",
"label_encoding": "BIO",
"use_unified_datasets": true,
"dropout": 0.3,
"include_start_end_transitions": false,
"text_field_embedder": {
"token_embedders": {
"tokens": {
"type": "embedding",
"embedding_dim": 50,
"pretrained_file": "https://s3-us-west-2.amazonaws.com/allennlp/datasets/glove/glove.6B.50d.txt.gz",
"trainable": true
},
"token_characters": {
"type": "character_encoding",
"embedding": {
"embedding_dim": 16
},
"encoder": {
"type": "cnn",
"embedding_dim": 16,
"num_filters": 128,
"ngram_filter_sizes": [3],
"conv_layer_activation": "relu"
}
}
}
},
"encoder": {
"type": "lstm",
"input_size": 178,
"hidden_size": 200,
"num_layers": 1,
"dropout": 0.5,
"bidirectional": true
},
"intent_encoder": {
"type": "lstm",
"input_size": 400,
"hidden_size": 200,
"num_layers": 1,
"dropout": 0.5,
"bidirectional": true
},
"attention": {
"type": "bilinear",
"vector_dim": 400,
"matrix_dim": 400
},
"context_for_intent": true,
"context_for_tag": false,
"attention_for_intent": false,
"attention_for_tag": false,
"regularizer": [
[
"scalar_parameters",
{
"type": "l2",
"alpha": 0.1
}
]
]
},
"iterator": {
"type": "basic",
"batch_size": 64
},
"trainer": {
"optimizer": {
"type": "adam",
"lr": 0.001
},
"validation_metric": "+f1-measure",
"num_serialized_models_to_keep": 3,
"num_epochs": 40,
"grad_norm": 5.0,
"patience": 75,
"cuda_device": 0
},
"evaluate_on_test": true
}
{
"dataset_reader": {
"type": "milu",
"token_indexers": {
"tokens": {
"type": "single_id",
"lowercase_tokens": true
},
"token_characters": {
"type": "characters",
"min_padding_length": 3
},
},
"context_size": 3,
"agent": "user",
"use_unified_datasets": true,
"dataset_name": "tm1",
"random_context_size": false
},
"train_data_path": "train",
"validation_data_path": "validation",
"test_data_path": "test",
"model": {
"type": "milu",
"label_encoding": "BIO",
"use_unified_datasets": true,
"dropout": 0.3,
"include_start_end_transitions": false,
"text_field_embedder": {
"token_embedders": {
"tokens": {
"type": "embedding",
"embedding_dim": 50,
"pretrained_file": "https://s3-us-west-2.amazonaws.com/allennlp/datasets/glove/glove.6B.50d.txt.gz",
"trainable": true
},
"token_characters": {
"type": "character_encoding",
"embedding": {
"embedding_dim": 16
},
"encoder": {
"type": "cnn",
"embedding_dim": 16,
"num_filters": 128,
"ngram_filter_sizes": [3],
"conv_layer_activation": "relu"
}
}
}
},
"encoder": {
"type": "lstm",
"input_size": 178,
"hidden_size": 200,
"num_layers": 1,
"dropout": 0.5,
"bidirectional": true
},
"intent_encoder": {
"type": "lstm",
"input_size": 400,
"hidden_size": 200,
"num_layers": 1,
"dropout": 0.5,
"bidirectional": true
},
"attention": {
"type": "bilinear",
"vector_dim": 400,
"matrix_dim": 400
},
"context_for_intent": true,
"context_for_tag": false,
"attention_for_intent": false,
"attention_for_tag": false,
"regularizer": [
[
"scalar_parameters",
{
"type": "l2",
"alpha": 0.1
}
]
]
},
"iterator": {
"type": "basic",
"batch_size": 64
},
"trainer": {
"optimizer": {
"type": "adam",
"lr": 0.001
},
"validation_metric": "+f1-measure",
"num_serialized_models_to_keep": 3,
"num_epochs": 40,
"grad_norm": 5.0,
"patience": 75,
"cuda_device": 0
},
"evaluate_on_test": true
}
{
"dataset_reader": {
"type": "milu",
"token_indexers": {
"tokens": {
"type": "single_id",
"lowercase_tokens": true
},
"token_characters": {
"type": "characters",
"min_padding_length": 3
},
},
"context_size": 0,
"agent": "user",
"use_unified_datasets": true,
"dataset_name": "tm2",
"random_context_size": false
},
"train_data_path": "train",
"validation_data_path": "validation",
"test_data_path": "test",
"model": {
"type": "milu",
"label_encoding": "BIO",
"use_unified_datasets": true,
"dropout": 0.3,
"include_start_end_transitions": false,
"text_field_embedder": {
"token_embedders": {
"tokens": {
"type": "embedding",
"embedding_dim": 50,
"pretrained_file": "https://s3-us-west-2.amazonaws.com/allennlp/datasets/glove/glove.6B.50d.txt.gz",
"trainable": true
},
"token_characters": {
"type": "character_encoding",
"embedding": {
"embedding_dim": 16
},
"encoder": {
"type": "cnn",
"embedding_dim": 16,
"num_filters": 128,
"ngram_filter_sizes": [3],
"conv_layer_activation": "relu"
}
}
}
},
"encoder": {
"type": "lstm",
"input_size": 178,
"hidden_size": 200,
"num_layers": 1,
"dropout": 0.5,
"bidirectional": true
},
"intent_encoder": {
"type": "lstm",
"input_size": 400,
"hidden_size": 200,
"num_layers": 1,
"dropout": 0.5,
"bidirectional": true
},
"attention": {
"type": "bilinear",
"vector_dim": 400,
"matrix_dim": 400
},
"context_for_intent": true,
"context_for_tag": false,
"attention_for_intent": false,
"attention_for_tag": false,
"regularizer": [
[
"scalar_parameters",
{
"type": "l2",
"alpha": 0.1
}
]
]
},
"iterator": {
"type": "basic",
"batch_size": 64
},
"trainer": {
"optimizer": {
"type": "adam",
"lr": 0.001
},
"validation_metric": "+f1-measure",
"num_serialized_models_to_keep": 3,
"num_epochs": 40,
"grad_norm": 5.0,
"patience": 75,
"cuda_device": 0
},
"evaluate_on_test": true
}
{
"dataset_reader": {
"type": "milu",
"token_indexers": {
"tokens": {
"type": "single_id",
"lowercase_tokens": true
},
"token_characters": {
"type": "characters",
"min_padding_length": 3
},
},
"context_size": 3,
"agent": "user",
"use_unified_datasets": true,
"dataset_name": "tm2",
"random_context_size": false
},
"train_data_path": "train",
"validation_data_path": "validation",
"test_data_path": "test",
"model": {
"type": "milu",
"label_encoding": "BIO",
"use_unified_datasets": true,
"dropout": 0.3,
"include_start_end_transitions": false,
"text_field_embedder": {
"token_embedders": {
"tokens": {
"type": "embedding",
"embedding_dim": 50,
"pretrained_file": "https://s3-us-west-2.amazonaws.com/allennlp/datasets/glove/glove.6B.50d.txt.gz",
"trainable": true
},
"token_characters": {
"type": "character_encoding",
"embedding": {
"embedding_dim": 16
},
"encoder": {
"type": "cnn",
"embedding_dim": 16,
"num_filters": 128,
"ngram_filter_sizes": [3],
"conv_layer_activation": "relu"
}
}
}
},
"encoder": {
"type": "lstm",
"input_size": 178,
"hidden_size": 200,
"num_layers": 1,
"dropout": 0.5,
"bidirectional": true
},
"intent_encoder": {
"type": "lstm",
"input_size": 400,
"hidden_size": 200,
"num_layers": 1,
"dropout": 0.5,
"bidirectional": true
},
"attention": {
"type": "bilinear",
"vector_dim": 400,
"matrix_dim": 400
},
"context_for_intent": true,
"context_for_tag": false,
"attention_for_intent": false,
"attention_for_tag": false,
"regularizer": [
[
"scalar_parameters",
{
"type": "l2",
"alpha": 0.1
}
]
]
},
"iterator": {
"type": "basic",
"batch_size": 64
},
"trainer": {
"optimizer": {
"type": "adam",
"lr": 0.001
},
"validation_metric": "+f1-measure",
"num_serialized_models_to_keep": 3,
"num_epochs": 40,
"grad_norm": 5.0,
"patience": 75,
"cuda_device": 0
},
"evaluate_on_test": true
}
{
"dataset_reader": {
"type": "milu",
"token_indexers": {
"tokens": {
"type": "single_id",
"lowercase_tokens": true
},
"token_characters": {
"type": "characters",
"min_padding_length": 3
},
},
"context_size": 0,
"agent": "user",
"use_unified_datasets": true,
"dataset_name": "tm3",
"random_context_size": false
},
"train_data_path": "train",
"validation_data_path": "validation",
"test_data_path": "test",
"model": {
"type": "milu",
"label_encoding": "BIO",
"use_unified_datasets": true,
"dropout": 0.3,
"include_start_end_transitions": false,
"text_field_embedder": {
"token_embedders": {
"tokens": {
"type": "embedding",
"embedding_dim": 50,
"pretrained_file": "https://s3-us-west-2.amazonaws.com/allennlp/datasets/glove/glove.6B.50d.txt.gz",
"trainable": true
},
"token_characters": {
"type": "character_encoding",
"embedding": {
"embedding_dim": 16
},
"encoder": {
"type": "cnn",
"embedding_dim": 16,
"num_filters": 128,
"ngram_filter_sizes": [3],
"conv_layer_activation": "relu"
}
}
}
},
"encoder": {
"type": "lstm",
"input_size": 178,
"hidden_size": 200,
"num_layers": 1,
"dropout": 0.5,
"bidirectional": true
},
"intent_encoder": {
"type": "lstm",
"input_size": 400,
"hidden_size": 200,
"num_layers": 1,
"dropout": 0.5,
"bidirectional": true
},
"attention": {
"type": "bilinear",
"vector_dim": 400,
"matrix_dim": 400
},
"context_for_intent": true,
"context_for_tag": false,
"attention_for_intent": false,
"attention_for_tag": false,
"regularizer": [
[
"scalar_parameters",
{
"type": "l2",
"alpha": 0.1
}
]
]
},
"iterator": {
"type": "basic",
"batch_size": 64
},
"trainer": {
"optimizer": {
"type": "adam",
"lr": 0.001
},
"validation_metric": "+f1-measure",
"num_serialized_models_to_keep": 3,
"num_epochs": 40,
"grad_norm": 5.0,
"patience": 75,
"cuda_device": 0
},
"evaluate_on_test": true
}
{
"dataset_reader": {
"type": "milu",
"token_indexers": {
"tokens": {
"type": "single_id",
"lowercase_tokens": true
},
"token_characters": {
"type": "characters",
"min_padding_length": 3
},
},
"context_size": 3,
"agent": "user",
"use_unified_datasets": true,
"dataset_name": "tm3",
"random_context_size": false
},
"train_data_path": "train",
"validation_data_path": "validation",
"test_data_path": "test",
"model": {
"type": "milu",
"label_encoding": "BIO",
"use_unified_datasets": true,
"dropout": 0.3,
"include_start_end_transitions": false,
"text_field_embedder": {
"token_embedders": {
"tokens": {
"type": "embedding",
"embedding_dim": 50,
"pretrained_file": "https://s3-us-west-2.amazonaws.com/allennlp/datasets/glove/glove.6B.50d.txt.gz",
"trainable": true
},
"token_characters": {
"type": "character_encoding",
"embedding": {
"embedding_dim": 16
},
"encoder": {
"type": "cnn",
"embedding_dim": 16,
"num_filters": 128,
"ngram_filter_sizes": [3],
"conv_layer_activation": "relu"
}
}
}
},
"encoder": {
"type": "lstm",
"input_size": 178,
"hidden_size": 200,
"num_layers": 1,
"dropout": 0.5,
"bidirectional": true
},
"intent_encoder": {
"type": "lstm",
"input_size": 400,
"hidden_size": 200,
"num_layers": 1,
"dropout": 0.5,
"bidirectional": true
},
"attention": {
"type": "bilinear",
"vector_dim": 400,
"matrix_dim": 400
},
"context_for_intent": true,
"context_for_tag": false,
"attention_for_intent": false,
"attention_for_tag": false,
"regularizer": [
[
"scalar_parameters",
{
"type": "l2",
"alpha": 0.1
}
]
]
},
"iterator": {
"type": "basic",
"batch_size": 64
},
"trainer": {
"optimizer": {
"type": "adam",
"lr": 0.001
},
"validation_metric": "+f1-measure",
"num_serialized_models_to_keep": 3,
"num_epochs": 40,
"grad_norm": 5.0,
"patience": 75,
"cuda_device": 0
},
"evaluate_on_test": true
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment