diff --git a/.gitignore b/.gitignore index 16ca3096dc83b11b3870d7d07c7197b10d8f0152..9d78dc0f1dcdbf2fb092d4d69d5e75ee5100bace 100644 --- a/.gitignore +++ b/.gitignore @@ -97,8 +97,8 @@ convlab/dst/trade/crosswoz_config/ convlab/dst/trade/multiwoz/data/ convlab/dst/trade/multiwoz/model/ convlab/dst/trade/multiwoz_config/ -deploy/bert_multiwoz_all.zip -deploy/templates/dialog_eg.html +convlab/deploy/bert_multiwoz_all.zip +convlab/deploy/templates/dialog_eg.html test.py *convlab/policy/vector/action_dicts diff --git a/MANIFEST.in b/MANIFEST.in index 2fc765c4fd5fc98eb010040a610d46fb175098a7..6006776dbcf6d766b4ffe7e17fb31929fa692c31 100755 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,2 @@ - include LICENSE.txt include README.md -prune convlab/*/__pycache__ diff --git a/tests/qex_convlab_eval.sh b/agent_examples/qex_convlab_eval.sh similarity index 100% rename from tests/qex_convlab_eval.sh rename to agent_examples/qex_convlab_eval.sh diff --git a/tests/test_BERTNLU-RuleDST-DQNPolicy-TemplateNLG.py b/agent_examples/test_BERTNLU-RuleDST-DQNPolicy-TemplateNLG.py similarity index 100% rename from tests/test_BERTNLU-RuleDST-DQNPolicy-TemplateNLG.py rename to agent_examples/test_BERTNLU-RuleDST-DQNPolicy-TemplateNLG.py diff --git a/tests/test_BERTNLU-RuleDST-GDPLPolicy-TemplateNLG.py b/agent_examples/test_BERTNLU-RuleDST-GDPLPolicy-TemplateNLG.py similarity index 100% rename from tests/test_BERTNLU-RuleDST-GDPLPolicy-TemplateNLG.py rename to agent_examples/test_BERTNLU-RuleDST-GDPLPolicy-TemplateNLG.py diff --git a/tests/test_BERTNLU-RuleDST-HDSA.py b/agent_examples/test_BERTNLU-RuleDST-HDSA.py similarity index 100% rename from tests/test_BERTNLU-RuleDST-HDSA.py rename to agent_examples/test_BERTNLU-RuleDST-HDSA.py diff --git a/tests/test_BERTNLU-RuleDST-LaRL.py b/agent_examples/test_BERTNLU-RuleDST-LaRL.py similarity index 100% rename from tests/test_BERTNLU-RuleDST-LaRL.py rename to agent_examples/test_BERTNLU-RuleDST-LaRL.py diff --git a/tests/test_BERTNLU-RuleDST-MDRG.py b/agent_examples/test_BERTNLU-RuleDST-MDRG.py similarity index 100% rename from tests/test_BERTNLU-RuleDST-MDRG.py rename to agent_examples/test_BERTNLU-RuleDST-MDRG.py diff --git a/tests/test_BERTNLU-RuleDST-MLEPolicy-TemplateNLG.py b/agent_examples/test_BERTNLU-RuleDST-MLEPolicy-TemplateNLG.py similarity index 100% rename from tests/test_BERTNLU-RuleDST-MLEPolicy-TemplateNLG.py rename to agent_examples/test_BERTNLU-RuleDST-MLEPolicy-TemplateNLG.py diff --git a/tests/test_BERTNLU-RuleDST-PGPolicy-TemplateNLG.py b/agent_examples/test_BERTNLU-RuleDST-PGPolicy-TemplateNLG.py similarity index 100% rename from tests/test_BERTNLU-RuleDST-PGPolicy-TemplateNLG.py rename to agent_examples/test_BERTNLU-RuleDST-PGPolicy-TemplateNLG.py diff --git a/tests/test_BERTNLU-RuleDST-PPOPolicy-TemplateNLG.py b/agent_examples/test_BERTNLU-RuleDST-PPOPolicy-TemplateNLG.py similarity index 100% rename from tests/test_BERTNLU-RuleDST-PPOPolicy-TemplateNLG.py rename to agent_examples/test_BERTNLU-RuleDST-PPOPolicy-TemplateNLG.py diff --git a/tests/test_BERTNLU-RuleDST-RulePolicy-SCLSTM.py b/agent_examples/test_BERTNLU-RuleDST-RulePolicy-SCLSTM.py similarity index 100% rename from tests/test_BERTNLU-RuleDST-RulePolicy-SCLSTM.py rename to agent_examples/test_BERTNLU-RuleDST-RulePolicy-SCLSTM.py diff --git a/tests/test_BERTNLU-RuleDST-RulePolicy-TemplateNLG.py b/agent_examples/test_BERTNLU-RuleDST-RulePolicy-TemplateNLG.py similarity index 100% rename from tests/test_BERTNLU-RuleDST-RulePolicy-TemplateNLG.py rename to agent_examples/test_BERTNLU-RuleDST-RulePolicy-TemplateNLG.py diff --git a/tests/test_BERTNLU-RuleDST-TUS-TemplateNLG.py b/agent_examples/test_BERTNLU-RuleDST-TUS-TemplateNLG.py similarity index 100% rename from tests/test_BERTNLU-RuleDST-TUS-TemplateNLG.py rename to agent_examples/test_BERTNLU-RuleDST-TUS-TemplateNLG.py diff --git a/tests/test_BERTNLU-RuleDST-VtraceRNN-TemplateNLG.py b/agent_examples/test_BERTNLU-RuleDST-VtraceRNN-TemplateNLG.py similarity index 100% rename from tests/test_BERTNLU-RuleDST-VtraceRNN-TemplateNLG.py rename to agent_examples/test_BERTNLU-RuleDST-VtraceRNN-TemplateNLG.py diff --git a/tests/test_COMER_RulePolicy_TemplateNLG.py b/agent_examples/test_COMER_RulePolicy_TemplateNLG.py similarity index 100% rename from tests/test_COMER_RulePolicy_TemplateNLG.py rename to agent_examples/test_COMER_RulePolicy_TemplateNLG.py diff --git a/tests/test_DAMD.py b/agent_examples/test_DAMD.py similarity index 100% rename from tests/test_DAMD.py rename to agent_examples/test_DAMD.py diff --git a/tests/test_LAVA.py b/agent_examples/test_LAVA.py similarity index 100% rename from tests/test_LAVA.py rename to agent_examples/test_LAVA.py diff --git a/tests/test_MDBT_RulePolicy_TemplateNLG.py b/agent_examples/test_MDBT_RulePolicy_TemplateNLG.py similarity index 100% rename from tests/test_MDBT_RulePolicy_TemplateNLG.py rename to agent_examples/test_MDBT_RulePolicy_TemplateNLG.py diff --git a/tests/test_MILU-RuleDST-RulePolicy-TemplateNLG.py b/agent_examples/test_MILU-RuleDST-RulePolicy-TemplateNLG.py similarity index 100% rename from tests/test_MILU-RuleDST-RulePolicy-TemplateNLG.py rename to agent_examples/test_MILU-RuleDST-RulePolicy-TemplateNLG.py diff --git a/tests/test_NLG_end2end.py b/agent_examples/test_NLG_end2end.py similarity index 100% rename from tests/test_NLG_end2end.py rename to agent_examples/test_NLG_end2end.py diff --git a/tests/test_PPO.py b/agent_examples/test_PPO.py similarity index 100% rename from tests/test_PPO.py rename to agent_examples/test_PPO.py diff --git a/tests/test_RulePolicy.py b/agent_examples/test_RulePolicy.py similarity index 100% rename from tests/test_RulePolicy.py rename to agent_examples/test_RulePolicy.py diff --git a/tests/test_SUMBT-LaRL.py b/agent_examples/test_SUMBT-LaRL.py similarity index 100% rename from tests/test_SUMBT-LaRL.py rename to agent_examples/test_SUMBT-LaRL.py diff --git a/tests/test_SUMBT-RulePolicy-TemplateNLG.py b/agent_examples/test_SUMBT-RulePolicy-TemplateNLG.py similarity index 100% rename from tests/test_SUMBT-RulePolicy-TemplateNLG.py rename to agent_examples/test_SUMBT-RulePolicy-TemplateNLG.py diff --git a/tests/test_SVMNLU-RuleDST-RulePolicy-TemplateNLG.py b/agent_examples/test_SVMNLU-RuleDST-RulePolicy-TemplateNLG.py similarity index 100% rename from tests/test_SVMNLU-RuleDST-RulePolicy-TemplateNLG.py rename to agent_examples/test_SVMNLU-RuleDST-RulePolicy-TemplateNLG.py diff --git a/tests/test_Sequicity.py b/agent_examples/test_Sequicity.py similarity index 100% rename from tests/test_Sequicity.py rename to agent_examples/test_Sequicity.py diff --git a/tests/test_SetSUMBT-END-PPO-TemplateNLG.py b/agent_examples/test_SetSUMBT-END-PPO-TemplateNLG.py similarity index 100% rename from tests/test_SetSUMBT-END-PPO-TemplateNLG.py rename to agent_examples/test_SetSUMBT-END-PPO-TemplateNLG.py diff --git a/tests/test_SetSUMBT-LaRL.py b/agent_examples/test_SetSUMBT-LaRL.py similarity index 100% rename from tests/test_SetSUMBT-LaRL.py rename to agent_examples/test_SetSUMBT-LaRL.py diff --git a/tests/test_SetSUMBT-RulePolicy-TemplateNLG.py b/agent_examples/test_SetSUMBT-RulePolicy-TemplateNLG.py similarity index 100% rename from tests/test_SetSUMBT-RulePolicy-TemplateNLG.py rename to agent_examples/test_SetSUMBT-RulePolicy-TemplateNLG.py diff --git a/tests/test_SetSUMBT-VtraceRNN-TemplateNLG.py b/agent_examples/test_SetSUMBT-VtraceRNN-TemplateNLG.py similarity index 100% rename from tests/test_SetSUMBT-VtraceRNN-TemplateNLG.py rename to agent_examples/test_SetSUMBT-VtraceRNN-TemplateNLG.py diff --git a/tests/test_TRADE_RulePolicy_TemplateNLG.py b/agent_examples/test_TRADE_RulePolicy_TemplateNLG.py similarity index 100% rename from tests/test_TRADE_RulePolicy_TemplateNLG.py rename to agent_examples/test_TRADE_RulePolicy_TemplateNLG.py diff --git a/tests/test_VtraceRNN.py b/agent_examples/test_VtraceRNN.py similarity index 100% rename from tests/test_VtraceRNN.py rename to agent_examples/test_VtraceRNN.py diff --git a/tests/test_end2end.py b/agent_examples/test_end2end.py similarity index 100% rename from tests/test_end2end.py rename to agent_examples/test_end2end.py diff --git a/tests/test_vtrace_masking.py b/agent_examples/test_vtrace_masking.py similarity index 100% rename from tests/test_vtrace_masking.py rename to agent_examples/test_vtrace_masking.py diff --git a/tests/__init__.py b/convlab/base_models/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from tests/__init__.py rename to convlab/base_models/__init__.py diff --git a/deploy/README.md b/convlab/deploy/README.md similarity index 100% rename from deploy/README.md rename to convlab/deploy/README.md diff --git a/deploy/__init__.py b/convlab/deploy/__init__.py similarity index 100% rename from deploy/__init__.py rename to convlab/deploy/__init__.py diff --git a/deploy/config.py b/convlab/deploy/config.py similarity index 100% rename from deploy/config.py rename to convlab/deploy/config.py diff --git a/deploy/ctrl/__init__.py b/convlab/deploy/ctrl/__init__.py similarity index 100% rename from deploy/ctrl/__init__.py rename to convlab/deploy/ctrl/__init__.py diff --git a/deploy/ctrl/model.py b/convlab/deploy/ctrl/model.py similarity index 100% rename from deploy/ctrl/model.py rename to convlab/deploy/ctrl/model.py diff --git a/deploy/ctrl/module.py b/convlab/deploy/ctrl/module.py similarity index 100% rename from deploy/ctrl/module.py rename to convlab/deploy/ctrl/module.py diff --git a/deploy/ctrl/session.py b/convlab/deploy/ctrl/session.py similarity index 100% rename from deploy/ctrl/session.py rename to convlab/deploy/ctrl/session.py diff --git a/deploy/dep_config.json b/convlab/deploy/dep_config.json similarity index 100% rename from deploy/dep_config.json rename to convlab/deploy/dep_config.json diff --git a/deploy/run.py b/convlab/deploy/run.py similarity index 100% rename from deploy/run.py rename to convlab/deploy/run.py diff --git a/deploy/server.py b/convlab/deploy/server.py similarity index 100% rename from deploy/server.py rename to convlab/deploy/server.py diff --git a/deploy/static/css/15071191L1.css b/convlab/deploy/static/css/15071191L1.css similarity index 100% rename from deploy/static/css/15071191L1.css rename to convlab/deploy/static/css/15071191L1.css diff --git a/deploy/static/css/bootstrap-theme.css b/convlab/deploy/static/css/bootstrap-theme.css similarity index 100% rename from deploy/static/css/bootstrap-theme.css rename to convlab/deploy/static/css/bootstrap-theme.css diff --git a/deploy/static/css/bootstrap-theme.min.css b/convlab/deploy/static/css/bootstrap-theme.min.css similarity index 100% rename from deploy/static/css/bootstrap-theme.min.css rename to convlab/deploy/static/css/bootstrap-theme.min.css diff --git a/deploy/static/css/bootstrap.css b/convlab/deploy/static/css/bootstrap.css similarity index 100% rename from deploy/static/css/bootstrap.css rename to convlab/deploy/static/css/bootstrap.css diff --git a/deploy/static/css/bootstrap.min.css b/convlab/deploy/static/css/bootstrap.min.css similarity index 100% rename from deploy/static/css/bootstrap.min.css rename to convlab/deploy/static/css/bootstrap.min.css diff --git a/deploy/static/css/specific.css b/convlab/deploy/static/css/specific.css similarity index 100% rename from deploy/static/css/specific.css rename to convlab/deploy/static/css/specific.css diff --git a/deploy/static/js/bootstrap.js b/convlab/deploy/static/js/bootstrap.js similarity index 100% rename from deploy/static/js/bootstrap.js rename to convlab/deploy/static/js/bootstrap.js diff --git a/deploy/static/js/bootstrap.min.js b/convlab/deploy/static/js/bootstrap.min.js similarity index 100% rename from deploy/static/js/bootstrap.min.js rename to convlab/deploy/static/js/bootstrap.min.js diff --git a/deploy/static/js/jquery-1.11.1.min.js b/convlab/deploy/static/js/jquery-1.11.1.min.js similarity index 100% rename from deploy/static/js/jquery-1.11.1.min.js rename to convlab/deploy/static/js/jquery-1.11.1.min.js diff --git a/deploy/static/js/jquery.min.js b/convlab/deploy/static/js/jquery.min.js similarity index 100% rename from deploy/static/js/jquery.min.js rename to convlab/deploy/static/js/jquery.min.js diff --git a/deploy/static/js/npm.js b/convlab/deploy/static/js/npm.js similarity index 100% rename from deploy/static/js/npm.js rename to convlab/deploy/static/js/npm.js diff --git a/deploy/static/js/vue.min.js b/convlab/deploy/static/js/vue.min.js similarity index 100% rename from deploy/static/js/vue.min.js rename to convlab/deploy/static/js/vue.min.js diff --git a/deploy/templates/dialog.html b/convlab/deploy/templates/dialog.html similarity index 100% rename from deploy/templates/dialog.html rename to convlab/deploy/templates/dialog.html diff --git a/deploy/utils/__init__.py b/convlab/deploy/utils/__init__.py similarity index 100% rename from deploy/utils/__init__.py rename to convlab/deploy/utils/__init__.py diff --git a/deploy/utils/error.py b/convlab/deploy/utils/error.py similarity index 100% rename from deploy/utils/error.py rename to convlab/deploy/utils/error.py diff --git a/deploy/utils/expire.py b/convlab/deploy/utils/expire.py similarity index 100% rename from deploy/utils/expire.py rename to convlab/deploy/utils/expire.py diff --git a/deploy/utils/lock.py b/convlab/deploy/utils/lock.py similarity index 100% rename from deploy/utils/lock.py rename to convlab/deploy/utils/lock.py diff --git a/requirements.txt b/requirements.txt index b9fecf8056275f8430c88d292bbb26565cba59a5..b6f4eef9cae94c001ae0608e673f5574dcec46fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,213 +1,102 @@ -absl-py==1.0.0 +absl-py==1.1.0 aiohttp==3.8.1 aiosignal==1.2.0 -allennlp==2.8.0 -argon2-cffi==21.1.0 -argon2-cffi-bindings==21.2.0 -async-timeout==4.0.1 -attrs==21.2.0 -autopep8==1.6.0 -backcall==0.2.0 -backports.csv==1.0.7 -base58==2.1.1 -beautifulsoup4==4.10.0 -bert-score==0.3.11 -bleach==4.1.0 -blis==0.7.5 -boto3==1.20.14 -botocore==1.23.14 -cached-path==0.3.2 -cachetools==4.2.4 -catalogue==2.0.6 -certifi==2021.10.8 -cffi==1.15.0 -chardet==4.0.0 -charset-normalizer==2.0.8 -checklist==0.0.11 -cheroot==8.5.2 -CherryPy==18.6.1 -click==8.0.3 -colorama==0.4.4 -configparser==5.1.0 -cryptography==36.0.0 +async-timeout==4.0.2 +attrs==21.4.0 +blis==0.7.7 +cachetools==5.2.0 +catalogue==2.0.7 +certifi==2022.5.18.1 +charset-normalizer==2.0.12 +click==8.1.3 cycler==0.11.0 cymem==2.0.6 -datasets==1.16.1 -debugpy==1.5.1 -decorator==5.1.0 +datasets==2.3.2 deepspeech==0.9.3 -defusedxml==0.7.1 -dill==0.3.4 -docker-pycreds==0.4.0 +dill==0.3.5.1 embeddings==0.0.8 -entrypoints==0.3 -fairscale==0.4.0 -feedparser==6.0.8 -filelock==3.3.2 -fonttools==4.28.5 -frozenlist==1.2.0 -fsspec==2021.11.1 -future==0.18.2 +filelock==3.7.1 +fonttools==4.33.3 +frozenlist==1.3.0 +fsspec==2022.5.0 fuzzywuzzy==0.18.0 -gitdb==4.0.9 -GitPython==3.1.24 -google-api-core==2.2.2 -google-auth==2.3.3 -google-cloud-core==2.2.1 -google-cloud-storage==1.43.0 -google-crc32c==1.3.0 -google-resumable-media==2.1.0 -googleapis-common-protos==1.53.0 -gTTS==2.2.3 -h5py==3.6.0 -huggingface-hub==0.1.2 +google-auth==2.8.0 +google-auth-oauthlib==0.4.6 +grpcio==1.46.3 +gTTS==2.2.4 +huggingface-hub==0.8.1 idna==3.3 -iniconfig==1.1.1 -ipykernel==6.5.1 -ipython==7.30.0 -ipython-genutils==0.2.0 -ipywidgets==7.6.5 -iso-639==0.4.5 -jaraco.classes==3.2.1 -jaraco.collections==3.4.0 -jaraco.functools==3.4.0 -jaraco.text==3.6.0 -jedi==0.18.1 +importlib-metadata==4.11.4 jieba==0.42.1 -Jinja2==3.0.3 -jmespath==0.10.0 +Jinja2==3.1.2 joblib==1.1.0 json-lines==0.5.0 -jsonnet==0.17.0 jsonpatch==1.32 -jsonpointer==2.2 -jsonschema==4.2.1 -jupyter==1.0.0 -jupyter-client==7.1.0 -jupyter-console==6.4.0 -jupyter-core==4.9.1 -jupyterlab-pygments==0.1.2 -jupyterlab-widgets==1.0.2 -kiwisolver==1.3.2 +jsonpointer==2.3 +kiwisolver==1.4.3 langcodes==3.3.0 -lmdb==1.2.1 -lxml==4.6.4 -MarkupSafe==2.0.1 -matplotlib==3.5.1 -matplotlib-inline==0.1.3 -mistune==0.8.4 -more-itertools==8.12.0 -multidict==5.2.0 -multiprocess==0.70.12.2 -munch==2.5.0 -murmurhash==1.0.6 -nbclient==0.5.9 -nbconvert==6.3.0 -nbformat==5.1.3 -nest-asyncio==1.5.1 -nltk==3.6.5 -notebook==6.4.6 -numpy==1.21.4 -overrides==3.1.0 +Markdown==3.3.7 +MarkupSafe==2.1.1 +matplotlib==3.5.2 +multidict==6.0.2 +multiprocess==0.70.13 +murmurhash==1.0.7 +nltk==3.7 +numpy==1.22.4 +oauthlib==3.2.0 packaging==21.3 -pandas==1.3.4 -pandocfilters==1.5.0 -parso==0.8.2 -pathtools==0.1.2 +pandas==1.4.2 pathy==0.6.1 -patternfork-nosql==3.6 -pdfminer.six==20211012 -pexpect==4.8.0 -pickleshare==0.7.5 -Pillow==8.4.0 -pipdeptree==2.2.0 -pluggy==1.0.0 -portalocker==2.3.2 -portend==3.1.0 +Pillow==9.1.1 preshed==3.0.6 -prometheus-client==0.12.0 -promise==2.3 -prompt-toolkit==3.0.23 -protobuf==3.19.1 -psutil==5.8.0 -ptyprocess==0.7.0 -py==1.11.0 -pyarrow==6.0.1 +protobuf==3.19.4 +pyarrow==8.0.0 pyasn1==0.4.8 pyasn1-modules==0.2.8 -pycodestyle==2.8.0 -pycparser==2.21 pydantic==1.8.2 pydub==0.25.1 -Pygments==2.10.0 -pyparsing==3.0.6 -pyrsistent==0.18.0 -pytest==6.2.5 +pyparsing==3.0.9 python-dateutil==2.8.2 -python-docx==0.8.11 python-Levenshtein==0.12.2 -pytokenizations==0.8.4 -pytz==2021.3 +pytz==2022.1 PyYAML==6.0 -pyzmq==22.3.0 -qtconsole==5.2.1 -QtPy==1.11.2 -quadprog==0.1.10 -regex==2021.11.10 -requests==2.26.0 -rouge-score==0.0.4 +pyzmq==23.2.0 +quadprog==0.1.11 +regex==2022.6.2 +requests==2.28.0 +requests-oauthlib==1.3.1 +responses==0.18.0 rsa==4.8 -s3transfer==0.5.0 -sacrebleu==2.0.0 -sacremoses==0.0.46 -scikit-learn==1.0.1 -scipy==1.7.3 -Send2Trash==1.8.0 -sentencepiece==0.1.96 -sentry-sdk==1.5.0 +scikit-learn==1.1.1 +scipy==1.8.1 seqeval==1.2.2 -sgmllib3k==1.0.0 -shortuuid==1.0.8 simplejson==3.17.6 six==1.16.0 smart-open==5.2.1 -smmap==5.0.0 -soupsieve==2.3.1 -spacy==3.1.4 -spacy-legacy==3.0.8 -spacy-loggers==1.0.1 -sqlitedict==1.7.0 -srsly==2.4.2 -subprocess32==3.5.4 -tabulate==0.8.9 -tempora==4.1.2 -tensorboardX==2.4.1 -termcolor==1.1.0 -terminado==0.12.1 -testpath==0.5.0 -thinc==8.0.13 -threadpoolctl==3.0.0 -tokenizers==0.10.3 -toml==0.10.2 -torch==1.8.1+cu101 +spacy==3.3.1 +spacy-legacy==3.0.9 +spacy-loggers==1.0.2 +srsly==2.4.3 +tabulate==0.8.10 +tensorboard==2.9.1 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +thinc==8.0.17 +threadpoolctl==3.1.0 +tokenizers==0.12.1 +torch==1.11.0 torchfile==0.1.0 -torchvision==0.9.1+cu101 tornado==6.1 -tqdm==4.62.3 -traitlets==5.1.1 -transformers==4.12.5 -typer==0.4.0 -typing_extensions==4.0.0 -Unidecode==1.3.2 -urllib3==1.26.7 +tqdm==4.64.0 +transformers==4.20.1 +typer==0.4.1 +typing_extensions==4.2.0 +Unidecode==1.3.4 +urllib3==1.26.9 visdom==0.1.8.9 -wandb==0.12.7 -wasabi==0.8.2 -wcwidth==0.2.5 -webencodings==0.5.1 -websocket-client==1.2.1 -widgetsnbextension==3.5.2 -xxhash==2.0.2 +wasabi==0.9.1 +websocket-client==1.3.3 +Werkzeug==2.1.2 +xxhash==3.0.0 yarl==1.7.2 -yaspin==2.1.0 -zc.lockfile==2.0 +zipp==3.8.0 diff --git a/setup.cfg b/setup.cfg old mode 100755 new mode 100644 diff --git a/setup.py b/setup.py index ef798d8ece195266dcda23b9ae8a1b3612822b8c..731e044b0e2b1dfbc529a651ed782e63f98a4543 100755 --- a/setup.py +++ b/setup.py @@ -1,44 +1,33 @@ ''' setup.py for ConvLab-3 ''' -import sys -import os from setuptools import setup, find_packages -from setuptools.command.test import test as TestCommand - - -class LibTest(TestCommand): - - def run_tests(self): - # import here, cause outside the eggs aren't loaded - ret = os.system("pytest --cov=ConvLab-3 tests/ --cov-report term-missing") - sys.exit(ret >> 8) - setup( - name='ConvLab', + name='convlab', version='3.0.0', - packages=find_packages(exclude=[]), + packages=find_packages(), license='Apache', - description='An Open-source Dialog System Toolkits', + description='An Open-source Dialog System Platform', long_description=open('README.md', encoding='UTF-8').read(), long_description_content_type="text/markdown", classifiers=[ - 'Development Status :: 3 - Alpha', - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Intended Audience :: Science/Research', - 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Development Status :: 3 - Alpha', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Intended Audience :: Science/Research', + 'Intended Audience :: Developers', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', ], install_requires=[ 'matplotlib', 'tabulate', 'python-Levenshtein', 'requests', - 'numpy', + 'numpy>=1.22', 'nltk', 'scipy', 'tensorboard', @@ -72,12 +61,12 @@ setup( "lxml", ] }, - cmdclass={'test': LibTest}, + cmdclass={}, entry_points={}, include_package_data=True, url='https://github.com/ConvLab/ConvLab-3', - author='thu-coai', - author_email='thu-coai-developer@googlegroups.com', + author='convlab', + author_email='convlab@googlegroups.com', python_requires='>=3.6', zip_safe=False ) diff --git a/tests/results/BERTNLU-RuleDST-GDPLPolicy-TemplateNLG/Frequency_of_domain.png b/tests/results/BERTNLU-RuleDST-GDPLPolicy-TemplateNLG/Frequency_of_domain.png deleted file mode 100755 index 08425accf37ad68275aba67a145db0df84957185..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-GDPLPolicy-TemplateNLG/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-GDPLPolicy-TemplateNLG/Performance_for_each_domain.png b/tests/results/BERTNLU-RuleDST-GDPLPolicy-TemplateNLG/Performance_for_each_domain.png deleted file mode 100755 index a98fe2757cd56f5603e2fe4525e45f6c19876a01..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-GDPLPolicy-TemplateNLG/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-GDPLPolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png b/tests/results/BERTNLU-RuleDST-GDPLPolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index 295a3da96650057c1160910d2dc9fc5d7da38976..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-GDPLPolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-GDPLPolicy-TemplateNLG/report_multiwoz.html b/tests/results/BERTNLU-RuleDST-GDPLPolicy-TemplateNLG/report_multiwoz.html deleted file mode 100755 index 44bbabf8c7447f49f6524c0e7b311dcae48a8b66..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-GDPLPolicy-TemplateNLG/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: BERTNLU-RuleDST-GDPLPolicy-TemplateNLG</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-27 02:32:42</h3><h3>Overall Results</h3><p> Success Rate: 49.5 %</p><p> (Precision, Recall, F1) : (0.670, 0.764, 0.682) </p><p> Average Dialog Turn (Succ): 11.479 </p><p> Average Dialog Turn (All): 24.328 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">400</td><td align="center">0.378</td><td align="center">0.406</td><td align="center">0.478</td><td align="center">0.420</td><td align="center">0.532</td><td align="center">7.338</td><td align="center">21.630</td></tr><tr><td align="center">restaurant</td><td align="center">390</td><td align="center">0.674</td><td align="center">0.531</td><td align="center">0.779</td><td align="center">0.605</td><td align="center">0.310</td><td align="center">7.658</td><td align="center">15.036</td></tr><tr><td align="center">attraction</td><td align="center">302</td><td align="center">0.881</td><td align="center">0.764</td><td align="center">0.920</td><td align="center">0.807</td><td align="center">0.099</td><td align="center">6.541</td><td align="center">9.642</td></tr><tr><td align="center">taxi</td><td align="center">104</td><td align="center">0.558</td><td align="center">0.596</td><td align="center">0.577</td><td align="center">0.583</td><td align="center">0.433</td><td align="center">7.276</td><td align="center">15.308</td></tr><tr><td align="center">train</td><td align="center">344</td><td align="center">0.820</td><td align="center">0.904</td><td align="center">0.914</td><td align="center">0.906</td><td align="center">0.102</td><td align="center">5.809</td><td align="center">8.331</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">1.000</td><td align="center">0.587</td><td align="center">1.000</td><td align="center">0.733</td><td align="center">0.000</td><td align="center">2.000</td><td align="center">2.000</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">0.933</td><td align="center">0.933</td><td align="center">0.933</td><td align="center">0.933</td><td align="center">0.067</td><td align="center">4.000</td><td align="center">6.400</td></tr></table><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 37.8 %</p><p> (Precision, Recall, F1) : (0.406, 0.478, 0.420) </p><p> Average Dialog Turn (Succ): 7.338 </p><p> Average Dialog Turn (All): 21.630 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 368</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-hotel Occur Num: 66</li><li>Inform-Hotel-Internet-yes Occur Num: 60</li><li>Inform-Hotel-Type-guesthouse Occur Num: 44</li><li>Inform-Hotel-Stars-4 Occur Num: 43</li><li>Inform-Hotel-Price-moderate Occur Num: 32</li></ul></ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 355</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 53</li><li>Inform-Hotel-Type-hotel Occur Num: 51</li><li>Inform-Hotel-Type-guesthouse Occur Num: 43</li><li>Inform-Hotel-Stars-4 Occur Num: 41</li><li>Inform-Hotel-Price-moderate Occur Num: 33</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 219</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 43</li><li>Inform-Hotel-Parking-yes Occur Num: 41</li><li>Inform-Hotel-Stars-4 Occur Num: 22</li><li>Inform-Hotel-Type-hotel Occur Num: 21</li><li>Inform-Hotel-Area-north Occur Num: 15</li></ul></ul><li>Inform-Hotel-Stars-4<ul><li>Occur Num: 210</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 41</li><li>Inform-Hotel-Parking-yes Occur Num: 41</li><li>Inform-Hotel-Type-hotel Occur Num: 36</li><li>Inform-Hotel-Type-guesthouse Occur Num: 22</li><li>Inform-Hotel-Price-cheap Occur Num: 11</li></ul></ul><li>Inform-Hotel-Price-moderate<ul><li>Occur Num: 185</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-hotel Occur Num: 38</li><li>Inform-Hotel-Internet-yes Occur Num: 34</li><li>Inform-Hotel-Parking-yes Occur Num: 31</li><li>Inform-Hotel-Type-guesthouse Occur Num: 15</li><li>Inform-Hotel-Stars-4 Occur Num: 11</li></ul></ul><li>Request-Hotel-Addr-?<ul><li>Occur Num: 148</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 25</li><li>Request-Hotel-Post-? Occur Num: 22</li><li>Request-Hotel-Phone-? Occur Num: 18</li><li>Request-Attraction-Addr-? Occur Num: 13</li><li>Request-Hotel-Internet-? Occur Num: 10</li></ul></ul><li>Inform-Hotel-Area-north<ul><li>Occur Num: 130</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 25</li><li>Inform-Hotel-Parking-yes Occur Num: 24</li><li>Inform-Hotel-Type-hotel Occur Num: 23</li><li>Inform-Hotel-Type-guesthouse Occur Num: 15</li><li>Inform-Hotel-Stars-4 Occur Num: 9</li></ul></ul><li>Inform-Hotel-Price-cheap<ul><li>Occur Num: 89</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-hotel Occur Num: 19</li><li>Inform-Hotel-Internet-yes Occur Num: 17</li><li>Inform-Hotel-Parking-yes Occur Num: 15</li><li>Inform-Hotel-Stars-4 Occur Num: 11</li><li>Inform-Hotel-Type-guesthouse Occur Num: 11</li></ul></ul><li>Inform-Hotel-Parking-no<ul><li>Occur Num: 88</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 19</li><li>Inform-Hotel-Internet-yes Occur Num: 12</li><li>Inform-Hotel-Type-guesthouse Occur Num: 10</li><li>Inform-Hotel-Type-hotel Occur Num: 10</li><li>Inform-Hotel-Area-north Occur Num: 8</li></ul></ul><li>Inform-Hotel-Area-centre<ul><li>Occur Num: 87</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 20</li><li>Inform-Hotel-Type-hotel Occur Num: 13</li><li>Inform-Hotel-Type-guesthouse Occur Num: 12</li><li>Inform-Hotel-Parking-yes Occur Num: 10</li><li>Inform-Hotel-Price-moderate Occur Num: 10</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Request-Hotel-Area-?<ul><li>Occur Num: 466</li><li>NLU Output</li><ul><li>Request-Hotel-Price-? Occur Num: 45</li><li>Inform-Hotel-Choice-3 Occur Num: 39</li><li>Inform-Hotel-Choice-0 Occur Num: 35</li><li>Inform-Hotel-Type-guesthouse Occur Num: 33</li><li>NoOffer-Hotel-none-none Occur Num: 22</li></ul></ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 431</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-none Occur Num: 69</li><li>Inform-Booking-none-none Occur Num: 66</li><li>Request-Booking-Day-? Occur Num: 17</li><li>Request-Booking-Stay-? Occur Num: 15</li><li>Inform-Hotel-Parking-none Occur Num: 13</li></ul></ul><li>Inform-Hotel-Name-acorn guest house<ul><li>Occur Num: 427</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 57</li><li>Recommend-Hotel-Name-acorn guest house Occur Num: 53</li><li>Inform-Hotel-Area-north Occur Num: 33</li><li>reqmore-general-none-none Occur Num: 26</li><li>Recommend-Hotel-Name-Acorn guest house Occur Num: 21</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 389</li><li>NLU Output</li><ul><li>Request-Hotel-Area-? Occur Num: 43</li><li>Inform-Booking-none-none Occur Num: 43</li><li>Recommend-Hotel-Type-guesthouse Occur Num: 29</li><li>Inform-Hotel-Area-north Occur Num: 21</li><li>Inform-Hotel-Choice-2 Occur Num: 19</li></ul></ul><li>Inform-Hotel-Name-huntingdon marriott hotel<ul><li>Occur Num: 323</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 30</li><li>Recommend-Hotel-Name-huntingdon marriott hotel Occur Num: 29</li><li>Inform-Hotel-Choice-3 Occur Num: 23</li><li>reqmore-general-none-none Occur Num: 17</li><li>Request-Hotel-Area-? Occur Num: 16</li></ul></ul><li>Inform-Hotel-Type-hotel<ul><li>Occur Num: 306</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 38</li><li>Inform-Hotel-Choice-2 Occur Num: 29</li><li>Inform-Hotel-Choice-3 Occur Num: 22</li><li>Request-Hotel-Area-? Occur Num: 21</li><li>Recommend-Hotel-Name-huntingdon marriott hotel Occur Num: 13</li></ul></ul><li>Inform-Hotel-Area-north<ul><li>Occur Num: 297</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 50</li><li>Recommend-Hotel-Name-acorn guest house Occur Num: 32</li><li>Inform-Hotel-Price-moderate Occur Num: 19</li><li>Recommend-Hotel-Name-ashley hotel Occur Num: 15</li><li>Recommend-Hotel-Type-guesthouse Occur Num: 15</li></ul></ul><li>Inform-Hotel-Name-ashley hotel<ul><li>Occur Num: 261</li><li>NLU Output</li><ul><li>Recommend-Hotel-Name-ashley hotel Occur Num: 26</li><li>Inform-Booking-none-none Occur Num: 25</li><li>reqmore-general-none-none Occur Num: 17</li><li>Inform-Hotel-Addr-74 chesterton road Occur Num: 17</li><li>Inform-Hotel-Choice-2 Occur Num: 16</li></ul></ul><li>Inform-Hotel-Price-moderate<ul><li>Occur Num: 258</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 38</li><li>Inform-Hotel-Area-north Occur Num: 21</li><li>Recommend-Hotel-Price-moderate Occur Num: 17</li><li>Recommend-Hotel-Name-acorn guest house Occur Num: 16</li><li>Request-Hotel-Area-? Occur Num: 12</li></ul></ul><li>Inform-Hotel-Choice-2<ul><li>Occur Num: 206</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 33</li><li>Inform-Hotel-Type-guesthouse Occur Num: 15</li><li>Request-Hotel-Area-? Occur Num: 12</li><li>Inform-Hotel-Internet-none Occur Num: 9</li><li>Inform-Hotel-Price-moderate Occur Num: 9</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Hotel-Addr-? Occur Num: 29</li><li>Inform-Hotel-Stay-4 Occur Num: 19</li><li>Request-Hotel-Stars-? Occur Num: 16</li><li>Request-Hotel-Price-? Occur Num: 14</li><li>Request-Hotel-Type-? Occur Num: 12</li><li>Request-Hotel-Post-? Occur Num: 12</li><li>Inform-Hotel-Stay-3 Occur Num: 12</li><li>Request-Hotel-Area-? Occur Num: 10</li><li>Request-Hotel-Phone-? Occur Num: 10</li><li>Inform-Hotel-Stay-5 Occur Num: 9</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-internet Occur Num: 72</li><li>inform-hotel-parking Occur Num: 36</li><li>inform-hotel-area Occur Num: 35</li><li>inform-hotel-stars Occur Num: 27</li><li>recommend-hotel-parking Occur Num: 14</li><li>recommend-hotel-internet Occur Num: 13</li><li>inform-hotel-postcode Occur Num: 2</li><li>recommend-hotel-stars Occur Num: 2</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-address Occur Num: 41</li><li>request-hotel-internet Occur Num: 25</li><li>request-hotel-parking Occur Num: 22</li><li>request-hotel-stars Occur Num: 22</li><li>request-hotel-postcode Occur Num: 19</li><li>request-hotel-phone Occur Num: 18</li><li>request-hotel-pricerange Occur Num: 17</li><li>request-hotel-type Occur Num: 15</li><li>request-hotel-area Occur Num: 13</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-type Occur Num: 45</li><li>inform-hotel-area Occur Num: 37</li><li>inform-hotel-stars Occur Num: 3</li><li>inform-hotel-phone Occur Num: 1</li></ul></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 67.4 %</p><p> (Precision, Recall, F1) : (0.531, 0.779, 0.605) </p><p> Average Dialog Turn (Succ): 7.658 </p><p> Average Dialog Turn (All): 15.036 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Restaurant-Addr-?<ul><li>Occur Num: 84</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 35</li><li>Request-Restaurant-Post-? Occur Num: 22</li><li>Request-Attraction-Addr-? Occur Num: 11</li><li>Request-Attraction-Phone-? Occur Num: 6</li><li>Request-Attraction-Post-? Occur Num: 4</li></ul></ul><li>Request-Restaurant-Phone-?<ul><li>Occur Num: 83</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 36</li><li>Request-Attraction-Phone-? Occur Num: 20</li><li>Request-Restaurant-Post-? Occur Num: 14</li><li>Request-Attraction-Addr-? Occur Num: 6</li><li>Request-Attraction-Post-? Occur Num: 4</li></ul></ul><li>Inform-Restaurant-Food-dontcare<ul><li>Occur Num: 73</li><li>NLU Output</li><ul><li>Inform-Restaurant-Food-do n't care Occur Num: 51</li><li>Inform-Restaurant-Price-expensive Occur Num: 12</li><li>Inform-Restaurant-Price-moderate Occur Num: 8</li><li>Inform-Restaurant-Price-expensively Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 68</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensively Occur Num: 32</li><li>Inform-Restaurant-Food-do n't care Occur Num: 14</li><li>Inform-Restaurant-Area-do n't care Occur Num: 3</li><li>Inform-Restaurant-Area-east Occur Num: 3</li><li>Inform-Restaurant-Area-centre Occur Num: 3</li></ul></ul><li>Inform-Restaurant-Price-moderate<ul><li>Occur Num: 57</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 26</li><li>Inform-Restaurant-Food-do n't care Occur Num: 8</li><li>Inform-Restaurant-Area-centre Occur Num: 7</li><li>Inform-Restaurant-Area-do n't care Occur Num: 7</li><li>Inform-Restaurant-Food-british Occur Num: 2</li></ul></ul><li>Request-Restaurant-Post-?<ul><li>Occur Num: 53</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 22</li><li>Request-Restaurant-Phone-? Occur Num: 14</li><li>Request-Attraction-Post-? Occur Num: 8</li><li>Request-Attraction-Phone-? Occur Num: 3</li><li>Request-Attraction-Addr-? Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Area-dontcare<ul><li>Occur Num: 39</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-do n't care Occur Num: 26</li><li>Inform-Restaurant-Price-moderate Occur Num: 6</li><li>Inform-Restaurant-Price-cheap Occur Num: 3</li><li>Inform-Restaurant-Price-expensive Occur Num: 2</li><li>Inform-Restaurant-Price-expensively Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 25</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 5</li><li>Inform-Restaurant-Price-cheap Occur Num: 4</li><li>Inform-Restaurant-Price-expensive Occur Num: 3</li><li>Inform-Restaurant-Food-british Occur Num: 3</li><li>Inform-Restaurant-Price-moderate Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Price-cheap<ul><li>Occur Num: 24</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-cheaply Occur Num: 10</li><li>Inform-Restaurant-Area-do n't care Occur Num: 3</li><li>Request-Train-Ticket-? Occur Num: 2</li><li>Inform-Restaurant-Food-italian Occur Num: 2</li><li>Inform-Restaurant-Area-centre Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Food-portuguese<ul><li>Occur Num: 11</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 3</li><li>Inform-Restaurant-Price-expensive Occur Num: 3</li><li>Inform-Restaurant-Price-moderate Occur Num: 2</li><li>Inform-Restaurant-Area-north Occur Num: 1</li><li>Inform-Restaurant-Price-cheap Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 1259</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 87</li><li>Recommend-Restaurant-Price-cheap Occur Num: 68</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 65</li><li>Inform-Restaurant-Food-italian Occur Num: 53</li><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 51</li></ul></ul><li>Inform-Restaurant-Price-cheap<ul><li>Occur Num: 958</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-cheap Occur Num: 101</li><li>Request-Restaurant-Food-? Occur Num: 76</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 71</li><li>Inform-Restaurant-Area-centre Occur Num: 70</li><li>Inform-Restaurant-Food-italian Occur Num: 51</li></ul></ul><li>Inform-Restaurant-Name-pizza hut city centre<ul><li>Occur Num: 916</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 95</li><li>reqmore-general-none-none Occur Num: 89</li><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 83</li><li>Recommend-Restaurant-Price-cheap Occur Num: 74</li><li>Inform-Restaurant-Phone-01223323737 Occur Num: 71</li></ul></ul><li>Inform-Restaurant-Food-italian<ul><li>Occur Num: 577</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 62</li><li>Recommend-Restaurant-Price-cheap Occur Num: 60</li><li>Inform-Restaurant-Area-centre Occur Num: 51</li><li>Recommend-Restaurant-Food-italian Occur Num: 46</li><li>Inform-Booking-none-none Occur Num: 45</li></ul></ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 564</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-expensive Occur Num: 50</li><li>Inform-Booking-none-none Occur Num: 48</li><li>Inform-Restaurant-Food-indian Occur Num: 31</li><li>Request-Restaurant-Area-? Occur Num: 26</li><li>Inform-Restaurant-Area-centre Occur Num: 26</li></ul></ul><li>Inform-Restaurant-Phone-01223323737<ul><li>Occur Num: 558</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 93</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 75</li><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 67</li><li>Recommend-Restaurant-Phone-01223323737 Occur Num: 51</li><li>Inform-Restaurant-Food-italian Occur Num: 30</li></ul></ul><li>Inform-Restaurant-Addr-Regent Street City Centre<ul><li>Occur Num: 548</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 83</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 79</li><li>Inform-Restaurant-Phone-01223323737 Occur Num: 61</li><li>Recommend-Restaurant-Price-cheap Occur Num: 39</li><li>Inform-Restaurant-Food-italian Occur Num: 33</li></ul></ul><li>Request-Restaurant-Food-?<ul><li>Occur Num: 382</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 46</li><li>Inform-Restaurant-Price-cheap Occur Num: 44</li><li>Inform-Restaurant-Choice-69 Occur Num: 43</li><li>Inform-Restaurant-Choice-110 Occur Num: 36</li><li>Recommend-Restaurant-Price-cheap Occur Num: 33</li></ul></ul><li>Recommend-Restaurant-Name-pizza hut city centre<ul><li>Occur Num: 359</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-cheap Occur Num: 64</li><li>Inform-Restaurant-Area-centre Occur Num: 40</li><li>Inform-Booking-none-none Occur Num: 39</li><li>Inform-Restaurant-Food-italian Occur Num: 36</li><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 27</li></ul></ul><li>Inform-Restaurant-Price-moderate<ul><li>Occur Num: 318</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-moderate Occur Num: 30</li><li>Inform-Restaurant-Area-centre Occur Num: 25</li><li>Inform-Booking-none-none Occur Num: 24</li><li>Inform-Restaurant-Choice-2 Occur Num: 20</li><li>Inform-Restaurant-Food-british Occur Num: 20</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Inform-Restaurant-Price-expensive Occur Num: 21</li><li>Inform-Restaurant-Price-moderate Occur Num: 21</li><li>Request-Restaurant-Phone-? Occur Num: 19</li><li>Request-Restaurant-Addr-? Occur Num: 6</li><li>Request-Restaurant-Area-? Occur Num: 5</li><li>Inform-Restaurant-Day-saturday Occur Num: 3</li><li>Request-Restaurant-Food-? Occur Num: 3</li><li>Inform-Restaurant-Day-sunday Occur Num: 3</li><li>Inform-Restaurant-Time-18:30 Occur Num: 2</li><li>Inform-Restaurant-Day-tuesday Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-restaurant-area Occur Num: 6</li><li>inform-restaurant-postcode Occur Num: 2</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-phone Occur Num: 18</li><li>request-restaurant-address Occur Num: 13</li><li>request-restaurant-postcode Occur Num: 12</li><li>request-restaurant-area Occur Num: 5</li><li>request-restaurant-food Occur Num: 3</li><li>request-restaurant-pricerange Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-phone Occur Num: 190</li><li>inform-restaurant-food Occur Num: 76</li><li>inform-restaurant-area Occur Num: 70</li></ul></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 88.1 %</p><p> (Precision, Recall, F1) : (0.764, 0.920, 0.807) </p><p> Average Dialog Turn (Succ): 6.541 </p><p> Average Dialog Turn (All): 9.642 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Attraction-Post-?<ul><li>Occur Num: 153</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 58</li><li>Request-Attraction-Addr-? Occur Num: 40</li><li>Request-Attraction-Fee-? Occur Num: 32</li><li>Request-Restaurant-Post-? Occur Num: 14</li><li>Request-Restaurant-Phone-? Occur Num: 3</li></ul></ul><li>Request-Attraction-Phone-?<ul><li>Occur Num: 149</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 58</li><li>Request-Attraction-Addr-? Occur Num: 35</li><li>Request-Attraction-Fee-? Occur Num: 26</li><li>Request-Restaurant-Phone-? Occur Num: 14</li><li>Request-Restaurant-Addr-? Occur Num: 6</li></ul></ul><li>Request-Attraction-Addr-?<ul><li>Occur Num: 132</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 40</li><li>Request-Attraction-Phone-? Occur Num: 36</li><li>Request-Attraction-Fee-? Occur Num: 17</li><li>Request-Restaurant-Addr-? Occur Num: 15</li><li>Request-Restaurant-Post-? Occur Num: 10</li></ul></ul><li>Request-Attraction-Fee-?<ul><li>Occur Num: 73</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 32</li><li>Request-Attraction-Phone-? Occur Num: 26</li><li>Request-Attraction-Addr-? Occur Num: 15</li></ul></ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 38</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-entertainment Occur Num: 14</li><li>Inform-Attraction-Type-do n't care Occur Num: 4</li><li>Inform-Restaurant-Area-centre Occur Num: 3</li><li>Inform-Attraction-Type-museum Occur Num: 3</li><li>Inform-Attraction-Type-museums Occur Num: 3</li></ul></ul><li>Inform-Attraction-Area-dontcare<ul><li>Occur Num: 36</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-do n't care Occur Num: 22</li><li>Request-Attraction-Area-? Occur Num: 13</li><li>Inform-Attraction-Type-museum Occur Num: 1</li></ul></ul><li>Inform-Attraction-Type-dontcare<ul><li>Occur Num: 35</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-do n't care Occur Num: 22</li><li>Request-Attraction-Type-? Occur Num: 7</li><li>Inform-Attraction-Area-centre Occur Num: 4</li><li>Inform-Attraction-Area-west Occur Num: 1</li><li>Inform-Attraction-Area-east Occur Num: 1</li></ul></ul><li>Inform-Attraction-Type-museum<ul><li>Occur Num: 30</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 12</li><li>Inform-Attraction-Type-museums Occur Num: 9</li><li>Inform-Attraction-Type-entertainment Occur Num: 5</li><li>Inform-Attraction-Area-west Occur Num: 3</li><li>Inform-Attraction-Area-do n't care Occur Num: 1</li></ul></ul><li>Inform-Attraction-Type-swimmingpool<ul><li>Occur Num: 17</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 6</li><li>Inform-Attraction-none-none Occur Num: 5</li><li>Inform-Attraction-Type-entertainment Occur Num: 2</li><li>Inform-Attraction-Area-east Occur Num: 2</li><li>Inform-Attraction-Area-north Occur Num: 2</li></ul></ul><li>Request-Attraction-Area-?<ul><li>Occur Num: 16</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 5</li><li>Request-Attraction-Addr-? Occur Num: 4</li><li>Request-Restaurant-Area-? Occur Num: 4</li><li>Request-Attraction-Post-? Occur Num: 2</li><li>Request-Attraction-Fee-? Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 796</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 51</li><li>Request-Attraction-Type-? Occur Num: 43</li><li>Inform-Attraction-Choice-44 Occur Num: 43</li><li>Inform-Attraction-Type-architecture Occur Num: 43</li><li>Inform-Attraction-Type-theatre Occur Num: 39</li></ul></ul><li>Inform-Attraction-Fee-?<ul><li>Occur Num: 502</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 68</li><li>Inform-Attraction-Addr-park street Occur Num: 18</li><li>Inform-Attraction-Choice-44 Occur Num: 15</li><li>Inform-Attraction-Phone-01223300085 Occur Num: 15</li><li>Inform-Attraction-Area-centre Occur Num: 15</li></ul></ul><li>Inform-Attraction-Fee-free<ul><li>Occur Num: 496</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 80</li><li>Inform-Attraction-Area-centre Occur Num: 25</li><li>Inform-Attraction-Addr-98 king street Occur Num: 19</li><li>Inform-Attraction-Type-park Occur Num: 16</li><li>Inform-Attraction-Name-broughton house gallery Occur Num: 13</li></ul></ul><li>Inform-Attraction-Type-museum<ul><li>Occur Num: 346</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 33</li><li>Inform-Attraction-Choice-11 Occur Num: 27</li><li>Request-Attraction-Type-? Occur Num: 19</li><li>Inform-Attraction-Area-west Occur Num: 19</li><li>reqmore-general-none-none Occur Num: 16</li></ul></ul><li>Request-Attraction-Type-?<ul><li>Occur Num: 344</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 48</li><li>Inform-Attraction-Choice-44 Occur Num: 42</li><li>Inform-Attraction-Type-theatre Occur Num: 38</li><li>Inform-Attraction-Type-architecture Occur Num: 33</li><li>Inform-Attraction-Type-museum Occur Num: 18</li></ul></ul><li>Inform-Attraction-Name-adc theatre<ul><li>Occur Num: 307</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 46</li><li>Inform-Attraction-Area-centre Occur Num: 27</li><li>Recommend-Attraction-Name-adc theatre Occur Num: 27</li><li>Inform-Attraction-Addr-park street Occur Num: 24</li><li>Inform-Attraction-Choice-44 Occur Num: 19</li></ul></ul><li>Inform-Attraction-Name-broughton house gallery<ul><li>Occur Num: 305</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 35</li><li>Inform-Attraction-Area-centre Occur Num: 29</li><li>Inform-Attraction-Addr-98 king street Occur Num: 27</li><li>Inform-Attraction-Choice-11 Occur Num: 27</li><li>Inform-Attraction-Fee-free Occur Num: 26</li></ul></ul><li>Inform-Attraction-Type-theatre<ul><li>Occur Num: 279</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 46</li><li>Inform-Attraction-Choice-44 Occur Num: 42</li><li>Request-Attraction-Type-? Occur Num: 37</li><li>Inform-Attraction-Type-architecture Occur Num: 33</li><li>Inform-Attraction-Name-adc theatre Occur Num: 12</li></ul></ul><li>Inform-Attraction-Type-architecture<ul><li>Occur Num: 253</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 48</li><li>Inform-Attraction-Choice-44 Occur Num: 37</li><li>Inform-Attraction-Type-theatre Occur Num: 35</li><li>Request-Attraction-Type-? Occur Num: 33</li><li>Inform-Attraction-Choice-5 Occur Num: 10</li></ul></ul><li>Inform-Attraction-Name-christ's college<ul><li>Occur Num: 248</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 28</li><li>Inform-Attraction-Area-centre Occur Num: 22</li><li>Inform-Attraction-Fee-free Occur Num: 21</li><li>Inform-Attraction-Name-christ 's college Occur Num: 19</li><li>Recommend-Attraction-Name-christ 's college Occur Num: 18</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Attraction-Post-? Occur Num: 12</li><li>Request-Attraction-Addr-? Occur Num: 10</li><li>Request-Attraction-Area-? Occur Num: 4</li><li>Request-Attraction-Fee-? Occur Num: 3</li><li>Request-Attraction-Phone-? Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-attraction-area Occur Num: 47</li><li>recommend-attraction-area Occur Num: 1</li><li>inform-attraction-postcode Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-address Occur Num: 18</li><li>request-attraction-postcode Occur Num: 15</li><li>request-attraction-area Occur Num: 7</li><li>request-attraction-phone Occur Num: 5</li><li>request-attraction-entrance fee Occur Num: 3</li><li>request-attraction-type Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-type Occur Num: 58</li><li>inform-attraction-area Occur Num: 45</li><li>inform-attraction-phone Occur Num: 40</li></ul></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 55.8 %</p><p> (Precision, Recall, F1) : (0.596, 0.577, 0.583) </p><p> Average Dialog Turn (Succ): 7.276 </p><p> Average Dialog Turn (All): 15.308 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Phone-?<ul><li>Occur Num: 45</li><li>NLU Output</li><ul><li>Request-Taxi-Car-? Occur Num: 45</li></ul></ul><li>Inform-Taxi-Depart-the fitzwilliam museum<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 2</li><li>Inform-Taxi-Dest-golden house Occur Num: 1</li><li>Inform-Taxi-Leave-13:45 Occur Num: 1</li><li>Inform-Taxi-Arrive-13:45 Occur Num: 1</li><li>Inform-Taxi-Dest-la mimosa Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-home from home<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-home from home Occur Num: 1</li><li>Inform-Taxi-Depart-arbury lodge guesthouse Occur Num: 1</li><li>Inform-Taxi-Leave-02:45 Occur Num: 1</li><li>Inform-Taxi-Depart-thanh binh Occur Num: 1</li><li>Inform-Taxi-Dest-home Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the lensfield hotel<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li><li>Inform-Taxi-Dest-williams art and antiques Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-thanh binh<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li><li>Inform-Taxi-Dest-thanh Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-02:00<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-efes restaurant<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-broxbourne train station Occur Num: 1</li><li>Inform-Restaurant-Name-efes Occur Num: 1</li><li>Inform-Taxi-Arrive-16:15 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-kohinoor<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-grafton hotel restaurant Occur Num: 1</li><li>Inform-Restaurant-Name-kohinoor Occur Num: 1</li><li>Inform-Taxi-Arrive-18:30 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the good luck chinese food takeaway<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-nandos Occur Num: 1</li><li>Inform-Taxi-Leave-05:15 Occur Num: 1</li><li>Inform-Taxi-Depart-good luck chinese food takeaway Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-15:15<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-15:15 Occur Num: 1</li><li>Inform-Taxi-Depart-rajmahal Occur Num: 1</li><li>Inform-Taxi-Dest-good luck chinese food takeaway Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Dest-?<ul><li>Occur Num: 34</li><li>NLU Output</li><ul><li>Inform-Taxi-Car-lexus Occur Num: 2</li><li>Inform-Taxi-Car-audi Occur Num: 2</li><li>Inform-Taxi-Car-volkswagen Occur Num: 2</li><li>Inform-Taxi-Car-skoda Occur Num: 2</li><li>Inform-Taxi-Phone-93793671822 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-ford<ul><li>Occur Num: 33</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 12</li><li>Request-Taxi-Dest-? Occur Num: 3</li><li>Inform-Taxi-Phone-93793671822 Occur Num: 1</li><li>Inform-Taxi-Phone-93595337635 Occur Num: 1</li><li>Request-Taxi-Depart-? Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-bmw<ul><li>Occur Num: 29</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 13</li><li>Request-Taxi-Dest-? Occur Num: 2</li><li>Inform-Taxi-Phone-29268185547 Occur Num: 1</li><li>Inform-Taxi-Phone-31857656946 Occur Num: 1</li><li>Inform-Taxi-Phone-89881559925 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-honda<ul><li>Occur Num: 27</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 10</li><li>Request-Taxi-Dest-? Occur Num: 2</li><li>Inform-Taxi-Phone-43785244638 Occur Num: 1</li><li>Inform-Taxi-Phone-87187563117 Occur Num: 1</li><li>Inform-Taxi-Phone-14488657765 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-lexus<ul><li>Occur Num: 27</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 7</li><li>Request-Taxi-Dest-? Occur Num: 4</li><li>Inform-Taxi-Phone-41948119556 Occur Num: 1</li><li>Inform-Taxi-Phone-51157786596 Occur Num: 1</li><li>Inform-Taxi-Phone-55381758639 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-volkswagen<ul><li>Occur Num: 27</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 9</li><li>Request-Taxi-Dest-? Occur Num: 4</li><li>Inform-Taxi-Phone-24639761748 Occur Num: 1</li><li>Inform-Taxi-Phone-95286317471 Occur Num: 1</li><li>Inform-Taxi-Phone-85574585429 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-toyota<ul><li>Occur Num: 25</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 11</li><li>Request-Taxi-Dest-? Occur Num: 2</li><li>Inform-Taxi-Phone-37424256488 Occur Num: 1</li><li>Inform-Taxi-Phone-19159546163 Occur Num: 1</li><li>Inform-Taxi-Phone-73638356465 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-audi<ul><li>Occur Num: 23</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 8</li><li>Request-Taxi-Dest-? Occur Num: 2</li><li>Inform-Taxi-Phone-23616226613 Occur Num: 1</li><li>Inform-Taxi-Phone-32349919343 Occur Num: 1</li><li>Inform-Taxi-Phone-54929313192 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-skoda<ul><li>Occur Num: 22</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 8</li><li>Request-Taxi-Dest-? Occur Num: 3</li><li>Inform-Taxi-Phone-91988538946 Occur Num: 1</li><li>Inform-Taxi-Phone-15315982942 Occur Num: 1</li><li>Inform-Taxi-Phone-17381338962 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-volvo<ul><li>Occur Num: 20</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 10</li><li>Inform-Taxi-Phone-96979485925 Occur Num: 1</li><li>Inform-Taxi-Phone-61262159536 Occur Num: 1</li><li>Inform-Taxi-Phone-65418143719 Occur Num: 1</li><li>Inform-Taxi-Phone-66148921575 Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Taxi-Car-? Occur Num: 45</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-taxi-car type Occur Num: 46</li><li>request-taxi-phone Occur Num: 42</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 82.0 %</p><p> (Precision, Recall, F1) : (0.904, 0.914, 0.906) </p><p> Average Dialog Turn (Succ): 5.809 </p><p> Average Dialog Turn (All): 8.331 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Train-Dest-cambridge<ul><li>Occur Num: 120</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 12</li><li>Inform-Train-Day-wednesday Occur Num: 8</li><li>Inform-Train-Day-thursday Occur Num: 8</li><li>Inform-Train-Day-friday Occur Num: 6</li><li>Inform-Train-Day-monday Occur Num: 6</li></ul></ul><li>Inform-Train-Depart-cambridge<ul><li>Occur Num: 117</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 31</li><li>Inform-Train-Day-thursday Occur Num: 9</li><li>Inform-Train-Day-friday Occur Num: 7</li><li>Inform-Train-Day-tuesday Occur Num: 6</li><li>Inform-Train-Day-wednesday Occur Num: 5</li></ul></ul><li>Inform-Train-Day-monday<ul><li>Occur Num: 38</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 5</li><li>Inform-Train-Day-Saturday Occur Num: 3</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Dest-leicester Occur Num: 3</li><li>Inform-Train-Dest-norwich Occur Num: 2</li></ul></ul><li>Inform-Train-Day-friday<ul><li>Occur Num: 32</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 4</li><li>Inform-Train-Dest-cambridge Occur Num: 4</li><li>Inform-Train-Day-Saturday Occur Num: 3</li><li>Inform-Train-Arrive-11:15 Occur Num: 2</li><li>Inform-Restaurant-Day-friday Occur Num: 2</li></ul></ul><li>Inform-Train-Day-tuesday<ul><li>Occur Num: 32</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 4</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Day-Saturday Occur Num: 3</li><li>Inform-Train-Dest-stansted airport Occur Num: 2</li><li>Inform-Train-Dest-peterborough Occur Num: 1</li></ul></ul><li>Inform-Train-Day-wednesday<ul><li>Occur Num: 28</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 6</li><li>Inform-Train-Day-Saturday Occur Num: 3</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Depart-broxbourne Occur Num: 1</li><li>Inform-Train-Leave-18:15 Occur Num: 1</li></ul></ul><li>Request-Train-Ticket-?<ul><li>Occur Num: 26</li><li>NLU Output</li><ul><li>Request-Train-Time-? Occur Num: 24</li><li>Request-Train-Leave-? Occur Num: 2</li></ul></ul><li>Request-Train-Time-?<ul><li>Occur Num: 26</li><li>NLU Output</li><ul><li>Request-Train-Ticket-? Occur Num: 24</li><li>Request-Train-Id-? Occur Num: 1</li><li>Inform-Train-none-none Occur Num: 1</li></ul></ul><li>Inform-Train-Day-thursday<ul><li>Occur Num: 25</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 7</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Leave-15:30 Occur Num: 2</li><li>Inform-Train-Leave-21:15 Occur Num: 1</li><li>Inform-Train-Leave-do Occur Num: 1</li></ul></ul><li>Inform-Train-Dest-leicester<ul><li>Occur Num: 19</li><li>NLU Output</li><ul><li>Inform-Train-Day-monday Occur Num: 3</li><li>Inform-Train-Day-Saturday Occur Num: 2</li><li>Inform-Train-Dest-peterborough Occur Num: 1</li><li>Inform-Train-Day-tuesday Occur Num: 1</li><li>Inform-Train-Arrive-18:30 Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>OfferBook-Train-none-none<ul><li>Occur Num: 1870</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-05:51 Occur Num: 324</li><li>Inform-Train-Choice-404 Occur Num: 292</li><li>Inform-Train-Leave-09:00 Occur Num: 259</li><li>Inform-Train-Leave-05:00 Occur Num: 210</li><li>Inform-Train-Ticket-23.60 pounds Occur Num: 138</li></ul></ul><li>Inform-Train-Arrive-05:51<ul><li>Occur Num: 1543</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 318</li><li>Inform-Train-Choice-404 Occur Num: 278</li><li>Inform-Train-Leave-09:00 Occur Num: 245</li><li>Inform-Train-Leave-05:00 Occur Num: 177</li><li>Inform-Train-Id-TR0899 Occur Num: 61</li></ul></ul><li>Inform-Train-Leave-09:00<ul><li>Occur Num: 1418</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 282</li><li>Inform-Train-Arrive-05:51 Occur Num: 277</li><li>Inform-Train-Choice-404 Occur Num: 269</li><li>Inform-Train-Leave-05:00 Occur Num: 168</li><li>Inform-Train-Id-TR0899 Occur Num: 57</li></ul></ul><li>Inform-Train-Choice-404<ul><li>Occur Num: 1396</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-05:51 Occur Num: 279</li><li>OfferBook-Train-none-none Occur Num: 272</li><li>Inform-Train-Leave-09:00 Occur Num: 236</li><li>Inform-Train-Leave-05:00 Occur Num: 146</li><li>Request-Train-Leave-? Occur Num: 56</li></ul></ul><li>Inform-Train-Leave-05:00<ul><li>Occur Num: 1037</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 211</li><li>Inform-Train-Arrive-05:51 Occur Num: 183</li><li>Inform-Train-Leave-09:00 Occur Num: 156</li><li>Inform-Train-Choice-404 Occur Num: 150</li><li>Inform-Train-Id-TR0899 Occur Num: 51</li></ul></ul><li>Inform-Train-Ticket-23.60 pounds<ul><li>Occur Num: 524</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 138</li><li>reqmore-general-none-none Occur Num: 106</li><li>Inform-Train-Time-51 minutes Occur Num: 81</li><li>Inform-Train-Id-TR8272 Occur Num: 27</li><li>Inform-Train-Id-TR5767 Occur Num: 26</li></ul></ul><li>Inform-Train-Time-51 minutes<ul><li>Occur Num: 360</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 126</li><li>Inform-Train-Ticket-23.60 pounds Occur Num: 85</li><li>Inform-Train-Ticket-18.88 pounds Occur Num: 41</li><li>OfferBook-Train-none-none Occur Num: 32</li><li>welcome-general-none-none Occur Num: 23</li></ul></ul><li>Inform-Train-Id-TR0899<ul><li>Occur Num: 326</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 61</li><li>Inform-Train-Arrive-05:51 Occur Num: 61</li><li>Inform-Train-Leave-05:00 Occur Num: 48</li><li>Inform-Train-Leave-09:00 Occur Num: 46</li><li>Inform-Train-Choice-404 Occur Num: 42</li></ul></ul><li>Request-Train-Leave-?<ul><li>Occur Num: 321</li><li>NLU Output</li><ul><li>Inform-Train-Choice-404 Occur Num: 65</li><li>Inform-Train-Arrive-05:51 Occur Num: 60</li><li>OfferBook-Train-none-none Occur Num: 50</li><li>Inform-Train-Leave-09:00 Occur Num: 48</li><li>Inform-Train-Leave-05:00 Occur Num: 22</li></ul></ul><li>Inform-Train-Id-TR8272<ul><li>Occur Num: 302</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 55</li><li>Inform-Train-Arrive-05:51 Occur Num: 50</li><li>Inform-Train-Choice-404 Occur Num: 50</li><li>Inform-Train-Leave-09:00 Occur Num: 46</li><li>Inform-Train-Leave-05:00 Occur Num: 37</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Inform-Train-People-2 Occur Num: 5</li><li>Inform-Train-People-6 Occur Num: 5</li><li>Inform-Train-People-3 Occur Num: 4</li><li>Inform-Train-People-1 Occur Num: 4</li><li>Inform-Train-People-8 Occur Num: 3</li><li>Inform-Train-People-4 Occur Num: 2</li><li>Inform-Train-People-5 Occur Num: 2</li><li>Inform-Train-People-7 Occur Num: 2</li><li>Inform-Train-Leave-09:30 Occur Num: 2</li><li>Inform-Train-Leave-18:00 Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-train-duration Occur Num: 10</li><li>inform-train-price Occur Num: 6</li><li>inform-train-leaveAt Occur Num: 2</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-train-leaveAt Occur Num: 6</li><li>request-train-price Occur Num: 1</li><li>request-train-duration Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 100.0 %</p><p> (Precision, Recall, F1) : (0.587, 1.000, 0.733) </p><p> Average Dialog Turn (Succ): 2.000 </p><p> Average Dialog Turn (All): 2.000 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Police-none-none<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Police-Name-Parkside Police Station Occur Num: 6</li><li>Request-Police-Addr-? Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Police-Addr-Parkside, Cambridge<ul><li>Occur Num: 68</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 23</li><li>Inform-Police-Phone-01223358966 Occur Num: 23</li><li>Inform-Police-Post-none Occur Num: 22</li></ul></ul><li>Inform-Police-Post-none<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 23</li><li>Inform-Police-Phone-01223358966 Occur Num: 23</li></ul></ul><li>Inform-Police-Phone-01223358966<ul><li>Occur Num: 45</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 23</li><li>Inform-Police-Post-none Occur Num: 22</li></ul></ul><li>Inform-Police-Name-none<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 1</li><li>Inform-Police-Phone-01223358966 Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><p>Nothing</p><b> Bad Inform Dialog Act</b><ul><li>inform-police-postcode Occur Num: 22</li></ul><b> Request But Not Inform Dialog Act</b><p>Nothing</p><b> Inform But Not Request Dialog Act</b><ul><li>inform-police-phone Occur Num: 3</li></ul></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 93.3 %</p><p> (Precision, Recall, F1) : (0.933, 0.933, 0.933) </p><p> Average Dialog Turn (Succ): 4.000 </p><p> Average Dialog Turn (All): 6.400 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Hospital-Phone-?<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Request-Police-Phone-? Occur Num: 2</li><li>Request-Restaurant-Phone-? Occur Num: 2</li><li>Request-Attraction-Phone-? Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-plastic and vascular surgery plastics<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li><li>Inform-Hospital-Department-plastic and vascular surgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-haematology and haematological oncology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurosurgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-oncology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-surgery<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-intermediate dependancy area<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-intermediate dependancy Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-children's oncology and haematology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-emergency department<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-emergency department department Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hospital-Addr-none<ul><li>Occur Num: 84</li><li>NLU Output</li><ul><li>Request-Hospital-Department-? Occur Num: 28</li><li>Inform-Hospital-Department-neurosciences critical care unit Occur Num: 28</li><li>Inform-Hospital-Phone-01223216297 Occur Num: 28</li></ul></ul><li>Inform-Hospital-Post-none<ul><li>Occur Num: 84</li><li>NLU Output</li><ul><li>Request-Hospital-Department-? Occur Num: 28</li><li>Inform-Hospital-Department-neurosciences critical care unit Occur Num: 28</li><li>Inform-Hospital-Phone-01223216297 Occur Num: 28</li></ul></ul><li>Inform-Hospital-Department-neurosciences critical care unit<ul><li>Occur Num: 56</li><li>NLU Output</li><ul><li>Request-Hospital-Department-? Occur Num: 28</li><li>Inform-Hospital-Phone-01223216297 Occur Num: 28</li></ul></ul><li>Inform-Hospital-Phone-01223216297<ul><li>Occur Num: 56</li><li>NLU Output</li><ul><li>Request-Hospital-Department-? Occur Num: 28</li><li>Inform-Hospital-Department-neurosciences critical care unit Occur Num: 28</li></ul></ul><li>Request-Hospital-Department-?<ul><li>Occur Num: 56</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurosciences critical care unit Occur Num: 28</li><li>Inform-Hospital-Phone-01223216297 Occur Num: 28</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Hospital-Phone-? Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-hospital-phone Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/BERTNLU-RuleDST-GDPLPolicy-TemplateNLG/res.txt b/tests/results/BERTNLU-RuleDST-GDPLPolicy-TemplateNLG/res.txt deleted file mode 100755 index 05fbfdc3e73524facb77836c951599b9d9ccd873..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-GDPLPolicy-TemplateNLG/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.579 -success number of dialogs/tot: 0.495 -average precision: 0.6698519692353618 -average recall: 0.7639243270863209 -average f1: 0.6823902525712193 -average book rate: 0.3348931623931624 -average turn (succ): 11.478787878787879 -average turn (all): 24.328 diff --git a/tests/results/BERTNLU-RuleDST-HDSA/Frequency_of_domain.png b/tests/results/BERTNLU-RuleDST-HDSA/Frequency_of_domain.png deleted file mode 100755 index 3bc5164c8b8ae9b143a3d37333930e5079b0cc9b..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-HDSA/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-HDSA/Performance_for_each_domain.png b/tests/results/BERTNLU-RuleDST-HDSA/Performance_for_each_domain.png deleted file mode 100755 index 2981c47c6617f1b97e3738e6d047d79082268d9c..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-HDSA/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-HDSA/Proportions_of_the_dialogue_loop.png b/tests/results/BERTNLU-RuleDST-HDSA/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index 9446fe6651182d9465cdfa60ab063d7d7f26e4a5..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-HDSA/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-HDSA/report_multiwoz.html b/tests/results/BERTNLU-RuleDST-HDSA/report_multiwoz.html deleted file mode 100755 index b43a5783cd89fb51d6f5144f28ce2f69ce8cf347..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-HDSA/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: BERTNLU-RuleDST-HDSA</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-26 07:28:10</h3><h3>Overall Results</h3><p> Success Rate: 27.5 %</p><p> (Precision, Recall, F1) : (0.478, 0.572, 0.488) </p><p> Average Dialog Turn (Succ): 12.996 </p><p> Average Dialog Turn (All): 31.536 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">327</td><td align="center">0.333</td><td align="center">0.362</td><td align="center">0.521</td><td align="center">0.408</td><td align="center">0.599</td><td align="center">7.321</td><td align="center">20.856</td></tr><tr><td align="center">attraction</td><td align="center">285</td><td align="center">0.442</td><td align="center">0.592</td><td align="center">0.571</td><td align="center">0.562</td><td align="center">0.540</td><td align="center">10.603</td><td align="center">25.046</td></tr><tr><td align="center">restaurant</td><td align="center">368</td><td align="center">0.535</td><td align="center">0.466</td><td align="center">0.714</td><td align="center">0.536</td><td align="center">0.454</td><td align="center">6.579</td><td align="center">16.848</td></tr><tr><td align="center">train</td><td align="center">314</td><td align="center">0.459</td><td align="center">0.617</td><td align="center">0.596</td><td align="center">0.593</td><td align="center">0.525</td><td align="center">11.472</td><td align="center">22.656</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">0.870</td><td align="center">0.920</td><td align="center">0.913</td><td align="center">0.904</td><td align="center">0.130</td><td align="center">10.700</td><td align="center">14.522</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">0.867</td><td align="center">0.867</td><td align="center">0.867</td><td align="center">0.867</td><td align="center">0.133</td><td align="center">8.000</td><td align="center">12.267</td></tr><tr><td align="center">taxi</td><td align="center">76</td><td align="center">0.342</td><td align="center">0.684</td><td align="center">0.513</td><td align="center">0.570</td><td align="center">0.632</td><td align="center">2.154</td><td align="center">18.763</td></tr></table><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 33.3 %</p><p> (Precision, Recall, F1) : (0.362, 0.521, 0.408) </p><p> Average Dialog Turn (Succ): 7.321 </p><p> Average Dialog Turn (All): 20.856 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 291</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 46</li><li>Inform-Hotel-Type-guesthouse Occur Num: 35</li><li>Inform-Hotel-Type-hotel Occur Num: 32</li><li>Inform-Hotel-Stars-4 Occur Num: 32</li><li>Inform-Hotel-Price-moderate Occur Num: 24</li></ul></ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 273</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 47</li><li>Inform-Hotel-Type-guesthouse Occur Num: 31</li><li>Inform-Hotel-Type-hotel Occur Num: 31</li><li>Inform-Hotel-Price-moderate Occur Num: 29</li><li>Inform-Hotel-Stars-4 Occur Num: 28</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 166</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 32</li><li>Inform-Hotel-Internet-yes Occur Num: 30</li><li>Inform-Hotel-Stars-4 Occur Num: 15</li><li>Inform-Hotel-Area-north Occur Num: 14</li><li>Inform-Hotel-Price-moderate Occur Num: 12</li></ul></ul><li>Inform-Hotel-Price-moderate<ul><li>Occur Num: 147</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 30</li><li>Inform-Hotel-Parking-yes Occur Num: 26</li><li>Inform-Hotel-Type-hotel Occur Num: 19</li><li>Inform-Hotel-Type-guesthouse Occur Num: 12</li><li>Inform-Hotel-Stars-4 Occur Num: 7</li></ul></ul><li>Inform-Hotel-Stars-4<ul><li>Occur Num: 146</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 33</li><li>Inform-Hotel-Internet-yes Occur Num: 28</li><li>Inform-Hotel-Type-hotel Occur Num: 18</li><li>Inform-Hotel-Type-guesthouse Occur Num: 15</li><li>Inform-Hotel-Price-cheap Occur Num: 9</li></ul></ul><li>Request-Hotel-Addr-?<ul><li>Occur Num: 131</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 26</li><li>Request-Hotel-Phone-? Occur Num: 18</li><li>Request-Hotel-Post-? Occur Num: 18</li><li>Request-Attraction-Addr-? Occur Num: 8</li><li>Inform-Hotel-Parking-yes Occur Num: 7</li></ul></ul><li>Inform-Hotel-Area-north<ul><li>Occur Num: 109</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 25</li><li>Inform-Hotel-Internet-yes Occur Num: 22</li><li>Inform-Hotel-Type-guesthouse Occur Num: 14</li><li>Inform-Hotel-Type-hotel Occur Num: 12</li><li>Inform-Hotel-Stars-4 Occur Num: 8</li></ul></ul><li>Inform-Hotel-Type-hotel<ul><li>Occur Num: 83</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 12</li><li>Inform-Hotel-Internet-yes Occur Num: 12</li><li>Inform-Hotel-Price-moderate Occur Num: 9</li><li>Inform-Hotel-Stars-4 Occur Num: 8</li><li>Inform-Hotel-Stars-3 Occur Num: 5</li></ul></ul><li>Request-Hotel-Post-?<ul><li>Occur Num: 77</li><li>NLU Output</li><ul><li>Request-Hotel-Addr-? Occur Num: 18</li><li>Request-Restaurant-Post-? Occur Num: 11</li><li>Request-Hotel-Phone-? Occur Num: 8</li><li>Request-Restaurant-Addr-? Occur Num: 5</li><li>Inform-Hotel-Parking-yes Occur Num: 4</li></ul></ul><li>Inform-Hotel-Parking-no<ul><li>Occur Num: 74</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 16</li><li>Inform-Hotel-Internet-yes Occur Num: 8</li><li>Inform-Hotel-Price-moderate Occur Num: 8</li><li>Inform-Hotel-Type-guesthouse Occur Num: 7</li><li>Inform-Hotel-Type-hotel Occur Num: 7</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Inform-Hotel-Stay-4 Occur Num: 22</li><li>Inform-Hotel-Stay-5 Occur Num: 16</li><li>Request-Hotel-Addr-? Occur Num: 16</li><li>Request-Hotel-Internet-? Occur Num: 15</li><li>Inform-Hotel-Stay-3 Occur Num: 15</li><li>Request-Hotel-Phone-? Occur Num: 14</li><li>Request-Hotel-Stars-? Occur Num: 12</li><li>Request-Hotel-Post-? Occur Num: 11</li><li>Inform-Hotel-Stay-2 Occur Num: 11</li><li>Request-Hotel-Parking-? Occur Num: 9</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-pricerange Occur Num: 103</li><li>inform-hotel-internet Occur Num: 25</li><li>inform-hotel-parking Occur Num: 24</li><li>inform-hotel-area Occur Num: 13</li><li>inform-hotel-postcode Occur Num: 5</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-address Occur Num: 25</li><li>request-hotel-internet Occur Num: 20</li><li>request-hotel-phone Occur Num: 20</li><li>request-hotel-parking Occur Num: 19</li><li>request-hotel-postcode Occur Num: 14</li><li>request-hotel-stars Occur Num: 13</li><li>request-hotel-type Occur Num: 9</li><li>request-hotel-pricerange Occur Num: 8</li><li>request-hotel-area Occur Num: 3</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-phone Occur Num: 96</li><li>inform-hotel-area Occur Num: 64</li><li>inform-hotel-type Occur Num: 47</li><li>inform-hotel-stars Occur Num: 6</li></ul></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 44.2 %</p><p> (Precision, Recall, F1) : (0.592, 0.571, 0.562) </p><p> Average Dialog Turn (Succ): 10.603 </p><p> Average Dialog Turn (All): 25.046 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Attraction-Post-?<ul><li>Occur Num: 389</li><li>NLU Output</li><ul><li>Request-Restaurant-Post-? Occur Num: 105</li><li>Request-Attraction-Phone-? Occur Num: 47</li><li>Request-Attraction-Addr-? Occur Num: 46</li><li>Request-Restaurant-Addr-? Occur Num: 42</li><li>Request-Attraction-Fee-? Occur Num: 40</li></ul></ul><li>Request-Attraction-Addr-?<ul><li>Occur Num: 350</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 89</li><li>Request-Restaurant-Post-? Occur Num: 49</li><li>Request-Attraction-Post-? Occur Num: 49</li><li>Request-Attraction-Phone-? Occur Num: 42</li><li>Request-Attraction-Fee-? Occur Num: 40</li></ul></ul><li>Request-Attraction-Phone-?<ul><li>Occur Num: 347</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 87</li><li>Request-Attraction-Post-? Occur Num: 52</li><li>Request-Attraction-Addr-? Occur Num: 49</li><li>Request-Restaurant-Addr-? Occur Num: 40</li><li>Request-Attraction-Fee-? Occur Num: 39</li></ul></ul><li>Request-Attraction-Fee-?<ul><li>Occur Num: 120</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 40</li><li>Request-Attraction-Addr-? Occur Num: 39</li><li>Request-Attraction-Phone-? Occur Num: 31</li><li>Request-Attraction-Type-? Occur Num: 5</li><li>Request-Restaurant-Post-? Occur Num: 3</li></ul></ul><li>Request-Attraction-Area-?<ul><li>Occur Num: 53</li><li>NLU Output</li><ul><li>Request-Restaurant-Area-? Occur Num: 14</li><li>Request-Restaurant-Addr-? Occur Num: 9</li><li>Request-Attraction-Phone-? Occur Num: 7</li><li>Request-Restaurant-Phone-? Occur Num: 7</li><li>Request-Attraction-Addr-? Occur Num: 4</li></ul></ul><li>Request-Attraction-Type-?<ul><li>Occur Num: 31</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 8</li><li>Request-Attraction-Phone-? Occur Num: 8</li><li>Request-Attraction-Addr-? Occur Num: 7</li><li>Request-Attraction-Fee-? Occur Num: 6</li><li>Request-Restaurant-Post-? Occur Num: 2</li></ul></ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 29</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-entertainment Occur Num: 14</li><li>Inform-Attraction-Type-park Occur Num: 2</li><li>Inform-Attraction-Type-museums Occur Num: 2</li><li>Inform-Attraction-Type-boats Occur Num: 2</li><li>Inform-Attraction-Type-museum Occur Num: 2</li></ul></ul><li>Inform-Attraction-Type-museum<ul><li>Occur Num: 17</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 7</li><li>Inform-Attraction-Type-museums Occur Num: 5</li><li>Inform-Attraction-Type-entertainment Occur Num: 3</li><li>Inform-Attraction-Area-west Occur Num: 2</li></ul></ul><li>Inform-Attraction-Type-boat<ul><li>Occur Num: 16</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 6</li><li>Inform-Attraction-Type-boats Occur Num: 5</li><li>Inform-Attraction-Type-entertainment Occur Num: 3</li><li>Inform-Attraction-Area-west Occur Num: 1</li><li>Inform-Attraction-Area-north Occur Num: 1</li></ul></ul><li>Inform-Attraction-Type-nightclub<ul><li>Occur Num: 15</li><li>NLU Output</li><ul><li>Inform-Attraction-none-none Occur Num: 5</li><li>Inform-Attraction-Area-west Occur Num: 4</li><li>Inform-Attraction-Area-centre Occur Num: 4</li><li>Inform-Attraction-Type-entertainment Occur Num: 2</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Attraction-Addr-? Occur Num: 63</li><li>Request-Attraction-Post-? Occur Num: 55</li><li>Request-Attraction-Fee-? Occur Num: 19</li><li>Request-Attraction-Type-? Occur Num: 9</li><li>Request-Attraction-Phone-? Occur Num: 6</li><li>Request-Attraction-Area-? Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-attraction-area Occur Num: 31</li><li>inform-attraction-phone Occur Num: 6</li><li>inform-attraction-postcode Occur Num: 2</li><li>recommend-attraction-area Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-postcode Occur Num: 92</li><li>request-attraction-address Occur Num: 85</li><li>request-attraction-phone Occur Num: 45</li><li>request-attraction-area Occur Num: 15</li><li>request-attraction-type Occur Num: 13</li><li>request-attraction-entrance fee Occur Num: 4</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-phone Occur Num: 36</li><li>inform-attraction-area Occur Num: 6</li><li>inform-attraction-type Occur Num: 5</li></ul></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 53.5 %</p><p> (Precision, Recall, F1) : (0.466, 0.714, 0.536) </p><p> Average Dialog Turn (Succ): 6.579 </p><p> Average Dialog Turn (All): 16.848 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Restaurant-Addr-?<ul><li>Occur Num: 190</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 47</li><li>Request-Restaurant-Post-? Occur Num: 39</li><li>Request-Hospital-Addr-? Occur Num: 25</li><li>Request-Attraction-Addr-? Occur Num: 25</li><li>Request-Hotel-Addr-? Occur Num: 14</li></ul></ul><li>Request-Restaurant-Phone-?<ul><li>Occur Num: 164</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 47</li><li>Request-Attraction-Phone-? Occur Num: 35</li><li>Request-Restaurant-Post-? Occur Num: 22</li><li>Request-Hotel-Phone-? Occur Num: 15</li><li>Request-Hospital-Phone-? Occur Num: 13</li></ul></ul><li>Request-Restaurant-Post-?<ul><li>Occur Num: 120</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 36</li><li>Request-Restaurant-Phone-? Occur Num: 20</li><li>Request-Attraction-Post-? Occur Num: 17</li><li>Request-Hospital-Post-? Occur Num: 16</li><li>Request-Hotel-Post-? Occur Num: 9</li></ul></ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 26</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensively Occur Num: 10</li><li>Inform-Restaurant-Area-centre Occur Num: 4</li><li>Inform-Restaurant-Area-south Occur Num: 2</li><li>Inform-Restaurant-Food-indian Occur Num: 1</li><li>Inform-Restaurant-Food-mexican Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Price-moderate<ul><li>Occur Num: 25</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 11</li><li>Inform-Restaurant-Area-centre Occur Num: 6</li><li>Inform-Restaurant-Food-british Occur Num: 2</li><li>Inform-Restaurant-Food-mediterranean Occur Num: 1</li><li>Inform-Restaurant-Area-east Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 16</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 6</li><li>Inform-Restaurant-Food-british Occur Num: 3</li><li>Inform-Restaurant-Price-expensively Occur Num: 2</li><li>Inform-Restaurant-Food-modern european Occur Num: 1</li><li>Inform-Restaurant-Food-chinese Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Food-portuguese<ul><li>Occur Num: 11</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 3</li><li>Inform-Restaurant-none-none Occur Num: 3</li><li>Inform-Restaurant-Price-expensive Occur Num: 2</li><li>Inform-Restaurant-Price-moderately Occur Num: 1</li><li>Inform-Restaurant-Price-moderate Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Price-cheap<ul><li>Occur Num: 10</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-cheaply Occur Num: 4</li><li>Inform-Restaurant-Food-italian Occur Num: 3</li><li>Inform-Restaurant-Area-east Occur Num: 1</li><li>Inform-Restaurant-Area-north Occur Num: 1</li><li>Inform-Restaurant-Area-centre Occur Num: 1</li></ul></ul><li>Request-Restaurant-Area-?<ul><li>Occur Num: 8</li><li>NLU Output</li><ul><li>Request-Attraction-Area-? Occur Num: 4</li><li>Request-Hotel-Area-? Occur Num: 3</li><li>Request-Restaurant-Addr-? Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Food-italian<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-cheaply Occur Num: 3</li><li>Inform-Restaurant-Area-east Occur Num: 1</li><li>Inform-Restaurant-Area-north Occur Num: 1</li><li>Inform-Restaurant-Price-moderately Occur Num: 1</li><li>Inform-Restaurant-Area-west Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Restaurant-Phone-? Occur Num: 35</li><li>Request-Restaurant-Post-? Occur Num: 15</li><li>Request-Restaurant-Addr-? Occur Num: 13</li><li>Inform-Restaurant-Day-friday Occur Num: 5</li><li>Inform-Restaurant-Day-sunday Occur Num: 5</li><li>Inform-Restaurant-Day-saturday Occur Num: 5</li><li>Inform-Restaurant-Time-13:45 Occur Num: 4</li><li>Inform-Restaurant-Time-18:30 Occur Num: 4</li><li>Inform-Restaurant-People-2 Occur Num: 4</li><li>Inform-Restaurant-Time-12:30 Occur Num: 3</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-restaurant-area Occur Num: 110</li><li>inform-restaurant-pricerange Occur Num: 68</li><li>inform-restaurant-postcode Occur Num: 8</li><li>recommend-restaurant-area Occur Num: 4</li><li>recommend-restaurant-pricerange Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-phone Occur Num: 33</li><li>request-restaurant-address Occur Num: 29</li><li>request-restaurant-postcode Occur Num: 17</li><li>request-restaurant-area Occur Num: 6</li><li>request-restaurant-pricerange Occur Num: 2</li><li>request-restaurant-food Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-phone Occur Num: 84</li><li>inform-restaurant-food Occur Num: 45</li><li>inform-restaurant-area Occur Num: 36</li></ul></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 45.9 %</p><p> (Precision, Recall, F1) : (0.617, 0.596, 0.593) </p><p> Average Dialog Turn (Succ): 11.472 </p><p> Average Dialog Turn (All): 22.656 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Train-Dest-cambridge<ul><li>Occur Num: 210</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 21</li><li>Inform-Train-Leave-do n't care Occur Num: 19</li><li>Inform-Train-Day-wednesday Occur Num: 9</li><li>Inform-Train-Day-Saturday Occur Num: 8</li><li>Inform-Train-Dest-london liverpool street Occur Num: 7</li></ul></ul><li>Inform-Train-Leave-dontcare<ul><li>Occur Num: 144</li><li>NLU Output</li><ul><li>Inform-Train-Leave-do n't care Occur Num: 50</li><li>Inform-Train-Dest-cambridge Occur Num: 19</li><li>Inform-Train-Depart-kings lynn Occur Num: 5</li><li>Inform-Train-Depart-birmingham new street Occur Num: 5</li><li>Inform-Train-Depart-peterborough Occur Num: 5</li></ul></ul><li>Inform-Train-Depart-cambridge<ul><li>Occur Num: 135</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 25</li><li>Inform-Train-Day-Saturday Occur Num: 7</li><li>Inform-Train-Day-thursday Occur Num: 7</li><li>Inform-Train-Day-friday Occur Num: 6</li><li>Inform-Train-Day-monday Occur Num: 6</li></ul></ul><li>Request-Train-Time-?<ul><li>Occur Num: 127</li><li>NLU Output</li><ul><li>Inform-Train-none-none Occur Num: 57</li><li>Request-Train-Id-? Occur Num: 33</li><li>Request-Train-Ticket-? Occur Num: 30</li><li>Request-Train-Arrive-? Occur Num: 6</li><li>Request-Train-Leave-? Occur Num: 1</li></ul></ul><li>Request-Train-Id-?<ul><li>Occur Num: 98</li><li>NLU Output</li><ul><li>Inform-Train-none-none Occur Num: 41</li><li>Request-Train-Time-? Occur Num: 33</li><li>Request-Train-Ticket-? Occur Num: 20</li><li>Request-Train-Arrive-? Occur Num: 3</li><li>Request-Train-Leave-? Occur Num: 1</li></ul></ul><li>Request-Train-Ticket-?<ul><li>Occur Num: 48</li><li>NLU Output</li><ul><li>Request-Train-Time-? Occur Num: 28</li><li>Request-Train-Id-? Occur Num: 16</li><li>Request-Train-Leave-? Occur Num: 2</li><li>Request-Train-Arrive-? Occur Num: 1</li><li>Request-Hotel-Price-? Occur Num: 1</li></ul></ul><li>Inform-Train-Depart-london liverpool street<ul><li>Occur Num: 41</li><li>NLU Output</li><ul><li>Inform-Train-Dest-london liverpool street Occur Num: 11</li><li>Inform-Train-Dest-cambridge Occur Num: 6</li><li>Inform-Train-Leave-do n't care Occur Num: 4</li><li>Inform-Train-Day-wednesday Occur Num: 3</li><li>Inform-Train-Depart-cambridge Occur Num: 2</li></ul></ul><li>Inform-Train-Day-wednesday<ul><li>Occur Num: 37</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 8</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Dest-london liverpool street Occur Num: 2</li><li>Inform-Train-Depart-broxbourne Occur Num: 2</li><li>Inform-Train-Day-Saturday Occur Num: 2</li></ul></ul><li>Inform-Train-Day-tuesday<ul><li>Occur Num: 35</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 5</li><li>Inform-Train-Day-Saturday Occur Num: 4</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Dest-norwich Occur Num: 2</li><li>Inform-Train-Depart-london kings cross Occur Num: 2</li></ul></ul><li>Inform-Train-Day-monday<ul><li>Occur Num: 34</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 5</li><li>Inform-Train-Dest-cambridge Occur Num: 4</li><li>Inform-Train-Dest-norwich Occur Num: 2</li><li>Inform-Train-Day-Saturday Occur Num: 2</li><li>Inform-Train-Leave-13:00 Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Train-Time-? Occur Num: 60</li><li>Request-Train-Id-? Occur Num: 15</li><li>Inform-Train-Dest-cambridge Occur Num: 9</li><li>Inform-Train-Leave-dontcare Occur Num: 7</li><li>Request-Train-Leave-? Occur Num: 7</li><li>Inform-Train-People-1 Occur Num: 5</li><li>Inform-Train-People-3 Occur Num: 4</li><li>Inform-Train-Depart-peterborough Occur Num: 4</li><li>Inform-Train-People-4 Occur Num: 3</li><li>Inform-Train-Depart-birmingham new street Occur Num: 3</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-train-leaveAt Occur Num: 20</li><li>inform-train-price Occur Num: 14</li><li>offerbooked-train-price Occur Num: 9</li><li>inform-train-duration Occur Num: 8</li><li>inform-train-trainID Occur Num: 4</li><li>inform-train-arriveBy Occur Num: 1</li><li>offerbooked-train-trainID Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-train-duration Occur Num: 90</li><li>request-train-trainID Occur Num: 39</li><li>request-train-arriveBy Occur Num: 9</li><li>request-train-leaveAt Occur Num: 9</li><li>request-train-price Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 87.0 %</p><p> (Precision, Recall, F1) : (0.920, 0.913, 0.904) </p><p> Average Dialog Turn (Succ): 10.700 </p><p> Average Dialog Turn (All): 14.522 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Police-Phone-?<ul><li>Occur Num: 34</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 10</li><li>Request-Police-Addr-? Occur Num: 7</li><li>Request-Restaurant-Addr-? Occur Num: 4</li><li>Request-Attraction-Phone-? Occur Num: 4</li><li>Request-Hospital-Phone-? Occur Num: 3</li></ul></ul><li>Request-Police-Addr-?<ul><li>Occur Num: 20</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 6</li><li>Request-Restaurant-Phone-? Occur Num: 5</li><li>Request-Police-Phone-? Occur Num: 5</li><li>Request-Attraction-Addr-? Occur Num: 2</li><li>Request-Attraction-Phone-? Occur Num: 2</li></ul></ul><li>Inform-Police-none-none<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Police-Name-Parkside Police Station Occur Num: 6</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Police-Addr-? Occur Num: 3</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-police-postcode Occur Num: 2</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-police-address Occur Num: 3</li><li>request-police-phone Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 86.7 %</p><p> (Precision, Recall, F1) : (0.867, 0.867, 0.867) </p><p> Average Dialog Turn (Succ): 8.000 </p><p> Average Dialog Turn (All): 12.267 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Hospital-Phone-?<ul><li>Occur Num: 18</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 9</li><li>Request-Attraction-Phone-? Occur Num: 7</li><li>Inform-Hospital-none-none Occur Num: 2</li></ul></ul><li>Inform-Hospital-Department-oncology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-plastic and vascular surgery plastics<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-plastic and vascular surgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-surgery<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-intermediate dependancy area<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-intermediate dependancy Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-children's oncology and haematology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Hospital-Phone-? Occur Num: 4</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-hospital-phone Occur Num: 4</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 34.2 %</p><p> (Precision, Recall, F1) : (0.684, 0.513, 0.570) </p><p> Average Dialog Turn (Succ): 2.154 </p><p> Average Dialog Turn (All): 18.763 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Phone-?<ul><li>Occur Num: 90</li><li>NLU Output</li><ul><li>Request-Taxi-Car-? Occur Num: 33</li><li>Request-Attraction-Phone-? Occur Num: 24</li><li>Request-Restaurant-Phone-? Occur Num: 21</li><li>Request-Hotel-Phone-? Occur Num: 7</li><li>Request-Police-Phone-? Occur Num: 3</li></ul></ul><li>Request-Taxi-Car-?<ul><li>Occur Num: 10</li><li>NLU Output</li><ul><li>Request-Hotel-Phone-? Occur Num: 7</li><li>Request-Hotel-Type-? Occur Num: 3</li></ul></ul><li>Inform-Taxi-Depart-the lensfield hotel<ul><li>Occur Num: 8</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-the Occur Num: 2</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 2</li><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li><li>Inform-Taxi-Dest-williams art and antiques Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the fitzwilliam museum<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 2</li><li>Inform-Taxi-Dest-golden house Occur Num: 1</li><li>Inform-Taxi-Leave-13:45 Occur Num: 1</li><li>Inform-Taxi-Arrive-13:45 Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-home from home<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-home from home Occur Num: 2</li><li>Inform-Taxi-Depart-arbury lodge guesthouse Occur Num: 1</li><li>Inform-Taxi-Leave-02:45 Occur Num: 1</li><li>Inform-Taxi-Depart-thanh binh Occur Num: 1</li><li>Inform-Taxi-Leave-02:30 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-02:00<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-parkside police station<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-lynne strover gallery Occur Num: 1</li><li>Inform-Police-Name-parkside police station Occur Num: 1</li><li>Inform-Taxi-Arrive-12:15 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-13:45<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 1</li><li>Inform-Taxi-Dest-la mimosa Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-la mimosa<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Arrive-13:45 Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-efes restaurant<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-broxbourne train station Occur Num: 1</li><li>Inform-Taxi-Dest-efes Occur Num: 1</li><li>Inform-Taxi-Arrive-16:15 Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Taxi-Car-? Occur Num: 31</li><li>Request-Taxi-Phone-? Occur Num: 17</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-taxi-phone Occur Num: 49</li><li>request-taxi-car type Occur Num: 25</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/BERTNLU-RuleDST-HDSA/res.txt b/tests/results/BERTNLU-RuleDST-HDSA/res.txt deleted file mode 100755 index 57c086c610bc96f8602faf5265ea0323dfd2bc98..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-HDSA/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.356 -success number of dialogs/tot: 0.275 -average precision: 0.4777462168038492 -average recall: 0.5723429593367289 -average f1: 0.4877892521286327 -average book rate: 0.05416666666666667 -average turn (succ): 12.996363636363636 -average turn (all): 31.536 diff --git a/tests/results/BERTNLU-RuleDST-LaRL/Frequency_of_domain.png b/tests/results/BERTNLU-RuleDST-LaRL/Frequency_of_domain.png deleted file mode 100755 index 50972177d4bdd2396c24bcdac076b3f6890f63bf..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-LaRL/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-LaRL/Performance_for_each_domain.png b/tests/results/BERTNLU-RuleDST-LaRL/Performance_for_each_domain.png deleted file mode 100755 index 416dcd6858b83f9349e4f3ed3c0b81ec8285798c..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-LaRL/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-LaRL/Proportions_of_the_dialogue_loop.png b/tests/results/BERTNLU-RuleDST-LaRL/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index ebceef21d2b24ea1eec6a5b48ee115b9f54e8a60..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-LaRL/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-LaRL/report_multiwoz.html b/tests/results/BERTNLU-RuleDST-LaRL/report_multiwoz.html deleted file mode 100755 index 2158e7a799c90a64a50855db6799c2da1a70f932..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-LaRL/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: BERTNLU-RuleDST-HDSA</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-26 00:12:41</h3><h3>Overall Results</h3><p> Success Rate: 34.0 %</p><p> (Precision, Recall, F1) : (0.478, 0.541, 0.476) </p><p> Average Dialog Turn (Succ): 15.000 </p><p> Average Dialog Turn (All): 28.600 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">334</td><td align="center">0.377</td><td align="center">0.277</td><td align="center">0.448</td><td align="center">0.317</td><td align="center">0.440</td><td align="center">9.857</td><td align="center">17.437</td></tr><tr><td align="center">attraction</td><td align="center">285</td><td align="center">0.614</td><td align="center">0.623</td><td align="center">0.719</td><td align="center">0.642</td><td align="center">0.337</td><td align="center">8.651</td><td align="center">17.684</td></tr><tr><td align="center">restaurant</td><td align="center">377</td><td align="center">0.584</td><td align="center">0.482</td><td align="center">0.559</td><td align="center">0.502</td><td align="center">0.347</td><td align="center">9.618</td><td align="center">17.416</td></tr><tr><td align="center">train</td><td align="center">320</td><td align="center">0.631</td><td align="center">0.754</td><td align="center">0.670</td><td align="center">0.697</td><td align="center">0.362</td><td align="center">9.356</td><td align="center">18.812</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">0.870</td><td align="center">0.913</td><td align="center">0.913</td><td align="center">0.901</td><td align="center">0.130</td><td align="center">9.700</td><td align="center">13.652</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">0.933</td><td align="center">0.933</td><td align="center">0.933</td><td align="center">0.933</td><td align="center">0.067</td><td align="center">3.286</td><td align="center">5.733</td></tr><tr><td align="center">taxi</td><td align="center">88</td><td align="center">0.136</td><td align="center">0.182</td><td align="center">0.159</td><td align="center">0.167</td><td align="center">0.830</td><td align="center">8.167</td><td align="center">26.500</td></tr></table><h3>Dialogue Loop</h3><img src="Proportions_of_the_dialogue_loop.png" width=800 /><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 37.7 %</p><p> (Precision, Recall, F1) : (0.277, 0.448, 0.317) </p><p> Average Dialog Turn (Succ): 9.857 </p><p> Average Dialog Turn (All): 17.437 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 306</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 50</li><li>Inform-Hotel-Type-hotel Occur Num: 37</li><li>Inform-Hotel-Type-guesthouse Occur Num: 34</li><li>Inform-Hotel-Stars-4 Occur Num: 34</li><li>Inform-Hotel-Price-moderate Occur Num: 28</li></ul></ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 299</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 52</li><li>Inform-Hotel-Type-guesthouse Occur Num: 38</li><li>Inform-Hotel-Stars-4 Occur Num: 33</li><li>Inform-Hotel-Type-hotel Occur Num: 32</li><li>Inform-Hotel-Price-moderate Occur Num: 25</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 187</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 37</li><li>Inform-Hotel-Parking-yes Occur Num: 35</li><li>Inform-Hotel-Stars-4 Occur Num: 17</li><li>Inform-Hotel-Area-north Occur Num: 14</li><li>Inform-Hotel-Price-moderate Occur Num: 13</li></ul></ul><li>Inform-Hotel-Stars-4<ul><li>Occur Num: 159</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 38</li><li>Inform-Hotel-Internet-yes Occur Num: 33</li><li>Inform-Hotel-Type-hotel Occur Num: 18</li><li>Inform-Hotel-Type-guesthouse Occur Num: 17</li><li>Inform-Hotel-Price-cheap Occur Num: 9</li></ul></ul><li>Inform-Hotel-Price-moderate<ul><li>Occur Num: 147</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 29</li><li>Inform-Hotel-Internet-yes Occur Num: 26</li><li>Inform-Hotel-Type-hotel Occur Num: 19</li><li>Inform-Hotel-Type-guesthouse Occur Num: 13</li><li>Inform-Hotel-Stars-4 Occur Num: 9</li></ul></ul><li>Request-Hotel-Addr-?<ul><li>Occur Num: 130</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 28</li><li>Request-Hotel-Post-? Occur Num: 18</li><li>Request-Hotel-Phone-? Occur Num: 16</li><li>Request-Attraction-Addr-? Occur Num: 9</li><li>Request-Hotel-Internet-? Occur Num: 9</li></ul></ul><li>Inform-Hotel-Type-hotel<ul><li>Occur Num: 99</li><li>NLU Output</li><ul><li>Inform-Hotel-none-none Occur Num: 19</li><li>Inform-Hotel-Parking-yes Occur Num: 13</li><li>Inform-Hotel-Internet-yes Occur Num: 12</li><li>Inform-Hotel-Stars-4 Occur Num: 8</li><li>Inform-Hotel-Price-moderate Occur Num: 8</li></ul></ul><li>Inform-Hotel-Area-north<ul><li>Occur Num: 99</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 22</li><li>Inform-Hotel-Parking-yes Occur Num: 20</li><li>Inform-Hotel-Type-guesthouse Occur Num: 14</li><li>Inform-Hotel-Type-hotel Occur Num: 13</li><li>Inform-Hotel-Stars-4 Occur Num: 6</li></ul></ul><li>Request-Hotel-Phone-?<ul><li>Occur Num: 94</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 22</li><li>Request-Restaurant-Phone-? Occur Num: 21</li><li>Request-Hotel-Addr-? Occur Num: 13</li><li>Request-Hotel-Post-? Occur Num: 8</li><li>Request-Attraction-Addr-? Occur Num: 6</li></ul></ul><li>Request-Hotel-Post-?<ul><li>Occur Num: 80</li><li>NLU Output</li><ul><li>Request-Hotel-Addr-? Occur Num: 16</li><li>Request-Restaurant-Post-? Occur Num: 13</li><li>Request-Attraction-Post-? Occur Num: 10</li><li>Request-Hotel-Phone-? Occur Num: 8</li><li>Request-Restaurant-Addr-? Occur Num: 6</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Hotel-Addr-? Occur Num: 27</li><li>Request-Hotel-Post-? Occur Num: 24</li><li>Request-Hotel-Phone-? Occur Num: 21</li><li>Inform-Hotel-Type-hotel Occur Num: 8</li><li>Inform-Hotel-Stay-4 Occur Num: 7</li><li>Request-Hotel-Internet-? Occur Num: 7</li><li>Inform-Hotel-Stay-5 Occur Num: 7</li><li>Inform-Hotel-Parking-yes Occur Num: 4</li><li>Request-Hotel-Stars-? Occur Num: 4</li><li>Request-Hotel-Area-? Occur Num: 3</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-pricerange Occur Num: 140</li><li>recommend-hotel-pricerange Occur Num: 61</li><li>inform-hotel-parking Occur Num: 41</li><li>inform-hotel-postcode Occur Num: 29</li><li>inform-hotel-internet Occur Num: 27</li><li>inform-hotel-stars Occur Num: 14</li><li>recommend-hotel-stars Occur Num: 8</li><li>recommend-hotel-internet Occur Num: 2</li><li>recommend-hotel-parking Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-postcode Occur Num: 42</li><li>request-hotel-phone Occur Num: 36</li><li>request-hotel-address Occur Num: 28</li><li>request-hotel-internet Occur Num: 21</li><li>request-hotel-parking Occur Num: 17</li><li>request-hotel-pricerange Occur Num: 16</li><li>request-hotel-stars Occur Num: 8</li><li>request-hotel-area Occur Num: 7</li><li>request-hotel-type Occur Num: 3</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-type Occur Num: 103</li><li>inform-hotel-area Occur Num: 55</li><li>inform-hotel-stars Occur Num: 16</li><li>inform-hotel-phone Occur Num: 10</li></ul></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 61.4 %</p><p> (Precision, Recall, F1) : (0.623, 0.719, 0.642) </p><p> Average Dialog Turn (Succ): 8.651 </p><p> Average Dialog Turn (All): 17.684 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Attraction-Post-?<ul><li>Occur Num: 108</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 38</li><li>Request-Attraction-Addr-? Occur Num: 31</li><li>Request-Attraction-Fee-? Occur Num: 20</li><li>Request-Attraction-Area-? Occur Num: 4</li><li>Request-Restaurant-Post-? Occur Num: 4</li></ul></ul><li>Request-Attraction-Phone-?<ul><li>Occur Num: 101</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 38</li><li>Request-Attraction-Addr-? Occur Num: 30</li><li>Request-Attraction-Fee-? Occur Num: 14</li><li>Request-Restaurant-Phone-? Occur Num: 6</li><li>Request-Attraction-Type-? Occur Num: 4</li></ul></ul><li>Request-Attraction-Addr-?<ul><li>Occur Num: 100</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 34</li><li>Request-Attraction-Phone-? Occur Num: 30</li><li>Request-Attraction-Fee-? Occur Num: 16</li><li>Request-Restaurant-Addr-? Occur Num: 7</li><li>Request-Attraction-Area-? Occur Num: 5</li></ul></ul><li>Request-Attraction-Fee-?<ul><li>Occur Num: 51</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 20</li><li>Request-Attraction-Addr-? Occur Num: 15</li><li>Request-Attraction-Phone-? Occur Num: 14</li><li>Request-Attraction-Type-? Occur Num: 1</li><li>Request-Restaurant-Post-? Occur Num: 1</li></ul></ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 28</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-entertainment Occur Num: 12</li><li>Inform-Attraction-Type-museums Occur Num: 3</li><li>Inform-Attraction-Type-boats Occur Num: 2</li><li>Inform-Attraction-Type-museum Occur Num: 2</li><li>Inform-Attraction-Type-colleges Occur Num: 1</li></ul></ul><li>Request-Attraction-Type-?<ul><li>Occur Num: 23</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 8</li><li>Request-Attraction-Phone-? Occur Num: 6</li><li>Request-Attraction-Fee-? Occur Num: 5</li><li>Inform-Attraction-none-none Occur Num: 2</li><li>Request-Attraction-Addr-? Occur Num: 2</li></ul></ul><li>Inform-Attraction-Type-museum<ul><li>Occur Num: 21</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 9</li><li>Inform-Attraction-Type-museums Occur Num: 7</li><li>Inform-Attraction-Type-entertainment Occur Num: 3</li><li>Inform-Attraction-Area-west Occur Num: 2</li></ul></ul><li>Inform-Attraction-Type-nightclub<ul><li>Occur Num: 18</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 6</li><li>Inform-Attraction-none-none Occur Num: 5</li><li>Inform-Attraction-Area-west Occur Num: 4</li><li>Inform-Attraction-Type-entertainment Occur Num: 3</li></ul></ul><li>Request-Attraction-Area-?<ul><li>Occur Num: 17</li><li>NLU Output</li><ul><li>Request-Attraction-Addr-? Occur Num: 4</li><li>Request-Attraction-Post-? Occur Num: 4</li><li>Request-Attraction-Phone-? Occur Num: 3</li><li>Request-Hotel-Area-? Occur Num: 3</li><li>Request-Restaurant-Area-? Occur Num: 1</li></ul></ul><li>Inform-Attraction-Type-boat<ul><li>Occur Num: 16</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 6</li><li>Inform-Attraction-Type-boats Occur Num: 5</li><li>Inform-Attraction-Type-entertainment Occur Num: 3</li><li>Inform-Attraction-Area-west Occur Num: 1</li><li>Inform-Attraction-Area-north Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Attraction-Fee-? Occur Num: 32</li><li>Request-Attraction-Addr-? Occur Num: 26</li><li>Request-Attraction-Post-? Occur Num: 20</li><li>Request-Attraction-Phone-? Occur Num: 11</li><li>Request-Attraction-Type-? Occur Num: 5</li><li>Request-Attraction-Area-? Occur Num: 1</li><li>Inform-Attraction-Type-museum Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-attraction-postcode Occur Num: 100</li><li>recommend-attraction-postcode Occur Num: 14</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-postcode Occur Num: 62</li><li>request-attraction-phone Occur Num: 50</li><li>request-attraction-address Occur Num: 32</li><li>request-attraction-entrance fee Occur Num: 25</li><li>request-attraction-area Occur Num: 9</li><li>request-attraction-type Occur Num: 6</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-phone Occur Num: 50</li><li>inform-attraction-area Occur Num: 11</li><li>inform-attraction-type Occur Num: 10</li></ul></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 58.4 %</p><p> (Precision, Recall, F1) : (0.482, 0.559, 0.502) </p><p> Average Dialog Turn (Succ): 9.618 </p><p> Average Dialog Turn (All): 17.416 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Restaurant-Phone-?<ul><li>Occur Num: 126</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 41</li><li>Request-Attraction-Phone-? Occur Num: 32</li><li>Request-Restaurant-Post-? Occur Num: 25</li><li>Request-Attraction-Addr-? Occur Num: 10</li><li>Request-Attraction-Post-? Occur Num: 5</li></ul></ul><li>Request-Restaurant-Addr-?<ul><li>Occur Num: 125</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 39</li><li>Request-Restaurant-Post-? Occur Num: 35</li><li>Request-Attraction-Addr-? Occur Num: 24</li><li>Request-Attraction-Phone-? Occur Num: 11</li><li>Request-Attraction-Post-? Occur Num: 6</li></ul></ul><li>Request-Restaurant-Post-?<ul><li>Occur Num: 86</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 35</li><li>Request-Restaurant-Phone-? Occur Num: 23</li><li>Request-Attraction-Post-? Occur Num: 13</li><li>Request-Attraction-Addr-? Occur Num: 6</li><li>Request-Attraction-Phone-? Occur Num: 5</li></ul></ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 28</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensively Occur Num: 11</li><li>Inform-Restaurant-Area-centre Occur Num: 4</li><li>Inform-Restaurant-Area-south Occur Num: 2</li><li>Inform-Restaurant-Area-do n't care Occur Num: 2</li><li>Inform-Restaurant-Food-indian Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Price-moderate<ul><li>Occur Num: 22</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 9</li><li>Inform-Restaurant-Area-centre Occur Num: 5</li><li>Inform-Restaurant-Food-italian Occur Num: 2</li><li>Inform-Restaurant-Food-british Occur Num: 2</li><li>Inform-Restaurant-Food-mediterranean Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 19</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 5</li><li>Inform-Restaurant-Food-british Occur Num: 3</li><li>Inform-Restaurant-Food-modern european Occur Num: 2</li><li>Inform-Restaurant-Price-expensively Occur Num: 2</li><li>Inform-Restaurant-Food-indian Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Price-cheap<ul><li>Occur Num: 17</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-cheaply Occur Num: 7</li><li>Inform-Restaurant-Food-italian Occur Num: 3</li><li>Inform-Restaurant-Area-centre Occur Num: 2</li><li>Inform-Restaurant-Food-european Occur Num: 1</li><li>Inform-Restaurant-Area-east Occur Num: 1</li></ul></ul><li>Request-Restaurant-Area-?<ul><li>Occur Num: 14</li><li>NLU Output</li><ul><li>Request-Attraction-Area-? Occur Num: 5</li><li>Request-Hotel-Area-? Occur Num: 4</li><li>Request-Restaurant-Addr-? Occur Num: 3</li><li>Request-Restaurant-Post-? Occur Num: 1</li><li>Request-Attraction-Post-? Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Area-dontcare<ul><li>Occur Num: 10</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-do n't care Occur Num: 7</li><li>Inform-Restaurant-Price-expensive Occur Num: 2</li><li>Inform-Restaurant-Price-cheap Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Food-portuguese<ul><li>Occur Num: 9</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 3</li><li>Inform-Restaurant-none-none Occur Num: 2</li><li>Inform-Restaurant-Area-centre Occur Num: 2</li><li>Inform-Restaurant-Price-moderate Occur Num: 1</li><li>Inform-Restaurant-Price-cheap Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Restaurant-Addr-? Occur Num: 44</li><li>Request-Restaurant-Phone-? Occur Num: 22</li><li>Request-Restaurant-Post-? Occur Num: 8</li><li>Inform-Restaurant-Food-italian Occur Num: 4</li><li>Inform-Restaurant-Food-indian Occur Num: 4</li><li>Inform-Restaurant-Food-chinese Occur Num: 4</li><li>Inform-Restaurant-Food-modern european Occur Num: 3</li><li>Inform-Restaurant-Food-international Occur Num: 3</li><li>Request-Restaurant-Area-? Occur Num: 3</li><li>Inform-Restaurant-Time-17:00 Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-restaurant-postcode Occur Num: 93</li><li>recommend-restaurant-postcode Occur Num: 30</li><li>inform-restaurant-pricerange Occur Num: 16</li><li>inform-restaurant-area Occur Num: 13</li><li>recommend-restaurant-pricerange Occur Num: 2</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-address Occur Num: 73</li><li>request-restaurant-postcode Occur Num: 56</li><li>request-restaurant-phone Occur Num: 23</li><li>request-restaurant-area Occur Num: 7</li><li>request-restaurant-pricerange Occur Num: 5</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-food Occur Num: 62</li><li>inform-restaurant-phone Occur Num: 17</li><li>inform-restaurant-area Occur Num: 11</li></ul></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 63.1 %</p><p> (Precision, Recall, F1) : (0.754, 0.670, 0.697) </p><p> Average Dialog Turn (Succ): 9.356 </p><p> Average Dialog Turn (All): 18.812 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Train-Dest-cambridge<ul><li>Occur Num: 136</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 21</li><li>Inform-Train-Day-friday Occur Num: 8</li><li>Inform-Train-Day-Saturday Occur Num: 7</li><li>Inform-Train-Day-wednesday Occur Num: 7</li><li>Inform-Train-Day-monday Occur Num: 7</li></ul></ul><li>Inform-Train-Depart-cambridge<ul><li>Occur Num: 117</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 20</li><li>Inform-Train-Day-friday Occur Num: 7</li><li>Inform-Train-Day-wednesday Occur Num: 7</li><li>Inform-Train-Day-monday Occur Num: 6</li><li>Inform-Train-Day-Saturday Occur Num: 5</li></ul></ul><li>Request-Train-Ticket-?<ul><li>Occur Num: 60</li><li>NLU Output</li><ul><li>Request-Train-Time-? Occur Num: 33</li><li>Request-Restaurant-Price-? Occur Num: 12</li><li>Request-Hotel-Price-? Occur Num: 9</li><li>Request-Train-Id-? Occur Num: 3</li><li>Request-Train-Leave-? Occur Num: 2</li></ul></ul><li>Request-Train-Time-?<ul><li>Occur Num: 50</li><li>NLU Output</li><ul><li>Request-Train-Ticket-? Occur Num: 33</li><li>Inform-Train-none-none Occur Num: 8</li><li>Request-Train-Id-? Occur Num: 7</li><li>Request-Train-Arrive-? Occur Num: 2</li></ul></ul><li>Inform-Train-Day-wednesday<ul><li>Occur Num: 37</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 6</li><li>Inform-Train-Dest-cambridge Occur Num: 5</li><li>Inform-Train-Depart-broxbourne Occur Num: 2</li><li>Inform-Train-Day-Saturday Occur Num: 2</li><li>Inform-Train-Depart-london liverpool street Occur Num: 2</li></ul></ul><li>Inform-Train-Day-monday<ul><li>Occur Num: 36</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 7</li><li>Inform-Train-Dest-norwich Occur Num: 3</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Dest-kings lynn Occur Num: 2</li><li>Inform-Train-Dest-leicester Occur Num: 2</li></ul></ul><li>Inform-Train-Day-friday<ul><li>Occur Num: 29</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 5</li><li>Inform-Train-Dest-cambridge Occur Num: 5</li><li>Inform-Train-Day-Saturday Occur Num: 3</li><li>Inform-Train-Arrive-11:15 Occur Num: 2</li><li>Inform-Restaurant-Day-friday Occur Num: 1</li></ul></ul><li>Inform-Train-Day-thursday<ul><li>Occur Num: 24</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 5</li><li>Inform-Train-Dest-cambridge Occur Num: 3</li><li>Inform-Train-Dest-stevenage Occur Num: 2</li><li>Inform-Train-Leave-15:30 Occur Num: 2</li><li>Inform-Train-Depart-kings lynn Occur Num: 1</li></ul></ul><li>Inform-Train-Day-tuesday<ul><li>Occur Num: 23</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Day-Saturday Occur Num: 2</li><li>Inform-Train-Dest-cambridge Occur Num: 2</li><li>Inform-Train-Leave-16:15 Occur Num: 1</li><li>Inform-Train-Arrive-09:15 Occur Num: 1</li></ul></ul><li>Inform-Train-Depart-norwich<ul><li>Occur Num: 19</li><li>NLU Output</li><ul><li>Inform-Train-Dest-norwich Occur Num: 4</li><li>Inform-Train-Dest-cambridge Occur Num: 3</li><li>Inform-Train-Day-monday Occur Num: 2</li><li>Inform-Train-Day-Saturday Occur Num: 2</li><li>Inform-Train-Arrive-08:00 Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Train-Ticket-? Occur Num: 75</li><li>Request-Train-Time-? Occur Num: 31</li><li>Inform-Train-People-1 Occur Num: 2</li><li>Request-Train-Leave-? Occur Num: 2</li><li>Inform-Train-People-7 Occur Num: 2</li><li>Request-Train-Arrive-? Occur Num: 2</li><li>Inform-Train-People-4 Occur Num: 1</li><li>Request-Train-Id-? Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-train-price Occur Num: 2</li><li>offerbooked-train-price Occur Num: 1</li><li>inform-train-duration Occur Num: 1</li><li>offerbook-train-leaveAt Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-train-price Occur Num: 108</li><li>request-train-trainID Occur Num: 3</li><li>request-train-arriveBy Occur Num: 3</li><li>request-train-leaveAt Occur Num: 2</li><li>request-train-duration Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 87.0 %</p><p> (Precision, Recall, F1) : (0.913, 0.913, 0.901) </p><p> Average Dialog Turn (Succ): 9.700 </p><p> Average Dialog Turn (All): 13.652 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Police-Phone-?<ul><li>Occur Num: 30</li><li>NLU Output</li><ul><li>Request-Police-Addr-? Occur Num: 7</li><li>Request-Restaurant-Phone-? Occur Num: 5</li><li>Request-Attraction-Phone-? Occur Num: 4</li><li>Inform-Police-none-none Occur Num: 4</li><li>Request-Attraction-Addr-? Occur Num: 2</li></ul></ul><li>Request-Police-Addr-?<ul><li>Occur Num: 19</li><li>NLU Output</li><ul><li>Request-Police-Phone-? Occur Num: 6</li><li>Request-Restaurant-Addr-? Occur Num: 4</li><li>Request-Attraction-Addr-? Occur Num: 2</li><li>Request-Attraction-Phone-? Occur Num: 2</li><li>Request-Restaurant-Phone-? Occur Num: 2</li></ul></ul><li>Inform-Police-none-none<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Police-Name-Parkside Police Station Occur Num: 6</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Police-Addr-? Occur Num: 3</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-police-postcode Occur Num: 3</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-police-address Occur Num: 3</li><li>request-police-phone Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 93.3 %</p><p> (Precision, Recall, F1) : (0.933, 0.933, 0.933) </p><p> Average Dialog Turn (Succ): 3.286 </p><p> Average Dialog Turn (All): 5.733 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Hospital-Phone-?<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 2</li><li>Request-Hotel-Phone-? Occur Num: 1</li><li>Request-Police-Phone-? Occur Num: 1</li><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-oncology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-plastic and vascular surgery plastics<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-plastic and vascular surgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-surgery<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-children's oncology and haematology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Hospital-Phone-? Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-hospital-phone Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 13.6 %</p><p> (Precision, Recall, F1) : (0.182, 0.159, 0.167) </p><p> Average Dialog Turn (Succ): 8.167 </p><p> Average Dialog Turn (All): 26.500 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Phone-?<ul><li>Occur Num: 86</li><li>NLU Output</li><ul><li>Request-Taxi-Car-? Occur Num: 69</li><li>Request-Hotel-Phone-? Occur Num: 6</li><li>Request-Police-Phone-? Occur Num: 3</li><li>Request-Restaurant-Phone-? Occur Num: 3</li><li>Request-Hospital-Phone-? Occur Num: 2</li></ul></ul><li>Inform-Taxi-Depart-the lensfield hotel<ul><li>Occur Num: 8</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-the Occur Num: 2</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 2</li><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li><li>Inform-Taxi-Dest-williams art and antiques Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the fitzwilliam museum<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 2</li><li>Inform-Taxi-Dest-golden house Occur Num: 1</li><li>Inform-Taxi-Leave-13:45 Occur Num: 1</li><li>Inform-Taxi-Arrive-13:45 Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li></ul></ul><li>Request-Taxi-Car-?<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Request-Hotel-Phone-? Occur Num: 6</li><li>Request-Hotel-Type-? Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-home from home<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-home from home Occur Num: 2</li><li>Inform-Taxi-Depart-arbury lodge guesthouse Occur Num: 1</li><li>Inform-Taxi-Leave-02:45 Occur Num: 1</li><li>Inform-Taxi-Depart-thanh binh Occur Num: 1</li><li>Inform-Taxi-Leave-02:30 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Leave-dontcare<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Inform-Taxi-Leave-do Occur Num: 3</li><li>Inform-Taxi-Leave-do n't care Occur Num: 1</li><li>Inform-Taxi-Depart-rajmahal Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-19:30<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-19:30 Occur Num: 2</li><li>Inform-Taxi-Depart-warkworth house Occur Num: 1</li><li>Inform-Taxi-Dest-the gardenia Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-great saint mary's church<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-broughton house gallery Occur Num: 1</li><li>Inform-Taxi-Arrive-01:15 Occur Num: 1</li><li>Inform-Taxi-none-none Occur Num: 1</li><li>Inform-Taxi-Leave-01:15 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-13:45<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 1</li><li>Inform-Taxi-Dest-la mimosa Occur Num: 1</li><li>Inform-Train-Arrive-13:45 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the good luck chinese food takeaway<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-nandos Occur Num: 1</li><li>Inform-Taxi-Leave-05:15 Occur Num: 1</li><li>Inform-Taxi-Depart-good luck Occur Num: 1</li><li>Inform-Taxi-Depart-good luck chinese food Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Taxi-Car-? Occur Num: 68</li><li>Request-Taxi-Phone-? Occur Num: 4</li><li>Inform-Taxi-Depart-London Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-taxi-phone Occur Num: 76</li><li>request-taxi-car type Occur Num: 72</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/BERTNLU-RuleDST-LaRL/res.txt b/tests/results/BERTNLU-RuleDST-LaRL/res.txt deleted file mode 100755 index a575df7c90bbe4e8b46972f13899ec2f0570175e..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-LaRL/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.406 -success number of dialogs/tot: 0.34 -average precision: 0.4775414345324553 -average recall: 0.5413267072145577 -average f1: 0.4764173612317243 -average book rate: 0.4560897435897436 -average turn (succ): 15.0 -average turn (all): 28.6 diff --git a/tests/results/BERTNLU-RuleDST-MDRG/Frequency_of_domain.png b/tests/results/BERTNLU-RuleDST-MDRG/Frequency_of_domain.png deleted file mode 100755 index d6e888b4e82a6c0fef5f21d674b1cf35983bee23..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-MDRG/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-MDRG/Performance_for_each_domain.png b/tests/results/BERTNLU-RuleDST-MDRG/Performance_for_each_domain.png deleted file mode 100755 index 7792cac38494c9664cc3391a21eeeb1d2d7bcdad..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-MDRG/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-MDRG/Proportions_of_the_dialogue_loop.png b/tests/results/BERTNLU-RuleDST-MDRG/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index 55df730962f8a81fffec94c871e99855b896af08..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-MDRG/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-MDRG/report_multiwoz.html b/tests/results/BERTNLU-RuleDST-MDRG/report_multiwoz.html deleted file mode 100755 index a1e5dcc13841a9234715104e37966f1f142021ac..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-MDRG/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: BERTNLU-RuleDST-MDRG</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-28 06:00:57</h3><h3>Overall Results</h3><p> Success Rate: 25.2 %</p><p> (Precision, Recall, F1) : (0.466, 0.431, 0.420) </p><p> Average Dialog Turn (Succ): 13.635 </p><p> Average Dialog Turn (All): 33.580 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">332</td><td align="center">0.449</td><td align="center">0.306</td><td align="center">0.414</td><td align="center">0.339</td><td align="center">0.497</td><td align="center">7.960</td><td align="center">20.464</td></tr><tr><td align="center">attraction</td><td align="center">285</td><td align="center">0.512</td><td align="center">0.623</td><td align="center">0.603</td><td align="center">0.592</td><td align="center">0.467</td><td align="center">11.082</td><td align="center">21.551</td></tr><tr><td align="center">restaurant</td><td align="center">378</td><td align="center">0.706</td><td align="center">0.505</td><td align="center">0.652</td><td align="center">0.552</td><td align="center">0.288</td><td align="center">6.412</td><td align="center">15.106</td></tr><tr><td align="center">train</td><td align="center">329</td><td align="center">0.377</td><td align="center">0.515</td><td align="center">0.411</td><td align="center">0.445</td><td align="center">0.614</td><td align="center">8.000</td><td align="center">25.088</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">1.000</td><td align="center">0.000</td><td align="center">40.000</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">1.000</td><td align="center">0.000</td><td align="center">40.000</td></tr><tr><td align="center">taxi</td><td align="center">74</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.946</td><td align="center">0.000</td><td align="center">30.973</td></tr></table><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 44.9 %</p><p> (Precision, Recall, F1) : (0.306, 0.414, 0.339) </p><p> Average Dialog Turn (Succ): 7.960 </p><p> Average Dialog Turn (All): 20.464 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 320</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 52</li><li>Inform-Hotel-Type-hotel Occur Num: 38</li><li>Inform-Hotel-Stars-4 Occur Num: 38</li><li>Inform-Hotel-Type-guesthouse Occur Num: 37</li><li>Inform-Hotel-Price-moderate Occur Num: 28</li></ul></ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 297</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 53</li><li>Inform-Hotel-Stars-4 Occur Num: 35</li><li>Inform-Hotel-Type-guesthouse Occur Num: 35</li><li>Inform-Hotel-Type-hotel Occur Num: 34</li><li>Inform-Hotel-Price-moderate Occur Num: 25</li></ul></ul><li>Request-Hotel-Addr-?<ul><li>Occur Num: 196</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 61</li><li>Request-Hotel-Post-? Occur Num: 24</li><li>Request-Hotel-Phone-? Occur Num: 21</li><li>Request-Attraction-Addr-? Occur Num: 18</li><li>Request-Hotel-Parking-? Occur Num: 10</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 184</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 37</li><li>Inform-Hotel-Internet-yes Occur Num: 34</li><li>Inform-Hotel-Stars-4 Occur Num: 16</li><li>Inform-Hotel-Area-north Occur Num: 14</li><li>Inform-Hotel-Price-moderate Occur Num: 14</li></ul></ul><li>Inform-Hotel-Stars-4<ul><li>Occur Num: 172</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 40</li><li>Inform-Hotel-Internet-yes Occur Num: 35</li><li>Inform-Hotel-Type-hotel Occur Num: 24</li><li>Inform-Hotel-Type-guesthouse Occur Num: 16</li><li>Inform-Hotel-Price-moderate Occur Num: 11</li></ul></ul><li>Inform-Hotel-Price-moderate<ul><li>Occur Num: 150</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 28</li><li>Inform-Hotel-Internet-yes Occur Num: 26</li><li>Inform-Hotel-Type-hotel Occur Num: 20</li><li>Inform-Hotel-Type-guesthouse Occur Num: 14</li><li>Inform-Hotel-Stars-4 Occur Num: 11</li></ul></ul><li>Request-Hotel-Phone-?<ul><li>Occur Num: 96</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 26</li><li>Request-Hotel-Addr-? Occur Num: 20</li><li>Request-Attraction-Phone-? Occur Num: 11</li><li>Request-Hotel-Post-? Occur Num: 10</li><li>Request-Restaurant-Addr-? Occur Num: 9</li></ul></ul><li>Inform-Hotel-Area-north<ul><li>Occur Num: 96</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 23</li><li>Inform-Hotel-Parking-yes Occur Num: 22</li><li>Inform-Hotel-Type-guesthouse Occur Num: 14</li><li>Inform-Hotel-Stars-4 Occur Num: 8</li><li>Inform-Hotel-Type-hotel Occur Num: 7</li></ul></ul><li>Request-Hotel-Post-?<ul><li>Occur Num: 89</li><li>NLU Output</li><ul><li>Request-Hotel-Addr-? Occur Num: 23</li><li>Request-Restaurant-Post-? Occur Num: 16</li><li>Request-Hotel-Phone-? Occur Num: 10</li><li>Request-Restaurant-Addr-? Occur Num: 7</li><li>Request-Hospital-Post-? Occur Num: 5</li></ul></ul><li>Inform-Hotel-Parking-no<ul><li>Occur Num: 79</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 17</li><li>Inform-Hotel-Type-guesthouse Occur Num: 11</li><li>Inform-Hotel-Internet-yes Occur Num: 11</li><li>Inform-Hotel-Price-moderate Occur Num: 8</li><li>Inform-Hotel-Stars-4 Occur Num: 6</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Hotel-Addr-? Occur Num: 47</li><li>Request-Hotel-Phone-? Occur Num: 26</li><li>Request-Hotel-Post-? Occur Num: 22</li><li>Request-Hotel-Stars-? Occur Num: 12</li><li>Request-Hotel-Internet-? Occur Num: 8</li><li>Request-Hotel-Type-? Occur Num: 7</li><li>Inform-Hotel-Stay-4 Occur Num: 6</li><li>Inform-Hotel-Stay-5 Occur Num: 5</li><li>Request-Hotel-Price-? Occur Num: 5</li><li>Inform-Hotel-Type-guesthouse Occur Num: 5</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-parking Occur Num: 33</li><li>inform-hotel-pricerange Occur Num: 18</li><li>inform-hotel-internet Occur Num: 13</li><li>inform-hotel-postcode Occur Num: 1</li><li>recommend-hotel-pricerange Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-address Occur Num: 59</li><li>request-hotel-phone Occur Num: 30</li><li>request-hotel-postcode Occur Num: 25</li><li>request-hotel-parking Occur Num: 20</li><li>request-hotel-stars Occur Num: 17</li><li>request-hotel-internet Occur Num: 16</li><li>request-hotel-type Occur Num: 11</li><li>request-hotel-pricerange Occur Num: 11</li><li>request-hotel-area Occur Num: 4</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-area Occur Num: 120</li><li>inform-hotel-type Occur Num: 52</li><li>inform-hotel-phone Occur Num: 2</li></ul></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 51.2 %</p><p> (Precision, Recall, F1) : (0.623, 0.603, 0.592) </p><p> Average Dialog Turn (Succ): 11.082 </p><p> Average Dialog Turn (All): 21.551 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Attraction-Addr-?<ul><li>Occur Num: 263</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 65</li><li>Request-Attraction-Post-? Occur Num: 64</li><li>Request-Attraction-Fee-? Occur Num: 45</li><li>Request-Restaurant-Addr-? Occur Num: 33</li><li>Request-Restaurant-Post-? Occur Num: 22</li></ul></ul><li>Request-Attraction-Post-?<ul><li>Occur Num: 255</li><li>NLU Output</li><ul><li>Request-Attraction-Addr-? Occur Num: 62</li><li>Request-Attraction-Phone-? Occur Num: 58</li><li>Request-Attraction-Fee-? Occur Num: 44</li><li>Request-Restaurant-Post-? Occur Num: 40</li><li>Request-Restaurant-Phone-? Occur Num: 17</li></ul></ul><li>Request-Attraction-Phone-?<ul><li>Occur Num: 245</li><li>NLU Output</li><ul><li>Request-Attraction-Addr-? Occur Num: 67</li><li>Request-Attraction-Post-? Occur Num: 60</li><li>Request-Attraction-Fee-? Occur Num: 38</li><li>Request-Restaurant-Phone-? Occur Num: 35</li><li>Request-Restaurant-Post-? Occur Num: 13</li></ul></ul><li>Request-Attraction-Fee-?<ul><li>Occur Num: 136</li><li>NLU Output</li><ul><li>Request-Attraction-Addr-? Occur Num: 45</li><li>Request-Attraction-Post-? Occur Num: 44</li><li>Request-Attraction-Phone-? Occur Num: 37</li><li>Request-Attraction-Type-? Occur Num: 7</li><li>Request-Attraction-Area-? Occur Num: 2</li></ul></ul><li>Request-Attraction-Type-?<ul><li>Occur Num: 36</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 11</li><li>Request-Attraction-Addr-? Occur Num: 9</li><li>Request-Attraction-Post-? Occur Num: 8</li><li>Request-Attraction-Fee-? Occur Num: 7</li><li>Inform-Attraction-Type-dontcare Occur Num: 1</li></ul></ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 36</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-entertainment Occur Num: 14</li><li>Inform-Attraction-Type-do n't care Occur Num: 4</li><li>Inform-Attraction-Type-colleges Occur Num: 3</li><li>Inform-Restaurant-Area-centre Occur Num: 2</li><li>Inform-Attraction-Type-museums Occur Num: 2</li></ul></ul><li>Inform-Attraction-Type-dontcare<ul><li>Occur Num: 18</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-do n't care Occur Num: 12</li><li>Inform-Attraction-Area-centre Occur Num: 4</li><li>Inform-Attraction-Type-entertainment Occur Num: 1</li><li>Inform-Attraction-Area-south Occur Num: 1</li></ul></ul><li>Inform-Attraction-Type-boat<ul><li>Occur Num: 16</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 6</li><li>Inform-Attraction-Type-boats Occur Num: 5</li><li>Inform-Attraction-Type-entertainment Occur Num: 3</li><li>Inform-Attraction-Area-west Occur Num: 1</li><li>Inform-Attraction-Area-north Occur Num: 1</li></ul></ul><li>Inform-Attraction-Type-nightclub<ul><li>Occur Num: 16</li><li>NLU Output</li><ul><li>Inform-Attraction-none-none Occur Num: 5</li><li>Inform-Attraction-Area-centre Occur Num: 5</li><li>Inform-Attraction-Area-west Occur Num: 4</li><li>Inform-Attraction-Type-entertainment Occur Num: 2</li></ul></ul><li>Request-Attraction-Area-?<ul><li>Occur Num: 16</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 3</li><li>Request-Attraction-Fee-? Occur Num: 3</li><li>Request-Attraction-Addr-? Occur Num: 3</li><li>Request-Restaurant-Area-? Occur Num: 2</li><li>Request-Restaurant-Post-? Occur Num: 2</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Attraction-Fee-? Occur Num: 74</li><li>Request-Attraction-Post-? Occur Num: 24</li><li>Request-Attraction-Addr-? Occur Num: 19</li><li>Request-Attraction-Type-? Occur Num: 8</li><li>Request-Attraction-Phone-? Occur Num: 5</li><li>Request-Attraction-Area-? Occur Num: 1</li><li>Inform-Attraction-Name-all saints church Occur Num: 1</li><li>Inform-Attraction-Name-broughton house gallery Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-attraction-area Occur Num: 4</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-entrance fee Occur Num: 80</li><li>request-attraction-postcode Occur Num: 56</li><li>request-attraction-phone Occur Num: 54</li><li>request-attraction-address Occur Num: 53</li><li>request-attraction-type Occur Num: 15</li><li>request-attraction-area Occur Num: 3</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-area Occur Num: 60</li></ul></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 70.6 %</p><p> (Precision, Recall, F1) : (0.505, 0.652, 0.552) </p><p> Average Dialog Turn (Succ): 6.412 </p><p> Average Dialog Turn (All): 15.106 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Restaurant-Addr-?<ul><li>Occur Num: 110</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 55</li><li>Request-Restaurant-Post-? Occur Num: 44</li><li>Request-Attraction-Addr-? Occur Num: 5</li><li>Request-Restaurant-Price-? Occur Num: 3</li><li>Request-Attraction-Post-? Occur Num: 2</li></ul></ul><li>Request-Restaurant-Phone-?<ul><li>Occur Num: 97</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 57</li><li>Request-Restaurant-Post-? Occur Num: 26</li><li>Request-Attraction-Phone-? Occur Num: 5</li><li>Request-Hospital-Post-? Occur Num: 3</li><li>Request-Restaurant-Price-? Occur Num: 3</li></ul></ul><li>Request-Restaurant-Post-?<ul><li>Occur Num: 82</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 44</li><li>Request-Restaurant-Phone-? Occur Num: 25</li><li>Request-Hospital-Post-? Occur Num: 6</li><li>Request-Attraction-Post-? Occur Num: 4</li><li>Request-Restaurant-Food-? Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 29</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensively Occur Num: 11</li><li>Inform-Restaurant-Area-centre Occur Num: 4</li><li>Inform-Restaurant-Area-south Occur Num: 2</li><li>Inform-Restaurant-Area-east Occur Num: 2</li><li>Inform-Restaurant-Food-indian Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Price-moderate<ul><li>Occur Num: 23</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 9</li><li>Inform-Restaurant-Area-centre Occur Num: 7</li><li>Inform-Restaurant-Food-british Occur Num: 2</li><li>Inform-Restaurant-Food-mediterranean Occur Num: 1</li><li>Inform-Restaurant-Food-italian Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 17</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 6</li><li>Inform-Restaurant-Food-british Occur Num: 3</li><li>Inform-Restaurant-Price-expensively Occur Num: 2</li><li>Inform-Restaurant-Price-moderate Occur Num: 1</li><li>Inform-Restaurant-Food-modern european Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Area-dontcare<ul><li>Occur Num: 12</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-do n't care Occur Num: 12</li></ul></ul><li>Inform-Restaurant-Food-portuguese<ul><li>Occur Num: 12</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 3</li><li>Inform-Restaurant-Price-expensive Occur Num: 3</li><li>Inform-Restaurant-Price-moderate Occur Num: 2</li><li>Inform-Restaurant-none-none Occur Num: 2</li><li>Inform-Restaurant-Area-north Occur Num: 1</li></ul></ul><li>Request-Restaurant-Price-?<ul><li>Occur Num: 12</li><li>NLU Output</li><ul><li>Request-Hotel-Price-? Occur Num: 3</li><li>Inform-Restaurant-none-none Occur Num: 3</li><li>Request-Restaurant-Phone-? Occur Num: 3</li><li>Request-Restaurant-Addr-? Occur Num: 3</li></ul></ul><li>Inform-Restaurant-Food-dontcare<ul><li>Occur Num: 10</li><li>NLU Output</li><ul><li>Inform-Restaurant-Food-do n't care Occur Num: 10</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Restaurant-Phone-? Occur Num: 48</li><li>Request-Restaurant-Addr-? Occur Num: 22</li><li>Request-Restaurant-Post-? Occur Num: 15</li><li>Request-Restaurant-Food-? Occur Num: 4</li><li>Request-Restaurant-Price-? Occur Num: 3</li><li>Inform-Restaurant-Day-sunday Occur Num: 2</li><li>Inform-Restaurant-Time-12:30 Occur Num: 1</li><li>Inform-Restaurant-Time-10:45 Occur Num: 1</li><li>Inform-Restaurant-Day-tuesday Occur Num: 1</li><li>Inform-Restaurant-Time-15:30 Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-restaurant-area Occur Num: 37</li><li>inform-restaurant-pricerange Occur Num: 10</li><li>recommend-restaurant-area Occur Num: 2</li><li>inform-restaurant-postcode Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-phone Occur Num: 47</li><li>request-restaurant-address Occur Num: 39</li><li>request-restaurant-postcode Occur Num: 20</li><li>request-restaurant-pricerange Occur Num: 7</li><li>request-restaurant-food Occur Num: 6</li><li>request-restaurant-area Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-area Occur Num: 98</li><li>inform-restaurant-food Occur Num: 23</li><li>inform-restaurant-phone Occur Num: 4</li></ul></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 37.7 %</p><p> (Precision, Recall, F1) : (0.515, 0.411, 0.445) </p><p> Average Dialog Turn (Succ): 8.000 </p><p> Average Dialog Turn (All): 25.088 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Train-Dest-cambridge<ul><li>Occur Num: 155</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 29</li><li>Inform-Train-Day-Saturday Occur Num: 9</li><li>Inform-Train-Day-friday Occur Num: 8</li><li>Inform-Train-Day-thursday Occur Num: 8</li><li>Inform-Train-Day-wednesday Occur Num: 6</li></ul></ul><li>Inform-Train-Depart-cambridge<ul><li>Occur Num: 127</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 24</li><li>Inform-Train-Day-friday Occur Num: 9</li><li>Inform-Train-Day-thursday Occur Num: 9</li><li>Inform-Train-Day-Saturday Occur Num: 6</li><li>Inform-Train-Day-tuesday Occur Num: 6</li></ul></ul><li>Request-Train-Time-?<ul><li>Occur Num: 121</li><li>NLU Output</li><ul><li>Inform-Train-none-none Occur Num: 59</li><li>Request-Train-Ticket-? Occur Num: 35</li><li>Request-Train-Id-? Occur Num: 16</li><li>Request-Train-Arrive-? Occur Num: 8</li><li>Request-Train-Leave-? Occur Num: 3</li></ul></ul><li>Request-Train-Ticket-?<ul><li>Occur Num: 69</li><li>NLU Output</li><ul><li>Request-Train-Time-? Occur Num: 35</li><li>Request-Train-Id-? Occur Num: 11</li><li>Request-Restaurant-Price-? Occur Num: 7</li><li>Request-Train-Leave-? Occur Num: 7</li><li>Request-Train-Arrive-? Occur Num: 5</li></ul></ul><li>Request-Train-Id-?<ul><li>Occur Num: 59</li><li>NLU Output</li><ul><li>Request-Train-Time-? Occur Num: 23</li><li>Request-Train-Ticket-? Occur Num: 15</li><li>Inform-Train-none-none Occur Num: 12</li><li>Request-Train-Arrive-? Occur Num: 5</li><li>Request-Train-Leave-? Occur Num: 4</li></ul></ul><li>Request-Train-Leave-?<ul><li>Occur Num: 40</li><li>NLU Output</li><ul><li>Inform-Train-none-none Occur Num: 23</li><li>Request-Train-Time-? Occur Num: 7</li><li>Request-Train-Ticket-? Occur Num: 7</li><li>Request-Train-Id-? Occur Num: 3</li></ul></ul><li>Inform-Train-Day-friday<ul><li>Occur Num: 37</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 6</li><li>Inform-Train-Dest-cambridge Occur Num: 5</li><li>Inform-Train-Day-Saturday Occur Num: 3</li><li>Inform-Train-Arrive-11:15 Occur Num: 2</li><li>Inform-Train-Depart-leicester Occur Num: 2</li></ul></ul><li>Inform-Train-Day-thursday<ul><li>Occur Num: 34</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 7</li><li>Inform-Train-Dest-cambridge Occur Num: 5</li><li>Inform-Train-Leave-15:30 Occur Num: 2</li><li>Inform-Train-Depart-kings lynn Occur Num: 1</li><li>Inform-Train-Arrive-19:15 Occur Num: 1</li></ul></ul><li>Inform-Train-Day-monday<ul><li>Occur Num: 29</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 4</li><li>Inform-Train-Depart-cambridge Occur Num: 4</li><li>Inform-Train-Dest-kings lynn Occur Num: 2</li><li>Inform-Train-Day-Saturday Occur Num: 2</li><li>Inform-Train-Arrive-13:30 Occur Num: 1</li></ul></ul><li>Inform-Train-Day-tuesday<ul><li>Occur Num: 27</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 4</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Dest-norwich Occur Num: 2</li><li>Inform-Train-Arrive-09:15 Occur Num: 2</li><li>Inform-Train-Day-Saturday Occur Num: 2</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Train-Time-? Occur Num: 96</li><li>Request-Train-Ticket-? Occur Num: 67</li><li>Request-Train-Leave-? Occur Num: 25</li><li>Request-Train-Arrive-? Occur Num: 14</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-train-duration Occur Num: 2</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-train-price Occur Num: 104</li><li>request-train-duration Occur Num: 85</li><li>request-train-leaveAt Occur Num: 23</li><li>request-train-arriveBy Occur Num: 19</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 0.0 %</p><p> (Precision, Recall, F1) : (0.000, 0.000, 0.000) </p><p> Average Dialog Turn (Succ): 0.000 </p><p> Average Dialog Turn (All): 40.000 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Police-Phone-?<ul><li>Occur Num: 49</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 19</li><li>Request-Police-Addr-? Occur Num: 14</li><li>Request-Restaurant-Addr-? Occur Num: 8</li><li>Inform-Police-none-none Occur Num: 5</li><li>Request-Hospital-Phone-? Occur Num: 3</li></ul></ul><li>Request-Police-Addr-?<ul><li>Occur Num: 36</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 13</li><li>Request-Police-Phone-? Occur Num: 12</li><li>Request-Restaurant-Addr-? Occur Num: 11</li></ul></ul><li>Inform-Police-none-none<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Police-Name-Parkside Police Station Occur Num: 6</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Police-Addr-? Occur Num: 17</li><li>Request-Police-Phone-? Occur Num: 6</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-police-phone Occur Num: 20</li><li>request-police-address Occur Num: 17</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 0.0 %</p><p> (Precision, Recall, F1) : (0.000, 0.000, 0.000) </p><p> Average Dialog Turn (Succ): 0.000 </p><p> Average Dialog Turn (All): 40.000 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Hospital-Phone-?<ul><li>Occur Num: 28</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 28</li></ul></ul><li>Inform-Hospital-Department-oncology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-plastic and vascular surgery plastics<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-plastic and vascular surgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-surgery<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-children's oncology and haematology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Hospital-Phone-? Occur Num: 30</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-hospital-phone Occur Num: 30</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 0.0 %</p><p> (Precision, Recall, F1) : (0.000, 0.000, 0.000) </p><p> Average Dialog Turn (Succ): 0.000 </p><p> Average Dialog Turn (All): 30.973 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Phone-?<ul><li>Occur Num: 89</li><li>NLU Output</li><ul><li>Request-Taxi-Car-? Occur Num: 71</li><li>Request-Hotel-Type-? Occur Num: 14</li><li>Request-Hotel-Phone-? Occur Num: 2</li><li>Request-Attraction-Type-? Occur Num: 2</li></ul></ul><li>Request-Taxi-Car-?<ul><li>Occur Num: 18</li><li>NLU Output</li><ul><li>Request-Hotel-Type-? Occur Num: 14</li><li>Request-Hotel-Phone-? Occur Num: 2</li><li>Request-Attraction-Type-? Occur Num: 2</li></ul></ul><li>Inform-Taxi-Depart-the lensfield hotel<ul><li>Occur Num: 8</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-the Occur Num: 2</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 2</li><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li><li>Inform-Taxi-Dest-williams art and antiques Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the fitzwilliam museum<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 2</li><li>Inform-Taxi-Dest-golden house Occur Num: 1</li><li>Inform-Taxi-Leave-13:45 Occur Num: 1</li><li>Inform-Taxi-Arrive-13:45 Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-home from home<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-home from home Occur Num: 2</li><li>Inform-Taxi-Depart-arbury lodge guesthouse Occur Num: 1</li><li>Inform-Taxi-Leave-02:45 Occur Num: 1</li><li>Inform-Taxi-Depart-thanh binh Occur Num: 1</li><li>Inform-Taxi-Leave-02:30 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-13:45<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 1</li><li>Inform-Taxi-Dest-la mimosa Occur Num: 1</li><li>Inform-Train-Arrive-13:45 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-02:00<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-parkside police station<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-lynne strover gallery Occur Num: 1</li><li>Inform-Police-Name-parkside police station Occur Num: 1</li><li>Inform-Taxi-Arrive-12:15 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-la mimosa<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Arrive-13:45 Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-efes restaurant<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-broxbourne train station Occur Num: 1</li><li>Inform-Taxi-Dest-efes Occur Num: 1</li><li>Inform-Taxi-Arrive-16:15 Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Taxi-Car-? Occur Num: 70</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-taxi-car type Occur Num: 74</li><li>request-taxi-phone Occur Num: 74</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/BERTNLU-RuleDST-MDRG/res.txt b/tests/results/BERTNLU-RuleDST-MDRG/res.txt deleted file mode 100755 index b4689d30d89afe4aac0159049eb5e125d4bf814c..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-MDRG/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.27 -success number of dialogs/tot: 0.252 -average precision: 0.46578760260101465 -average recall: 0.4306643425802305 -average f1: 0.4197515091550275 -average book rate: 0.4897435897435897 -average turn (succ): 13.634920634920634 -average turn (all): 33.58 diff --git a/tests/results/BERTNLU-RuleDST-MLEPolicy-TemplateNLG/Frequency_of_domain.png b/tests/results/BERTNLU-RuleDST-MLEPolicy-TemplateNLG/Frequency_of_domain.png deleted file mode 100755 index e3b6d58e14743a35e4294b3a9c99403f19ee2195..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-MLEPolicy-TemplateNLG/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-MLEPolicy-TemplateNLG/Performance_for_each_domain.png b/tests/results/BERTNLU-RuleDST-MLEPolicy-TemplateNLG/Performance_for_each_domain.png deleted file mode 100755 index 924d1b82e520474bc62d22f800da474bac5a790e..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-MLEPolicy-TemplateNLG/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-MLEPolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png b/tests/results/BERTNLU-RuleDST-MLEPolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index 42aa54acaaaf689a62722b4b06a65fad07cce913..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-MLEPolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-MLEPolicy-TemplateNLG/report_multiwoz.html b/tests/results/BERTNLU-RuleDST-MLEPolicy-TemplateNLG/report_multiwoz.html deleted file mode 100755 index e311b468c7a268e8e985c538f0f240496be5fdcc..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-MLEPolicy-TemplateNLG/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: BERTNLU-RuleDST-MLEPolicy-TemplateNLG</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-27 02:37:46</h3><h3>Overall Results</h3><p> Success Rate: 48.4 %</p><p> (Precision, Recall, F1) : (0.663, 0.727, 0.660) </p><p> Average Dialog Turn (Succ): 12.455 </p><p> Average Dialog Turn (All): 26.270 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">375</td><td align="center">0.432</td><td align="center">0.422</td><td align="center">0.503</td><td align="center">0.434</td><td align="center">0.469</td><td align="center">8.074</td><td align="center">19.781</td></tr><tr><td align="center">restaurant</td><td align="center">394</td><td align="center">0.609</td><td align="center">0.492</td><td align="center">0.722</td><td align="center">0.558</td><td align="center">0.386</td><td align="center">10.183</td><td align="center">20.365</td></tr><tr><td align="center">attraction</td><td align="center">289</td><td align="center">0.875</td><td align="center">0.770</td><td align="center">0.916</td><td align="center">0.812</td><td align="center">0.107</td><td align="center">6.427</td><td align="center">9.723</td></tr><tr><td align="center">taxi</td><td align="center">99</td><td align="center">0.677</td><td align="center">0.747</td><td align="center">0.712</td><td align="center">0.724</td><td align="center">0.323</td><td align="center">7.313</td><td align="center">13.192</td></tr><tr><td align="center">train</td><td align="center">327</td><td align="center">0.795</td><td align="center">0.867</td><td align="center">0.868</td><td align="center">0.862</td><td align="center">0.202</td><td align="center">7.646</td><td align="center">13.125</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">1.000</td><td align="center">0.935</td><td align="center">1.000</td><td align="center">0.957</td><td align="center">0.000</td><td align="center">2.000</td><td align="center">2.000</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">0.933</td><td align="center">0.933</td><td align="center">0.933</td><td align="center">0.933</td><td align="center">0.067</td><td align="center">4.000</td><td align="center">6.400</td></tr></table><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 43.2 %</p><p> (Precision, Recall, F1) : (0.422, 0.503, 0.434) </p><p> Average Dialog Turn (Succ): 8.074 </p><p> Average Dialog Turn (All): 19.781 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 343</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 55</li><li>Inform-Hotel-Type-hotel Occur Num: 54</li><li>Inform-Hotel-Type-guesthouse Occur Num: 41</li><li>Inform-Hotel-Stars-4 Occur Num: 36</li><li>Inform-Hotel-Price-moderate Occur Num: 33</li></ul></ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 340</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-hotel Occur Num: 64</li><li>Inform-Hotel-Internet-yes Occur Num: 57</li><li>Inform-Hotel-Type-guesthouse Occur Num: 39</li><li>Inform-Hotel-Stars-4 Occur Num: 38</li><li>Inform-Hotel-Price-moderate Occur Num: 28</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 211</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 41</li><li>Inform-Hotel-Parking-yes Occur Num: 37</li><li>Inform-Hotel-Stars-4 Occur Num: 21</li><li>Inform-Hotel-Type-hotel Occur Num: 20</li><li>Inform-Hotel-Price-moderate Occur Num: 15</li></ul></ul><li>Inform-Hotel-Stars-4<ul><li>Occur Num: 191</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 38</li><li>Inform-Hotel-Internet-yes Occur Num: 36</li><li>Inform-Hotel-Type-hotel Occur Num: 35</li><li>Inform-Hotel-Type-guesthouse Occur Num: 21</li><li>Inform-Hotel-Price-cheap Occur Num: 11</li></ul></ul><li>Inform-Hotel-Price-moderate<ul><li>Occur Num: 165</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 34</li><li>Inform-Hotel-Type-hotel Occur Num: 33</li><li>Inform-Hotel-Parking-yes Occur Num: 26</li><li>Inform-Hotel-Type-guesthouse Occur Num: 15</li><li>Inform-Hotel-Area-centre Occur Num: 10</li></ul></ul><li>Request-Hotel-Addr-?<ul><li>Occur Num: 141</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 21</li><li>Request-Hotel-Post-? Occur Num: 19</li><li>Request-Attraction-Addr-? Occur Num: 18</li><li>Request-Hotel-Phone-? Occur Num: 14</li><li>Request-Hotel-Internet-? Occur Num: 10</li></ul></ul><li>Inform-Hotel-Area-north<ul><li>Occur Num: 110</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 24</li><li>Inform-Hotel-Parking-yes Occur Num: 20</li><li>Inform-Hotel-Type-hotel Occur Num: 17</li><li>Inform-Hotel-Type-guesthouse Occur Num: 14</li><li>Inform-Hotel-Price-do n't care Occur Num: 8</li></ul></ul><li>Inform-Hotel-Area-centre<ul><li>Occur Num: 91</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 20</li><li>Inform-Hotel-Type-hotel Occur Num: 15</li><li>Inform-Hotel-Type-guesthouse Occur Num: 13</li><li>Inform-Hotel-Parking-yes Occur Num: 10</li><li>Inform-Hotel-Price-moderate Occur Num: 10</li></ul></ul><li>Inform-Hotel-Price-cheap<ul><li>Occur Num: 86</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-hotel Occur Num: 18</li><li>Inform-Hotel-Internet-yes Occur Num: 16</li><li>Inform-Hotel-Parking-yes Occur Num: 16</li><li>Inform-Hotel-Stars-4 Occur Num: 11</li><li>Inform-Hotel-Type-guesthouse Occur Num: 11</li></ul></ul><li>Inform-Hotel-Parking-no<ul><li>Occur Num: 78</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 18</li><li>Inform-Hotel-Internet-yes Occur Num: 12</li><li>Inform-Hotel-Type-guesthouse Occur Num: 10</li><li>Inform-Hotel-Type-hotel Occur Num: 8</li><li>Inform-Hotel-Area-north Occur Num: 6</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Name-acorn guest house<ul><li>Occur Num: 380</li><li>NLU Output</li><ul><li>Recommend-Hotel-Name-acorn guest house Occur Num: 51</li><li>Inform-Booking-none-none Occur Num: 49</li><li>Inform-Hotel-Area-north Occur Num: 29</li><li>reqmore-general-none-none Occur Num: 27</li><li>Inform-Hotel-Addr-154 chesterton road Occur Num: 23</li></ul></ul><li>Request-Hotel-Area-?<ul><li>Occur Num: 342</li><li>NLU Output</li><ul><li>Request-Hotel-Price-? Occur Num: 44</li><li>Inform-Hotel-Choice-3 Occur Num: 32</li><li>Inform-Hotel-Choice-0 Occur Num: 31</li><li>NoOffer-Hotel-none-none Occur Num: 24</li><li>Inform-Hotel-Choice-8 Occur Num: 21</li></ul></ul><li>Inform-Hotel-Price-moderate<ul><li>Occur Num: 308</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 51</li><li>Recommend-Hotel-Price-moderate Occur Num: 27</li><li>Recommend-Hotel-Name-acorn guest house Occur Num: 19</li><li>Inform-Hotel-Area-north Occur Num: 16</li><li>Inform-Hotel-Choice-2 Occur Num: 10</li></ul></ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 295</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 44</li><li>Inform-Booking-none-none Occur Num: 38</li><li>Recommend-Hotel-Parking-none Occur Num: 24</li><li>Inform-Hotel-Choice-2 Occur Num: 17</li><li>Inform-Hotel-Internet-none Occur Num: 15</li></ul></ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 280</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-none Occur Num: 48</li><li>Inform-Booking-none-none Occur Num: 46</li><li>Inform-Hotel-Parking-none Occur Num: 16</li><li>Recommend-Hotel-Internet-none Occur Num: 10</li><li>Request-Booking-Day-? Occur Num: 8</li></ul></ul><li>Inform-Hotel-Name-huntingdon marriott hotel<ul><li>Occur Num: 273</li><li>NLU Output</li><ul><li>Recommend-Hotel-Name-huntingdon marriott hotel Occur Num: 27</li><li>Inform-Booking-none-none Occur Num: 24</li><li>Inform-Hotel-Choice-3 Occur Num: 20</li><li>reqmore-general-none-none Occur Num: 16</li><li>Inform-Hotel-Area-west Occur Num: 14</li></ul></ul><li>Inform-Hotel-Area-north<ul><li>Occur Num: 263</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 35</li><li>Recommend-Hotel-Name-acorn guest house Occur Num: 31</li><li>Request-Hotel-Price-? Occur Num: 23</li><li>Inform-Hotel-Price-moderate Occur Num: 16</li><li>Recommend-Hotel-Name-ashley hotel Occur Num: 13</li></ul></ul><li>Inform-Hotel-Name-ashley hotel<ul><li>Occur Num: 263</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 29</li><li>Recommend-Hotel-Name-ashley hotel Occur Num: 24</li><li>reqmore-general-none-none Occur Num: 16</li><li>Inform-Hotel-Price-moderate Occur Num: 16</li><li>Inform-Hotel-Addr-74 chesterton road Occur Num: 15</li></ul></ul><li>NoOffer-Hotel-Type-none<ul><li>Occur Num: 239</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 37</li><li>Request-Hotel-Area-? Occur Num: 34</li><li>Inform-Hotel-Choice-0 Occur Num: 30</li><li>NoOffer-Hotel-Stars-none Occur Num: 13</li><li>Request-Hotel-Price-? Occur Num: 12</li></ul></ul><li>Inform-Hotel-Choice-2<ul><li>Occur Num: 225</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 40</li><li>Inform-Hotel-Type-guesthouse Occur Num: 17</li><li>Inform-Hotel-Parking-none Occur Num: 16</li><li>Inform-Hotel-Area-centre Occur Num: 11</li><li>Inform-Hotel-Price-cheap Occur Num: 11</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Hotel-Addr-? Occur Num: 29</li><li>Request-Hotel-Stars-? Occur Num: 17</li><li>Request-Hotel-Type-? Occur Num: 15</li><li>Inform-Hotel-Stay-4 Occur Num: 15</li><li>Request-Hotel-Price-? Occur Num: 12</li><li>Request-Hotel-Post-? Occur Num: 9</li><li>Request-Hotel-Area-? Occur Num: 8</li><li>Request-Hotel-Phone-? Occur Num: 8</li><li>Request-Hotel-Internet-? Occur Num: 7</li><li>Inform-Hotel-Stay-3 Occur Num: 6</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-parking Occur Num: 49</li><li>inform-hotel-internet Occur Num: 48</li><li>inform-hotel-area Occur Num: 34</li><li>recommend-hotel-parking Occur Num: 24</li><li>inform-hotel-stars Occur Num: 15</li><li>recommend-hotel-internet Occur Num: 10</li><li>recommend-hotel-stars Occur Num: 2</li><li>recommend-hotel-pricerange Occur Num: 1</li><li>inform-hotel-pricerange Occur Num: 1</li><li>inform-hotel-postcode Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-address Occur Num: 32</li><li>request-hotel-internet Occur Num: 24</li><li>request-hotel-stars Occur Num: 21</li><li>request-hotel-parking Occur Num: 20</li><li>request-hotel-phone Occur Num: 17</li><li>request-hotel-postcode Occur Num: 15</li><li>request-hotel-type Occur Num: 15</li><li>request-hotel-pricerange Occur Num: 15</li><li>request-hotel-area Occur Num: 13</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-area Occur Num: 41</li><li>inform-hotel-type Occur Num: 27</li><li>inform-hotel-phone Occur Num: 5</li><li>inform-hotel-stars Occur Num: 2</li></ul></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 60.9 %</p><p> (Precision, Recall, F1) : (0.492, 0.722, 0.558) </p><p> Average Dialog Turn (Succ): 10.183 </p><p> Average Dialog Turn (All): 20.365 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 112</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensively Occur Num: 65</li><li>Inform-Restaurant-Food-do n't care Occur Num: 21</li><li>Request-Train-Ticket-? Occur Num: 3</li><li>Inform-Restaurant-Area-do n't care Occur Num: 3</li><li>Inform-Restaurant-Area-east Occur Num: 3</li></ul></ul><li>Inform-Restaurant-Food-dontcare<ul><li>Occur Num: 92</li><li>NLU Output</li><ul><li>Inform-Restaurant-Food-do n't care Occur Num: 59</li><li>Inform-Restaurant-Price-expensive Occur Num: 17</li><li>Inform-Restaurant-Price-moderate Occur Num: 11</li><li>Inform-Restaurant-Price-expensively Occur Num: 4</li><li>Inform-Restaurant-Price-moderately Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Price-moderate<ul><li>Occur Num: 87</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 54</li><li>Inform-Restaurant-Food-do n't care Occur Num: 12</li><li>Inform-Restaurant-Area-do n't care Occur Num: 7</li><li>Inform-Restaurant-Area-centre Occur Num: 6</li><li>Inform-Restaurant-Food-italian Occur Num: 3</li></ul></ul><li>Request-Restaurant-Addr-?<ul><li>Occur Num: 77</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 35</li><li>Request-Restaurant-Post-? Occur Num: 23</li><li>Request-Attraction-Addr-? Occur Num: 7</li><li>Request-Attraction-Phone-? Occur Num: 4</li><li>Request-Attraction-Post-? Occur Num: 3</li></ul></ul><li>Request-Restaurant-Phone-?<ul><li>Occur Num: 73</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 35</li><li>Request-Attraction-Phone-? Occur Num: 17</li><li>Request-Restaurant-Post-? Occur Num: 12</li><li>Request-Attraction-Post-? Occur Num: 3</li><li>Request-Attraction-Addr-? Occur Num: 3</li></ul></ul><li>Request-Restaurant-Post-?<ul><li>Occur Num: 48</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 23</li><li>Request-Restaurant-Phone-? Occur Num: 11</li><li>Request-Attraction-Post-? Occur Num: 6</li><li>Request-Attraction-Phone-? Occur Num: 2</li><li>Request-Attraction-Addr-? Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Price-cheap<ul><li>Occur Num: 35</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-cheaply Occur Num: 23</li><li>Inform-Restaurant-Area-do n't care Occur Num: 4</li><li>Inform-Restaurant-Area-north Occur Num: 2</li><li>Inform-Restaurant-Food-italian Occur Num: 2</li><li>Inform-Restaurant-Area-east Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Area-dontcare<ul><li>Occur Num: 34</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-do n't care Occur Num: 20</li><li>Inform-Restaurant-Price-moderate Occur Num: 6</li><li>Inform-Restaurant-Price-cheap Occur Num: 4</li><li>Inform-Restaurant-Price-expensive Occur Num: 2</li><li>Inform-Restaurant-Price-expensively Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 22</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 5</li><li>Inform-Restaurant-Price-cheap Occur Num: 3</li><li>Inform-Restaurant-Price-expensive Occur Num: 3</li><li>Inform-Restaurant-Food-british Occur Num: 3</li><li>Inform-Restaurant-Price-expensively Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Food-portuguese<ul><li>Occur Num: 12</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 3</li><li>Inform-Restaurant-Price-expensive Occur Num: 3</li><li>Inform-Restaurant-Price-moderate Occur Num: 2</li><li>Inform-Restaurant-Price-moderately Occur Num: 1</li><li>Inform-Restaurant-Area-north Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 1978</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 130</li><li>Recommend-Restaurant-Price-cheap Occur Num: 97</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 89</li><li>Inform-Restaurant-Food-italian Occur Num: 84</li><li>Inform-Booking-none-none Occur Num: 83</li></ul></ul><li>Inform-Restaurant-Price-cheap<ul><li>Occur Num: 1431</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-cheap Occur Num: 119</li><li>reqmore-general-none-none Occur Num: 104</li><li>Inform-Restaurant-Area-centre Occur Num: 101</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 88</li><li>Inform-Restaurant-Food-italian Occur Num: 87</li></ul></ul><li>Inform-Restaurant-Name-pizza hut city centre<ul><li>Occur Num: 1134</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 99</li><li>reqmore-general-none-none Occur Num: 98</li><li>Inform-Restaurant-Area-centre Occur Num: 90</li><li>Recommend-Restaurant-Price-cheap Occur Num: 87</li><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 87</li></ul></ul><li>Inform-Restaurant-Food-italian<ul><li>Occur Num: 970</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-cheap Occur Num: 90</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 87</li><li>Inform-Restaurant-Area-centre Occur Num: 82</li><li>reqmore-general-none-none Occur Num: 76</li><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 69</li></ul></ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 895</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-expensive Occur Num: 61</li><li>Inform-Booking-none-none Occur Num: 58</li><li>reqmore-general-none-none Occur Num: 56</li><li>Inform-Restaurant-Area-centre Occur Num: 43</li><li>Inform-Restaurant-Food-indian Occur Num: 36</li></ul></ul><li>Inform-Restaurant-Phone-01223323737<ul><li>Occur Num: 846</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 102</li><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 83</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 83</li><li>Recommend-Restaurant-Phone-01223323737 Occur Num: 70</li><li>Inform-Restaurant-Food-italian Occur Num: 65</li></ul></ul><li>Inform-Restaurant-Addr-Regent Street City Centre<ul><li>Occur Num: 833</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 109</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 84</li><li>Inform-Restaurant-Phone-01223323737 Occur Num: 77</li><li>Inform-Restaurant-Food-italian Occur Num: 67</li><li>Inform-Restaurant-Price-cheap Occur Num: 66</li></ul></ul><li>Inform-Restaurant-Price-moderate<ul><li>Occur Num: 516</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-moderate Occur Num: 42</li><li>Inform-Restaurant-Area-centre Occur Num: 31</li><li>reqmore-general-none-none Occur Num: 29</li><li>Inform-Booking-none-none Occur Num: 27</li><li>Inform-Restaurant-Food-british Occur Num: 21</li></ul></ul><li>Request-Restaurant-Food-?<ul><li>Occur Num: 430</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 66</li><li>Recommend-Restaurant-Price-cheap Occur Num: 50</li><li>Inform-Restaurant-Price-cheap Occur Num: 48</li><li>Inform-Restaurant-Choice-69 Occur Num: 46</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 37</li></ul></ul><li>Inform-Restaurant-Food-indian<ul><li>Occur Num: 368</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 32</li><li>Inform-Restaurant-Area-centre Occur Num: 24</li><li>reqmore-general-none-none Occur Num: 23</li><li>Recommend-Restaurant-Name-curry garden Occur Num: 23</li><li>Inform-Booking-none-none Occur Num: 21</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Inform-Restaurant-Price-moderate Occur Num: 43</li><li>Inform-Restaurant-Price-expensive Occur Num: 40</li><li>Request-Restaurant-Phone-? Occur Num: 18</li><li>Inform-Restaurant-Price-cheap Occur Num: 16</li><li>Request-Restaurant-Addr-? Occur Num: 5</li><li>Inform-Restaurant-Food-portuguese Occur Num: 3</li><li>Inform-Restaurant-Time-18:15 Occur Num: 2</li><li>Request-Restaurant-Area-? Occur Num: 2</li><li>Inform-Restaurant-Day-tuesday Occur Num: 2</li><li>Inform-Restaurant-Day-wednesday Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-restaurant-area Occur Num: 4</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-postcode Occur Num: 50</li><li>request-restaurant-phone Occur Num: 18</li><li>request-restaurant-address Occur Num: 12</li><li>request-restaurant-area Occur Num: 2</li><li>request-restaurant-pricerange Occur Num: 1</li><li>request-restaurant-food Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-phone Occur Num: 188</li><li>inform-restaurant-food Occur Num: 92</li><li>inform-restaurant-area Occur Num: 90</li></ul></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 87.5 %</p><p> (Precision, Recall, F1) : (0.770, 0.916, 0.812) </p><p> Average Dialog Turn (Succ): 6.427 </p><p> Average Dialog Turn (All): 9.723 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Attraction-Post-?<ul><li>Occur Num: 153</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 57</li><li>Request-Attraction-Addr-? Occur Num: 40</li><li>Request-Attraction-Fee-? Occur Num: 33</li><li>Request-Restaurant-Post-? Occur Num: 15</li><li>Request-Attraction-Area-? Occur Num: 3</li></ul></ul><li>Request-Attraction-Phone-?<ul><li>Occur Num: 148</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 57</li><li>Request-Attraction-Addr-? Occur Num: 38</li><li>Request-Attraction-Fee-? Occur Num: 26</li><li>Request-Restaurant-Phone-? Occur Num: 12</li><li>Request-Attraction-Area-? Occur Num: 6</li></ul></ul><li>Request-Attraction-Addr-?<ul><li>Occur Num: 137</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 41</li><li>Request-Attraction-Phone-? Occur Num: 38</li><li>Request-Attraction-Fee-? Occur Num: 18</li><li>Request-Restaurant-Addr-? Occur Num: 14</li><li>Request-Restaurant-Post-? Occur Num: 10</li></ul></ul><li>Request-Attraction-Fee-?<ul><li>Occur Num: 77</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 33</li><li>Request-Attraction-Phone-? Occur Num: 26</li><li>Request-Attraction-Addr-? Occur Num: 18</li></ul></ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 39</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-entertainment Occur Num: 15</li><li>Inform-Attraction-Type-do n't care Occur Num: 4</li><li>Inform-Restaurant-Area-centre Occur Num: 3</li><li>Inform-Attraction-Type-museum Occur Num: 3</li><li>Inform-Attraction-Type-museums Occur Num: 3</li></ul></ul><li>Inform-Attraction-Type-dontcare<ul><li>Occur Num: 34</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-do n't care Occur Num: 23</li><li>Request-Attraction-Type-? Occur Num: 5</li><li>Inform-Attraction-Area-centre Occur Num: 4</li><li>Inform-Attraction-Area-west Occur Num: 1</li><li>Inform-Attraction-Area-east Occur Num: 1</li></ul></ul><li>Inform-Attraction-Area-dontcare<ul><li>Occur Num: 32</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-do n't care Occur Num: 18</li><li>Request-Attraction-Area-? Occur Num: 12</li><li>Inform-Attraction-Type-college Occur Num: 1</li><li>Inform-Attraction-Type-museum Occur Num: 1</li></ul></ul><li>Inform-Attraction-Type-museum<ul><li>Occur Num: 26</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 10</li><li>Inform-Attraction-Type-museums Occur Num: 7</li><li>Inform-Attraction-Type-entertainment Occur Num: 5</li><li>Inform-Attraction-Area-west Occur Num: 2</li><li>Inform-Attraction-Area-north Occur Num: 1</li></ul></ul><li>Request-Attraction-Area-?<ul><li>Occur Num: 22</li><li>NLU Output</li><ul><li>Request-Restaurant-Area-? Occur Num: 7</li><li>Request-Attraction-Addr-? Occur Num: 6</li><li>Request-Attraction-Phone-? Occur Num: 6</li><li>Request-Attraction-Post-? Occur Num: 3</li></ul></ul><li>Inform-Attraction-Type-boat<ul><li>Occur Num: 17</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 7</li><li>Inform-Attraction-Type-boats Occur Num: 6</li><li>Inform-Attraction-Type-entertainment Occur Num: 2</li><li>Inform-Attraction-Area-west Occur Num: 1</li><li>Inform-Attraction-Area-north Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 730</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 57</li><li>Inform-Attraction-Choice-44 Occur Num: 42</li><li>Request-Attraction-Type-? Occur Num: 39</li><li>Inform-Attraction-Type-architecture Occur Num: 39</li><li>Inform-Attraction-Type-theatre Occur Num: 35</li></ul></ul><li>Inform-Attraction-Fee-?<ul><li>Occur Num: 487</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 64</li><li>Inform-Attraction-Addr-park street Occur Num: 18</li><li>Recommend-Attraction-Name-adc theatre Occur Num: 17</li><li>welcome-general-none-none Occur Num: 15</li><li>Inform-Attraction-Choice-44 Occur Num: 15</li></ul></ul><li>Inform-Attraction-Fee-free<ul><li>Occur Num: 444</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 77</li><li>Inform-Attraction-Area-centre Occur Num: 20</li><li>Inform-Attraction-Type-park Occur Num: 13</li><li>Inform-Attraction-Addr-saint andrew 's street Occur Num: 12</li><li>Inform-Attraction-Addr-98 king street Occur Num: 12</li></ul></ul><li>Request-Attraction-Type-?<ul><li>Occur Num: 319</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 41</li><li>Inform-Attraction-Choice-44 Occur Num: 41</li><li>Inform-Attraction-Type-theatre Occur Num: 34</li><li>Inform-Attraction-Type-architecture Occur Num: 31</li><li>Inform-Attraction-Type-museum Occur Num: 19</li></ul></ul><li>Inform-Attraction-Type-museum<ul><li>Occur Num: 315</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 26</li><li>Inform-Attraction-Choice-11 Occur Num: 23</li><li>Inform-Attraction-Area-west Occur Num: 19</li><li>Request-Attraction-Type-? Occur Num: 18</li><li>reqmore-general-none-none Occur Num: 17</li></ul></ul><li>Inform-Attraction-Name-adc theatre<ul><li>Occur Num: 313</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 43</li><li>Recommend-Attraction-Name-adc theatre Occur Num: 30</li><li>Inform-Attraction-Addr-park street Occur Num: 27</li><li>Inform-Attraction-Choice-44 Occur Num: 22</li><li>Inform-Attraction-Area-centre Occur Num: 20</li></ul></ul><li>Inform-Attraction-Type-theatre<ul><li>Occur Num: 276</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 44</li><li>Inform-Attraction-Choice-44 Occur Num: 42</li><li>Request-Attraction-Type-? Occur Num: 37</li><li>Inform-Attraction-Type-architecture Occur Num: 31</li><li>Inform-Attraction-Name-adc theatre Occur Num: 12</li></ul></ul><li>Inform-Attraction-Type-architecture<ul><li>Occur Num: 248</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 46</li><li>Inform-Attraction-Choice-44 Occur Num: 37</li><li>Request-Attraction-Type-? Occur Num: 34</li><li>Inform-Attraction-Type-theatre Occur Num: 33</li><li>Inform-Attraction-Choice-5 Occur Num: 10</li></ul></ul><li>Inform-Attraction-Name-christ's college<ul><li>Occur Num: 248</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 29</li><li>Inform-Attraction-Area-centre Occur Num: 21</li><li>Inform-Attraction-Fee-free Occur Num: 19</li><li>Recommend-Attraction-Name-christ 's college Occur Num: 17</li><li>Inform-Attraction-Name-christ 's college Occur Num: 16</li></ul></ul><li>Inform-Attraction-Name-broughton house gallery<ul><li>Occur Num: 247</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 31</li><li>Inform-Attraction-Choice-11 Occur Num: 23</li><li>Inform-Attraction-Area-centre Occur Num: 22</li><li>Inform-Attraction-Type-museum Occur Num: 22</li><li>Inform-Attraction-Addr-98 king street Occur Num: 20</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Attraction-Addr-? Occur Num: 11</li><li>Request-Attraction-Post-? Occur Num: 11</li><li>Request-Attraction-Area-? Occur Num: 5</li><li>Request-Attraction-Fee-? Occur Num: 3</li><li>Request-Attraction-Type-? Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-attraction-area Occur Num: 41</li><li>recommend-attraction-area Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-address Occur Num: 19</li><li>request-attraction-postcode Occur Num: 15</li><li>request-attraction-area Occur Num: 8</li><li>request-attraction-phone Occur Num: 6</li><li>request-attraction-type Occur Num: 3</li><li>request-attraction-entrance fee Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-type Occur Num: 57</li><li>inform-attraction-phone Occur Num: 37</li><li>inform-attraction-area Occur Num: 34</li></ul></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 67.7 %</p><p> (Precision, Recall, F1) : (0.747, 0.712, 0.724) </p><p> Average Dialog Turn (Succ): 7.313 </p><p> Average Dialog Turn (All): 13.192 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Phone-?<ul><li>Occur Num: 33</li><li>NLU Output</li><ul><li>Request-Taxi-Car-? Occur Num: 31</li><li>Request-Attraction-Phone-? Occur Num: 1</li><li>Request-Hotel-Phone-? Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the fitzwilliam museum<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 2</li><li>Inform-Taxi-Dest-golden house Occur Num: 1</li><li>Inform-Taxi-Leave-13:45 Occur Num: 1</li><li>Inform-Taxi-Arrive-13:45 Occur Num: 1</li><li>Inform-Taxi-Dest-la mimosa Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-dontcare<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Leave-do Occur Num: 3</li><li>Inform-Taxi-Leave-12:00 Occur Num: 1</li><li>Inform-Taxi-Leave-20:30 Occur Num: 1</li><li>Inform-Taxi-Leave-13:15 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-home from home<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-home from home Occur Num: 1</li><li>Inform-Taxi-Depart-arbury lodge guesthouse Occur Num: 1</li><li>Inform-Taxi-Leave-02:45 Occur Num: 1</li><li>Inform-Taxi-Depart-thanh binh Occur Num: 1</li><li>Inform-Taxi-Dest-home Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the lensfield hotel<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li><li>Inform-Taxi-Dest-williams art and antiques Occur Num: 1</li></ul></ul><li>Inform-Taxi-Leave-20:30<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Train-Leave-20:30 Occur Num: 1</li><li>Inform-Taxi-Dest-finches bed and breakfast Occur Num: 1</li><li>Inform-Taxi-Depart-sidney sussex college Occur Num: 1</li><li>Inform-Taxi-Leave-do Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-02:00<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-efes restaurant<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-broxbourne train station Occur Num: 1</li><li>Inform-Restaurant-Name-efes Occur Num: 1</li><li>Inform-Taxi-Arrive-16:15 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-kohinoor<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-grafton hotel restaurant Occur Num: 1</li><li>Inform-Restaurant-Name-kohinoor Occur Num: 1</li><li>Inform-Taxi-Arrive-18:30 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the good luck chinese food takeaway<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-nandos Occur Num: 1</li><li>Inform-Taxi-Leave-05:15 Occur Num: 1</li><li>Inform-Taxi-Depart-good luck chinese food takeaway Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Depart-?<ul><li>Occur Num: 62</li><li>NLU Output</li><ul><li>Request-Taxi-Dest-? Occur Num: 17</li><li>Inform-Taxi-Car-tesla Occur Num: 4</li><li>Inform-Taxi-Car-audi Occur Num: 2</li><li>Inform-Taxi-Car-lexus Occur Num: 2</li><li>Inform-Taxi-Phone-93793671822 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-tesla<ul><li>Occur Num: 56</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 21</li><li>Request-Taxi-Depart-? Occur Num: 5</li><li>Request-Taxi-Dest-? Occur Num: 1</li><li>Inform-Taxi-Phone-61488179956 Occur Num: 1</li><li>Inform-Taxi-Phone-99646126925 Occur Num: 1</li></ul></ul><li>Request-Taxi-Dest-?<ul><li>Occur Num: 51</li><li>NLU Output</li><ul><li>Request-Taxi-Depart-? Occur Num: 16</li><li>Inform-Taxi-Car-lexus Occur Num: 2</li><li>Inform-Taxi-Car-volkswagen Occur Num: 2</li><li>Inform-Taxi-Phone-93793671822 Occur Num: 1</li><li>Inform-Taxi-Car-tesla Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-toyota<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 18</li><li>Request-Taxi-Depart-? Occur Num: 3</li><li>Request-Taxi-Dest-? Occur Num: 3</li><li>Inform-Taxi-Phone-73636724163 Occur Num: 1</li><li>Inform-Taxi-Phone-85765694681 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-volkswagen<ul><li>Occur Num: 41</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 14</li><li>Request-Taxi-Depart-? Occur Num: 4</li><li>Request-Taxi-Dest-? Occur Num: 4</li><li>Inform-Taxi-Phone-69794859257 Occur Num: 1</li><li>Inform-Taxi-Phone-88155992586 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-honda<ul><li>Occur Num: 41</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 16</li><li>Request-Taxi-Depart-? Occur Num: 3</li><li>Request-Taxi-Dest-? Occur Num: 2</li><li>Inform-Taxi-Phone-43785244638 Occur Num: 1</li><li>Inform-Taxi-Phone-71199834421 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-ford<ul><li>Occur Num: 37</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 11</li><li>Request-Taxi-Depart-? Occur Num: 4</li><li>Request-Taxi-Dest-? Occur Num: 2</li><li>Inform-Taxi-Phone-93793671822 Occur Num: 1</li><li>Inform-Taxi-Phone-87965624366 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-bmw<ul><li>Occur Num: 33</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 12</li><li>Request-Taxi-Depart-? Occur Num: 2</li><li>Request-Taxi-Dest-? Occur Num: 2</li><li>Inform-Taxi-Phone-74242564888 Occur Num: 1</li><li>Inform-Taxi-Phone-36371815544 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-volvo<ul><li>Occur Num: 33</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 13</li><li>Request-Taxi-Dest-? Occur Num: 2</li><li>OfferBooked-Train-Id-76488729113 Occur Num: 1</li><li>Inform-Taxi-Phone-36951548116 Occur Num: 1</li><li>welcome-general-none-none Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-skoda<ul><li>Occur Num: 32</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 11</li><li>Request-Taxi-Depart-? Occur Num: 3</li><li>Request-Taxi-Dest-? Occur Num: 2</li><li>Inform-Taxi-Phone-89918998729 Occur Num: 1</li><li>Inform-Taxi-Phone-76618714713 Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Taxi-Car-? Occur Num: 32</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-taxi-car type Occur Num: 32</li><li>request-taxi-phone Occur Num: 25</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 79.5 %</p><p> (Precision, Recall, F1) : (0.867, 0.868, 0.862) </p><p> Average Dialog Turn (Succ): 7.646 </p><p> Average Dialog Turn (All): 13.125 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Train-Depart-cambridge<ul><li>Occur Num: 133</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 37</li><li>Inform-Train-Day-thursday Occur Num: 9</li><li>Inform-Train-Day-friday Occur Num: 7</li><li>Inform-Train-Day-tuesday Occur Num: 7</li><li>Inform-Train-Day-Saturday Occur Num: 6</li></ul></ul><li>Inform-Train-Dest-cambridge<ul><li>Occur Num: 128</li><li>NLU Output</li><ul><li>Inform-Train-Day-thursday Occur Num: 10</li><li>Inform-Train-Depart-cambridge Occur Num: 10</li><li>Request-Train-Arrive-? Occur Num: 8</li><li>Inform-Train-Day-friday Occur Num: 7</li><li>Inform-Train-Day-wednesday Occur Num: 7</li></ul></ul><li>Inform-Train-Day-monday<ul><li>Occur Num: 35</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 4</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Dest-leicester Occur Num: 3</li><li>Inform-Train-Dest-norwich Occur Num: 2</li><li>Inform-Train-Day-Saturday Occur Num: 2</li></ul></ul><li>Inform-Train-Day-thursday<ul><li>Occur Num: 33</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 10</li><li>Inform-Train-Dest-stevenage Occur Num: 2</li><li>Inform-Train-Leave-15:30 Occur Num: 2</li><li>Inform-Train-Depart-cambridge Occur Num: 2</li><li>Inform-Train-Leave-do n't care Occur Num: 2</li></ul></ul><li>Request-Train-Time-?<ul><li>Occur Num: 33</li><li>NLU Output</li><ul><li>Request-Train-Ticket-? Occur Num: 26</li><li>Inform-Train-none-none Occur Num: 4</li><li>Request-Train-Id-? Occur Num: 3</li></ul></ul><li>Inform-Train-Day-tuesday<ul><li>Occur Num: 31</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 5</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Day-Saturday Occur Num: 3</li><li>Inform-Train-Dest-peterborough Occur Num: 1</li><li>Inform-Train-Dest-leicester Occur Num: 1</li></ul></ul><li>Inform-Train-Day-wednesday<ul><li>Occur Num: 31</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 6</li><li>Inform-Train-Day-Saturday Occur Num: 4</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Depart-broxbourne Occur Num: 1</li><li>Inform-Train-Leave-18:15 Occur Num: 1</li></ul></ul><li>Inform-Train-Leave-dontcare<ul><li>Occur Num: 27</li><li>NLU Output</li><ul><li>Inform-Train-Leave-do n't care Occur Num: 16</li><li>Inform-Train-Day-thursday Occur Num: 2</li><li>Inform-Train-Arrive-17:30 Occur Num: 1</li><li>Inform-Train-Arrive-17:00 Occur Num: 1</li><li>Inform-Train-Depart-peterborough Occur Num: 1</li></ul></ul><li>Inform-Train-Day-friday<ul><li>Occur Num: 26</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 4</li><li>Inform-Train-Depart-cambridge Occur Num: 4</li><li>Inform-Train-Day-Saturday Occur Num: 3</li><li>Inform-Train-Arrive-11:15 Occur Num: 2</li><li>Inform-Train-Arrive-09:30 Occur Num: 1</li></ul></ul><li>Request-Train-Ticket-?<ul><li>Occur Num: 26</li><li>NLU Output</li><ul><li>Request-Train-Time-? Occur Num: 25</li><li>Request-Train-Id-? Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>OfferBook-Train-none-none<ul><li>Occur Num: 2121</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-05:51 Occur Num: 322</li><li>Inform-Train-Choice-404 Occur Num: 312</li><li>Inform-Train-Leave-05:00 Occur Num: 306</li><li>reqmore-general-none-none Occur Num: 165</li><li>Inform-Train-Ticket-23.60 pounds Occur Num: 159</li></ul></ul><li>Inform-Train-Leave-05:00<ul><li>Occur Num: 1881</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-05:51 Occur Num: 319</li><li>OfferBook-Train-none-none Occur Num: 318</li><li>Inform-Train-Choice-404 Occur Num: 306</li><li>Inform-Train-Ticket-23.60 pounds Occur Num: 93</li><li>Inform-Train-Id-TR0899 Occur Num: 90</li></ul></ul><li>Inform-Train-Arrive-05:51<ul><li>Occur Num: 1652</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 317</li><li>Inform-Train-Leave-05:00 Occur Num: 300</li><li>Inform-Train-Choice-404 Occur Num: 286</li><li>Inform-Train-Id-TR0899 Occur Num: 89</li><li>Inform-Train-Depart-cambridge Occur Num: 63</li></ul></ul><li>Inform-Train-Choice-404<ul><li>Occur Num: 1457</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-05:51 Occur Num: 286</li><li>Inform-Train-Leave-05:00 Occur Num: 282</li><li>OfferBook-Train-none-none Occur Num: 250</li><li>Request-Train-Leave-? Occur Num: 81</li><li>Inform-Train-Depart-cambridge Occur Num: 68</li></ul></ul><li>Inform-Train-Ticket-23.60 pounds<ul><li>Occur Num: 595</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 153</li><li>reqmore-general-none-none Occur Num: 118</li><li>Inform-Train-Leave-05:00 Occur Num: 92</li><li>Inform-Train-Time-51 minutes Occur Num: 73</li><li>Inform-Train-Arrive-05:51 Occur Num: 45</li></ul></ul><li>Request-Train-Leave-?<ul><li>Occur Num: 548</li><li>NLU Output</li><ul><li>Inform-Train-Choice-404 Occur Num: 113</li><li>Inform-Train-Leave-05:00 Occur Num: 101</li><li>Inform-Train-Arrive-05:51 Occur Num: 88</li><li>Request-Train-Arrive-? Occur Num: 50</li><li>OfferBook-Train-none-none Occur Num: 42</li></ul></ul><li>Inform-Train-Id-TR0899<ul><li>Occur Num: 446</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 90</li><li>Inform-Train-Arrive-05:51 Occur Num: 89</li><li>Inform-Train-Leave-05:00 Occur Num: 86</li><li>Inform-Train-Ticket-18.88 pounds Occur Num: 47</li><li>Inform-Train-Choice-404 Occur Num: 45</li></ul></ul><li>Inform-Train-Depart-cambridge<ul><li>Occur Num: 398</li><li>NLU Output</li><ul><li>Inform-Train-Choice-404 Occur Num: 68</li><li>Inform-Train-Leave-05:00 Occur Num: 64</li><li>Inform-Train-Arrive-05:51 Occur Num: 63</li><li>OfferBook-Train-none-none Occur Num: 56</li><li>Request-Train-Leave-? Occur Num: 24</li></ul></ul><li>Inform-Train-Ticket-18.88 pounds<ul><li>Occur Num: 342</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 81</li><li>Inform-Train-Leave-05:00 Occur Num: 64</li><li>reqmore-general-none-none Occur Num: 57</li><li>Inform-Train-Id-TR0899 Occur Num: 48</li><li>Inform-Train-Time-51 minutes Occur Num: 27</li></ul></ul><li>Inform-Train-Time-51 minutes<ul><li>Occur Num: 317</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 111</li><li>Inform-Train-Ticket-23.60 pounds Occur Num: 75</li><li>OfferBook-Train-none-none Occur Num: 50</li><li>Inform-Train-Ticket-18.88 pounds Occur Num: 33</li><li>welcome-general-none-none Occur Num: 10</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Inform-Train-People-3 Occur Num: 7</li><li>Inform-Train-People-7 Occur Num: 5</li><li>Inform-Train-People-1 Occur Num: 4</li><li>Inform-Train-People-6 Occur Num: 4</li><li>Inform-Train-People-5 Occur Num: 4</li><li>Inform-Train-People-4 Occur Num: 3</li><li>Inform-Train-People-2 Occur Num: 3</li><li>Inform-Train-Depart-london kings cross Occur Num: 2</li><li>Inform-Train-People-8 Occur Num: 2</li><li>Inform-Train-Leave-08:15 Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-train-duration Occur Num: 7</li><li>inform-train-price Occur Num: 6</li><li>inform-train-leaveAt Occur Num: 3</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-train-duration Occur Num: 17</li><li>request-train-price Occur Num: 12</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 100.0 %</p><p> (Precision, Recall, F1) : (0.935, 1.000, 0.957) </p><p> Average Dialog Turn (Succ): 2.000 </p><p> Average Dialog Turn (All): 2.000 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Police-none-none<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Police-Name-Parkside Police Station Occur Num: 6</li><li>Request-Police-Addr-? Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Police-Addr-Parkside, Cambridge<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 23</li><li>Inform-Police-Phone-01223358966 Occur Num: 23</li></ul></ul><li>Inform-Police-Name-none<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 23</li><li>Inform-Police-Phone-01223358966 Occur Num: 23</li></ul></ul><li>Inform-Police-Post-none<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 23</li><li>Inform-Police-Phone-01223358966 Occur Num: 23</li></ul></ul><li>Inform-Police-Phone-01223358966<ul><li>Occur Num: 23</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 23</li></ul></ul></ul><b> Dialog Loop</b><p>Nothing</p><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><p>Nothing</p><b> Inform But Not Request Dialog Act</b><ul><li>inform-police-phone Occur Num: 3</li></ul></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 93.3 %</p><p> (Precision, Recall, F1) : (0.933, 0.933, 0.933) </p><p> Average Dialog Turn (Succ): 4.000 </p><p> Average Dialog Turn (All): 6.400 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Hospital-Phone-?<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Request-Police-Phone-? Occur Num: 2</li><li>Request-Restaurant-Phone-? Occur Num: 2</li></ul></ul><li>Inform-Hospital-Department-plastic and vascular surgery plastics<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li><li>Inform-Hospital-Department-plastic and vascular surgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-haematology and haematological oncology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurosurgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-oncology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-surgery<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-intermediate dependancy area<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-intermediate dependancy Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-children's oncology and haematology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-emergency department<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-emergency department department Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hospital-Addr-none<ul><li>Occur Num: 58</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurosciences critical care unit Occur Num: 28</li><li>Inform-Hospital-Phone-01223216297 Occur Num: 28</li><li>Request-Hospital-Department-? Occur Num: 2</li></ul></ul><li>Inform-Hospital-Department-neurosciences critical care unit<ul><li>Occur Num: 30</li><li>NLU Output</li><ul><li>Inform-Hospital-Phone-01223216297 Occur Num: 28</li><li>Request-Hospital-Department-? Occur Num: 2</li></ul></ul><li>Inform-Hospital-Phone-01223216297<ul><li>Occur Num: 30</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurosciences critical care unit Occur Num: 28</li><li>Request-Hospital-Department-? Occur Num: 2</li></ul></ul><li>Inform-Hospital-Post-none<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Request-Hospital-Department-? Occur Num: 2</li><li>Inform-Hospital-Department-neurosciences critical care unit Occur Num: 2</li><li>Inform-Hospital-Phone-01223216297 Occur Num: 2</li></ul></ul><li>Request-Hospital-Department-?<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurosciences critical care unit Occur Num: 2</li><li>Inform-Hospital-Phone-01223216297 Occur Num: 2</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Hospital-Phone-? Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-hospital-phone Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/BERTNLU-RuleDST-MLEPolicy-TemplateNLG/res.txt b/tests/results/BERTNLU-RuleDST-MLEPolicy-TemplateNLG/res.txt deleted file mode 100755 index e91def6f1e7eb987cd06b3e2f1d4c44ad2c78450..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-MLEPolicy-TemplateNLG/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.526 -success number of dialogs/tot: 0.484 -average precision: 0.6628294459925087 -average recall: 0.7267286423049664 -average f1: 0.6602935206785587 -average book rate: 0.3551923076923077 -average turn (succ): 12.454545454545455 -average turn (all): 26.27 diff --git a/tests/results/BERTNLU-RuleDST-PGPolicy-TemplateNLG/Frequency_of_domain.png b/tests/results/BERTNLU-RuleDST-PGPolicy-TemplateNLG/Frequency_of_domain.png deleted file mode 100755 index 88a4bf1bd23751933bfde7fdd62e5eac0f74d365..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-PGPolicy-TemplateNLG/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-PGPolicy-TemplateNLG/Performance_for_each_domain.png b/tests/results/BERTNLU-RuleDST-PGPolicy-TemplateNLG/Performance_for_each_domain.png deleted file mode 100755 index c26b55d87829886d8aef61b2bbd9f62d5d4dd55f..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-PGPolicy-TemplateNLG/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-PGPolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png b/tests/results/BERTNLU-RuleDST-PGPolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index 26f38f76f2b005806dc3b7d25345d23a114deef0..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-PGPolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-PGPolicy-TemplateNLG/report_multiwoz.html b/tests/results/BERTNLU-RuleDST-PGPolicy-TemplateNLG/report_multiwoz.html deleted file mode 100755 index 47275c02af2c1f13a3387e76539f0af5d033975c..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-PGPolicy-TemplateNLG/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: BERTNLU-RuleDST-PGPolicy-TemplateNLG</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-27 13:55:52</h3><h3>Overall Results</h3><p> Success Rate: 43.3 %</p><p> (Precision, Recall, F1) : (0.619, 0.668, 0.604) </p><p> Average Dialog Turn (Succ): 14.693 </p><p> Average Dialog Turn (All): 29.068 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">345</td><td align="center">0.400</td><td align="center">0.323</td><td align="center">0.465</td><td align="center">0.353</td><td align="center">0.490</td><td align="center">7.522</td><td align="center">21.154</td></tr><tr><td align="center">restaurant</td><td align="center">383</td><td align="center">0.603</td><td align="center">0.497</td><td align="center">0.713</td><td align="center">0.556</td><td align="center">0.389</td><td align="center">11.247</td><td align="center">21.770</td></tr><tr><td align="center">train</td><td align="center">314</td><td align="center">0.710</td><td align="center">0.810</td><td align="center">0.790</td><td align="center">0.790</td><td align="center">0.280</td><td align="center">12.404</td><td align="center">18.930</td></tr><tr><td align="center">attraction</td><td align="center">269</td><td align="center">0.844</td><td align="center">0.764</td><td align="center">0.897</td><td align="center">0.799</td><td align="center">0.141</td><td align="center">6.907</td><td align="center">11.346</td></tr><tr><td align="center">taxi</td><td align="center">86</td><td align="center">0.721</td><td align="center">0.767</td><td align="center">0.744</td><td align="center">0.752</td><td align="center">0.267</td><td align="center">7.258</td><td align="center">12.558</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">1.000</td><td align="center">0.935</td><td align="center">1.000</td><td align="center">0.957</td><td align="center">0.000</td><td align="center">2.000</td><td align="center">2.000</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">0.933</td><td align="center">0.933</td><td align="center">0.933</td><td align="center">0.933</td><td align="center">0.067</td><td align="center">38.429</td><td align="center">38.533</td></tr></table><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 40.0 %</p><p> (Precision, Recall, F1) : (0.323, 0.465, 0.353) </p><p> Average Dialog Turn (Succ): 7.522 </p><p> Average Dialog Turn (All): 21.154 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 331</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 53</li><li>Inform-Hotel-Type-hotel Occur Num: 53</li><li>Inform-Hotel-Type-guesthouse Occur Num: 39</li><li>Inform-Hotel-Stars-4 Occur Num: 36</li><li>Inform-Hotel-Price-moderate Occur Num: 34</li></ul></ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 325</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-hotel Occur Num: 60</li><li>Inform-Hotel-Internet-yes Occur Num: 56</li><li>Inform-Hotel-Type-guesthouse Occur Num: 40</li><li>Inform-Hotel-Stars-4 Occur Num: 38</li><li>Inform-Hotel-Price-moderate Occur Num: 28</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 200</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 39</li><li>Inform-Hotel-Parking-yes Occur Num: 38</li><li>Inform-Hotel-Stars-4 Occur Num: 21</li><li>Inform-Hotel-Area-north Occur Num: 14</li><li>Inform-Hotel-Type-hotel Occur Num: 14</li></ul></ul><li>Inform-Hotel-Stars-4<ul><li>Occur Num: 185</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 36</li><li>Inform-Hotel-Parking-yes Occur Num: 36</li><li>Inform-Hotel-Type-hotel Occur Num: 31</li><li>Inform-Hotel-Type-guesthouse Occur Num: 21</li><li>Inform-Hotel-Price-cheap Occur Num: 11</li></ul></ul><li>Inform-Hotel-Price-moderate<ul><li>Occur Num: 165</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 35</li><li>Inform-Hotel-Type-hotel Occur Num: 32</li><li>Inform-Hotel-Parking-yes Occur Num: 28</li><li>Inform-Hotel-Type-guesthouse Occur Num: 14</li><li>Inform-Hotel-Area-centre Occur Num: 10</li></ul></ul><li>Request-Hotel-Addr-?<ul><li>Occur Num: 113</li><li>NLU Output</li><ul><li>Request-Hotel-Post-? Occur Num: 16</li><li>Request-Restaurant-Addr-? Occur Num: 15</li><li>Request-Hotel-Phone-? Occur Num: 15</li><li>Request-Attraction-Addr-? Occur Num: 13</li><li>Request-Hotel-Internet-? Occur Num: 10</li></ul></ul><li>Inform-Hotel-Area-north<ul><li>Occur Num: 108</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 23</li><li>Inform-Hotel-Parking-yes Occur Num: 18</li><li>Inform-Hotel-Type-hotel Occur Num: 16</li><li>Inform-Hotel-Type-guesthouse Occur Num: 14</li><li>Inform-Hotel-Price-do n't care Occur Num: 9</li></ul></ul><li>Inform-Hotel-Area-centre<ul><li>Occur Num: 79</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 19</li><li>Inform-Hotel-Type-guesthouse Occur Num: 11</li><li>Inform-Hotel-Price-moderate Occur Num: 10</li><li>Inform-Hotel-Parking-yes Occur Num: 9</li><li>Inform-Hotel-Type-hotel Occur Num: 9</li></ul></ul><li>Inform-Hotel-Price-cheap<ul><li>Occur Num: 78</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-hotel Occur Num: 16</li><li>Inform-Hotel-Internet-yes Occur Num: 15</li><li>Inform-Hotel-Parking-yes Occur Num: 12</li><li>Inform-Hotel-Stars-4 Occur Num: 11</li><li>Inform-Hotel-Type-guesthouse Occur Num: 10</li></ul></ul><li>Inform-Hotel-Parking-no<ul><li>Occur Num: 76</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 19</li><li>Inform-Hotel-Internet-yes Occur Num: 12</li><li>Inform-Hotel-Type-guesthouse Occur Num: 10</li><li>Inform-Hotel-Area-north Occur Num: 6</li><li>Inform-Hotel-Type-hotel Occur Num: 6</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 843</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 96</li><li>Inform-Hotel-Internet-none Occur Num: 95</li><li>Recommend-Hotel-Internet-none Occur Num: 27</li><li>Inform-Hotel-Parking-none Occur Num: 26</li><li>Inform-Hotel-Choice-2 Occur Num: 21</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 717</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 68</li><li>Request-Hotel-Area-? Occur Num: 56</li><li>Recommend-Hotel-Type-guesthouse Occur Num: 48</li><li>Inform-Hotel-Area-north Occur Num: 44</li><li>Request-Hotel-Price-? Occur Num: 31</li></ul></ul><li>Inform-Hotel-Name-acorn guest house<ul><li>Occur Num: 522</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 49</li><li>Recommend-Hotel-Name-acorn guest house Occur Num: 47</li><li>Inform-Hotel-Area-north Occur Num: 41</li><li>reqmore-general-none-none Occur Num: 31</li><li>Inform-Hotel-Addr-154 chesterton road Occur Num: 30</li></ul></ul><li>Inform-Hotel-Type-hotel<ul><li>Occur Num: 518</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 42</li><li>Request-Hotel-Area-? Occur Num: 40</li><li>Inform-Hotel-Choice-3 Occur Num: 34</li><li>Inform-Hotel-Area-west Occur Num: 29</li><li>Inform-Hotel-Choice-2 Occur Num: 28</li></ul></ul><li>Request-Hotel-Area-?<ul><li>Occur Num: 488</li><li>NLU Output</li><ul><li>Request-Hotel-Price-? Occur Num: 48</li><li>Inform-Hotel-Type-guesthouse Occur Num: 44</li><li>Inform-Hotel-Choice-3 Occur Num: 33</li><li>Inform-Hotel-Choice-0 Occur Num: 25</li><li>NoOffer-Hotel-none-none Occur Num: 20</li></ul></ul><li>Inform-Hotel-Area-north<ul><li>Occur Num: 482</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 57</li><li>Recommend-Hotel-Name-acorn guest house Occur Num: 36</li><li>Inform-Hotel-Type-guesthouse Occur Num: 30</li><li>Recommend-Hotel-Type-guesthouse Occur Num: 27</li><li>Inform-Hotel-Price-moderate Occur Num: 24</li></ul></ul><li>Inform-Hotel-Name-huntingdon marriott hotel<ul><li>Occur Num: 376</li><li>NLU Output</li><ul><li>Inform-Hotel-Area-west Occur Num: 33</li><li>Inform-Hotel-Choice-3 Occur Num: 26</li><li>Recommend-Hotel-Name-huntingdon marriott hotel Occur Num: 26</li><li>Inform-Booking-none-none Occur Num: 25</li><li>Request-Hotel-Area-? Occur Num: 22</li></ul></ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 324</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 44</li><li>Inform-Booking-none-none Occur Num: 39</li><li>Inform-Hotel-Internet-none Occur Num: 26</li><li>Recommend-Hotel-Parking-none Occur Num: 20</li><li>Inform-Hotel-Choice-2 Occur Num: 13</li></ul></ul><li>Inform-Hotel-Name-ashley hotel<ul><li>Occur Num: 299</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 27</li><li>Recommend-Hotel-Name-ashley hotel Occur Num: 23</li><li>Inform-Hotel-Area-north Occur Num: 21</li><li>reqmore-general-none-none Occur Num: 15</li><li>Inform-Hotel-Addr-74 chesterton road Occur Num: 15</li></ul></ul><li>Inform-Hotel-Area-centre<ul><li>Occur Num: 296</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 32</li><li>Inform-Hotel-Type-guesthouse Occur Num: 24</li><li>Inform-Hotel-Choice-2 Occur Num: 21</li><li>Inform-Hotel-Internet-none Occur Num: 19</li><li>Inform-Hotel-Choice-1 Occur Num: 12</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Inform-Hotel-Internet-yes Occur Num: 30</li><li>Request-Hotel-Addr-? Occur Num: 21</li><li>Request-Hotel-Stars-? Occur Num: 17</li><li>Request-Hotel-Price-? Occur Num: 12</li><li>Request-Hotel-Post-? Occur Num: 10</li><li>Inform-Hotel-Stay-4 Occur Num: 9</li><li>Request-Hotel-Phone-? Occur Num: 9</li><li>Request-Hotel-Type-? Occur Num: 8</li><li>Inform-Hotel-Stay-3 Occur Num: 8</li><li>Inform-Hotel-Stay-5 Occur Num: 6</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-internet Occur Num: 98</li><li>inform-hotel-area Occur Num: 55</li><li>inform-hotel-parking Occur Num: 46</li><li>recommend-hotel-internet Occur Num: 27</li><li>inform-hotel-stars Occur Num: 25</li><li>recommend-hotel-parking Occur Num: 20</li><li>recommend-hotel-stars Occur Num: 3</li><li>inform-hotel-pricerange Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-address Occur Num: 38</li><li>request-hotel-phone Occur Num: 25</li><li>request-hotel-internet Occur Num: 23</li><li>request-hotel-postcode Occur Num: 21</li><li>request-hotel-stars Occur Num: 19</li><li>request-hotel-parking Occur Num: 17</li><li>request-hotel-pricerange Occur Num: 14</li><li>request-hotel-type Occur Num: 10</li><li>request-hotel-area Occur Num: 9</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-area Occur Num: 69</li><li>inform-hotel-type Occur Num: 58</li><li>inform-hotel-stars Occur Num: 9</li><li>inform-hotel-phone Occur Num: 8</li></ul></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 60.3 %</p><p> (Precision, Recall, F1) : (0.497, 0.713, 0.556) </p><p> Average Dialog Turn (Succ): 11.247 </p><p> Average Dialog Turn (All): 21.770 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 95</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensively Occur Num: 61</li><li>Inform-Restaurant-Food-do n't care Occur Num: 17</li><li>Inform-Restaurant-Area-centre Occur Num: 3</li><li>Inform-Restaurant-Area-south Occur Num: 2</li><li>Inform-Restaurant-Food-italian Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Price-moderate<ul><li>Occur Num: 85</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 56</li><li>Inform-Restaurant-Food-do n't care Occur Num: 14</li><li>Inform-Restaurant-Area-centre Occur Num: 7</li><li>Inform-Restaurant-Food-british Occur Num: 2</li><li>Inform-Restaurant-Food-mediterranean Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Food-dontcare<ul><li>Occur Num: 84</li><li>NLU Output</li><ul><li>Inform-Restaurant-Food-do n't care Occur Num: 53</li><li>Inform-Restaurant-Price-expensive Occur Num: 16</li><li>Inform-Restaurant-Price-moderate Occur Num: 13</li><li>Inform-Restaurant-Price-expensively Occur Num: 1</li><li>Inform-Restaurant-Price-moderately Occur Num: 1</li></ul></ul><li>Request-Restaurant-Addr-?<ul><li>Occur Num: 73</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 33</li><li>Request-Restaurant-Post-? Occur Num: 21</li><li>Request-Attraction-Addr-? Occur Num: 6</li><li>Request-Attraction-Phone-? Occur Num: 4</li><li>Request-Attraction-Post-? Occur Num: 3</li></ul></ul><li>Request-Restaurant-Phone-?<ul><li>Occur Num: 68</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 33</li><li>Request-Attraction-Phone-? Occur Num: 15</li><li>Request-Restaurant-Post-? Occur Num: 11</li><li>Request-Attraction-Post-? Occur Num: 3</li><li>Request-Attraction-Addr-? Occur Num: 3</li></ul></ul><li>Request-Restaurant-Post-?<ul><li>Occur Num: 44</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 21</li><li>Request-Restaurant-Phone-? Occur Num: 11</li><li>Request-Attraction-Post-? Occur Num: 5</li><li>Request-Attraction-Phone-? Occur Num: 2</li><li>Request-Restaurant-Price-? Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Price-cheap<ul><li>Occur Num: 33</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-cheaply Occur Num: 24</li><li>Inform-Restaurant-Food-italian Occur Num: 2</li><li>Inform-Restaurant-Area-centre Occur Num: 2</li><li>Inform-Restaurant-Area-east Occur Num: 1</li><li>Inform-Hotel-Price-cheap Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 27</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 6</li><li>Inform-Restaurant-Price-cheap Occur Num: 4</li><li>Inform-Restaurant-Price-expensive Occur Num: 3</li><li>Inform-Restaurant-Food-british Occur Num: 3</li><li>Request-Train-Ticket-? Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Food-portuguese<ul><li>Occur Num: 13</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 3</li><li>Inform-Restaurant-Price-expensive Occur Num: 3</li><li>Inform-Restaurant-Price-moderate Occur Num: 2</li><li>Inform-Restaurant-Price-moderately Occur Num: 1</li><li>Inform-Restaurant-Area-north Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Area-east<ul><li>Occur Num: 9</li><li>NLU Output</li><ul><li>Inform-Restaurant-Food-italian Occur Num: 3</li><li>Inform-Restaurant-Price-expensive Occur Num: 2</li><li>Inform-Restaurant-Price-cheaply Occur Num: 1</li><li>Request-Train-Ticket-? Occur Num: 1</li><li>Inform-Restaurant-Food-turkish Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 1681</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 110</li><li>Recommend-Restaurant-Price-cheap Occur Num: 98</li><li>Inform-Restaurant-Food-italian Occur Num: 89</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 89</li><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 74</li></ul></ul><li>Inform-Restaurant-Price-cheap<ul><li>Occur Num: 1340</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-cheap Occur Num: 120</li><li>Inform-Restaurant-Food-italian Occur Num: 96</li><li>Inform-Restaurant-Area-centre Occur Num: 95</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 87</li><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 83</li></ul></ul><li>Inform-Restaurant-Name-pizza hut city centre<ul><li>Occur Num: 1106</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 102</li><li>Inform-Restaurant-Food-italian Occur Num: 97</li><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 94</li><li>Inform-Restaurant-Area-centre Occur Num: 90</li><li>Recommend-Restaurant-Price-cheap Occur Num: 88</li></ul></ul><li>Inform-Restaurant-Food-italian<ul><li>Occur Num: 1037</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-cheap Occur Num: 97</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 95</li><li>Inform-Restaurant-Area-centre Occur Num: 89</li><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 70</li><li>Inform-Restaurant-Price-cheap Occur Num: 66</li></ul></ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 802</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-expensive Occur Num: 59</li><li>Inform-Booking-none-none Occur Num: 55</li><li>Inform-Restaurant-Food-indian Occur Num: 36</li><li>reqmore-general-none-none Occur Num: 33</li><li>Inform-Restaurant-Area-centre Occur Num: 29</li></ul></ul><li>Inform-Restaurant-Addr-Regent Street City Centre<ul><li>Occur Num: 800</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 87</li><li>reqmore-general-none-none Occur Num: 79</li><li>Inform-Restaurant-Phone-01223323737 Occur Num: 75</li><li>Inform-Restaurant-Price-cheap Occur Num: 72</li><li>Inform-Restaurant-Food-italian Occur Num: 69</li></ul></ul><li>Inform-Restaurant-Phone-01223323737<ul><li>Occur Num: 686</li><li>NLU Output</li><ul><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 85</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 82</li><li>reqmore-general-none-none Occur Num: 66</li><li>Inform-Restaurant-Food-italian Occur Num: 66</li><li>Inform-Restaurant-Price-cheap Occur Num: 57</li></ul></ul><li>Request-Restaurant-Food-?<ul><li>Occur Num: 484</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-cheap Occur Num: 47</li><li>Inform-Restaurant-Food-italian Occur Num: 42</li><li>Inform-Restaurant-Area-centre Occur Num: 42</li><li>Inform-Restaurant-Choice-69 Occur Num: 41</li><li>Inform-Restaurant-Choice-110 Occur Num: 35</li></ul></ul><li>Inform-Restaurant-Price-moderate<ul><li>Occur Num: 448</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-moderate Occur Num: 37</li><li>Inform-Booking-none-none Occur Num: 34</li><li>Inform-Restaurant-Area-centre Occur Num: 27</li><li>Inform-Restaurant-Food-british Occur Num: 20</li><li>Inform-Restaurant-Choice-2 Occur Num: 17</li></ul></ul><li>Inform-Restaurant-Food-chinese<ul><li>Occur Num: 351</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-expensive Occur Num: 22</li><li>Recommend-Restaurant-Name-good luck chinese food takeaway Occur Num: 21</li><li>Inform-Booking-none-none Occur Num: 20</li><li>Inform-Restaurant-Area-south Occur Num: 18</li><li>Inform-Restaurant-Price-expensive Occur Num: 17</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Inform-Restaurant-Price-moderate Occur Num: 46</li><li>Inform-Restaurant-Price-expensive Occur Num: 37</li><li>Request-Restaurant-Phone-? Occur Num: 17</li><li>Inform-Restaurant-Price-cheap Occur Num: 16</li><li>Request-Restaurant-Addr-? Occur Num: 4</li><li>Inform-Restaurant-Food-portuguese Occur Num: 3</li><li>Inform-Restaurant-Day-friday Occur Num: 2</li><li>Inform-Restaurant-Time-18:15 Occur Num: 2</li><li>Inform-Restaurant-Day-tuesday Occur Num: 2</li><li>Inform-Restaurant-Time-19:00 Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-restaurant-area Occur Num: 11</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-postcode Occur Num: 51</li><li>request-restaurant-phone Occur Num: 25</li><li>request-restaurant-address Occur Num: 11</li><li>request-restaurant-pricerange Occur Num: 1</li><li>request-restaurant-area Occur Num: 1</li><li>request-restaurant-food Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-phone Occur Num: 168</li><li>inform-restaurant-food Occur Num: 108</li><li>inform-restaurant-area Occur Num: 80</li></ul></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 71.0 %</p><p> (Precision, Recall, F1) : (0.810, 0.790, 0.790) </p><p> Average Dialog Turn (Succ): 12.404 </p><p> Average Dialog Turn (All): 18.930 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Train-Depart-cambridge<ul><li>Occur Num: 138</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 39</li><li>Inform-Train-Day-thursday Occur Num: 8</li><li>Inform-Train-Day-wednesday Occur Num: 7</li><li>Inform-Train-Day-friday Occur Num: 6</li><li>Inform-Train-Day-Saturday Occur Num: 6</li></ul></ul><li>Inform-Train-Dest-cambridge<ul><li>Occur Num: 124</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 9</li><li>Inform-Train-Day-friday Occur Num: 7</li><li>Inform-Train-Day-wednesday Occur Num: 7</li><li>Inform-Train-Day-Saturday Occur Num: 7</li><li>Inform-Train-Day-thursday Occur Num: 7</li></ul></ul><li>Inform-Train-Day-monday<ul><li>Occur Num: 42</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 6</li><li>Inform-Train-Dest-norwich Occur Num: 3</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Day-Saturday Occur Num: 3</li><li>Inform-Train-Dest-kings lynn Occur Num: 2</li></ul></ul><li>Inform-Train-Depart-bishops stortford<ul><li>Occur Num: 40</li><li>NLU Output</li><ul><li>Inform-Train-Dest-bishops stortford Occur Num: 11</li><li>Inform-Train-Day-Saturday Occur Num: 7</li><li>Inform-Train-Dest-cambridge Occur Num: 4</li><li>Request-Train-Arrive-? Occur Num: 2</li><li>Inform-Train-Leave-09:45 Occur Num: 2</li></ul></ul><li>Inform-Train-Depart-norwich<ul><li>Occur Num: 36</li><li>NLU Output</li><ul><li>Inform-Train-Dest-norwich Occur Num: 8</li><li>Inform-Train-Day-Saturday Occur Num: 6</li><li>Inform-Train-Dest-cambridge Occur Num: 4</li><li>Inform-Train-Day-monday Occur Num: 2</li><li>Inform-Train-Leave-16:45 Occur Num: 1</li></ul></ul><li>Inform-Train-Day-wednesday<ul><li>Occur Num: 32</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 7</li><li>Inform-Train-Day-Saturday Occur Num: 3</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Dest-ely Occur Num: 1</li><li>Inform-Train-Leave-08:15 Occur Num: 1</li></ul></ul><li>Inform-Train-Depart-london liverpool street<ul><li>Occur Num: 27</li><li>NLU Output</li><ul><li>Inform-Train-Dest-london liverpool street Occur Num: 8</li><li>Inform-Train-Dest-cambridge Occur Num: 4</li><li>Inform-Train-Day-saturday Occur Num: 2</li><li>Inform-Train-Day-Saturday Occur Num: 1</li><li>Inform-Train-Leave-08:45 Occur Num: 1</li></ul></ul><li>Inform-Train-Leave-12:15<ul><li>Occur Num: 26</li><li>NLU Output</li><ul><li>Inform-Train-Day-Saturday Occur Num: 8</li><li>Inform-Train-Depart-london kings cross Occur Num: 3</li><li>Inform-Train-Dest-cambridge Occur Num: 3</li><li>Inform-Train-Dest-london kings cross Occur Num: 3</li><li>Inform-Train-Depart-cambridge Occur Num: 1</li></ul></ul><li>Inform-Train-Day-thursday<ul><li>Occur Num: 26</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 9</li><li>Inform-Train-Dest-leicester Occur Num: 2</li><li>Inform-Train-Leave-17:30 Occur Num: 2</li><li>Inform-Train-Leave-15:30 Occur Num: 2</li><li>Inform-Train-Dest-stevenage Occur Num: 1</li></ul></ul><li>Request-Train-Time-?<ul><li>Occur Num: 25</li><li>NLU Output</li><ul><li>Request-Train-Ticket-? Occur Num: 20</li><li>Inform-Train-none-none Occur Num: 4</li><li>Request-Train-Id-? Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>OfferBook-Train-none-none<ul><li>Occur Num: 2238</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-05:51 Occur Num: 313</li><li>Inform-Train-Choice-404 Occur Num: 306</li><li>Inform-Train-Leave-05:00 Occur Num: 299</li><li>reqmore-general-none-none Occur Num: 180</li><li>Inform-Train-Depart-cambridge Occur Num: 178</li></ul></ul><li>Inform-Train-Leave-05:00<ul><li>Occur Num: 2163</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-05:51 Occur Num: 313</li><li>OfferBook-Train-none-none Occur Num: 310</li><li>Inform-Train-Choice-404 Occur Num: 307</li><li>Inform-Train-Depart-cambridge Occur Num: 178</li><li>Inform-Train-Id-TR0899 Occur Num: 133</li></ul></ul><li>Inform-Train-Arrive-05:51<ul><li>Occur Num: 1743</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 309</li><li>Inform-Train-Choice-404 Occur Num: 299</li><li>Inform-Train-Leave-05:00 Occur Num: 295</li><li>Inform-Train-Depart-cambridge Occur Num: 171</li><li>Inform-Train-Id-TR0899 Occur Num: 134</li></ul></ul><li>Inform-Train-Choice-404<ul><li>Occur Num: 1606</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-05:51 Occur Num: 299</li><li>OfferBook-Train-none-none Occur Num: 296</li><li>Inform-Train-Leave-05:00 Occur Num: 282</li><li>Inform-Train-Depart-cambridge Occur Num: 177</li><li>Inform-Train-Id-TR0899 Occur Num: 96</li></ul></ul><li>Inform-Train-Depart-cambridge<ul><li>Occur Num: 1001</li><li>NLU Output</li><ul><li>Inform-Train-Choice-404 Occur Num: 177</li><li>OfferBook-Train-none-none Occur Num: 173</li><li>Inform-Train-Arrive-05:51 Occur Num: 171</li><li>Inform-Train-Leave-05:00 Occur Num: 168</li><li>Inform-Train-Id-TR0899 Occur Num: 65</li></ul></ul><li>Inform-Train-Id-TR0899<ul><li>Occur Num: 836</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-05:51 Occur Num: 134</li><li>OfferBook-Train-none-none Occur Num: 133</li><li>Inform-Train-Leave-05:00 Occur Num: 132</li><li>Inform-Train-Choice-404 Occur Num: 96</li><li>Inform-Train-Ticket-18.88 pounds Occur Num: 83</li></ul></ul><li>Inform-Train-Ticket-23.60 pounds<ul><li>Occur Num: 506</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 128</li><li>reqmore-general-none-none Occur Num: 108</li><li>Inform-Train-Leave-05:00 Occur Num: 84</li><li>Inform-Train-Time-51 minutes Occur Num: 52</li><li>Inform-Train-Arrive-05:51 Occur Num: 29</li></ul></ul><li>Inform-Train-Ticket-18.88 pounds<ul><li>Occur Num: 457</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 109</li><li>reqmore-general-none-none Occur Num: 95</li><li>Inform-Train-Leave-05:00 Occur Num: 94</li><li>Inform-Train-Id-TR0899 Occur Num: 84</li><li>Inform-Train-Time-51 minutes Occur Num: 17</li></ul></ul><li>Request-Train-Leave-?<ul><li>Occur Num: 325</li><li>NLU Output</li><ul><li>Inform-Train-Choice-404 Occur Num: 61</li><li>Inform-Train-Leave-05:00 Occur Num: 57</li><li>Inform-Train-Arrive-05:51 Occur Num: 51</li><li>Inform-Train-Depart-cambridge Occur Num: 46</li><li>OfferBook-Train-none-none Occur Num: 44</li></ul></ul><li>Inform-Train-Id-TR8272<ul><li>Occur Num: 303</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 52</li><li>Inform-Train-Arrive-05:51 Occur Num: 52</li><li>Inform-Train-Leave-05:00 Occur Num: 49</li><li>Inform-Train-Choice-404 Occur Num: 46</li><li>Inform-Train-Depart-cambridge Occur Num: 31</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Inform-Train-Depart-bishops stortford Occur Num: 5</li><li>Inform-Train-Depart-london liverpool street Occur Num: 5</li><li>Inform-Train-Depart-london kings cross Occur Num: 5</li><li>Inform-Train-Depart-peterborough Occur Num: 4</li><li>Inform-Train-Depart-norwich Occur Num: 4</li><li>Inform-Train-Leave-16:00 Occur Num: 3</li><li>Inform-Train-Leave-16:30 Occur Num: 3</li><li>Inform-Train-People-1 Occur Num: 3</li><li>Inform-Train-Depart-stansted airport Occur Num: 3</li><li>Inform-Train-Depart-kings lynn Occur Num: 3</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-train-duration Occur Num: 15</li><li>inform-train-leaveAt Occur Num: 6</li><li>inform-train-price Occur Num: 2</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-train-duration Occur Num: 47</li><li>request-train-price Occur Num: 12</li><li>request-train-leaveAt Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 84.4 %</p><p> (Precision, Recall, F1) : (0.764, 0.897, 0.799) </p><p> Average Dialog Turn (Succ): 6.907 </p><p> Average Dialog Turn (All): 11.346 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Attraction-Post-?<ul><li>Occur Num: 143</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 53</li><li>Request-Attraction-Addr-? Occur Num: 32</li><li>Request-Attraction-Fee-? Occur Num: 27</li><li>Request-Restaurant-Post-? Occur Num: 17</li><li>Request-Restaurant-Addr-? Occur Num: 4</li></ul></ul><li>Request-Attraction-Phone-?<ul><li>Occur Num: 140</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 53</li><li>Request-Attraction-Addr-? Occur Num: 31</li><li>Request-Attraction-Fee-? Occur Num: 23</li><li>Request-Restaurant-Phone-? Occur Num: 15</li><li>Request-Restaurant-Addr-? Occur Num: 9</li></ul></ul><li>Request-Attraction-Addr-?<ul><li>Occur Num: 128</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 32</li><li>Request-Attraction-Post-? Occur Num: 31</li><li>Request-Restaurant-Addr-? Occur Num: 21</li><li>Request-Restaurant-Post-? Occur Num: 13</li><li>Request-Attraction-Fee-? Occur Num: 11</li></ul></ul><li>Request-Attraction-Fee-?<ul><li>Occur Num: 58</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 27</li><li>Request-Attraction-Phone-? Occur Num: 23</li><li>Request-Attraction-Addr-? Occur Num: 8</li></ul></ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 38</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-entertainment Occur Num: 16</li><li>Inform-Restaurant-Area-centre Occur Num: 3</li><li>Inform-Attraction-Type-college Occur Num: 3</li><li>Inform-Attraction-Type-museums Occur Num: 3</li><li>Inform-Attraction-Type-do n't care Occur Num: 2</li></ul></ul><li>Inform-Attraction-Area-dontcare<ul><li>Occur Num: 34</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-do n't care Occur Num: 20</li><li>Request-Attraction-Area-? Occur Num: 13</li><li>Inform-Attraction-Type-museum Occur Num: 1</li></ul></ul><li>Inform-Attraction-Type-dontcare<ul><li>Occur Num: 33</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-do n't care Occur Num: 23</li><li>Request-Attraction-Type-? Occur Num: 8</li><li>Inform-Attraction-Area-centre Occur Num: 2</li></ul></ul><li>Request-Attraction-Area-?<ul><li>Occur Num: 21</li><li>NLU Output</li><ul><li>Request-Restaurant-Area-? Occur Num: 7</li><li>Request-Attraction-Addr-? Occur Num: 6</li><li>Request-Attraction-Phone-? Occur Num: 5</li><li>Request-Attraction-Post-? Occur Num: 2</li><li>Request-Attraction-Fee-? Occur Num: 1</li></ul></ul><li>Inform-Attraction-Type-museum<ul><li>Occur Num: 20</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 7</li><li>Inform-Attraction-Type-museums Occur Num: 5</li><li>Inform-Attraction-Type-entertainment Occur Num: 4</li><li>Inform-Attraction-Area-west Occur Num: 3</li><li>Inform-Attraction-Area-do n't care Occur Num: 1</li></ul></ul><li>Inform-Attraction-Type-boat<ul><li>Occur Num: 15</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 6</li><li>Inform-Attraction-Type-boats Occur Num: 5</li><li>Inform-Attraction-Type-entertainment Occur Num: 2</li><li>Inform-Attraction-Area-west Occur Num: 1</li><li>Inform-Attraction-Area-north Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 703</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 76</li><li>Inform-Attraction-Fee-free Occur Num: 46</li><li>Request-Attraction-Area-? Occur Num: 39</li><li>Inform-Attraction-Area-centre of the city Occur Num: 23</li><li>Inform-Attraction-Type-museum Occur Num: 23</li></ul></ul><li>Inform-Attraction-Fee-free<ul><li>Occur Num: 543</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 85</li><li>Inform-Attraction-Area-centre Occur Num: 36</li><li>Inform-Attraction-Type-museum Occur Num: 23</li><li>Inform-Attraction-Addr-98 king street Occur Num: 21</li><li>Inform-Attraction-Type-park Occur Num: 17</li></ul></ul><li>Inform-Attraction-Fee-?<ul><li>Occur Num: 488</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 57</li><li>Inform-Attraction-Addr-park street Occur Num: 21</li><li>Inform-Attraction-Area-centre Occur Num: 20</li><li>Inform-Attraction-Name-adc theatre Occur Num: 18</li><li>Inform-Attraction-Choice-44 Occur Num: 16</li></ul></ul><li>Inform-Attraction-Type-museum<ul><li>Occur Num: 419</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 41</li><li>Inform-Attraction-Fee-free Occur Num: 36</li><li>Request-Attraction-Area-? Occur Num: 35</li><li>Inform-Attraction-Area-centre Occur Num: 25</li><li>Inform-Attraction-Addr-98 king street Occur Num: 22</li></ul></ul><li>Request-Attraction-Area-?<ul><li>Occur Num: 404</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 35</li><li>NoOffer-Attraction-none-none Occur Num: 33</li><li>Inform-Attraction-Type-museum Occur Num: 29</li><li>Inform-Attraction-Choice-5 Occur Num: 22</li><li>Inform-Attraction-Type-college Occur Num: 20</li></ul></ul><li>Inform-Attraction-Name-adc theatre<ul><li>Occur Num: 338</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 43</li><li>Inform-Attraction-Name-Adc theatre Occur Num: 28</li><li>Recommend-Attraction-Name-adc theatre Occur Num: 28</li><li>Inform-Attraction-Addr-park street Occur Num: 28</li><li>Inform-Attraction-Area-centre Occur Num: 24</li></ul></ul><li>Inform-Attraction-Name-broughton house gallery<ul><li>Occur Num: 285</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 32</li><li>Inform-Attraction-Type-museum Occur Num: 27</li><li>Inform-Attraction-Addr-98 king street Occur Num: 26</li><li>Inform-Attraction-Fee-free Occur Num: 25</li><li>Inform-Attraction-Area-centre Occur Num: 22</li></ul></ul><li>Inform-Attraction-Type-theatre<ul><li>Occur Num: 280</li><li>NLU Output</li><ul><li>Request-Attraction-Type-? Occur Num: 36</li><li>Inform-Attraction-Choice-44 Occur Num: 36</li><li>Inform-Attraction-Type-architecture Occur Num: 25</li><li>Inform-Attraction-Area-centre Occur Num: 24</li><li>Recommend-Attraction-Name-adc theatre Occur Num: 20</li></ul></ul><li>Inform-Attraction-Name-christ's college<ul><li>Occur Num: 266</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 25</li><li>Inform-Attraction-Fee-free Occur Num: 23</li><li>Inform-Attraction-Area-centre Occur Num: 20</li><li>Inform-Attraction-Name-christ 's college Occur Num: 19</li><li>Inform-Attraction-Addr-saint andrew 's street Occur Num: 18</li></ul></ul><li>Request-Attraction-Type-?<ul><li>Occur Num: 262</li><li>NLU Output</li><ul><li>Inform-Attraction-Choice-44 Occur Num: 37</li><li>Inform-Attraction-Type-theatre Occur Num: 33</li><li>Inform-Attraction-Type-architecture Occur Num: 26</li><li>Inform-Attraction-Choice-13 Occur Num: 16</li><li>Inform-Attraction-Type-museum Occur Num: 16</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Attraction-Post-? Occur Num: 13</li><li>Request-Attraction-Addr-? Occur Num: 11</li><li>Request-Attraction-Fee-? Occur Num: 6</li><li>Request-Attraction-Area-? Occur Num: 5</li><li>Request-Attraction-Phone-? Occur Num: 2</li><li>Request-Attraction-Type-? Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-attraction-area Occur Num: 42</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-address Occur Num: 23</li><li>request-attraction-postcode Occur Num: 13</li><li>request-attraction-phone Occur Num: 9</li><li>request-attraction-area Occur Num: 8</li><li>request-attraction-type Occur Num: 3</li><li>request-attraction-entrance fee Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-type Occur Num: 53</li><li>inform-attraction-phone Occur Num: 28</li><li>inform-attraction-area Occur Num: 27</li></ul></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 72.1 %</p><p> (Precision, Recall, F1) : (0.767, 0.744, 0.752) </p><p> Average Dialog Turn (Succ): 7.258 </p><p> Average Dialog Turn (All): 12.558 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Phone-?<ul><li>Occur Num: 36</li><li>NLU Output</li><ul><li>Request-Taxi-Car-? Occur Num: 28</li><li>Request-Attraction-Phone-? Occur Num: 5</li><li>Request-Police-Phone-? Occur Num: 1</li><li>Request-Attraction-Type-? Occur Num: 1</li><li>Request-Hotel-Phone-? Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-dontcare<ul><li>Occur Num: 11</li><li>NLU Output</li><ul><li>Inform-Taxi-Leave-do Occur Num: 3</li><li>Inform-Taxi-Dest-golden house Occur Num: 1</li><li>Inform-Taxi-Leave-12:00 Occur Num: 1</li><li>Inform-Taxi-Dest-clare college Occur Num: 1</li><li>Inform-Taxi-Dest-shiraz restaurant Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the fitzwilliam museum<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 2</li><li>Inform-Taxi-Dest-golden house Occur Num: 1</li><li>Inform-Taxi-Leave-13:45 Occur Num: 1</li><li>Inform-Taxi-Arrive-13:45 Occur Num: 1</li><li>Inform-Taxi-Dest-la mimosa Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-home from home<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-home from home Occur Num: 1</li><li>Inform-Taxi-Depart-arbury lodge guesthouse Occur Num: 1</li><li>Inform-Taxi-Leave-02:45 Occur Num: 1</li><li>Inform-Taxi-Depart-thanh binh Occur Num: 1</li><li>Inform-Taxi-Dest-home Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the lensfield hotel<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li><li>Inform-Taxi-Dest-williams art and antiques Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-02:00<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li></ul></ul><li>Request-Taxi-Car-?<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 2</li><li>Request-Attraction-Type-? Occur Num: 1</li><li>Request-Taxi-Phone-? Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-parkside police station<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Arrive-12:15 Occur Num: 1</li><li>Inform-Police-Name-parkside police station Occur Num: 1</li><li>Inform-Taxi-Dest-clare college Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-efes restaurant<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-broxbourne train station Occur Num: 1</li><li>Inform-Restaurant-Name-efes Occur Num: 1</li><li>Inform-Taxi-Arrive-16:15 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-kohinoor<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-grafton hotel restaurant Occur Num: 1</li><li>Inform-Restaurant-Name-kohinoor Occur Num: 1</li><li>Inform-Taxi-Arrive-18:30 Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Arrive-?<ul><li>Occur Num: 76</li><li>NLU Output</li><ul><li>Request-Taxi-Dest-? Occur Num: 19</li><li>Request-Taxi-Depart-? Occur Num: 12</li><li>Inform-Taxi-Car-tesla Occur Num: 4</li><li>Inform-Taxi-Car-audi Occur Num: 2</li><li>Inform-Taxi-Car-toyota Occur Num: 2</li></ul></ul><li>Request-Taxi-Dest-?<ul><li>Occur Num: 43</li><li>NLU Output</li><ul><li>Request-Taxi-Arrive-? Occur Num: 17</li><li>Inform-Taxi-Car-lexus Occur Num: 2</li><li>Inform-Taxi-Phone-93793671822 Occur Num: 1</li><li>Inform-Taxi-Car-tesla Occur Num: 1</li><li>Inform-Taxi-Phone-61488179956 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-honda<ul><li>Occur Num: 41</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 15</li><li>Request-Taxi-Arrive-? Occur Num: 3</li><li>Request-Taxi-Dest-? Occur Num: 2</li><li>Inform-Taxi-Phone-18881852346 Occur Num: 1</li><li>Request-Taxi-Depart-? Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-toyota<ul><li>Occur Num: 40</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 14</li><li>Request-Taxi-Dest-? Occur Num: 3</li><li>Request-Taxi-Arrive-? Occur Num: 3</li><li>Inform-Taxi-Phone-11282911436 Occur Num: 1</li><li>Inform-Taxi-Phone-34659982464 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-tesla<ul><li>Occur Num: 40</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 13</li><li>Request-Taxi-Arrive-? Occur Num: 5</li><li>Request-Taxi-Depart-? Occur Num: 3</li><li>Inform-Taxi-Phone-79656243666 Occur Num: 1</li><li>Inform-Taxi-Phone-79253567944 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-ford<ul><li>Occur Num: 38</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 12</li><li>Request-Taxi-Arrive-? Occur Num: 4</li><li>Request-Taxi-Depart-? Occur Num: 3</li><li>Request-Taxi-Dest-? Occur Num: 2</li><li>Inform-Taxi-Phone-93793671822 Occur Num: 1</li></ul></ul><li>Request-Taxi-Depart-?<ul><li>Occur Num: 37</li><li>NLU Output</li><ul><li>Request-Taxi-Arrive-? Occur Num: 13</li><li>Inform-Taxi-Car-tesla Occur Num: 3</li><li>Request-Taxi-Dest-? Occur Num: 2</li><li>Inform-Taxi-Car-audi Occur Num: 1</li><li>Inform-Taxi-Phone-68383131866 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-lexus<ul><li>Occur Num: 37</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 11</li><li>Request-Taxi-Arrive-? Occur Num: 5</li><li>Request-Taxi-Dest-? Occur Num: 4</li><li>Inform-Taxi-Phone-42425648888 Occur Num: 1</li><li>Inform-Taxi-Phone-41948119556 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-audi<ul><li>Occur Num: 35</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 14</li><li>Request-Taxi-Arrive-? Occur Num: 2</li><li>Inform-Taxi-Phone-97948592576 Occur Num: 1</li><li>Request-Taxi-Depart-? Occur Num: 1</li><li>Inform-Taxi-Phone-68383131866 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-skoda<ul><li>Occur Num: 35</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 10</li><li>Request-Taxi-Arrive-? Occur Num: 3</li><li>Request-Taxi-Dest-? Occur Num: 3</li><li>Request-Taxi-Depart-? Occur Num: 2</li><li>Inform-Taxi-Phone-36162266133 Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Taxi-Car-? Occur Num: 19</li><li>Request-Taxi-Phone-? Occur Num: 4</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-taxi-phone Occur Num: 24</li><li>request-taxi-car type Occur Num: 20</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 100.0 %</p><p> (Precision, Recall, F1) : (0.935, 1.000, 0.957) </p><p> Average Dialog Turn (Succ): 2.000 </p><p> Average Dialog Turn (All): 2.000 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Police-none-none<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Police-Name-Parkside Police Station Occur Num: 6</li><li>Request-Police-Addr-? Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Police-Addr-Parkside, Cambridge<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 23</li><li>Inform-Police-Phone-01223358966 Occur Num: 23</li></ul></ul><li>Inform-Police-Name-none<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 23</li><li>Inform-Police-Phone-01223358966 Occur Num: 23</li></ul></ul><li>Inform-Police-Post-none<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 23</li><li>Inform-Police-Phone-01223358966 Occur Num: 23</li></ul></ul><li>Inform-Police-Phone-01223358966<ul><li>Occur Num: 23</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 23</li></ul></ul></ul><b> Dialog Loop</b><p>Nothing</p><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><p>Nothing</p><b> Inform But Not Request Dialog Act</b><ul><li>inform-police-phone Occur Num: 3</li></ul></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 93.3 %</p><p> (Precision, Recall, F1) : (0.933, 0.933, 0.933) </p><p> Average Dialog Turn (Succ): 38.429 </p><p> Average Dialog Turn (All): 38.533 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Hospital-Department-plastic and vascular surgery plastics<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li><li>Inform-Hospital-Department-plastic and vascular surgery Occur Num: 1</li><li>Inform-Hospital-Department-plastic Occur Num: 1</li><li>Inform-Hospital-Department-vascular surgery Occur Num: 1</li><li>Inform-Hospital-Department-neurosurgery Occur Num: 1</li></ul></ul><li>Request-Hospital-Phone-?<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Request-Police-Phone-? Occur Num: 2</li><li>Request-Restaurant-Phone-? Occur Num: 2</li></ul></ul><li>Inform-Hospital-Department-emergency department<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-emergency department department Occur Num: 2</li><li>Inform-Hospital-Department-neurosurgery Occur Num: 2</li></ul></ul><li>Inform-Hospital-Department-hepatobillary and gastrointestinal surgery regional referral centre<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurosurgery Occur Num: 1</li><li>Inform-Hospital-Department-hepatobillary Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-haematology and haematological oncology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurosurgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-acute medicine for the elderly<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurosurgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-neonatal unit<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurosurgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-oncology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-trauma and orthopaedics<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-trauma and orthopaedics neurosurgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-transplant high dependency unit<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurosurgery Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hospital-Addr-none<ul><li>Occur Num: 58</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurosciences critical care unit Occur Num: 28</li><li>Inform-Hospital-Phone-01223216297 Occur Num: 28</li><li>Request-Hospital-Department-? Occur Num: 2</li></ul></ul><li>Inform-Hospital-Post-none<ul><li>Occur Num: 58</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurosciences critical care unit Occur Num: 28</li><li>Inform-Hospital-Phone-01223216297 Occur Num: 28</li><li>Request-Hospital-Department-? Occur Num: 2</li></ul></ul><li>Inform-Hospital-Department-neurosciences critical care unit<ul><li>Occur Num: 30</li><li>NLU Output</li><ul><li>Inform-Hospital-Phone-01223216297 Occur Num: 28</li><li>Request-Hospital-Department-? Occur Num: 2</li></ul></ul><li>Inform-Hospital-Phone-01223216297<ul><li>Occur Num: 30</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurosciences critical care unit Occur Num: 28</li><li>Request-Hospital-Department-? Occur Num: 2</li></ul></ul><li>Request-Hospital-Department-?<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurosciences critical care unit Occur Num: 2</li><li>Inform-Hospital-Phone-01223216297 Occur Num: 2</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Hospital-Phone-? Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-hospital-phone Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/BERTNLU-RuleDST-PGPolicy-TemplateNLG/res.txt b/tests/results/BERTNLU-RuleDST-PGPolicy-TemplateNLG/res.txt deleted file mode 100755 index 202f8c080c1baa6dfc03ecae374da1e0b73adbf0..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-PGPolicy-TemplateNLG/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.429 -success number of dialogs/tot: 0.433 -average precision: 0.6187398586642582 -average recall: 0.6681233414099458 -average f1: 0.6039577413844464 -average book rate: 0.30991452991452995 -average turn (succ): 14.69284064665127 -average turn (all): 29.068 diff --git a/tests/results/BERTNLU-RuleDST-PPOPolicy-TemplateNLG/Frequency_of_domain.png b/tests/results/BERTNLU-RuleDST-PPOPolicy-TemplateNLG/Frequency_of_domain.png deleted file mode 100755 index 093539916ae361c475c55922aa99cfa893c252a1..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-PPOPolicy-TemplateNLG/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-PPOPolicy-TemplateNLG/Performance_for_each_domain.png b/tests/results/BERTNLU-RuleDST-PPOPolicy-TemplateNLG/Performance_for_each_domain.png deleted file mode 100755 index 02c3000adb94b0f14b48037c49dd9736d5a206e2..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-PPOPolicy-TemplateNLG/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-PPOPolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png b/tests/results/BERTNLU-RuleDST-PPOPolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index aa9f150e85ee6a58204d77609ca559f4067eb6c3..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-PPOPolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-PPOPolicy-TemplateNLG/report_multiwoz.html b/tests/results/BERTNLU-RuleDST-PPOPolicy-TemplateNLG/report_multiwoz.html deleted file mode 100755 index 5aaa35eb51119a10a23bf70c0b157de5c83b7571..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-PPOPolicy-TemplateNLG/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: BERTNLU-RuleDST-PPOPolicy-TemplateNLG</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-27 02:25:52</h3><h3>Overall Results</h3><p> Success Rate: 56.6 %</p><p> (Precision, Recall, F1) : (0.648, 0.790, 0.681) </p><p> Average Dialog Turn (Succ): 12.880 </p><p> Average Dialog Turn (All): 22.110 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">402</td><td align="center">0.500</td><td align="center">0.357</td><td align="center">0.515</td><td align="center">0.391</td><td align="center">0.343</td><td align="center">7.851</td><td align="center">16.343</td></tr><tr><td align="center">restaurant</td><td align="center">402</td><td align="center">0.677</td><td align="center">0.534</td><td align="center">0.739</td><td align="center">0.594</td><td align="center">0.296</td><td align="center">8.213</td><td align="center">14.756</td></tr><tr><td align="center">attraction</td><td align="center">309</td><td align="center">0.883</td><td align="center">0.753</td><td align="center">0.920</td><td align="center">0.800</td><td align="center">0.100</td><td align="center">6.388</td><td align="center">9.502</td></tr><tr><td align="center">taxi</td><td align="center">121</td><td align="center">0.802</td><td align="center">0.884</td><td align="center">0.843</td><td align="center">0.857</td><td align="center">0.174</td><td align="center">7.052</td><td align="center">9.868</td></tr><tr><td align="center">train</td><td align="center">349</td><td align="center">0.857</td><td align="center">0.900</td><td align="center">0.919</td><td align="center">0.907</td><td align="center">0.129</td><td align="center">7.391</td><td align="center">8.395</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">1.000</td><td align="center">0.935</td><td align="center">1.000</td><td align="center">0.957</td><td align="center">0.000</td><td align="center">2.000</td><td align="center">2.000</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">0.933</td><td align="center">0.933</td><td align="center">0.933</td><td align="center">0.933</td><td align="center">0.067</td><td align="center">2.000</td><td align="center">4.533</td></tr></table><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 50.0 %</p><p> (Precision, Recall, F1) : (0.357, 0.515, 0.391) </p><p> Average Dialog Turn (Succ): 7.851 </p><p> Average Dialog Turn (All): 16.343 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 393</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-hotel Occur Num: 81</li><li>Inform-Hotel-Internet-yes Occur Num: 66</li><li>Inform-Hotel-Stars-4 Occur Num: 43</li><li>Inform-Hotel-Type-guesthouse Occur Num: 41</li><li>Inform-Hotel-Price-moderate Occur Num: 32</li></ul></ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 375</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 65</li><li>Inform-Hotel-Type-hotel Occur Num: 61</li><li>Inform-Hotel-Type-guesthouse Occur Num: 44</li><li>Inform-Hotel-Stars-4 Occur Num: 38</li><li>Inform-Hotel-Price-moderate Occur Num: 34</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 216</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 45</li><li>Inform-Hotel-Internet-yes Occur Num: 44</li><li>Inform-Hotel-Stars-4 Occur Num: 22</li><li>Inform-Hotel-Type-hotel Occur Num: 18</li><li>Inform-Hotel-Area-north Occur Num: 14</li></ul></ul><li>Inform-Hotel-Stars-4<ul><li>Occur Num: 215</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 45</li><li>Inform-Hotel-Type-hotel Occur Num: 39</li><li>Inform-Hotel-Internet-yes Occur Num: 38</li><li>Inform-Hotel-Type-guesthouse Occur Num: 22</li><li>Inform-Hotel-Price-moderate Occur Num: 13</li></ul></ul><li>Inform-Hotel-Price-moderate<ul><li>Occur Num: 185</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-hotel Occur Num: 37</li><li>Inform-Hotel-Internet-yes Occur Num: 35</li><li>Inform-Hotel-Parking-yes Occur Num: 34</li><li>Inform-Hotel-Type-guesthouse Occur Num: 14</li><li>Inform-Hotel-Stars-4 Occur Num: 13</li></ul></ul><li>Inform-Hotel-Area-north<ul><li>Occur Num: 131</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 26</li><li>Inform-Hotel-Internet-yes Occur Num: 25</li><li>Inform-Hotel-Type-hotel Occur Num: 23</li><li>Inform-Hotel-Type-guesthouse Occur Num: 14</li><li>Inform-Hotel-Price-do n't care Occur Num: 12</li></ul></ul><li>Request-Hotel-Addr-?<ul><li>Occur Num: 130</li><li>NLU Output</li><ul><li>Request-Hotel-Post-? Occur Num: 21</li><li>Request-Hotel-Phone-? Occur Num: 18</li><li>Request-Restaurant-Addr-? Occur Num: 17</li><li>Request-Attraction-Addr-? Occur Num: 11</li><li>Request-Hotel-Internet-? Occur Num: 11</li></ul></ul><li>Inform-Hotel-Parking-no<ul><li>Occur Num: 102</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 25</li><li>Inform-Hotel-Type-hotel Occur Num: 13</li><li>Inform-Hotel-Type-guesthouse Occur Num: 12</li><li>Inform-Hotel-Internet-yes Occur Num: 12</li><li>Inform-Hotel-Price-moderate Occur Num: 9</li></ul></ul><li>Inform-Hotel-Area-centre<ul><li>Occur Num: 91</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 21</li><li>Inform-Hotel-Type-hotel Occur Num: 15</li><li>Inform-Hotel-Type-guesthouse Occur Num: 12</li><li>Inform-Hotel-Parking-yes Occur Num: 12</li><li>Inform-Hotel-Price-moderate Occur Num: 11</li></ul></ul><li>Inform-Hotel-Area-dontcare<ul><li>Occur Num: 88</li><li>NLU Output</li><ul><li>Inform-Hotel-Area-do n't care Occur Num: 75</li><li>Inform-Hotel-Price-do n't care Occur Num: 5</li><li>Inform-Hotel-Internet-yes Occur Num: 3</li><li>Inform-Hotel-Type-guesthouse Occur Num: 3</li><li>Inform-Hotel-Parking-yes Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 727</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 82</li><li>Recommend-Hotel-Type-guesthouse Occur Num: 66</li><li>Request-Hotel-Area-? Occur Num: 66</li><li>Inform-Hotel-Area-north Occur Num: 40</li><li>Request-Hotel-Price-? Occur Num: 37</li></ul></ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 691</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-none Occur Num: 89</li><li>Inform-Booking-none-none Occur Num: 84</li><li>Inform-Hotel-Parking-none Occur Num: 45</li><li>Request-Booking-Day-? Occur Num: 32</li><li>Request-Booking-Stay-? Occur Num: 23</li></ul></ul><li>Inform-Hotel-Name-acorn guest house<ul><li>Occur Num: 581</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 70</li><li>Recommend-Hotel-Name-acorn guest house Occur Num: 62</li><li>Inform-Hotel-Area-north Occur Num: 44</li><li>reqmore-general-none-none Occur Num: 43</li><li>Recommend-Hotel-Type-guesthouse Occur Num: 38</li></ul></ul><li>Request-Hotel-Area-?<ul><li>Occur Num: 534</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-guesthouse Occur Num: 61</li><li>Request-Hotel-Price-? Occur Num: 54</li><li>Inform-Hotel-Choice-0 Occur Num: 40</li><li>Inform-Hotel-Choice-3 Occur Num: 36</li><li>NoOffer-Hotel-none-none Occur Num: 32</li></ul></ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 531</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 71</li><li>Inform-Booking-none-none Occur Num: 54</li><li>Inform-Hotel-Internet-none Occur Num: 39</li><li>Inform-Hotel-Choice-2 Occur Num: 23</li><li>Recommend-Hotel-Parking-none Occur Num: 23</li></ul></ul><li>Inform-Hotel-Type-hotel<ul><li>Occur Num: 514</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 51</li><li>Request-Hotel-Area-? Occur Num: 41</li><li>Inform-Hotel-Choice-3 Occur Num: 39</li><li>Inform-Hotel-Choice-2 Occur Num: 34</li><li>Inform-Hotel-Parking-none Occur Num: 24</li></ul></ul><li>Inform-Hotel-Area-north<ul><li>Occur Num: 434</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 65</li><li>Recommend-Hotel-Name-acorn guest house Occur Num: 43</li><li>Recommend-Hotel-Type-guesthouse Occur Num: 39</li><li>Recommend-Hotel-Name-ashley hotel Occur Num: 21</li><li>Inform-Hotel-Choice-2 Occur Num: 14</li></ul></ul><li>Inform-Hotel-Name-huntingdon marriott hotel<ul><li>Occur Num: 366</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 34</li><li>Recommend-Hotel-Name-huntingdon marriott hotel Occur Num: 27</li><li>Inform-Hotel-Area-west Occur Num: 25</li><li>Inform-Hotel-Choice-3 Occur Num: 23</li><li>reqmore-general-none-none Occur Num: 22</li></ul></ul><li>Inform-Hotel-Name-ashley hotel<ul><li>Occur Num: 320</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 28</li><li>Recommend-Hotel-Name-ashley hotel Occur Num: 28</li><li>Inform-Hotel-Area-north Occur Num: 21</li><li>reqmore-general-none-none Occur Num: 20</li><li>Inform-Hotel-Addr-74 chesterton road Occur Num: 17</li></ul></ul><li>Inform-Hotel-Choice-2<ul><li>Occur Num: 264</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 37</li><li>Inform-Hotel-Parking-none Occur Num: 23</li><li>Inform-Hotel-Internet-none Occur Num: 18</li><li>Inform-Hotel-Area-centre Occur Num: 17</li><li>Inform-Hotel-Type-guesthouse Occur Num: 16</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Hotel-Addr-? Occur Num: 22</li><li>Request-Hotel-Stars-? Occur Num: 17</li><li>Request-Hotel-Price-? Occur Num: 15</li><li>Request-Hotel-Post-? Occur Num: 12</li><li>Inform-Hotel-Internet-yes Occur Num: 12</li><li>Inform-Hotel-Parking-yes Occur Num: 10</li><li>Request-Hotel-Type-? Occur Num: 9</li><li>Request-Hotel-Area-? Occur Num: 6</li><li>Request-Hotel-Phone-? Occur Num: 6</li><li>Request-Hotel-Internet-? Occur Num: 6</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-internet Occur Num: 90</li><li>inform-hotel-parking Occur Num: 83</li><li>inform-hotel-area Occur Num: 47</li><li>inform-hotel-stars Occur Num: 25</li><li>recommend-hotel-parking Occur Num: 23</li><li>recommend-hotel-internet Occur Num: 21</li><li>recommend-hotel-stars Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-address Occur Num: 31</li><li>request-hotel-internet Occur Num: 28</li><li>request-hotel-parking Occur Num: 21</li><li>request-hotel-stars Occur Num: 21</li><li>request-hotel-postcode Occur Num: 18</li><li>request-hotel-pricerange Occur Num: 17</li><li>request-hotel-phone Occur Num: 12</li><li>request-hotel-area Occur Num: 11</li><li>request-hotel-type Occur Num: 9</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-type Occur Num: 76</li><li>inform-hotel-area Occur Num: 61</li><li>inform-hotel-phone Occur Num: 7</li><li>inform-hotel-stars Occur Num: 4</li></ul></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 67.7 %</p><p> (Precision, Recall, F1) : (0.534, 0.739, 0.594) </p><p> Average Dialog Turn (Succ): 8.213 </p><p> Average Dialog Turn (All): 14.756 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 92</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensively Occur Num: 46</li><li>Inform-Restaurant-Food-do n't care Occur Num: 19</li><li>Inform-Restaurant-Area-centre Occur Num: 4</li><li>Inform-Restaurant-Area-do n't care Occur Num: 3</li><li>Inform-Restaurant-Area-north Occur Num: 2</li></ul></ul><li>Request-Restaurant-Phone-?<ul><li>Occur Num: 84</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 37</li><li>Request-Attraction-Phone-? Occur Num: 22</li><li>Request-Restaurant-Post-? Occur Num: 13</li><li>Request-Attraction-Addr-? Occur Num: 6</li><li>Request-Police-Phone-? Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Food-dontcare<ul><li>Occur Num: 82</li><li>NLU Output</li><ul><li>Inform-Restaurant-Food-do n't care Occur Num: 49</li><li>Inform-Restaurant-Price-expensive Occur Num: 17</li><li>Inform-Restaurant-Price-moderate Occur Num: 14</li><li>Inform-Restaurant-Price-expensively Occur Num: 2</li></ul></ul><li>Request-Restaurant-Addr-?<ul><li>Occur Num: 81</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 35</li><li>Request-Restaurant-Post-? Occur Num: 23</li><li>Request-Attraction-Addr-? Occur Num: 10</li><li>Request-Attraction-Phone-? Occur Num: 6</li><li>Request-Police-Addr-? Occur Num: 3</li></ul></ul><li>Inform-Restaurant-Price-moderate<ul><li>Occur Num: 79</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 41</li><li>Inform-Restaurant-Food-do n't care Occur Num: 14</li><li>Inform-Restaurant-Area-centre Occur Num: 7</li><li>Inform-Restaurant-Area-do n't care Occur Num: 7</li><li>Inform-Restaurant-Food-italian Occur Num: 2</li></ul></ul><li>Request-Restaurant-Post-?<ul><li>Occur Num: 51</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 23</li><li>Request-Restaurant-Phone-? Occur Num: 13</li><li>Request-Attraction-Post-? Occur Num: 10</li><li>Request-Attraction-Phone-? Occur Num: 2</li><li>Request-Restaurant-Price-? Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Area-dontcare<ul><li>Occur Num: 31</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-do n't care Occur Num: 18</li><li>Inform-Restaurant-Price-moderate Occur Num: 6</li><li>Inform-Restaurant-Price-cheap Occur Num: 3</li><li>Inform-Restaurant-Price-expensive Occur Num: 2</li><li>Inform-Restaurant-Price-expensively Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Price-cheap<ul><li>Occur Num: 30</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-cheaply Occur Num: 17</li><li>Inform-Restaurant-Area-do n't care Occur Num: 3</li><li>Inform-Restaurant-Food-italian Occur Num: 2</li><li>Inform-Restaurant-Food-indian Occur Num: 1</li><li>Request-Train-Ticket-? Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 26</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 5</li><li>Inform-Restaurant-Price-expensive Occur Num: 4</li><li>Inform-Restaurant-Price-cheap Occur Num: 3</li><li>Inform-Restaurant-Food-british Occur Num: 3</li><li>Inform-Restaurant-Price-moderate Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Food-portuguese<ul><li>Occur Num: 11</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 3</li><li>Inform-Restaurant-Price-expensive Occur Num: 3</li><li>Inform-Restaurant-Price-moderate Occur Num: 2</li><li>Inform-Restaurant-Area-north Occur Num: 1</li><li>Inform-Restaurant-Price-cheap Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 1593</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 119</li><li>Recommend-Restaurant-Price-cheap Occur Num: 90</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 82</li><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 68</li><li>Inform-Booking-none-none Occur Num: 68</li></ul></ul><li>Inform-Restaurant-Price-cheap<ul><li>Occur Num: 1129</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-cheap Occur Num: 104</li><li>Inform-Restaurant-Area-centre Occur Num: 94</li><li>reqmore-general-none-none Occur Num: 84</li><li>Request-Restaurant-Food-? Occur Num: 82</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 79</li></ul></ul><li>Inform-Restaurant-Name-pizza hut city centre<ul><li>Occur Num: 971</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 97</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 93</li><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 85</li><li>Inform-Restaurant-Area-centre Occur Num: 80</li><li>Recommend-Restaurant-Price-cheap Occur Num: 78</li></ul></ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 707</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-expensive Occur Num: 59</li><li>Inform-Booking-none-none Occur Num: 52</li><li>Inform-Restaurant-Area-centre Occur Num: 39</li><li>reqmore-general-none-none Occur Num: 38</li><li>Inform-Restaurant-Food-indian Occur Num: 34</li></ul></ul><li>Inform-Restaurant-Food-italian<ul><li>Occur Num: 650</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-cheap Occur Num: 68</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 67</li><li>Inform-Restaurant-Area-centre Occur Num: 58</li><li>reqmore-general-none-none Occur Num: 51</li><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 49</li></ul></ul><li>Inform-Restaurant-Phone-01223323737<ul><li>Occur Num: 636</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 93</li><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 74</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 74</li><li>Recommend-Restaurant-Phone-01223323737 Occur Num: 61</li><li>Inform-Restaurant-Price-cheap Occur Num: 48</li></ul></ul><li>Inform-Restaurant-Addr-Regent Street City Centre<ul><li>Occur Num: 621</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 110</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 74</li><li>Inform-Restaurant-Phone-01223323737 Occur Num: 69</li><li>Inform-Restaurant-Price-cheap Occur Num: 53</li><li>Inform-Restaurant-Food-italian Occur Num: 47</li></ul></ul><li>Inform-Restaurant-Price-moderate<ul><li>Occur Num: 429</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Price-moderate Occur Num: 33</li><li>reqmore-general-none-none Occur Num: 26</li><li>Inform-Booking-none-none Occur Num: 25</li><li>Inform-Restaurant-Area-centre Occur Num: 25</li><li>Inform-Restaurant-Choice-2 Occur Num: 19</li></ul></ul><li>Request-Restaurant-Food-?<ul><li>Occur Num: 356</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 51</li><li>Recommend-Restaurant-Price-cheap Occur Num: 48</li><li>Inform-Restaurant-Price-cheap Occur Num: 47</li><li>Recommend-Restaurant-Name-pizza hut city centre Occur Num: 40</li><li>Inform-Restaurant-Choice-69 Occur Num: 33</li></ul></ul><li>Inform-Restaurant-Food-indian<ul><li>Occur Num: 301</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 29</li><li>Recommend-Restaurant-Name-curry garden Occur Num: 20</li><li>Inform-Restaurant-Area-centre Occur Num: 20</li><li>reqmore-general-none-none Occur Num: 18</li><li>Inform-Booking-none-none Occur Num: 16</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Inform-Restaurant-Price-moderate Occur Num: 32</li><li>Inform-Restaurant-Price-expensive Occur Num: 28</li><li>Request-Restaurant-Phone-? Occur Num: 26</li><li>Request-Restaurant-Addr-? Occur Num: 7</li><li>Inform-Restaurant-Price-cheap Occur Num: 7</li><li>Request-Restaurant-Food-? Occur Num: 5</li><li>Request-Restaurant-Price-? Occur Num: 3</li><li>Request-Restaurant-Area-? Occur Num: 2</li><li>Request-Restaurant-Post-? Occur Num: 2</li><li>Inform-Restaurant-Name-eraina Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-restaurant-area Occur Num: 6</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-postcode Occur Num: 26</li><li>request-restaurant-phone Occur Num: 25</li><li>request-restaurant-address Occur Num: 15</li><li>request-restaurant-food Occur Num: 4</li><li>request-restaurant-area Occur Num: 3</li><li>request-restaurant-pricerange Occur Num: 3</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-phone Occur Num: 167</li><li>inform-restaurant-area Occur Num: 73</li><li>inform-restaurant-food Occur Num: 68</li></ul></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 88.3 %</p><p> (Precision, Recall, F1) : (0.753, 0.920, 0.800) </p><p> Average Dialog Turn (Succ): 6.388 </p><p> Average Dialog Turn (All): 9.502 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Attraction-Post-?<ul><li>Occur Num: 168</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 59</li><li>Request-Attraction-Addr-? Occur Num: 45</li><li>Request-Attraction-Fee-? Occur Num: 35</li><li>Request-Restaurant-Post-? Occur Num: 14</li><li>Request-Restaurant-Addr-? Occur Num: 5</li></ul></ul><li>Request-Attraction-Phone-?<ul><li>Occur Num: 162</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 59</li><li>Request-Attraction-Addr-? Occur Num: 40</li><li>Request-Attraction-Fee-? Occur Num: 29</li><li>Request-Restaurant-Phone-? Occur Num: 14</li><li>Request-Attraction-Area-? Occur Num: 8</li></ul></ul><li>Request-Attraction-Addr-?<ul><li>Occur Num: 153</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 46</li><li>Request-Attraction-Phone-? Occur Num: 41</li><li>Request-Attraction-Fee-? Occur Num: 21</li><li>Request-Restaurant-Addr-? Occur Num: 16</li><li>Request-Restaurant-Post-? Occur Num: 10</li></ul></ul><li>Request-Attraction-Fee-?<ul><li>Occur Num: 86</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 35</li><li>Request-Attraction-Phone-? Occur Num: 29</li><li>Request-Attraction-Addr-? Occur Num: 20</li><li>Request-Attraction-Type-? Occur Num: 2</li></ul></ul><li>Inform-Attraction-Type-dontcare<ul><li>Occur Num: 39</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-do n't care Occur Num: 25</li><li>Request-Attraction-Type-? Occur Num: 8</li><li>Inform-Attraction-Area-centre Occur Num: 4</li><li>Inform-Attraction-Area-west Occur Num: 1</li><li>Inform-Attraction-Area-east Occur Num: 1</li></ul></ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 39</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-entertainment Occur Num: 13</li><li>Inform-Attraction-Type-do n't care Occur Num: 4</li><li>Inform-Restaurant-Area-centre Occur Num: 4</li><li>Inform-Attraction-Type-colleges Occur Num: 3</li><li>Inform-Attraction-Type-museums Occur Num: 3</li></ul></ul><li>Inform-Attraction-Area-dontcare<ul><li>Occur Num: 30</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-do n't care Occur Num: 18</li><li>Request-Attraction-Area-? Occur Num: 11</li><li>Inform-Attraction-Type-museum Occur Num: 1</li></ul></ul><li>Inform-Attraction-Type-museum<ul><li>Occur Num: 27</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 10</li><li>Inform-Attraction-Type-museums Occur Num: 9</li><li>Inform-Attraction-Type-entertainment Occur Num: 3</li><li>Inform-Attraction-Area-west Occur Num: 2</li><li>Inform-Attraction-Area-south Occur Num: 1</li></ul></ul><li>Request-Attraction-Area-?<ul><li>Occur Num: 25</li><li>NLU Output</li><ul><li>Request-Attraction-Addr-? Occur Num: 8</li><li>Request-Attraction-Phone-? Occur Num: 8</li><li>Request-Restaurant-Area-? Occur Num: 6</li><li>Request-Attraction-Post-? Occur Num: 3</li></ul></ul><li>Inform-Attraction-Type-college<ul><li>Occur Num: 25</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 11</li><li>Inform-Attraction-Type-colleges Occur Num: 9</li><li>Inform-Attraction-Type-entertainment Occur Num: 3</li><li>Inform-Attraction-Area-south Occur Num: 1</li><li>Inform-Attraction-Area-west Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 778</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 87</li><li>Inform-Attraction-Choice-44 Occur Num: 46</li><li>Request-Attraction-Type-? Occur Num: 43</li><li>Inform-Attraction-Type-architecture Occur Num: 43</li><li>Inform-Attraction-Type-theatre Occur Num: 40</li></ul></ul><li>Inform-Attraction-Fee-?<ul><li>Occur Num: 501</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 78</li><li>Inform-Attraction-Phone-01223300085 Occur Num: 19</li><li>Inform-Attraction-Area-centre Occur Num: 18</li><li>Inform-Attraction-Choice-44 Occur Num: 17</li><li>Inform-Attraction-Addr-park street Occur Num: 16</li></ul></ul><li>Inform-Attraction-Fee-free<ul><li>Occur Num: 381</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 79</li><li>Inform-Attraction-Area-centre Occur Num: 15</li><li>Inform-Attraction-Addr-98 king street Occur Num: 14</li><li>Inform-Attraction-Type-park Occur Num: 12</li><li>Inform-Attraction-Phone-01223314960 Occur Num: 12</li></ul></ul><li>Request-Attraction-Type-?<ul><li>Occur Num: 346</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 47</li><li>Inform-Attraction-Choice-44 Occur Num: 45</li><li>Inform-Attraction-Type-theatre Occur Num: 39</li><li>Inform-Attraction-Type-architecture Occur Num: 36</li><li>Inform-Attraction-Type-museum Occur Num: 19</li></ul></ul><li>Inform-Attraction-Name-adc theatre<ul><li>Occur Num: 309</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 51</li><li>Recommend-Attraction-Name-adc theatre Occur Num: 29</li><li>Inform-Attraction-Addr-park street Occur Num: 24</li><li>Inform-Attraction-Area-centre Occur Num: 22</li><li>Inform-Attraction-Choice-44 Occur Num: 22</li></ul></ul><li>Inform-Attraction-Type-theatre<ul><li>Occur Num: 304</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 46</li><li>Inform-Attraction-Choice-44 Occur Num: 46</li><li>Request-Attraction-Type-? Occur Num: 38</li><li>Inform-Attraction-Type-architecture Occur Num: 36</li><li>reqmore-general-none-none Occur Num: 23</li></ul></ul><li>Inform-Attraction-Type-museum<ul><li>Occur Num: 290</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 27</li><li>Inform-Attraction-Choice-11 Occur Num: 26</li><li>reqmore-general-none-none Occur Num: 22</li><li>Request-Attraction-Type-? Occur Num: 18</li><li>Inform-Attraction-Area-west Occur Num: 18</li></ul></ul><li>Inform-Attraction-Name-broughton house gallery<ul><li>Occur Num: 278</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 37</li><li>Inform-Attraction-Choice-11 Occur Num: 26</li><li>Inform-Attraction-Addr-98 king street Occur Num: 25</li><li>Inform-Attraction-Area-centre Occur Num: 25</li><li>Inform-Attraction-Type-museum Occur Num: 23</li></ul></ul><li>Inform-Attraction-Choice-44<ul><li>Occur Num: 259</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 42</li><li>Request-Attraction-Type-? Occur Num: 40</li><li>Inform-Attraction-Type-theatre Occur Num: 38</li><li>Inform-Attraction-Type-architecture Occur Num: 36</li><li>reqmore-general-none-none Occur Num: 21</li></ul></ul><li>Inform-Attraction-Type-architecture<ul><li>Occur Num: 249</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 46</li><li>Inform-Attraction-Choice-44 Occur Num: 41</li><li>Inform-Attraction-Type-theatre Occur Num: 37</li><li>Request-Attraction-Type-? Occur Num: 35</li><li>reqmore-general-none-none Occur Num: 11</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Attraction-Addr-? Occur Num: 12</li><li>Request-Attraction-Post-? Occur Num: 11</li><li>Request-Attraction-Area-? Occur Num: 4</li><li>Request-Attraction-Fee-? Occur Num: 3</li><li>Request-Attraction-Type-? Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-attraction-area Occur Num: 48</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-address Occur Num: 19</li><li>request-attraction-postcode Occur Num: 16</li><li>request-attraction-area Occur Num: 8</li><li>request-attraction-phone Occur Num: 7</li><li>request-attraction-type Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-phone Occur Num: 67</li><li>inform-attraction-type Occur Num: 53</li><li>inform-attraction-area Occur Num: 40</li></ul></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 80.2 %</p><p> (Precision, Recall, F1) : (0.884, 0.843, 0.857) </p><p> Average Dialog Turn (Succ): 7.052 </p><p> Average Dialog Turn (All): 9.868 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Phone-?<ul><li>Occur Num: 24</li><li>NLU Output</li><ul><li>Request-Taxi-Car-? Occur Num: 24</li></ul></ul><li>Inform-Taxi-Depart-the fitzwilliam museum<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 2</li><li>Inform-Taxi-Dest-golden house Occur Num: 1</li><li>Inform-Taxi-Leave-13:45 Occur Num: 1</li><li>Inform-Taxi-Arrive-13:45 Occur Num: 1</li><li>Inform-Taxi-Dest-la mimosa Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-home from home<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-home from home Occur Num: 1</li><li>Inform-Taxi-Depart-arbury lodge guesthouse Occur Num: 1</li><li>Inform-Taxi-Leave-02:45 Occur Num: 1</li><li>Inform-Taxi-Depart-thanh binh Occur Num: 1</li><li>Inform-Taxi-Dest-home Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the lensfield hotel<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li><li>Inform-Taxi-Dest-williams art and antiques Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-02:00<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-efes restaurant<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-broxbourne train station Occur Num: 1</li><li>Inform-Restaurant-Name-efes Occur Num: 1</li><li>Inform-Taxi-Arrive-16:15 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-kohinoor<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-grafton hotel restaurant Occur Num: 1</li><li>Inform-Restaurant-Name-kohinoor Occur Num: 1</li><li>Inform-Taxi-Arrive-18:30 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the good luck chinese food takeaway<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-nandos Occur Num: 1</li><li>Inform-Taxi-Leave-05:15 Occur Num: 1</li><li>Inform-Taxi-Depart-good luck chinese food takeaway Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the cambridge chop house<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-churchill college Occur Num: 1</li><li>Inform-Taxi-Leave-22:15 Occur Num: 1</li><li>Inform-Taxi-Depart-cambridge chop house Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-thanh binh<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Depart-?<ul><li>Occur Num: 69</li><li>NLU Output</li><ul><li>Request-Taxi-Dest-? Occur Num: 17</li><li>Inform-Taxi-Car-tesla Occur Num: 4</li><li>reqmore-general-none-none Occur Num: 3</li><li>Inform-Taxi-Car-audi Occur Num: 2</li><li>Inform-Taxi-Car-lexus Occur Num: 2</li></ul></ul><li>Inform-Taxi-Car-bmw<ul><li>Occur Num: 60</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 25</li><li>Request-Taxi-Depart-? Occur Num: 2</li><li>Inform-Taxi-Phone-75932759348 Occur Num: 1</li><li>Inform-Taxi-Phone-74242564888 Occur Num: 1</li><li>Inform-Taxi-Phone-36371815544 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-tesla<ul><li>Occur Num: 59</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 22</li><li>Request-Taxi-Depart-? Occur Num: 5</li><li>Request-Taxi-Dest-? Occur Num: 1</li><li>Inform-Taxi-Phone-61488179956 Occur Num: 1</li><li>Inform-Taxi-Phone-99646126925 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-honda<ul><li>Occur Num: 54</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 19</li><li>Request-Taxi-Depart-? Occur Num: 3</li><li>Request-Taxi-Dest-? Occur Num: 2</li><li>Inform-Taxi-Phone-43785244638 Occur Num: 1</li><li>Inform-Taxi-Phone-71199834421 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-volkswagen<ul><li>Occur Num: 49</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 18</li><li>Request-Taxi-Depart-? Occur Num: 4</li><li>Request-Taxi-Dest-? Occur Num: 3</li><li>Inform-Taxi-Phone-69794859257 Occur Num: 1</li><li>Inform-Taxi-Phone-69476343616 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-ford<ul><li>Occur Num: 48</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 15</li><li>Request-Taxi-Depart-? Occur Num: 4</li><li>Request-Taxi-Dest-? Occur Num: 2</li><li>Inform-Taxi-Phone-61262683724 Occur Num: 1</li><li>Inform-Taxi-Phone-45377172757 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-volvo<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 20</li><li>Inform-Taxi-Phone-88185234639 Occur Num: 1</li><li>Inform-Taxi-Phone-75748995731 Occur Num: 1</li><li>Inform-Taxi-Phone-35537263179 Occur Num: 1</li><li>OfferBooked-Train-Id-76488729113 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-toyota<ul><li>Occur Num: 45</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 18</li><li>Request-Taxi-Depart-? Occur Num: 3</li><li>Request-Taxi-Dest-? Occur Num: 2</li><li>Inform-Taxi-Phone-11282911436 Occur Num: 1</li><li>Inform-Taxi-Phone-34659982464 Occur Num: 1</li></ul></ul><li>Request-Taxi-Dest-?<ul><li>Occur Num: 43</li><li>NLU Output</li><ul><li>Request-Taxi-Depart-? Occur Num: 17</li><li>Inform-Taxi-Car-lexus Occur Num: 2</li><li>Inform-Taxi-Phone-93793671822 Occur Num: 1</li><li>Inform-Taxi-Car-tesla Occur Num: 1</li><li>Inform-Taxi-Phone-61488179956 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-lexus<ul><li>Occur Num: 42</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 13</li><li>Request-Taxi-Dest-? Occur Num: 4</li><li>Request-Taxi-Depart-? Occur Num: 4</li><li>Inform-Taxi-Phone-33432618555 Occur Num: 1</li><li>Inform-Taxi-Phone-39342698845 Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Taxi-Car-? Occur Num: 21</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-taxi-car type Occur Num: 24</li><li>request-taxi-phone Occur Num: 14</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 85.7 %</p><p> (Precision, Recall, F1) : (0.900, 0.919, 0.907) </p><p> Average Dialog Turn (Succ): 7.391 </p><p> Average Dialog Turn (All): 8.395 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Train-Depart-cambridge<ul><li>Occur Num: 143</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 40</li><li>Inform-Train-Day-thursday Occur Num: 9</li><li>Inform-Train-Day-friday Occur Num: 7</li><li>Inform-Train-Day-Saturday Occur Num: 7</li><li>Inform-Train-Day-monday Occur Num: 7</li></ul></ul><li>Inform-Train-Dest-cambridge<ul><li>Occur Num: 143</li><li>NLU Output</li><ul><li>Request-Train-Arrive-? Occur Num: 12</li><li>Inform-Train-Day-Saturday Occur Num: 12</li><li>Inform-Train-Depart-cambridge Occur Num: 12</li><li>Inform-Train-Day-thursday Occur Num: 8</li><li>Inform-Train-Day-monday Occur Num: 7</li></ul></ul><li>Inform-Train-Day-monday<ul><li>Occur Num: 48</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 8</li><li>Inform-Train-Day-Saturday Occur Num: 5</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Dest-leicester Occur Num: 3</li><li>Inform-Train-Dest-norwich Occur Num: 2</li></ul></ul><li>Inform-Train-Day-tuesday<ul><li>Occur Num: 31</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 5</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Day-Saturday Occur Num: 3</li><li>Inform-Train-Dest-peterborough Occur Num: 1</li><li>Inform-Train-Dest-leicester Occur Num: 1</li></ul></ul><li>Inform-Train-Day-wednesday<ul><li>Occur Num: 29</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 5</li><li>Inform-Train-Day-Saturday Occur Num: 4</li><li>Inform-Train-Depart-cambridge Occur Num: 4</li><li>Inform-Train-Depart-broxbourne Occur Num: 1</li><li>Inform-Train-Leave-18:15 Occur Num: 1</li></ul></ul><li>Inform-Train-Leave-dontcare<ul><li>Occur Num: 27</li><li>NLU Output</li><ul><li>Inform-Train-Leave-do n't care Occur Num: 22</li><li>Inform-Train-Dest-cambridge Occur Num: 3</li><li>Inform-Train-Depart-london liverpool street Occur Num: 1</li><li>Inform-Train-Dest-stevenage Occur Num: 1</li></ul></ul><li>Inform-Train-Dest-leicester<ul><li>Occur Num: 23</li><li>NLU Output</li><ul><li>Inform-Train-Day-Saturday Occur Num: 3</li><li>Inform-Train-Day-monday Occur Num: 3</li><li>Inform-Train-Dest-cambridge Occur Num: 2</li><li>Inform-Train-Dest-peterborough Occur Num: 1</li><li>Inform-Train-Day-tuesday Occur Num: 1</li></ul></ul><li>Inform-Train-Day-thursday<ul><li>Occur Num: 23</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 7</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Leave-15:30 Occur Num: 2</li><li>Inform-Train-Dest-stevenage Occur Num: 1</li><li>Inform-Train-Arrive-21:45 Occur Num: 1</li></ul></ul><li>Inform-Train-Day-friday<ul><li>Occur Num: 21</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 4</li><li>Inform-Train-Depart-cambridge Occur Num: 4</li><li>Inform-Train-Arrive-11:15 Occur Num: 2</li><li>Inform-Train-Dest-peterborough Occur Num: 2</li><li>Inform-Train-Day-Saturday Occur Num: 2</li></ul></ul><li>Inform-Train-Depart-london liverpool street<ul><li>Occur Num: 20</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 4</li><li>Inform-Train-Dest-london liverpool street Occur Num: 4</li><li>Inform-Train-Leave-13:00 Occur Num: 1</li><li>Inform-Train-Day-monday Occur Num: 1</li><li>Inform-Train-Day-wednesday Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>OfferBook-Train-none-none<ul><li>Occur Num: 2324</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-05:51 Occur Num: 349</li><li>Inform-Train-Choice-404 Occur Num: 341</li><li>Inform-Train-Leave-05:00 Occur Num: 317</li><li>Inform-Train-Ticket-23.60 pounds Occur Num: 217</li><li>reqmore-general-none-none Occur Num: 193</li></ul></ul><li>Inform-Train-Arrive-05:51<ul><li>Occur Num: 2097</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 344</li><li>Inform-Train-Choice-404 Occur Num: 334</li><li>Inform-Train-Leave-05:00 Occur Num: 317</li><li>Inform-Train-Ticket-23.60 pounds Occur Num: 189</li><li>Inform-Train-Dest-london kings cross Occur Num: 152</li></ul></ul><li>Inform-Train-Leave-05:00<ul><li>Occur Num: 2020</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-05:51 Occur Num: 345</li><li>Inform-Train-Choice-404 Occur Num: 337</li><li>OfferBook-Train-none-none Occur Num: 335</li><li>Inform-Train-Dest-london kings cross Occur Num: 151</li><li>Inform-Train-Ticket-23.60 pounds Occur Num: 130</li></ul></ul><li>Inform-Train-Choice-404<ul><li>Occur Num: 1670</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-05:51 Occur Num: 334</li><li>OfferBook-Train-none-none Occur Num: 302</li><li>Inform-Train-Leave-05:00 Occur Num: 296</li><li>Inform-Train-Dest-london kings cross Occur Num: 147</li><li>Request-Train-Leave-? Occur Num: 99</li></ul></ul><li>Inform-Train-Ticket-23.60 pounds<ul><li>Occur Num: 1017</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 208</li><li>Inform-Train-Arrive-05:51 Occur Num: 189</li><li>reqmore-general-none-none Occur Num: 152</li><li>Inform-Train-Leave-05:00 Occur Num: 127</li><li>Inform-Train-Time-51 minutes Occur Num: 96</li></ul></ul><li>Inform-Train-Dest-london kings cross<ul><li>Occur Num: 800</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-05:51 Occur Num: 153</li><li>Inform-Train-Choice-404 Occur Num: 147</li><li>OfferBook-Train-none-none Occur Num: 142</li><li>Inform-Train-Leave-05:00 Occur Num: 117</li><li>Inform-Train-Id-TR0899 Occur Num: 31</li></ul></ul><li>Request-Train-Leave-?<ul><li>Occur Num: 581</li><li>NLU Output</li><ul><li>Inform-Train-Choice-404 Occur Num: 121</li><li>Inform-Train-Arrive-05:51 Occur Num: 107</li><li>Inform-Train-Leave-05:00 Occur Num: 105</li><li>OfferBook-Train-none-none Occur Num: 87</li><li>Inform-Train-Id-TR8272 Occur Num: 24</li></ul></ul><li>Inform-Train-Id-TR0899<ul><li>Occur Num: 490</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 84</li><li>Inform-Train-Arrive-05:51 Occur Num: 84</li><li>Inform-Train-Leave-05:00 Occur Num: 79</li><li>Inform-Train-Ticket-18.88 pounds Occur Num: 61</li><li>Inform-Train-Choice-404 Occur Num: 57</li></ul></ul><li>Inform-Train-Time-51 minutes<ul><li>Occur Num: 483</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 156</li><li>Inform-Train-Ticket-23.60 pounds Occur Num: 103</li><li>OfferBook-Train-none-none Occur Num: 41</li><li>Inform-Train-Ticket-18.88 pounds Occur Num: 40</li><li>Inform-Train-Arrive-05:51 Occur Num: 29</li></ul></ul><li>Inform-Train-Ticket-18.88 pounds<ul><li>Occur Num: 453</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 95</li><li>reqmore-general-none-none Occur Num: 79</li><li>Inform-Train-Id-TR0899 Occur Num: 67</li><li>Inform-Train-Arrive-05:51 Occur Num: 65</li><li>Inform-Train-Leave-05:00 Occur Num: 56</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Inform-Train-Dest-cambridge Occur Num: 7</li><li>Inform-Train-Leave-16:15 Occur Num: 3</li><li>Inform-Train-Leave-09:30 Occur Num: 3</li><li>Inform-Train-Leave-21:30 Occur Num: 2</li><li>Inform-Train-Leave-13:45 Occur Num: 2</li><li>Inform-Train-People-6 Occur Num: 2</li><li>Inform-Train-Leave-10:15 Occur Num: 2</li><li>Inform-Train-Leave-12:15 Occur Num: 2</li><li>Inform-Train-People-5 Occur Num: 2</li><li>Inform-Train-Leave-09:00 Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-train-duration Occur Num: 11</li><li>inform-train-price Occur Num: 6</li><li>inform-train-leaveAt Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-train-leaveAt Occur Num: 1</li><li>request-train-price Occur Num: 1</li><li>request-train-duration Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 100.0 %</p><p> (Precision, Recall, F1) : (0.935, 1.000, 0.957) </p><p> Average Dialog Turn (Succ): 2.000 </p><p> Average Dialog Turn (All): 2.000 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Police-none-none<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Police-Name-Parkside Police Station Occur Num: 6</li><li>Request-Police-Addr-? Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Police-Addr-Parkside, Cambridge<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 23</li><li>Inform-Police-Phone-01223358966 Occur Num: 23</li></ul></ul><li>Inform-Police-Name-none<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 23</li><li>Inform-Police-Phone-01223358966 Occur Num: 23</li></ul></ul><li>Inform-Police-Post-none<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 23</li><li>Inform-Police-Phone-01223358966 Occur Num: 23</li></ul></ul><li>Inform-Police-Phone-01223358966<ul><li>Occur Num: 23</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 23</li></ul></ul></ul><b> Dialog Loop</b><p>Nothing</p><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><p>Nothing</p><b> Inform But Not Request Dialog Act</b><ul><li>inform-police-phone Occur Num: 3</li></ul></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 93.3 %</p><p> (Precision, Recall, F1) : (0.933, 0.933, 0.933) </p><p> Average Dialog Turn (Succ): 2.000 </p><p> Average Dialog Turn (All): 4.533 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Hospital-Phone-?<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Request-Police-Phone-? Occur Num: 2</li><li>Request-Restaurant-Phone-? Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-plastic and vascular surgery plastics<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li><li>Inform-Hospital-Department-plastic and vascular surgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-oncology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-surgery<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-children's oncology and haematology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-emergency department<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-emergency department department Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hospital-Addr-none<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Phone-01223216297 Occur Num: 2</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Hospital-Phone-? Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-hospital-phone Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/BERTNLU-RuleDST-PPOPolicy-TemplateNLG/res.txt b/tests/results/BERTNLU-RuleDST-PPOPolicy-TemplateNLG/res.txt deleted file mode 100755 index 0df6bbd10ec6c6d977fe0c98ddc8da83c4ce5585..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-PPOPolicy-TemplateNLG/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.697 -success number of dialogs/tot: 0.566 -average precision: 0.6484364290254024 -average recall: 0.7901094463399759 -average f1: 0.6812068271789294 -average book rate: 0.5663461538461538 -average turn (succ): 12.879858657243815 -average turn (all): 22.11 diff --git a/tests/results/BERTNLU-RuleDST-RulePolicy-SCLSTM/Frequency_of_domain.png b/tests/results/BERTNLU-RuleDST-RulePolicy-SCLSTM/Frequency_of_domain.png deleted file mode 100755 index 1f02da6700e451ff695e1ae8ba9b8f952df0699d..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-RulePolicy-SCLSTM/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-RulePolicy-SCLSTM/Performance_for_each_domain.png b/tests/results/BERTNLU-RuleDST-RulePolicy-SCLSTM/Performance_for_each_domain.png deleted file mode 100755 index 8abf6e31e28ea32f43698bfa7d11cbe8d8828fa0..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-RulePolicy-SCLSTM/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-RulePolicy-SCLSTM/Proportions_of_the_dialogue_loop.png b/tests/results/BERTNLU-RuleDST-RulePolicy-SCLSTM/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index ac4d26d7b3b097fa695111e52e118b3379000000..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-RulePolicy-SCLSTM/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-RulePolicy-SCLSTM/report_multiwoz.html b/tests/results/BERTNLU-RuleDST-RulePolicy-SCLSTM/report_multiwoz.html deleted file mode 100755 index d526f00bacb547abd97c5eaf4c89d278db067b77..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-RulePolicy-SCLSTM/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: test</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-26 12:42:46</h3><h3>Overall Results</h3><p> Success Rate: 41.0 %</p><p> (Precision, Recall, F1) : (0.685, 0.565, 0.591) </p><p> Average Dialog Turn (Succ): 11.624 </p><p> Average Dialog Turn (All): 29.224 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">351</td><td align="center">0.328</td><td align="center">0.307</td><td align="center">0.262</td><td align="center">0.273</td><td align="center">0.655</td><td align="center">10.835</td><td align="center">27.875</td></tr><tr><td align="center">restaurant</td><td align="center">355</td><td align="center">0.780</td><td align="center">0.633</td><td align="center">0.694</td><td align="center">0.653</td><td align="center">0.214</td><td align="center">4.931</td><td align="center">12.039</td></tr><tr><td align="center">attraction</td><td align="center">280</td><td align="center">0.529</td><td align="center">0.742</td><td align="center">0.670</td><td align="center">0.693</td><td align="center">0.471</td><td align="center">6.297</td><td align="center">21.229</td></tr><tr><td align="center">taxi</td><td align="center">58</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">0.000</td><td align="center">4.552</td><td align="center">4.552</td></tr><tr><td align="center">train</td><td align="center">314</td><td align="center">0.583</td><td align="center">0.755</td><td align="center">0.644</td><td align="center">0.682</td><td align="center">0.404</td><td align="center">5.956</td><td align="center">18.510</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">0.174</td><td align="center">0.739</td><td align="center">0.457</td><td align="center">0.551</td><td align="center">0.826</td><td align="center">4.000</td><td align="center">33.739</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">0.000</td><td align="center">4.000</td><td align="center">4.000</td></tr></table><h3>Dialogue Loop</h3><img src="Proportions_of_the_dialogue_loop.png" width=800 /><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 32.8 %</p><p> (Precision, Recall, F1) : (0.307, 0.262, 0.273) </p><p> Average Dialog Turn (Succ): 10.835 </p><p> Average Dialog Turn (All): 27.875 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 309</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 50</li><li>Inform-Hotel-Type-hotel Occur Num: 38</li><li>Inform-Hotel-Type-guesthouse Occur Num: 36</li><li>Inform-Hotel-Stars-4 Occur Num: 36</li><li>Inform-Hotel-Price-moderate Occur Num: 25</li></ul></ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 306</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 55</li><li>Inform-Hotel-Type-hotel Occur Num: 38</li><li>Inform-Hotel-Stars-4 Occur Num: 35</li><li>Inform-Hotel-Type-guesthouse Occur Num: 32</li><li>Inform-Hotel-Price-moderate Occur Num: 29</li></ul></ul><li>Request-Hotel-Addr-?<ul><li>Occur Num: 237</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 112</li><li>Request-Hotel-Post-? Occur Num: 18</li><li>Request-Hotel-Phone-? Occur Num: 16</li><li>Request-Restaurant-Post-? Occur Num: 15</li><li>Request-Restaurant-Phone-? Occur Num: 14</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 183</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 37</li><li>Inform-Hotel-Internet-yes Occur Num: 31</li><li>Inform-Hotel-Stars-4 Occur Num: 17</li><li>Inform-Hotel-Area-north Occur Num: 14</li><li>Inform-Hotel-Price-moderate Occur Num: 13</li></ul></ul><li>Inform-Hotel-Stars-4<ul><li>Occur Num: 165</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 38</li><li>Inform-Hotel-Internet-yes Occur Num: 35</li><li>Inform-Hotel-Type-hotel Occur Num: 20</li><li>Inform-Hotel-Type-guesthouse Occur Num: 17</li><li>Inform-Hotel-Price-moderate Occur Num: 9</li></ul></ul><li>Inform-Hotel-Price-moderate<ul><li>Occur Num: 159</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 30</li><li>Inform-Hotel-Parking-yes Occur Num: 24</li><li>Inform-Hotel-Type-hotel Occur Num: 18</li><li>Inform-Hotel-Type-guesthouse Occur Num: 13</li><li>Inform-Restaurant-Price-moderate Occur Num: 12</li></ul></ul><li>Request-Hotel-Post-?<ul><li>Occur Num: 116</li><li>NLU Output</li><ul><li>Request-Restaurant-Post-? Occur Num: 45</li><li>Request-Hotel-Addr-? Occur Num: 18</li><li>Request-Restaurant-Addr-? Occur Num: 15</li><li>Request-Restaurant-Phone-? Occur Num: 6</li><li>Request-Hotel-Phone-? Occur Num: 6</li></ul></ul><li>Request-Hotel-Phone-?<ul><li>Occur Num: 99</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 49</li><li>Request-Restaurant-Addr-? Occur Num: 13</li><li>Request-Hotel-Addr-? Occur Num: 8</li><li>Request-Restaurant-Post-? Occur Num: 6</li><li>Request-Hotel-Post-? Occur Num: 5</li></ul></ul><li>Inform-Hotel-Area-north<ul><li>Occur Num: 99</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 22</li><li>Inform-Hotel-Internet-yes Occur Num: 21</li><li>Inform-Hotel-Type-guesthouse Occur Num: 14</li><li>Inform-Hotel-Type-hotel Occur Num: 9</li><li>Inform-Hotel-Stars-4 Occur Num: 7</li></ul></ul><li>Inform-Hotel-Parking-no<ul><li>Occur Num: 83</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 19</li><li>Inform-Hotel-Internet-yes Occur Num: 11</li><li>Inform-Hotel-Type-guesthouse Occur Num: 10</li><li>Inform-Hotel-Price-moderate Occur Num: 7</li><li>Inform-Hotel-Area-centre Occur Num: 6</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 182</li><li>NLU Output</li><ul><li>bye-general-none-none Occur Num: 16</li><li>Request-Restaurant-Food-? Occur Num: 10</li><li>Inform-Hotel-Price-cheap Occur Num: 10</li><li>Inform-Hotel-Price-moderate Occur Num: 10</li><li>Inform-Hotel-Type-guesthouse Occur Num: 9</li></ul></ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 182</li><li>NLU Output</li><ul><li>Request-Restaurant-Food-? Occur Num: 14</li><li>bye-general-none-none Occur Num: 13</li><li>Inform-Hotel-Price-cheap Occur Num: 12</li><li>Inform-Hotel-Type-guesthouse Occur Num: 8</li><li>Inform-Hotel-Price-moderate Occur Num: 8</li></ul></ul><li>NoOffer-Hotel-Internet-yes<ul><li>Occur Num: 110</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 39</li><li>Request-Hotel-Price-? Occur Num: 17</li><li>Request-Hotel-Area-? Occur Num: 11</li><li>NoOffer-Restaurant-none-none Occur Num: 11</li><li>reqmore-general-none-none Occur Num: 9</li></ul></ul><li>NoOffer-Hotel-Price-moderate<ul><li>Occur Num: 93</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 23</li><li>Request-Hotel-Price-? Occur Num: 17</li><li>NoOffer-Restaurant-none-none Occur Num: 12</li><li>reqmore-general-none-none Occur Num: 10</li><li>Request-Restaurant-Price-? Occur Num: 9</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 86</li><li>NLU Output</li><ul><li>Inform-Hotel-Price-cheap Occur Num: 10</li><li>reqmore-general-none-none Occur Num: 7</li><li>Inform-Hotel-Price-moderate Occur Num: 7</li><li>Inform-Hotel-Area-north Occur Num: 7</li><li>Inform-Hotel-Parking-none Occur Num: 6</li></ul></ul><li>NoOffer-Hotel-Parking-yes<ul><li>Occur Num: 84</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 28</li><li>Request-Hotel-Price-? Occur Num: 12</li><li>reqmore-general-none-none Occur Num: 10</li><li>Request-Hotel-Area-? Occur Num: 9</li><li>NoOffer-Restaurant-none-none Occur Num: 9</li></ul></ul><li>Inform-Hotel-Choice-2<ul><li>Occur Num: 82</li><li>NLU Output</li><ul><li>Request-Restaurant-Food-? Occur Num: 40</li><li>Inform-Restaurant-Choice-2 Occur Num: 40</li><li>reqmore-general-none-none Occur Num: 1</li><li>Inform-Train-Ticket-18.88 pounds Occur Num: 1</li></ul></ul><li>Request-Hotel-Area-?<ul><li>Occur Num: 81</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 29</li><li>Request-Hotel-Price-? Occur Num: 22</li><li>Request-Restaurant-Price-? Occur Num: 11</li><li>NoOffer-Restaurant-none-none Occur Num: 11</li><li>Request-Restaurant-Area-? Occur Num: 8</li></ul></ul><li>NoOffer-Hotel-Type-guesthouse<ul><li>Occur Num: 72</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 18</li><li>Request-Hotel-Price-? Occur Num: 13</li><li>NoOffer-Restaurant-none-none Occur Num: 9</li><li>Request-Hotel-Area-? Occur Num: 8</li><li>reqmore-general-none-none Occur Num: 7</li></ul></ul><li>NoOffer-Hotel-Type-hotel<ul><li>Occur Num: 70</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 27</li><li>Request-Hotel-Price-? Occur Num: 11</li><li>reqmore-general-none-none Occur Num: 8</li><li>NoOffer-Restaurant-none-none Occur Num: 8</li><li>Request-Hotel-Area-? Occur Num: 7</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Hotel-Addr-? Occur Num: 106</li><li>Request-Hotel-Post-? Occur Num: 37</li><li>Request-Hotel-Phone-? Occur Num: 36</li><li>Request-Hotel-Internet-? Occur Num: 13</li><li>Request-Hotel-Parking-? Occur Num: 11</li><li>Inform-Hotel-Price-moderate Occur Num: 6</li><li>Request-Hotel-Stars-? Occur Num: 6</li><li>Inform-Hotel-Area-centre Occur Num: 5</li><li>Request-Hotel-Type-? Occur Num: 3</li><li>Request-Hotel-Price-? Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-parking Occur Num: 12</li><li>inform-hotel-internet Occur Num: 12</li><li>inform-hotel-postcode Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-address Occur Num: 133</li><li>request-hotel-postcode Occur Num: 42</li><li>request-hotel-phone Occur Num: 38</li><li>request-hotel-parking Occur Num: 21</li><li>request-hotel-internet Occur Num: 19</li><li>request-hotel-type Occur Num: 10</li><li>request-hotel-stars Occur Num: 7</li><li>request-hotel-pricerange Occur Num: 2</li><li>request-hotel-area Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-area Occur Num: 17</li><li>inform-hotel-type Occur Num: 10</li><li>inform-hotel-stars Occur Num: 9</li></ul></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 78.0 %</p><p> (Precision, Recall, F1) : (0.633, 0.694, 0.653) </p><p> Average Dialog Turn (Succ): 4.931 </p><p> Average Dialog Turn (All): 12.039 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Restaurant-Addr-?<ul><li>Occur Num: 90</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 44</li><li>Request-Restaurant-Post-? Occur Num: 40</li><li>Request-Restaurant-Price-? Occur Num: 3</li><li>Request-Attraction-Addr-? Occur Num: 1</li><li>Request-Restaurant-Area-? Occur Num: 1</li></ul></ul><li>Request-Restaurant-Phone-?<ul><li>Occur Num: 69</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 44</li><li>Request-Restaurant-Post-? Occur Num: 21</li><li>Request-Restaurant-Price-? Occur Num: 3</li><li>Request-Attraction-Phone-? Occur Num: 1</li></ul></ul><li>Request-Restaurant-Post-?<ul><li>Occur Num: 62</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 40</li><li>Request-Restaurant-Phone-? Occur Num: 19</li><li>Request-Restaurant-Area-? Occur Num: 1</li><li>Request-Restaurant-Price-? Occur Num: 1</li><li>Request-Attraction-Post-? Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 28</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensively Occur Num: 11</li><li>Inform-Restaurant-Area-centre Occur Num: 4</li><li>Inform-Restaurant-Food-indian Occur Num: 2</li><li>Inform-Restaurant-Area-south Occur Num: 2</li><li>Inform-Restaurant-Food-mexican Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Price-moderate<ul><li>Occur Num: 22</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 10</li><li>Inform-Restaurant-Area-centre Occur Num: 6</li><li>Inform-Restaurant-Food-british Occur Num: 2</li><li>Inform-Restaurant-Food-mediterranean Occur Num: 1</li><li>Inform-Restaurant-Food-italian Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 18</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 6</li><li>Inform-Restaurant-Food-british Occur Num: 3</li><li>Inform-Attraction-Area-centre Occur Num: 2</li><li>Inform-Restaurant-Price-expensively Occur Num: 2</li><li>Inform-Restaurant-Food-modern european Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Price-cheap<ul><li>Occur Num: 11</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-cheaply Occur Num: 4</li><li>Inform-Restaurant-Food-italian Occur Num: 2</li><li>Inform-Restaurant-Area-south Occur Num: 1</li><li>Inform-Restaurant-Food-vietnamese Occur Num: 1</li><li>Inform-Restaurant-Area-east Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Food-portuguese<ul><li>Occur Num: 8</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 3</li><li>Inform-Restaurant-Area-centre Occur Num: 2</li><li>Inform-Restaurant-Price-moderate Occur Num: 1</li><li>Inform-Restaurant-Area-north Occur Num: 1</li><li>Inform-Restaurant-Price-cheap Occur Num: 1</li></ul></ul><li>Request-Restaurant-Price-?<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 3</li><li>Request-Restaurant-Addr-? Occur Num: 3</li><li>Request-Restaurant-Post-? Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Food-italian<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-cheaply Occur Num: 2</li><li>Inform-Restaurant-Area-east Occur Num: 1</li><li>Inform-Restaurant-Area-north Occur Num: 1</li><li>Inform-Restaurant-Price-moderately Occur Num: 1</li><li>Inform-Restaurant-Area-west Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Recommend-Restaurant-Area-centre<ul><li>Occur Num: 56</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 16</li><li>Inform-Restaurant-Choice-4 Occur Num: 4</li><li>Recommend-Restaurant-Name-mahal of cambridge Occur Num: 3</li><li>Inform-Restaurant-Choice-15 Occur Num: 2</li><li>Inform-Restaurant-Choice-21 Occur Num: 2</li></ul></ul><li>Request-Restaurant-Food-?<ul><li>Occur Num: 53</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 26</li><li>Request-Restaurant-Price-? Occur Num: 15</li><li>Request-Restaurant-Area-? Occur Num: 12</li></ul></ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 50</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 7</li><li>Inform-Attraction-Area-centre Occur Num: 5</li><li>Inform-Restaurant-Price-moderate Occur Num: 3</li><li>Inform-Restaurant-Food-modern european Occur Num: 2</li><li>Inform-Restaurant-Name-the cow pizza kitchen and bar Occur Num: 2</li></ul></ul><li>NoOffer-Restaurant-Area-centre<ul><li>Occur Num: 41</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 21</li><li>Request-Restaurant-Food-? Occur Num: 11</li><li>Request-Restaurant-Price-? Occur Num: 6</li><li>Request-Restaurant-Area-? Occur Num: 3</li></ul></ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 40</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 7</li><li>Inform-Restaurant-Area-south Occur Num: 3</li><li>Inform-Restaurant-Food-italian Occur Num: 2</li><li>Inform-Restaurant-Food-chinese Occur Num: 2</li><li>Inform-Restaurant-Area-east Occur Num: 2</li></ul></ul><li>NoOffer-Restaurant-Price-cheap<ul><li>Occur Num: 39</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 20</li><li>Request-Restaurant-Food-? Occur Num: 9</li><li>Request-Restaurant-Price-? Occur Num: 7</li><li>Request-Restaurant-Area-? Occur Num: 3</li></ul></ul><li>NoOffer-Restaurant-Price-expensive<ul><li>Occur Num: 38</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 23</li><li>Request-Restaurant-Food-? Occur Num: 6</li><li>Request-Restaurant-Price-? Occur Num: 5</li><li>Request-Restaurant-Area-? Occur Num: 4</li></ul></ul><li>Request-Restaurant-Price-?<ul><li>Occur Num: 38</li><li>NLU Output</li><ul><li>Request-Restaurant-Food-? Occur Num: 15</li><li>NoOffer-Restaurant-none-none Occur Num: 15</li><li>Request-Restaurant-Area-? Occur Num: 8</li></ul></ul><li>Request-Restaurant-Area-?<ul><li>Occur Num: 32</li><li>NLU Output</li><ul><li>Request-Restaurant-Food-? Occur Num: 12</li><li>NoOffer-Restaurant-none-none Occur Num: 12</li><li>Request-Restaurant-Price-? Occur Num: 8</li></ul></ul><li>Inform-Restaurant-Choice-4<ul><li>Occur Num: 30</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 4</li><li>Recommend-Restaurant-Name-mahal of cambridge Occur Num: 2</li><li>Recommend-Restaurant-Name-the nirala Occur Num: 2</li><li>Select-Restaurant-Name-meghna Occur Num: 1</li><li>Recommend-Restaurant-Name-j restaurant Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Restaurant-Phone-? Occur Num: 39</li><li>Request-Restaurant-Addr-? Occur Num: 13</li><li>Request-Restaurant-Post-? Occur Num: 13</li><li>Inform-Restaurant-Area-centre Occur Num: 3</li><li>Inform-Restaurant-Time-12:30 Occur Num: 1</li><li>Inform-Restaurant-Day-saturday Occur Num: 1</li><li>Inform-Restaurant-Time-21:00 Occur Num: 1</li><li>Inform-Restaurant-Time-19:15 Occur Num: 1</li><li>Inform-Restaurant-Food-gastropub Occur Num: 1</li><li>Inform-Restaurant-Area-north Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-restaurant-area Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-phone Occur Num: 41</li><li>request-restaurant-address Occur Num: 33</li><li>request-restaurant-postcode Occur Num: 19</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-area Occur Num: 23</li><li>inform-restaurant-food Occur Num: 23</li><li>inform-restaurant-phone Occur Num: 20</li></ul></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 52.9 %</p><p> (Precision, Recall, F1) : (0.742, 0.670, 0.693) </p><p> Average Dialog Turn (Succ): 6.297 </p><p> Average Dialog Turn (All): 21.229 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Attraction-Phone-?<ul><li>Occur Num: 224</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 55</li><li>Request-Attraction-Addr-? Occur Num: 47</li><li>Request-Restaurant-Phone-? Occur Num: 46</li><li>Request-Attraction-Fee-? Occur Num: 32</li><li>Request-Restaurant-Addr-? Occur Num: 20</li></ul></ul><li>Request-Attraction-Addr-?<ul><li>Occur Num: 223</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 57</li><li>Request-Attraction-Phone-? Occur Num: 43</li><li>Request-Restaurant-Addr-? Occur Num: 39</li><li>Request-Attraction-Fee-? Occur Num: 37</li><li>Request-Restaurant-Phone-? Occur Num: 23</li></ul></ul><li>Request-Attraction-Post-?<ul><li>Occur Num: 203</li><li>NLU Output</li><ul><li>Request-Attraction-Addr-? Occur Num: 56</li><li>Request-Attraction-Phone-? Occur Num: 49</li><li>Request-Attraction-Fee-? Occur Num: 39</li><li>Request-Restaurant-Post-? Occur Num: 30</li><li>Request-Restaurant-Addr-? Occur Num: 14</li></ul></ul><li>Request-Attraction-Fee-?<ul><li>Occur Num: 113</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 39</li><li>Request-Attraction-Addr-? Occur Num: 36</li><li>Request-Attraction-Phone-? Occur Num: 30</li><li>Request-Attraction-Type-? Occur Num: 6</li><li>Request-Attraction-Area-? Occur Num: 2</li></ul></ul><li>Request-Attraction-Area-?<ul><li>Occur Num: 49</li><li>NLU Output</li><ul><li>Request-Restaurant-Area-? Occur Num: 12</li><li>Request-Restaurant-Phone-? Occur Num: 7</li><li>Request-Restaurant-Addr-? Occur Num: 7</li><li>Request-Attraction-Addr-? Occur Num: 6</li><li>Request-Attraction-Phone-? Occur Num: 5</li></ul></ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 38</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-entertainment Occur Num: 18</li><li>Inform-Restaurant-Area-centre Occur Num: 4</li><li>Inform-Attraction-Type-museums Occur Num: 3</li><li>Inform-Attraction-Type-museum Occur Num: 3</li><li>Inform-Attraction-Type-colleges Occur Num: 2</li></ul></ul><li>Request-Attraction-Type-?<ul><li>Occur Num: 27</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 8</li><li>Request-Attraction-Addr-? Occur Num: 7</li><li>Request-Attraction-Post-? Occur Num: 6</li><li>Request-Attraction-Fee-? Occur Num: 6</li></ul></ul><li>Inform-Attraction-Type-museum<ul><li>Occur Num: 21</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 9</li><li>Inform-Attraction-Type-museums Occur Num: 6</li><li>Inform-Attraction-Type-entertainment Occur Num: 4</li><li>Inform-Attraction-Area-west Occur Num: 2</li></ul></ul><li>Inform-Attraction-Type-boat<ul><li>Occur Num: 18</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 7</li><li>Inform-Attraction-Type-boats Occur Num: 6</li><li>Inform-Attraction-Type-entertainment Occur Num: 3</li><li>Inform-Attraction-Area-west Occur Num: 1</li><li>Inform-Attraction-Area-north Occur Num: 1</li></ul></ul><li>Inform-Attraction-Area-west<ul><li>Occur Num: 16</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-entertainment Occur Num: 9</li><li>Inform-Attraction-Area-centre Occur Num: 3</li><li>Inform-Attraction-Type-swimmingpools Occur Num: 1</li><li>Inform-Attraction-Type-museums Occur Num: 1</li><li>Inform-Attraction-Type-boats Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Attraction-Fee-?<ul><li>Occur Num: 148</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 45</li><li>Inform-Attraction-Phone-01223300085 Occur Num: 14</li><li>Inform-Attraction-Post-cb58as Occur Num: 11</li><li>Inform-Attraction-Post-cb58nt Occur Num: 10</li><li>Inform-Attraction-Addr-pool way Occur Num: 10</li></ul></ul><li>Inform-Attraction-Fee-free<ul><li>Occur Num: 101</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 41</li><li>Inform-Attraction-Phone-01223314960 Occur Num: 5</li><li>Inform-Attraction-Post-cb11ln Occur Num: 4</li><li>Inform-Attraction-Phone-01223452587 Occur Num: 4</li><li>Inform-Attraction-Post-cb23bu Occur Num: 4</li></ul></ul><li>Request-Attraction-Type-?<ul><li>Occur Num: 80</li><li>NLU Output</li><ul><li>Request-Attraction-Area-? Occur Num: 36</li><li>NoOffer-Attraction-none-none Occur Num: 36</li><li>Request-Restaurant-Area-? Occur Num: 8</li></ul></ul><li>NoOffer-Attraction-Type-entertainment<ul><li>Occur Num: 74</li><li>NLU Output</li><ul><li>Request-Attraction-Area-? Occur Num: 22</li><li>Request-Restaurant-Area-? Occur Num: 22</li><li>NoOffer-Attraction-none-none Occur Num: 22</li><li>NoOffer-Restaurant-none-none Occur Num: 8</li></ul></ul><li>Inform-Attraction-Addr-pool way, whitehill road, off newmarket road<ul><li>Occur Num: 58</li><li>NLU Output</li><ul><li>Inform-Attraction-Addr-pool way Occur Num: 14</li><li>Inform-Attraction-Addr-whitehill road Occur Num: 14</li><li>Inform-Attraction-Addr-off newmarket road Occur Num: 14</li><li>Inform-Attraction-Post-cb58nt Occur Num: 11</li><li>Inform-Restaurant-Post-cb58nt Occur Num: 1</li></ul></ul><li>NoOffer-Attraction-Area-centre<ul><li>Occur Num: 52</li><li>NLU Output</li><ul><li>Request-Attraction-Area-? Occur Num: 16</li><li>Request-Restaurant-Area-? Occur Num: 16</li><li>NoOffer-Attraction-none-none Occur Num: 16</li><li>NoOffer-Restaurant-none-none Occur Num: 4</li></ul></ul><li>Inform-Attraction-Choice-44<ul><li>Occur Num: 40</li><li>NLU Output</li><ul><li>Inform-Restaurant-Choice-44 Occur Num: 34</li><li>Inform-Attraction-Type-college Occur Num: 3</li><li>Inform-Attraction-Type-theatre Occur Num: 1</li><li>Inform-Attraction-Type-museum Occur Num: 1</li><li>Inform-Attraction-Type-concerthall Occur Num: 1</li></ul></ul><li>Inform-Attraction-Phone-01223300085<ul><li>Occur Num: 40</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 13</li><li>Inform-Attraction-Post-cb58as Occur Num: 10</li><li>Inform-Attraction-Addr-park street Occur Num: 5</li><li>Inform-Attraction-Phone-01223323737 Occur Num: 4</li><li>Inform-Restaurant-Phone-01223300085 Occur Num: 3</li></ul></ul><li>Inform-Attraction-Post-cb58nt<ul><li>Occur Num: 40</li><li>NLU Output</li><ul><li>Inform-Attraction-Addr-pool way Occur Num: 12</li><li>Inform-Attraction-Addr-whitehill road Occur Num: 12</li><li>Inform-Attraction-Addr-off newmarket road Occur Num: 12</li><li>Inform-Restaurant-Post-cb58nt Occur Num: 1</li><li>Inform-Restaurant-Addr-Regent Street City Centre Occur Num: 1</li></ul></ul><li>Request-Attraction-Area-?<ul><li>Occur Num: 36</li><li>NLU Output</li><ul><li>NoOffer-Attraction-none-none Occur Num: 30</li><li>Request-Restaurant-Area-? Occur Num: 6</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Attraction-Fee-? Occur Num: 56</li><li>Request-Attraction-Addr-? Occur Num: 24</li><li>Request-Attraction-Phone-? Occur Num: 14</li><li>Inform-Attraction-Area-centre Occur Num: 13</li><li>Request-Attraction-Type-? Occur Num: 8</li><li>Request-Attraction-Post-? Occur Num: 6</li><li>Inform-Attraction-Area-north Occur Num: 5</li><li>Request-Attraction-Area-? Occur Num: 3</li><li>Inform-Attraction-Area-west Occur Num: 2</li><li>Inform-Attraction-Type-architecture Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-entrance fee Occur Num: 72</li><li>request-attraction-address Occur Num: 54</li><li>request-attraction-phone Occur Num: 35</li><li>request-attraction-postcode Occur Num: 26</li><li>request-attraction-type Occur Num: 12</li><li>request-attraction-area Occur Num: 10</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-type Occur Num: 10</li><li>inform-attraction-area Occur Num: 8</li><li>inform-attraction-phone Occur Num: 2</li></ul></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 100.0 %</p><p> (Precision, Recall, F1) : (1.000, 1.000, 1.000) </p><p> Average Dialog Turn (Succ): 4.552 </p><p> Average Dialog Turn (All): 4.552 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Taxi-Depart-the lensfield hotel<ul><li>Occur Num: 8</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-the Occur Num: 2</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 2</li><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li><li>Inform-Taxi-Dest-williams art and antiques Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the fitzwilliam museum<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 2</li><li>Inform-Taxi-Dest-golden house Occur Num: 1</li><li>Inform-Taxi-Leave-13:45 Occur Num: 1</li><li>Inform-Taxi-Arrive-13:45 Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-home from home<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-home from home Occur Num: 2</li><li>Inform-Taxi-Depart-arbury lodge guesthouse Occur Num: 1</li><li>Inform-Taxi-Leave-02:45 Occur Num: 1</li><li>Inform-Taxi-Depart-thanh binh Occur Num: 1</li><li>Inform-Taxi-Leave-02:30 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-02:00<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-parkside police station<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-lynne strover gallery Occur Num: 1</li><li>Inform-Police-Name-parkside police station Occur Num: 1</li><li>Inform-Taxi-Arrive-12:15 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-13:45<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 1</li><li>Inform-Taxi-Dest-la mimosa Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-la mimosa<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Arrive-13:45 Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-efes restaurant<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-broxbourne train station Occur Num: 1</li><li>Inform-Taxi-Dest-efes Occur Num: 1</li><li>Inform-Taxi-Arrive-16:15 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the good luck chinese food takeaway<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-nandos Occur Num: 1</li><li>Inform-Taxi-Leave-05:15 Occur Num: 1</li><li>Inform-Taxi-Depart-good luck Occur Num: 1</li></ul></ul><li>Request-Taxi-Phone-?<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Request-Taxi-Car-? Occur Num: 3</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Taxi-Car-toyota<ul><li>Occur Num: 30</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 15</li><li>Inform-Taxi-Phone-53134794492 Occur Num: 1</li><li>Inform-Taxi-Phone-46965660273 Occur Num: 1</li><li>Inform-Taxi-Phone-30056585132 Occur Num: 1</li><li>Inform-Taxi-Phone-24219248656 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-skoda<ul><li>Occur Num: 28</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 13</li><li>Inform-Taxi-Phone-47568258820 Occur Num: 1</li><li>Inform-Taxi-Phone-62490184653 Occur Num: 1</li><li>Inform-Taxi-Phone-73072570410 Occur Num: 1</li><li>Inform-Taxi-Phone-56513788991 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-bmw<ul><li>Occur Num: 26</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 11</li><li>Inform-Taxi-Phone-23438633852 Occur Num: 1</li><li>Inform-Taxi-Phone-35203019962 Occur Num: 1</li><li>Inform-Taxi-Phone-02613091667 Occur Num: 1</li><li>Inform-Taxi-Phone-79054617689 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-volvo<ul><li>Occur Num: 23</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 11</li><li>Inform-Taxi-Phone-26438028145 Occur Num: 1</li><li>Inform-Taxi-Phone-27749757303 Occur Num: 1</li><li>Inform-Taxi-Phone-70465889266 Occur Num: 1</li><li>Inform-Taxi-Phone-96368625434 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-ford<ul><li>Occur Num: 22</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 9</li><li>Inform-Taxi-Phone-15481979763 Occur Num: 1</li><li>Inform-Taxi-Phone-52166874792 Occur Num: 1</li><li>Inform-Taxi-Phone-34332387636 Occur Num: 1</li><li>Inform-Taxi-Phone-82736093617 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-honda<ul><li>Occur Num: 20</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 8</li><li>Inform-Taxi-Phone-52547806713 Occur Num: 1</li><li>Inform-Taxi-Phone-21059587741 Occur Num: 1</li><li>Inform-Taxi-Phone-26855777732 Occur Num: 1</li><li>Inform-Taxi-Phone-42223260114 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-volkswagen<ul><li>Occur Num: 16</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 7</li><li>Inform-Taxi-Phone-15214714504 Occur Num: 1</li><li>Inform-Taxi-Phone-26466745656 Occur Num: 1</li><li>Inform-Taxi-Phone-42490286783 Occur Num: 1</li><li>Inform-Taxi-Phone-81956642884 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-lexus<ul><li>Occur Num: 16</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 8</li><li>Inform-Taxi-Phone-83144288521 Occur Num: 1</li><li>Inform-Taxi-Phone-80542456465 Occur Num: 1</li><li>Inform-Taxi-Phone-67771199099 Occur Num: 1</li><li>Inform-Taxi-Phone-30893268569 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-tesla<ul><li>Occur Num: 14</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 7</li><li>Inform-Taxi-Phone-05465013303 Occur Num: 1</li><li>Inform-Taxi-Phone-27225654945 Occur Num: 1</li><li>Inform-Taxi-Phone-79050909210 Occur Num: 1</li><li>Inform-Taxi-Phone-59116714127 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-audi<ul><li>Occur Num: 14</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 7</li><li>Inform-Taxi-Phone-73297457818 Occur Num: 1</li><li>Inform-Taxi-Phone-02411795305 Occur Num: 1</li><li>Inform-Taxi-Phone-41082385801 Occur Num: 1</li><li>Inform-Taxi-Phone-73390690110 Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><p>Nothing</p><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><p>Nothing</p><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 58.3 %</p><p> (Precision, Recall, F1) : (0.755, 0.644, 0.682) </p><p> Average Dialog Turn (Succ): 5.956 </p><p> Average Dialog Turn (All): 18.510 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Train-Dest-cambridge<ul><li>Occur Num: 118</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 23</li><li>Inform-Train-Day-friday Occur Num: 7</li><li>Inform-Train-Day-Saturday Occur Num: 7</li><li>Inform-Train-Day-wednesday Occur Num: 6</li><li>Inform-Train-Day-thursday Occur Num: 6</li></ul></ul><li>Inform-Train-Depart-cambridge<ul><li>Occur Num: 118</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 22</li><li>Inform-Train-Day-friday Occur Num: 9</li><li>Inform-Train-Day-tuesday Occur Num: 7</li><li>Inform-Train-Day-monday Occur Num: 6</li><li>Inform-Train-Day-Saturday Occur Num: 5</li></ul></ul><li>Request-Train-Id-?<ul><li>Occur Num: 99</li><li>NLU Output</li><ul><li>Request-Train-Time-? Occur Num: 34</li><li>Inform-Train-none-none Occur Num: 26</li><li>Request-Train-Ticket-? Occur Num: 25</li><li>Request-Train-Leave-? Occur Num: 9</li><li>Request-Train-Arrive-? Occur Num: 5</li></ul></ul><li>Request-Train-Time-?<ul><li>Occur Num: 98</li><li>NLU Output</li><ul><li>Request-Train-Id-? Occur Num: 35</li><li>Request-Train-Ticket-? Occur Num: 33</li><li>Inform-Train-none-none Occur Num: 20</li><li>Request-Train-Arrive-? Occur Num: 6</li><li>Request-Train-Leave-? Occur Num: 4</li></ul></ul><li>Request-Train-Ticket-?<ul><li>Occur Num: 59</li><li>NLU Output</li><ul><li>Request-Train-Time-? Occur Num: 30</li><li>Request-Train-Id-? Occur Num: 17</li><li>Request-Train-Leave-? Occur Num: 7</li><li>Request-Train-Arrive-? Occur Num: 5</li></ul></ul><li>Inform-Train-Day-friday<ul><li>Occur Num: 33</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 6</li><li>Inform-Train-Dest-cambridge Occur Num: 4</li><li>Inform-Train-Arrive-11:15 Occur Num: 2</li><li>Inform-Restaurant-Day-friday Occur Num: 2</li><li>Inform-Train-Dest-london liverpool street Occur Num: 1</li></ul></ul><li>Inform-Train-Day-wednesday<ul><li>Occur Num: 29</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 4</li><li>Inform-Train-Dest-cambridge Occur Num: 4</li><li>Inform-Train-Day-Saturday Occur Num: 3</li><li>Inform-Train-Depart-broxbourne Occur Num: 2</li><li>Inform-Train-Depart-london liverpool street Occur Num: 2</li></ul></ul><li>Request-Train-Leave-?<ul><li>Occur Num: 27</li><li>NLU Output</li><ul><li>Request-Train-Ticket-? Occur Num: 9</li><li>Request-Train-Id-? Occur Num: 7</li><li>Request-Train-Time-? Occur Num: 6</li><li>Inform-Train-none-none Occur Num: 5</li></ul></ul><li>Inform-Train-Day-tuesday<ul><li>Occur Num: 27</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 5</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Day-Saturday Occur Num: 2</li><li>Inform-Restaurant-Day-tuesday Occur Num: 1</li><li>Inform-Train-Leave-15:15 Occur Num: 1</li></ul></ul><li>Inform-Train-Day-thursday<ul><li>Occur Num: 24</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 5</li><li>Inform-Train-Dest-cambridge Occur Num: 3</li><li>Inform-Train-Leave-15:30 Occur Num: 2</li><li>Inform-Train-Depart-kings lynn Occur Num: 1</li><li>Inform-Train-Arrive-19:15 Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Train-Time-51 minutes<ul><li>Occur Num: 153</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 33</li><li>OfferBook-Train-none-none Occur Num: 25</li><li>Inform-Train-Ticket-23.60 pounds Occur Num: 21</li><li>Inform-Booking-none-none Occur Num: 13</li><li>Inform-Train-Ticket-18.88 pounds Occur Num: 9</li></ul></ul><li>Request-Train-Dest-?<ul><li>Occur Num: 135</li><li>NLU Output</li><ul><li>Request-Train-Arrive-? Occur Num: 52</li><li>Request-Booking-People-? Occur Num: 43</li><li>Request-Train-People-? Occur Num: 38</li><li>Request-Taxi-Arrive-? Occur Num: 2</li></ul></ul><li>Request-Train-Depart-?<ul><li>Occur Num: 133</li><li>NLU Output</li><ul><li>Request-Train-Arrive-? Occur Num: 52</li><li>Request-Booking-People-? Occur Num: 43</li><li>Request-Train-People-? Occur Num: 33</li><li>Request-Taxi-Arrive-? Occur Num: 2</li><li>reqmore-general-none-none Occur Num: 1</li></ul></ul><li>OfferBook-Train-Dest-cambridge<ul><li>Occur Num: 118</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 108</li><li>Inform-Booking-none-none Occur Num: 6</li><li>reqmore-general-none-none Occur Num: 2</li><li>Inform-Train-Leave-05:00 Occur Num: 2</li></ul></ul><li>OfferBook-Train-Depart-cambridge<ul><li>Occur Num: 83</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 69</li><li>Inform-Booking-none-none Occur Num: 6</li><li>reqmore-general-none-none Occur Num: 4</li><li>Inform-Train-Leave-05:00 Occur Num: 4</li></ul></ul><li>Request-Train-Leave-?<ul><li>Occur Num: 50</li><li>NLU Output</li><ul><li>Request-Train-Arrive-? Occur Num: 31</li><li>Request-Booking-People-? Occur Num: 13</li><li>Request-Train-People-? Occur Num: 5</li><li>Request-Taxi-Arrive-? Occur Num: 1</li></ul></ul><li>OfferBook-Train-Day-monday<ul><li>Occur Num: 44</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 42</li><li>reqmore-general-none-none Occur Num: 1</li><li>Inform-Train-Leave-05:00 Occur Num: 1</li></ul></ul><li>OfferBook-Train-Day-thursday<ul><li>Occur Num: 37</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 33</li><li>Inform-Booking-none-none Occur Num: 4</li></ul></ul><li>OfferBook-Train-Day-wednesday<ul><li>Occur Num: 37</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 32</li><li>Inform-Booking-none-none Occur Num: 3</li><li>reqmore-general-none-none Occur Num: 1</li><li>Inform-Train-Leave-05:00 Occur Num: 1</li></ul></ul><li>OfferBook-Train-Day-saturday<ul><li>Occur Num: 36</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 33</li><li>Inform-Booking-none-none Occur Num: 1</li><li>reqmore-general-none-none Occur Num: 1</li><li>Inform-Train-Leave-05:00 Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Train-Time-? Occur Num: 126</li><li>Request-Train-Id-? Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-train-duration Occur Num: 126</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 17.4 %</p><p> (Precision, Recall, F1) : (0.739, 0.457, 0.551) </p><p> Average Dialog Turn (Succ): 4.000 </p><p> Average Dialog Turn (All): 33.739 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Police-Phone-?<ul><li>Occur Num: 65</li><li>NLU Output</li><ul><li>Inform-Police-none-none Occur Num: 18</li><li>Request-Restaurant-Phone-? Occur Num: 16</li><li>Request-Police-Addr-? Occur Num: 14</li><li>Request-Attraction-Phone-? Occur Num: 13</li><li>Request-Police-Post-? Occur Num: 4</li></ul></ul><li>Request-Police-Addr-?<ul><li>Occur Num: 17</li><li>NLU Output</li><ul><li>Request-Police-Phone-? Occur Num: 12</li><li>Request-Police-Post-? Occur Num: 4</li><li>Request-Restaurant-Phone-? Occur Num: 1</li></ul></ul><li>Inform-Police-none-none<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Police-Name-Parkside Police Station Occur Num: 6</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Police-Phone-01223358966<ul><li>Occur Num: 34</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 19</li><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 12</li><li>Inform-Attraction-Phone-01223323737 Occur Num: 2</li><li>Inform-Police-Phone-01223323737 Occur Num: 1</li></ul></ul><li>Inform-Police-Addr-Parkside, Cambridge<ul><li>Occur Num: 18</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 17</li><li>Inform-Police-Phone-01223323737 Occur Num: 1</li></ul></ul><li>Inform-Police-Post-unknown<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 4</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Police-Phone-? Occur Num: 19</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-police-phone Occur Num: 19</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 100.0 %</p><p> (Precision, Recall, F1) : (1.000, 1.000, 1.000) </p><p> Average Dialog Turn (Succ): 4.000 </p><p> Average Dialog Turn (All): 4.000 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Hospital-Department-oncology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-plastic and vascular surgery plastics<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-plastic and vascular surgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-surgery<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-children's oncology and haematology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hospital-Choice-66<ul><li>Occur Num: 29</li><li>NLU Output</li><ul><li>bye-general-none-none Occur Num: 29</li></ul></ul><li>Recommend-Hospital-Department-paediatric intensive care unit<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>bye-general-none-none Occur Num: 2</li></ul></ul><li>Recommend-Hospital-Phone-01223217715<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>bye-general-none-none Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Department-emergency department<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>bye-general-none-none Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Department-clinical decisions unit<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>bye-general-none-none Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Department-theatre admissions unit<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>bye-general-none-none Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223217216<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>bye-general-none-none Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223217667<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>bye-general-none-none Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223217231<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>bye-general-none-none Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223596066<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>bye-general-none-none Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><p>Nothing</p><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><p>Nothing</p><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/BERTNLU-RuleDST-RulePolicy-SCLSTM/res.txt b/tests/results/BERTNLU-RuleDST-RulePolicy-SCLSTM/res.txt deleted file mode 100755 index 238583fcf780b8108ed2cef6005e8f10c3b17594..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-RulePolicy-SCLSTM/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.401 -success number of dialogs/tot: 0.41 -average precision: 0.6854002434261492 -average recall: 0.5650410423774911 -average f1: 0.5905369753026531 -average book rate: 0.5152777777777777 -average turn (succ): 11.62439024390244 -average turn (all): 29.224 diff --git a/tests/results/BERTNLU-RuleDST-RulePolicy-TemplateNLG/Frequency_of_domain.png b/tests/results/BERTNLU-RuleDST-RulePolicy-TemplateNLG/Frequency_of_domain.png deleted file mode 100755 index 3650ce3529bad2cad49d88b354405d6c69bcc41e..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-RulePolicy-TemplateNLG/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-RulePolicy-TemplateNLG/Performance_for_each_domain.png b/tests/results/BERTNLU-RuleDST-RulePolicy-TemplateNLG/Performance_for_each_domain.png deleted file mode 100755 index 112f2b31c24eecbc0d5107ca29a9e1d828139f09..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-RulePolicy-TemplateNLG/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png b/tests/results/BERTNLU-RuleDST-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index d60bd9faae0e7776414ca0c628c5e1c866dcc0d7..0000000000000000000000000000000000000000 Binary files a/tests/results/BERTNLU-RuleDST-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/BERTNLU-RuleDST-RulePolicy-TemplateNLG/report_multiwoz.html b/tests/results/BERTNLU-RuleDST-RulePolicy-TemplateNLG/report_multiwoz.html deleted file mode 100755 index 2a93cdf59310b37d7f5fced73bf1503dec3ed392..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-RulePolicy-TemplateNLG/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: BERTNLU-RuleDST-RulePolicy-TemplateNLG</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-24 21:20:09</h3><h3>Overall Results</h3><p> Success Rate: 85.5 %</p><p> (Precision, Recall, F1) : (0.798, 0.928, 0.838) </p><p> Average Dialog Turn (Succ): 12.681 </p><p> Average Dialog Turn (All): 13.814 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">430</td><td align="center">0.758</td><td align="center">0.504</td><td align="center">0.669</td><td align="center">0.546</td><td align="center">0.051</td><td align="center">5.890</td><td align="center">6.586</td></tr><tr><td align="center">attraction</td><td align="center">334</td><td align="center">0.937</td><td align="center">0.888</td><td align="center">0.950</td><td align="center">0.910</td><td align="center">0.021</td><td align="center">5.974</td><td align="center">6.431</td></tr><tr><td align="center">taxi</td><td align="center">175</td><td align="center">0.994</td><td align="center">1.000</td><td align="center">0.997</td><td align="center">0.998</td><td align="center">0.006</td><td align="center">6.115</td><td align="center">6.149</td></tr><tr><td align="center">restaurant</td><td align="center">413</td><td align="center">0.959</td><td align="center">0.792</td><td align="center">0.840</td><td align="center">0.808</td><td align="center">0.036</td><td align="center">5.116</td><td align="center">6.223</td></tr><tr><td align="center">train</td><td align="center">387</td><td align="center">0.995</td><td align="center">0.845</td><td align="center">0.928</td><td align="center">0.869</td><td align="center">0.005</td><td align="center">6.478</td><td align="center">6.491</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">0.000</td><td align="center">4.087</td><td align="center">4.087</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">0.000</td><td align="center">4.067</td><td align="center">4.067</td></tr></table><h3>Dialogue Loop</h3><img src="Proportions_of_the_dialogue_loop.png" width=800 /><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 75.8 %</p><p> (Precision, Recall, F1) : (0.504, 0.669, 0.546) </p><p> Average Dialog Turn (Succ): 5.890 </p><p> Average Dialog Turn (All): 6.586 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 401</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 64</li><li>Inform-Hotel-Type-hotel Occur Num: 55</li><li>Inform-Hotel-Type-guesthouse Occur Num: 44</li><li>Inform-Hotel-Stars-4 Occur Num: 44</li><li>Inform-Hotel-Price-moderate Occur Num: 35</li></ul></ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 369</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 67</li><li>Inform-Hotel-Type-guesthouse Occur Num: 46</li><li>Inform-Hotel-Type-hotel Occur Num: 41</li><li>Inform-Hotel-Stars-4 Occur Num: 40</li><li>Inform-Hotel-Price-moderate Occur Num: 36</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 225</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 45</li><li>Inform-Hotel-Parking-yes Occur Num: 44</li><li>Inform-Hotel-Stars-4 Occur Num: 23</li><li>Inform-Hotel-Price-moderate Occur Num: 17</li><li>Inform-Hotel-Area-north Occur Num: 14</li></ul></ul><li>Inform-Hotel-Stars-4<ul><li>Occur Num: 202</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 43</li><li>Inform-Hotel-Internet-yes Occur Num: 40</li><li>Inform-Hotel-Type-guesthouse Occur Num: 23</li><li>Inform-Hotel-Type-hotel Occur Num: 22</li><li>Inform-Hotel-Price-moderate Occur Num: 14</li></ul></ul><li>Inform-Hotel-Price-moderate<ul><li>Occur Num: 196</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 37</li><li>Inform-Hotel-Internet-yes Occur Num: 37</li><li>Inform-Hotel-Type-hotel Occur Num: 22</li><li>Inform-Hotel-Type-guesthouse Occur Num: 17</li><li>Inform-Hotel-Stars-4 Occur Num: 14</li></ul></ul><li>Inform-Hotel-Area-north<ul><li>Occur Num: 117</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 28</li><li>Inform-Hotel-Internet-yes Occur Num: 26</li><li>Inform-Hotel-Type-guesthouse Occur Num: 14</li><li>Inform-Hotel-Type-hotel Occur Num: 11</li><li>Inform-Hotel-Stars-4 Occur Num: 9</li></ul></ul><li>Inform-Hotel-Parking-no<ul><li>Occur Num: 111</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 31</li><li>Inform-Hotel-Internet-yes Occur Num: 13</li><li>Inform-Hotel-Type-guesthouse Occur Num: 12</li><li>Inform-Hotel-Price-moderate Occur Num: 10</li><li>Inform-Hotel-Type-hotel Occur Num: 8</li></ul></ul><li>Inform-Hotel-Type-hotel<ul><li>Occur Num: 97</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 17</li><li>Inform-Hotel-Parking-yes Occur Num: 15</li><li>Inform-Hotel-Stars-4 Occur Num: 9</li><li>Inform-Hotel-Price-moderate Occur Num: 9</li><li>Inform-Hotel-Stars-3 Occur Num: 5</li></ul></ul><li>Request-Hotel-Addr-?<ul><li>Occur Num: 93</li><li>NLU Output</li><ul><li>Request-Hotel-Post-? Occur Num: 22</li><li>Request-Hotel-Phone-? Occur Num: 18</li><li>Request-Hotel-Internet-? Occur Num: 9</li><li>Request-Attraction-Addr-? Occur Num: 8</li><li>Inform-Hotel-Internet-yes Occur Num: 7</li></ul></ul><li>Inform-Hotel-Price-cheap<ul><li>Occur Num: 89</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 18</li><li>Inform-Hotel-Parking-yes Occur Num: 16</li><li>Inform-Hotel-Type-guesthouse Occur Num: 11</li><li>Inform-Hotel-Type-hotel Occur Num: 11</li><li>Inform-Hotel-Stars-4 Occur Num: 10</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 523</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-none Occur Num: 93</li><li>Inform-Hotel-Parking-none Occur Num: 58</li><li>Inform-Hotel-Type-guesthouse Occur Num: 32</li><li>Inform-Hotel-Stars-4 Occur Num: 31</li><li>Inform-Hotel-Type-hotel Occur Num: 23</li></ul></ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 448</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 79</li><li>Inform-Hotel-Internet-none Occur Num: 54</li><li>Inform-Hotel-Stars-4 Occur Num: 33</li><li>Inform-Hotel-Type-guesthouse Occur Num: 29</li><li>Inform-Hotel-Price-cheap Occur Num: 23</li></ul></ul><li>NoOffer-Hotel-Type-hotel<ul><li>Occur Num: 258</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 32</li><li>NoOffer-Hotel-Price-moderate Occur Num: 22</li><li>Request-Hotel-Stars-? Occur Num: 21</li><li>Request-Hotel-Price-? Occur Num: 20</li><li>NoOffer-Hotel-Parking-none Occur Num: 19</li></ul></ul><li>NoOffer-Hotel-Parking-yes<ul><li>Occur Num: 241</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 27</li><li>NoOffer-Hotel-Parking-none Occur Num: 27</li><li>NoOffer-Hotel-Type-hotel Occur Num: 19</li><li>Request-Hotel-Price-? Occur Num: 17</li><li>NoOffer-Hotel-Price-moderate Occur Num: 16</li></ul></ul><li>NoOffer-Hotel-Price-moderate<ul><li>Occur Num: 233</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 33</li><li>Request-Hotel-Price-? Occur Num: 20</li><li>NoOffer-Hotel-Type-hotels Occur Num: 18</li><li>NoOffer-Hotel-Type-hotel Occur Num: 16</li><li>Request-Hotel-Stars-? Occur Num: 16</li></ul></ul><li>NoOffer-Hotel-Internet-yes<ul><li>Occur Num: 232</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 29</li><li>NoOffer-Hotel-Internet-none Occur Num: 28</li><li>NoOffer-Hotel-Type-hotels Occur Num: 18</li><li>NoOffer-Hotel-Type-hotel Occur Num: 15</li><li>NoOffer-Hotel-Price-moderate Occur Num: 13</li></ul></ul><li>Inform-Hotel-Stars-4<ul><li>Occur Num: 214</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 34</li><li>Inform-Hotel-Internet-none Occur Num: 34</li><li>Inform-Hotel-Price-cheap Occur Num: 19</li><li>Inform-Hotel-Type-guesthouse Occur Num: 19</li><li>Inform-Hotel-Type-hotel Occur Num: 12</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 202</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-none Occur Num: 32</li><li>Inform-Hotel-Parking-none Occur Num: 29</li><li>Inform-Hotel-Stars-4 Occur Num: 16</li><li>Inform-Hotel-Price-moderate Occur Num: 15</li><li>Inform-Hotel-Price-cheap Occur Num: 14</li></ul></ul><li>Inform-Hotel-Type-hotel<ul><li>Occur Num: 186</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-none Occur Num: 28</li><li>Inform-Hotel-Parking-none Occur Num: 28</li><li>Inform-Hotel-Stars-4 Occur Num: 11</li><li>Inform-Hotel-Price-expensive Occur Num: 10</li><li>Inform-Hotel-Area-west Occur Num: 9</li></ul></ul><li>Request-Hotel-Area-?<ul><li>Occur Num: 169</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 22</li><li>Request-Hotel-Price-? Occur Num: 18</li><li>NoOffer-Hotel-Price-moderate Occur Num: 15</li><li>NoOffer-Hotel-Type-hotel Occur Num: 15</li><li>NoOffer-Hotel-Type-hotels Occur Num: 13</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Inform-Hotel-Price-moderate Occur Num: 3</li><li>Inform-Hotel-Internet-yes Occur Num: 3</li><li>Request-Hotel-Addr-? Occur Num: 2</li><li>Request-Hotel-Phone-? Occur Num: 2</li><li>Inform-Hotel-Parking-yes Occur Num: 2</li><li>Inform-Hotel-Stay-5 Occur Num: 2</li><li>Inform-Hotel-Stay-3 Occur Num: 1</li><li>Inform-Hotel-Name-alexander bed and breakfast Occur Num: 1</li><li>Inform-Hotel-People-1 Occur Num: 1</li><li>Request-Hotel-Post-? Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-parking Occur Num: 113</li><li>inform-hotel-internet Occur Num: 108</li><li>recommend-hotel-parking Occur Num: 21</li><li>inform-hotel-area Occur Num: 20</li><li>inform-hotel-stars Occur Num: 14</li><li>recommend-hotel-internet Occur Num: 14</li><li>recommend-hotel-phone Occur Num: 3</li><li>inform-hotel-postcode Occur Num: 2</li><li>recommend-hotel-stars Occur Num: 1</li><li>recommend-hotel-postcode Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-internet Occur Num: 29</li><li>request-hotel-parking Occur Num: 25</li><li>request-hotel-address Occur Num: 12</li><li>request-hotel-phone Occur Num: 11</li><li>request-hotel-postcode Occur Num: 8</li><li>request-hotel-stars Occur Num: 3</li><li>request-hotel-area Occur Num: 3</li><li>request-hotel-type Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-area Occur Num: 40</li><li>inform-hotel-stars Occur Num: 38</li><li>inform-hotel-type Occur Num: 37</li><li>inform-hotel-phone Occur Num: 22</li></ul></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 93.7 %</p><p> (Precision, Recall, F1) : (0.888, 0.950, 0.910) </p><p> Average Dialog Turn (Succ): 5.974 </p><p> Average Dialog Turn (All): 6.431 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Attraction-Addr-?<ul><li>Occur Num: 174</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 65</li><li>Request-Attraction-Phone-? Occur Num: 59</li><li>Request-Attraction-Fee-? Occur Num: 40</li><li>Request-Attraction-Type-? Occur Num: 4</li><li>Request-Attraction-Area-? Occur Num: 3</li></ul></ul><li>Request-Attraction-Phone-?<ul><li>Occur Num: 167</li><li>NLU Output</li><ul><li>Request-Attraction-Addr-? Occur Num: 61</li><li>Request-Attraction-Post-? Occur Num: 57</li><li>Request-Attraction-Fee-? Occur Num: 37</li><li>Request-Attraction-Type-? Occur Num: 5</li><li>Request-Attraction-Area-? Occur Num: 2</li></ul></ul><li>Request-Attraction-Post-?<ul><li>Occur Num: 167</li><li>NLU Output</li><ul><li>Request-Attraction-Addr-? Occur Num: 64</li><li>Request-Attraction-Phone-? Occur Num: 54</li><li>Request-Attraction-Fee-? Occur Num: 43</li><li>Request-Attraction-Area-? Occur Num: 2</li><li>Request-Attraction-Type-? Occur Num: 2</li></ul></ul><li>Request-Attraction-Fee-?<ul><li>Occur Num: 123</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 43</li><li>Request-Attraction-Addr-? Occur Num: 40</li><li>Request-Attraction-Phone-? Occur Num: 35</li><li>Request-Attraction-Type-? Occur Num: 4</li><li>Request-Attraction-Area-? Occur Num: 1</li></ul></ul><li>Request-Attraction-Type-?<ul><li>Occur Num: 35</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 11</li><li>Request-Attraction-Addr-? Occur Num: 9</li><li>Request-Attraction-Post-? Occur Num: 8</li><li>Request-Attraction-Fee-? Occur Num: 7</li></ul></ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 30</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-entertainment Occur Num: 14</li><li>Inform-Attraction-Type-museums Occur Num: 3</li><li>Inform-Attraction-Type-boats Occur Num: 2</li><li>Inform-Attraction-Type-museum Occur Num: 2</li><li>Inform-Attraction-Type-parks Occur Num: 1</li></ul></ul><li>Inform-Attraction-Type-museum<ul><li>Occur Num: 25</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 10</li><li>Inform-Attraction-Type-museums Occur Num: 8</li><li>Inform-Attraction-Type-entertainment Occur Num: 3</li><li>Inform-Attraction-Area-west Occur Num: 2</li><li>Inform-Attraction-Area-east Occur Num: 1</li></ul></ul><li>Request-Attraction-Area-?<ul><li>Occur Num: 20</li><li>NLU Output</li><ul><li>Request-Attraction-Addr-? Occur Num: 9</li><li>Request-Attraction-Phone-? Occur Num: 7</li><li>Request-Attraction-Post-? Occur Num: 3</li><li>Request-Attraction-Fee-? Occur Num: 1</li></ul></ul><li>Inform-Attraction-Type-boat<ul><li>Occur Num: 20</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 6</li><li>Inform-Attraction-Type-entertainment Occur Num: 5</li><li>Inform-Attraction-Type-boats Occur Num: 5</li><li>Inform-Attraction-Area-north Occur Num: 2</li><li>Inform-Attraction-Area-east Occur Num: 1</li></ul></ul><li>Inform-Attraction-Type-nightclub<ul><li>Occur Num: 16</li><li>NLU Output</li><ul><li>Inform-Attraction-none-none Occur Num: 6</li><li>Inform-Attraction-Area-west Occur Num: 4</li><li>Inform-Attraction-Area-centre Occur Num: 4</li><li>Inform-Attraction-Type-entertainment Occur Num: 2</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Attraction-Fee-?<ul><li>Occur Num: 216</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-park Occur Num: 26</li><li>Inform-Attraction-Fee-by our system Occur Num: 19</li><li>Inform-Attraction-Addr-whitehill road Occur Num: 15</li><li>Inform-Attraction-Addr-off newmarket road Occur Num: 15</li><li>Inform-Attraction-Phone-01223300085 Occur Num: 15</li></ul></ul><li>Inform-Attraction-Addr-pool way, whitehill road, off newmarket road<ul><li>Occur Num: 97</li><li>NLU Output</li><ul><li>Inform-Attraction-Addr-whitehill road Occur Num: 20</li><li>Inform-Attraction-Addr-off newmarket road Occur Num: 20</li><li>Inform-Attraction-Addr-pool way Occur Num: 16</li><li>Inform-Attraction-Post-cb58nt Occur Num: 13</li><li>Inform-Attraction-Phone-01223902088 Occur Num: 6</li></ul></ul><li>Inform-Attraction-Choice-44<ul><li>Occur Num: 81</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 6</li><li>Request-Attraction-Type-? Occur Num: 4</li><li>Recommend-Attraction-Name-king 's college Occur Num: 2</li><li>Recommend-Attraction-Name-christ 's college Occur Num: 2</li><li>Recommend-Attraction-Post-cb23bu Occur Num: 2</li></ul></ul><li>Inform-Attraction-Fee-free<ul><li>Occur Num: 69</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-park Occur Num: 14</li><li>Inform-Attraction-Addr-saint andrew 's street Occur Num: 6</li><li>Inform-Attraction-Post-cb23bu Occur Num: 4</li><li>Inform-Attraction-Addr-98 king street Occur Num: 3</li><li>Inform-Attraction-Addr-storey 's way Occur Num: 3</li></ul></ul><li>Inform-Attraction-Choice-5<ul><li>Occur Num: 62</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 7</li><li>Recommend-Attraction-Name-little saint mary 's church Occur Num: 4</li><li>Recommend-Attraction-Name-great saint mary 's church Occur Num: 3</li><li>Recommend-Attraction-Name-churchill college Occur Num: 3</li><li>Recommend-Attraction-Name-Old schools Occur Num: 2</li></ul></ul><li>Request-Attraction-Type-?<ul><li>Occur Num: 59</li><li>NLU Output</li><ul><li>NoOffer-Attraction-Type-entertainment Occur Num: 13</li><li>Request-Attraction-Area-? Occur Num: 8</li><li>NoOffer-Attraction-Area-centre Occur Num: 8</li><li>NoOffer-Attraction-Area-south Occur Num: 4</li><li>NoOffer-Attraction-Type-architecture Occur Num: 3</li></ul></ul><li>Inform-Attraction-Post-cb58nt<ul><li>Occur Num: 53</li><li>NLU Output</li><ul><li>Inform-Attraction-Addr-whitehill road Occur Num: 13</li><li>Inform-Attraction-Addr-off newmarket road Occur Num: 13</li><li>Inform-Attraction-Addr-pool way Occur Num: 11</li><li>Inform-Attraction-Phone-01223902088 Occur Num: 5</li><li>Inform-Attraction-Addr-Pool way Occur Num: 2</li></ul></ul><li>Inform-Attraction-Choice-13<ul><li>Occur Num: 50</li><li>NLU Output</li><ul><li>Recommend-Attraction-Name-hughes hall Occur Num: 2</li><li>Recommend-Attraction-Name-churchill college Occur Num: 2</li><li>Inform-Attraction-Name-lynne strover gallery Occur Num: 2</li><li>Recommend-Attraction-Name-Clare college Occur Num: 1</li><li>Recommend-Attraction-Addr-trinity lane Occur Num: 1</li></ul></ul><li>Inform-Attraction-Choice-11<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Recommend-Attraction-Type-museum Occur Num: 3</li><li>Recommend-Attraction-Name-regency gallery Occur Num: 2</li><li>reqmore-general-none-none Occur Num: 2</li><li>Recommend-Attraction-Name-fitzwilliam museum Occur Num: 2</li><li>Recommend-Attraction-Name-whipple museum of the history of science Occur Num: 2</li></ul></ul><li>Inform-Attraction-Addr-saint andrew's street<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Inform-Attraction-Addr-saint andrew 's street Occur Num: 14</li><li>Inform-Attraction-Fee-free Occur Num: 9</li><li>Inform-Attraction-Phone-01223334900 Occur Num: 8</li><li>Inform-Attraction-Post-cb23bu Occur Num: 6</li><li>Inform-Attraction-Addr-Saint andrew 's street Occur Num: 5</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Attraction-Fee-? Occur Num: 3</li><li>Request-Attraction-Addr-? Occur Num: 2</li><li>Request-Attraction-Post-? Occur Num: 1</li><li>Inform-Attraction-Type-boat Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-attraction-area Occur Num: 18</li><li>recommend-attraction-area Occur Num: 3</li><li>recommend-attraction-phone Occur Num: 2</li><li>inform-attraction-phone Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-entrance fee Occur Num: 10</li><li>request-attraction-phone Occur Num: 9</li><li>request-attraction-postcode Occur Num: 8</li><li>request-attraction-address Occur Num: 7</li><li>request-attraction-area Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-type Occur Num: 31</li><li>inform-attraction-area Occur Num: 15</li><li>inform-attraction-phone Occur Num: 10</li></ul></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 99.4 %</p><p> (Precision, Recall, F1) : (1.000, 0.997, 0.998) </p><p> Average Dialog Turn (Succ): 6.115 </p><p> Average Dialog Turn (All): 6.149 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Phone-?<ul><li>Occur Num: 56</li><li>NLU Output</li><ul><li>Request-Taxi-Car-? Occur Num: 52</li><li>Request-Hotel-Phone-? Occur Num: 3</li><li>Request-Hotel-Type-? Occur Num: 1</li></ul></ul><li>Inform-Taxi-Leave-dontcare<ul><li>Occur Num: 12</li><li>NLU Output</li><ul><li>Inform-Taxi-Leave-do Occur Num: 4</li><li>Inform-Taxi-Arrive-12:30 Occur Num: 2</li><li>Inform-Taxi-none-none Occur Num: 2</li><li>Inform-Taxi-Arrive-01:30 Occur Num: 1</li><li>Inform-Taxi-Arrive-12:15 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-dontcare<ul><li>Occur Num: 8</li><li>NLU Output</li><ul><li>Inform-Taxi-Arrive-do Occur Num: 4</li><li>Inform-Taxi-Leave-09:00 Occur Num: 1</li><li>Inform-Taxi-Leave-13:00 Occur Num: 1</li><li>Inform-Taxi-Leave-19:45 Occur Num: 1</li><li>Inform-Taxi-Leave-12:30 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the lensfield hotel<ul><li>Occur Num: 8</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-the Occur Num: 2</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 2</li><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li><li>Inform-Taxi-Dest-williams art and antiques Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the fitzwilliam museum<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 2</li><li>Inform-Taxi-Dest-golden house Occur Num: 1</li><li>Inform-Taxi-Leave-13:45 Occur Num: 1</li><li>Inform-Taxi-Arrive-13:45 Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-12:15<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-12:15 Occur Num: 4</li><li>Inform-Taxi-Dest-lynne strover gallery Occur Num: 1</li><li>Inform-Police-Name-parkside police station Occur Num: 1</li></ul></ul><li>Request-Taxi-Car-?<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Request-Hotel-Phone-? Occur Num: 3</li><li>Request-Hotel-Type-? Occur Num: 3</li></ul></ul><li>Inform-Taxi-Dest-home from home<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-home from home Occur Num: 2</li><li>Inform-Taxi-Depart-arbury lodge guesthouse Occur Num: 1</li><li>Inform-Taxi-Leave-02:45 Occur Num: 1</li><li>Inform-Taxi-Depart-thanh binh Occur Num: 1</li><li>Inform-Taxi-Leave-02:30 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-19:30<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-19:30 Occur Num: 3</li><li>Inform-Taxi-Depart-warkworth house Occur Num: 1</li><li>Inform-Taxi-Dest-the gardenia Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-02:00<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Depart-the Occur Num: 1</li><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 1</li><li>Inform-Taxi-Leave-02:00 Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Leave-?<ul><li>Occur Num: 78</li><li>NLU Output</li><ul><li>Request-Taxi-Arrive-? Occur Num: 39</li><li>Request-Train-Leave-? Occur Num: 17</li><li>Request-Train-Arrive-? Occur Num: 15</li><li>Inform-Hotel-Phone-01223353888 Occur Num: 2</li><li>Request-Train-Depart-? Occur Num: 1</li></ul></ul><li>Request-Taxi-Arrive-?<ul><li>Occur Num: 42</li><li>NLU Output</li><ul><li>Request-Taxi-Leave-? Occur Num: 21</li><li>Request-Train-Arrive-? Occur Num: 15</li><li>Inform-Hotel-Phone-01223353888 Occur Num: 2</li><li>Request-Train-Leave-? Occur Num: 2</li><li>Inform-Hotel-Type-guesthouse Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-honda<ul><li>Occur Num: 30</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-52547806713 Occur Num: 1</li><li>Inform-Taxi-Phone-43863385242 Occur Num: 1</li><li>Inform-Taxi-Phone-01996214154 Occur Num: 1</li><li>Inform-Taxi-Phone-72877208795 Occur Num: 1</li><li>Inform-Taxi-Phone-88297768151 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-ford<ul><li>Occur Num: 20</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-46089765412 Occur Num: 1</li><li>Inform-Taxi-Phone-49207435274 Occur Num: 1</li><li>Inform-Taxi-Phone-76911481339 Occur Num: 1</li><li>Inform-Taxi-Phone-11253585386 Occur Num: 1</li><li>Inform-Taxi-Phone-94896745615 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-toyota<ul><li>Occur Num: 20</li><li>NLU Output</li><ul><li>Book-Booking-Name-toyota Occur Num: 2</li><li>Inform-Taxi-Phone-11835164958 Occur Num: 1</li><li>Inform-Taxi-Phone-24219248656 Occur Num: 1</li><li>Inform-Taxi-Phone-87029902122 Occur Num: 1</li><li>Inform-Taxi-Phone-03811755419 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-lexus<ul><li>Occur Num: 19</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-55435186919 Occur Num: 1</li><li>Inform-Taxi-Phone-94093896163 Occur Num: 1</li><li>Inform-Taxi-Phone-83144288521 Occur Num: 1</li><li>Inform-Taxi-Phone-67771199099 Occur Num: 1</li><li>Inform-Taxi-Phone-33919142072 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-skoda<ul><li>Occur Num: 18</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-81080351148 Occur Num: 1</li><li>Inform-Taxi-Phone-95815019804 Occur Num: 1</li><li>Inform-Taxi-Phone-28321970261 Occur Num: 1</li><li>Inform-Taxi-Phone-62130883866 Occur Num: 1</li><li>Inform-Taxi-Phone-15366676097 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-volkswagen<ul><li>Occur Num: 18</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-15214714504 Occur Num: 1</li><li>Inform-Taxi-Phone-40241180583 Occur Num: 1</li><li>Inform-Taxi-Phone-37624820407 Occur Num: 1</li><li>Inform-Taxi-Phone-10567250292 Occur Num: 1</li><li>Inform-Taxi-Phone-93334449685 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-tesla<ul><li>Occur Num: 17</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-66842156860 Occur Num: 1</li><li>Inform-Taxi-Phone-11498147519 Occur Num: 1</li><li>Inform-Taxi-Phone-90591246686 Occur Num: 1</li><li>Inform-Taxi-Phone-54087301478 Occur Num: 1</li><li>Inform-Taxi-Phone-62981254942 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-bmw<ul><li>Occur Num: 16</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-98052904604 Occur Num: 1</li><li>Inform-Taxi-Phone-79054617689 Occur Num: 1</li><li>Inform-Taxi-Phone-01237054676 Occur Num: 1</li><li>Inform-Taxi-Phone-31068983551 Occur Num: 1</li><li>Inform-Taxi-Phone-36921391209 Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Taxi-Car-? Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-taxi-car type Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 95.9 %</p><p> (Precision, Recall, F1) : (0.792, 0.840, 0.808) </p><p> Average Dialog Turn (Succ): 5.116 </p><p> Average Dialog Turn (All): 6.223 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Restaurant-Addr-?<ul><li>Occur Num: 107</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 54</li><li>Request-Restaurant-Post-? Occur Num: 39</li><li>Request-Hotel-Addr-? Occur Num: 4</li><li>Request-Attraction-Addr-? Occur Num: 3</li><li>Request-Attraction-Phone-? Occur Num: 3</li></ul></ul><li>Request-Restaurant-Phone-?<ul><li>Occur Num: 100</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 52</li><li>Request-Restaurant-Post-? Occur Num: 24</li><li>Request-Attraction-Phone-? Occur Num: 9</li><li>Request-Hotel-Phone-? Occur Num: 7</li><li>Request-Attraction-Post-? Occur Num: 3</li></ul></ul><li>Request-Restaurant-Post-?<ul><li>Occur Num: 70</li><li>NLU Output</li><ul><li>Request-Restaurant-Addr-? Occur Num: 37</li><li>Request-Restaurant-Phone-? Occur Num: 21</li><li>Request-Attraction-Post-? Occur Num: 6</li><li>Request-Restaurant-Price-? Occur Num: 2</li><li>Request-Attraction-Addr-? Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Price-moderate<ul><li>Occur Num: 37</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 16</li><li>Inform-Restaurant-Area-centre Occur Num: 8</li><li>Inform-Hotel-Price-moderate Occur Num: 2</li><li>Inform-Restaurant-Food-british Occur Num: 2</li><li>Inform-Restaurant-Food-italian Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 32</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensively Occur Num: 11</li><li>Inform-Restaurant-Area-centre Occur Num: 4</li><li>Inform-Restaurant-Food-indian Occur Num: 2</li><li>Inform-Restaurant-Area-west Occur Num: 2</li><li>Inform-Restaurant-Area-south Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 26</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 8</li><li>Inform-Restaurant-Food-british Occur Num: 3</li><li>Inform-Attraction-Area-centre Occur Num: 2</li><li>Inform-Restaurant-Price-expensively Occur Num: 2</li><li>Inform-Hotel-Price-moderate Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Price-cheap<ul><li>Occur Num: 20</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-cheaply Occur Num: 7</li><li>Inform-Restaurant-Food-european Occur Num: 2</li><li>Inform-Restaurant-Food-italian Occur Num: 2</li><li>Inform-Restaurant-Area-centre Occur Num: 2</li><li>Inform-Restaurant-Food-indian Occur Num: 1</li></ul></ul><li>Request-Restaurant-Price-?<ul><li>Occur Num: 10</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 3</li><li>Request-Restaurant-Addr-? Occur Num: 3</li><li>Request-Restaurant-Post-? Occur Num: 2</li><li>Request-Hotel-Addr-? Occur Num: 1</li><li>Request-Hotel-Price-? Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Food-portuguese<ul><li>Occur Num: 9</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 3</li><li>Inform-Restaurant-Area-centre Occur Num: 2</li><li>Inform-Hotel-Price-moderate Occur Num: 1</li><li>Inform-Restaurant-Price-moderate Occur Num: 1</li><li>Inform-Restaurant-Area-north Occur Num: 1</li></ul></ul><li>Inform-Restaurant-Area-east<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderately Occur Num: 1</li><li>Inform-Restaurant-Price-cheaply Occur Num: 1</li><li>Inform-Restaurant-Food-italian Occur Num: 1</li><li>Inform-Hotel-Price-expensive Occur Num: 1</li><li>Inform-Hotel-Area-east Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Request-Restaurant-Food-?<ul><li>Occur Num: 95</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 13</li><li>Request-Restaurant-Price-? Occur Num: 11</li><li>NoOffer-Restaurant-Area-centre Occur Num: 9</li><li>NoOffer-Restaurant-Price-expensive Occur Num: 7</li><li>NoOffer-Restaurant-Price-cheap Occur Num: 7</li></ul></ul><li>NoOffer-Restaurant-Price-cheap<ul><li>Occur Num: 63</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 12</li><li>Request-Restaurant-Food-? Occur Num: 8</li><li>NoOffer-Restaurant-Area-centre Occur Num: 6</li><li>NoOffer-Restaurant-Area-east Occur Num: 5</li><li>NoOffer-Restaurant-Food-british Occur Num: 5</li></ul></ul><li>Inform-Restaurant-Choice-4<ul><li>Occur Num: 62</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 3</li><li>Recommend-Restaurant-Name-meghna Occur Num: 2</li><li>Inform-Booking-none-none Occur Num: 2</li><li>Recommend-Restaurant-Name-royal standard Occur Num: 2</li><li>Recommend-Restaurant-Area-centre Occur Num: 2</li></ul></ul><li>NoOffer-Restaurant-Area-centre<ul><li>Occur Num: 58</li><li>NLU Output</li><ul><li>Request-Restaurant-Food-? Occur Num: 9</li><li>NoOffer-Restaurant-none-none Occur Num: 6</li><li>NoOffer-Restaurant-Price-cheap Occur Num: 6</li><li>NoOffer-Restaurant-Food-indian Occur Num: 4</li><li>NoOffer-Restaurant-Price-moderate Occur Num: 4</li></ul></ul><li>NoOffer-Restaurant-Price-expensive<ul><li>Occur Num: 56</li><li>NLU Output</li><ul><li>Request-Restaurant-Food-? Occur Num: 8</li><li>NoOffer-Restaurant-none-none Occur Num: 8</li><li>Request-Restaurant-Price-? Occur Num: 7</li><li>NoOffer-Restaurant-Food-spanish Occur Num: 4</li><li>NoOffer-Restaurant-Area-in town Occur Num: 4</li></ul></ul><li>Request-Restaurant-Price-?<ul><li>Occur Num: 54</li><li>NLU Output</li><ul><li>Request-Restaurant-Food-? Occur Num: 12</li><li>NoOffer-Restaurant-none-none Occur Num: 7</li><li>NoOffer-Restaurant-Price-expensive Occur Num: 6</li><li>Request-Restaurant-Area-? Occur Num: 4</li><li>NoOffer-Restaurant-Area-east Occur Num: 4</li></ul></ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 52</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 5</li><li>Inform-Restaurant-Area-east Occur Num: 4</li><li>Recommend-Restaurant-Name-grafton hotel restaurant Occur Num: 3</li><li>Inform-Restaurant-Food-british Occur Num: 3</li><li>Recommend-Restaurant-Name-darrys cookhouse and wine shop Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 5</li><li>Recommend-Restaurant-Area-centre Occur Num: 4</li><li>Inform-Restaurant-Price-cheap Occur Num: 3</li><li>Inform-Restaurant-Price-moderate Occur Num: 3</li><li>Inform-Restaurant-Food-modern european Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Choice-2<ul><li>Occur Num: 42</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 3</li><li>Recommend-Restaurant-Name-hotel du vin and bistro Occur Num: 3</li><li>Recommend-Restaurant-Name-the nirala Occur Num: 2</li><li>Recommend-Restaurant-Name-The Occur Num: 2</li><li>Recommend-Restaurant-Name-anatolia Occur Num: 2</li></ul></ul><li>NoOffer-Restaurant-Price-moderate<ul><li>Occur Num: 41</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 7</li><li>NoOffer-Restaurant-Area-centre Occur Num: 6</li><li>Request-Restaurant-Food-? Occur Num: 6</li><li>NoOffer-Restaurant-Food-indian Occur Num: 4</li><li>NoOffer-Restaurant-Food-chinese Occur Num: 2</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Restaurant-Phone-? Occur Num: 12</li><li>Request-Restaurant-Addr-? Occur Num: 2</li><li>Inform-Restaurant-People-6 Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>recommend-restaurant-area Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-phone Occur Num: 12</li><li>request-restaurant-address Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-area Occur Num: 31</li><li>inform-restaurant-food Occur Num: 19</li><li>inform-restaurant-phone Occur Num: 16</li></ul></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 99.5 %</p><p> (Precision, Recall, F1) : (0.845, 0.928, 0.869) </p><p> Average Dialog Turn (Succ): 6.478 </p><p> Average Dialog Turn (All): 6.491 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Train-Depart-cambridge<ul><li>Occur Num: 165</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 29</li><li>Inform-Train-Day-friday Occur Num: 9</li><li>Inform-Train-Day-monday Occur Num: 8</li><li>Inform-Train-Day-tuesday Occur Num: 7</li><li>Inform-Train-Day-thursday Occur Num: 7</li></ul></ul><li>Inform-Train-Dest-cambridge<ul><li>Occur Num: 165</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 25</li><li>Inform-Train-Day-Saturday Occur Num: 10</li><li>Inform-Train-Day-friday Occur Num: 7</li><li>Inform-Train-Day-tuesday Occur Num: 7</li><li>Request-Train-Arrive-? Occur Num: 6</li></ul></ul><li>Request-Train-Id-?<ul><li>Occur Num: 107</li><li>NLU Output</li><ul><li>Request-Train-Time-? Occur Num: 37</li><li>Request-Train-Ticket-? Occur Num: 28</li><li>Inform-Train-none-none Occur Num: 28</li><li>Request-Train-Leave-? Occur Num: 9</li><li>Request-Train-Arrive-? Occur Num: 5</li></ul></ul><li>Request-Train-Time-?<ul><li>Occur Num: 82</li><li>NLU Output</li><ul><li>Request-Train-Ticket-? Occur Num: 40</li><li>Request-Train-Id-? Occur Num: 27</li><li>Inform-Train-none-none Occur Num: 8</li><li>Request-Train-Arrive-? Occur Num: 4</li><li>Request-Train-Leave-? Occur Num: 3</li></ul></ul><li>Request-Train-Ticket-?<ul><li>Occur Num: 64</li><li>NLU Output</li><ul><li>Request-Train-Time-? Occur Num: 37</li><li>Request-Train-Id-? Occur Num: 19</li><li>Request-Train-Leave-? Occur Num: 4</li><li>Request-Train-Arrive-? Occur Num: 4</li></ul></ul><li>Inform-Train-Day-monday<ul><li>Occur Num: 44</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 8</li><li>Inform-Train-Depart-cambridge Occur Num: 4</li><li>Inform-Train-Day-Saturday Occur Num: 3</li><li>Inform-Train-Dest-birmingham new street Occur Num: 3</li><li>Inform-Train-Dest-leicester Occur Num: 2</li></ul></ul><li>Inform-Train-Day-wednesday<ul><li>Occur Num: 34</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 6</li><li>Inform-Train-Dest-cambridge Occur Num: 4</li><li>Inform-Train-Depart-ely Occur Num: 3</li><li>Inform-Train-Day-Saturday Occur Num: 3</li><li>Inform-Train-Depart-broxbourne Occur Num: 2</li></ul></ul><li>Inform-Train-Day-tuesday<ul><li>Occur Num: 32</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 6</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Train-Day-Saturday Occur Num: 3</li><li>Inform-Train-Arrive-18:30 Occur Num: 1</li><li>Inform-Train-Dest-leicester Occur Num: 1</li></ul></ul><li>Inform-Train-Day-friday<ul><li>Occur Num: 31</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 7</li><li>Inform-Train-Dest-cambridge Occur Num: 3</li><li>Inform-Train-Arrive-11:15 Occur Num: 2</li><li>Inform-Train-Depart-london liverpool street Occur Num: 1</li><li>Inform-Train-Arrive-21:00 Occur Num: 1</li></ul></ul><li>Request-Train-Leave-?<ul><li>Occur Num: 29</li><li>NLU Output</li><ul><li>Request-Train-Id-? Occur Num: 9</li><li>Request-Train-Ticket-? Occur Num: 9</li><li>Request-Train-Time-? Occur Num: 6</li><li>Inform-Train-none-none Occur Num: 5</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>OfferBook-Train-Dest-cambridge<ul><li>Occur Num: 657</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 113</li><li>OfferBook-Train-Day-thursday Occur Num: 31</li><li>OfferBook-Train-Day-friday Occur Num: 25</li><li>OfferBook-Train-Depart-cambridge Occur Num: 25</li><li>OfferBook-Train-Day-saturday Occur Num: 24</li></ul></ul><li>OfferBook-Train-Depart-cambridge<ul><li>Occur Num: 529</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 78</li><li>OfferBook-Train-Dest-cambridge Occur Num: 39</li><li>OfferBook-Train-Day-friday Occur Num: 23</li><li>OfferBook-Train-Day-thursday Occur Num: 23</li><li>OfferBook-Train-Day-wednesday Occur Num: 21</li></ul></ul><li>OfferBook-Train-Day-thursday<ul><li>Occur Num: 219</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 35</li><li>OfferBook-Train-Dest-cambridge Occur Num: 27</li><li>OfferBook-Train-Depart-cambridge Occur Num: 20</li><li>Inform-Train-none-none Occur Num: 11</li><li>OfferBook-Train-Depart-peterborough Occur Num: 5</li></ul></ul><li>OfferBook-Train-Day-friday<ul><li>Occur Num: 204</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 29</li><li>OfferBook-Train-Dest-cambridge Occur Num: 25</li><li>OfferBook-Train-Depart-cambridge Occur Num: 20</li><li>OfferBook-Train-Depart-leicester Occur Num: 7</li><li>OfferBook-Train-Arrive-, Occur Num: 7</li></ul></ul><li>OfferBook-Train-Day-monday<ul><li>Occur Num: 202</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 35</li><li>OfferBook-Train-Dest-cambridge Occur Num: 20</li><li>OfferBook-Train-Depart-cambridge Occur Num: 12</li><li>OfferBook-Train-Arrive-that Occur Num: 10</li><li>OfferBook-Train-Arrive-, Occur Num: 9</li></ul></ul><li>Request-Train-Dest-?<ul><li>Occur Num: 200</li><li>NLU Output</li><ul><li>Request-Train-Depart-? Occur Num: 85</li><li>Request-Train-Arrive-? Occur Num: 59</li><li>Request-Train-Leave-? Occur Num: 38</li><li>Request-Taxi-Dest-? Occur Num: 9</li><li>Request-Taxi-Arrive-? Occur Num: 5</li></ul></ul><li>OfferBook-Train-Day-saturday<ul><li>Occur Num: 192</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 33</li><li>OfferBook-Train-Dest-cambridge Occur Num: 26</li><li>OfferBook-Train-Depart-cambridge Occur Num: 10</li><li>Select-Train-Day-saturday Occur Num: 9</li><li>OfferBook-Train-Arrive-that Occur Num: 8</li></ul></ul><li>OfferBook-Train-Day-tuesday<ul><li>Occur Num: 191</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 30</li><li>OfferBook-Train-Dest-cambridge Occur Num: 21</li><li>OfferBook-Train-Depart-cambridge Occur Num: 14</li><li>OfferBook-Train-Arrive-that Occur Num: 9</li><li>OfferBook-Train-Arrive-, Occur Num: 6</li></ul></ul><li>OfferBook-Train-Day-wednesday<ul><li>Occur Num: 179</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 30</li><li>OfferBook-Train-Dest-cambridge Occur Num: 19</li><li>OfferBook-Train-Depart-cambridge Occur Num: 18</li><li>OfferBook-Train-Arrive-, Occur Num: 6</li><li>OfferBook-Train-Arrive-that Occur Num: 6</li></ul></ul><li>Request-Train-Depart-?<ul><li>Occur Num: 167</li><li>NLU Output</li><ul><li>Request-Train-Dest-? Occur Num: 59</li><li>Request-Train-Arrive-? Occur Num: 55</li><li>Request-Train-Leave-? Occur Num: 39</li><li>Request-Taxi-Arrive-? Occur Num: 4</li><li>Request-Taxi-Depart-? Occur Num: 4</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Inform-Train-Day-thursday Occur Num: 1</li><li>Inform-Train-Day-sunday Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>offerbook-train-arriveBy Occur Num: 48</li><li>inform-train-duration Occur Num: 20</li></ul><b> Request But Not Inform Dialog Act</b><p>Nothing</p><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 100.0 %</p><p> (Precision, Recall, F1) : (1.000, 1.000, 1.000) </p><p> Average Dialog Turn (Succ): 4.087 </p><p> Average Dialog Turn (All): 4.087 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Police-Phone-?<ul><li>Occur Num: 20</li><li>NLU Output</li><ul><li>Request-Police-Addr-? Occur Num: 14</li><li>Request-Police-Post-? Occur Num: 5</li><li>Request-Attraction-Addr-? Occur Num: 1</li></ul></ul><li>Request-Police-Addr-?<ul><li>Occur Num: 19</li><li>NLU Output</li><ul><li>Request-Police-Phone-? Occur Num: 13</li><li>Request-Police-Post-? Occur Num: 5</li><li>Request-Attraction-Addr-? Occur Num: 1</li></ul></ul><li>Inform-Police-none-none<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Police-Name-Parkside Police Station Occur Num: 6</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Police-Addr-Parkside, Cambridge<ul><li>Occur Num: 33</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 17</li><li>Inform-Police-Phone-01223358966 Occur Num: 13</li><li>Inform-Attraction-Addr-pool way Occur Num: 1</li><li>Inform-Attraction-Addr-whitehill road Occur Num: 1</li><li>Inform-Attraction-Addr-off newmarket road Occur Num: 1</li></ul></ul><li>Inform-Police-Phone-01223358966<ul><li>Occur Num: 16</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 13</li><li>Inform-Attraction-Addr-pool way Occur Num: 1</li><li>Inform-Attraction-Addr-whitehill road Occur Num: 1</li><li>Inform-Attraction-Addr-off newmarket road Occur Num: 1</li></ul></ul><li>Inform-Police-Post-unknown<ul><li>Occur Num: 10</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 5</li><li>Inform-Police-Phone-01223358966 Occur Num: 5</li></ul></ul></ul><b> Dialog Loop</b><p>Nothing</p><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><p>Nothing</p><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 100.0 %</p><p> (Precision, Recall, F1) : (1.000, 1.000, 1.000) </p><p> Average Dialog Turn (Succ): 4.067 </p><p> Average Dialog Turn (All): 4.067 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Hospital-Department-neurology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurology neurosurgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-oncology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-plastic and vascular surgery plastics<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-plastic and vascular surgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-surgery<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-children's oncology and haematology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-oral and maxillofacial surgery and ent<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-oral and maxillofacial surgery and ent neurosurgery Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hospital-Choice-66<ul><li>Occur Num: 56</li><li>NLU Output</li><ul><li>Inform-Train-Ticket-66 Occur Num: 20</li><li>Inform-Train-Ref-66 Occur Num: 7</li><li>Inform-Hospital-Department-paediatric intensive care unit Occur Num: 2</li><li>Inform-Hospital-Department-66 Occur Num: 2</li><li>Inform-Hospital-Department-clinical decisions unit Occur Num: 2</li></ul></ul><li>Recommend-Hospital-Phone-01223217667<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Hospital-Phone-01223217667 Occur Num: 2</li><li>Inform-Train-Ticket-66 Occur Num: 1</li><li>Inform-Hospital-Department-postnatal Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Department-paediatric intensive care unit<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-paediatric intensive care unit Occur Num: 2</li><li>Inform-Hospital-Phone-01223217715 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223217715<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-paediatric intensive care unit Occur Num: 1</li><li>Inform-Hospital-Phone-01223217715 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Department-major trauma unit<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-66 Occur Num: 1</li><li>Inform-Hospital-Department-major trauma unit Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Department-clinical decisions unit<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-clinical decisions unit Occur Num: 2</li></ul></ul><li>Recommend-Hospital-Phone-01223217216<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Train-Ticket-66 Occur Num: 1</li><li>Inform-Hospital-Phone-01223217216 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223217231<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Train-Ticket-66 Occur Num: 1</li><li>Inform-Hospital-Phone-01223217231 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223596066<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Train-Ticket-66 Occur Num: 1</li><li>Inform-Hospital-Phone-01223596066 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223217297<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Phone-01223217297 Occur Num: 1</li><li>Inform-Hospital-Department-coronary care unit Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><p>Nothing</p><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><p>Nothing</p><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/BERTNLU-RuleDST-RulePolicy-TemplateNLG/res.txt b/tests/results/BERTNLU-RuleDST-RulePolicy-TemplateNLG/res.txt deleted file mode 100755 index ee82ca2b2f9f2e07c77840c74c6529e8757fab3d..0000000000000000000000000000000000000000 --- a/tests/results/BERTNLU-RuleDST-RulePolicy-TemplateNLG/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.921 -success number of dialogs/tot: 0.855 -average precision: 0.797874918435666 -average recall: 0.9279689792150851 -average f1: 0.8378418876894301 -average book rate: 0.91508547008547 -average turn (succ): 12.680701754385964 -average turn (all): 13.814 diff --git a/tests/results/DAMD/Frequency_of_domain.png b/tests/results/DAMD/Frequency_of_domain.png deleted file mode 100755 index 58abefe672d6165f883d52bb0669a225637f18ff..0000000000000000000000000000000000000000 Binary files a/tests/results/DAMD/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/DAMD/Performance_for_each_domain.png b/tests/results/DAMD/Performance_for_each_domain.png deleted file mode 100755 index 4e8d7fb7a2c45abe07d3b818dc2f5af79cdf4eb1..0000000000000000000000000000000000000000 Binary files a/tests/results/DAMD/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/DAMD/Proportions_of_the_dialogue_loop.png b/tests/results/DAMD/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index 1a64acfc0d98f856621b823ba85df63414dcc739..0000000000000000000000000000000000000000 Binary files a/tests/results/DAMD/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/DAMD/report_multiwoz.html b/tests/results/DAMD/report_multiwoz.html deleted file mode 100755 index 1fa9b2365a6c358cd892e79054b374c1a2c44c84..0000000000000000000000000000000000000000 --- a/tests/results/DAMD/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: DAMD</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-26 13:13:21</h3><h3>Overall Results</h3><p> Success Rate: 33.6 %</p><p> (Precision, Recall, F1) : (0.621, 0.607, 0.574) </p><p> Average Dialog Turn (Succ): 10.440 </p><p> Average Dialog Turn (All): 28.206 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">329</td><td align="center">0.587</td><td align="center">0.566</td><td align="center">0.668</td><td align="center">0.597</td><td align="center">0.185</td><td align="center">6.093</td><td align="center">9.909</td></tr><tr><td align="center">attraction</td><td align="center">289</td><td align="center">0.716</td><td align="center">0.746</td><td align="center">0.802</td><td align="center">0.754</td><td align="center">0.197</td><td align="center">6.251</td><td align="center">12.097</td></tr><tr><td align="center">restaurant</td><td align="center">381</td><td align="center">0.921</td><td align="center">0.693</td><td align="center">0.832</td><td align="center">0.737</td><td align="center">0.071</td><td align="center">5.516</td><td align="center">7.507</td></tr><tr><td align="center">taxi</td><td align="center">129</td><td align="center">0.140</td><td align="center">0.537</td><td align="center">0.341</td><td align="center">0.406</td><td align="center">0.845</td><td align="center">10.889</td><td align="center">26.388</td></tr><tr><td align="center">train</td><td align="center">373</td><td align="center">0.214</td><td align="center">0.293</td><td align="center">0.321</td><td align="center">0.272</td><td align="center">0.769</td><td align="center">18.150</td><td align="center">32.048</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">0.783</td><td align="center">0.783</td><td align="center">0.783</td><td align="center">0.783</td><td align="center">0.217</td><td align="center">3.556</td><td align="center">11.478</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">0.533</td><td align="center">0.533</td><td align="center">0.533</td><td align="center">0.533</td><td align="center">0.467</td><td align="center">7.250</td><td align="center">22.533</td></tr></table><h3>Dialogue Loop</h3><img src="Proportions_of_the_dialogue_loop.png" width=800 /><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 58.7 %</p><p> (Precision, Recall, F1) : (0.566, 0.668, 0.597) </p><p> Average Dialog Turn (Succ): 6.093 </p><p> Average Dialog Turn (All): 9.909 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Hotel-Addr-? Occur Num: 7</li><li>Request-Hotel-Internet-? Occur Num: 7</li><li>Inform-Hotel-Stay-2 Occur Num: 4</li><li>Inform-Hotel-Stay-4 Occur Num: 4</li><li>Inform-Hotel-Area-centre Occur Num: 4</li><li>Inform-Hotel-Area-dontcare Occur Num: 3</li><li>Inform-Hotel-Stay-6 Occur Num: 3</li><li>Request-Hotel-Type-? Occur Num: 3</li><li>Inform-Hotel-People-4 Occur Num: 2</li><li>Inform-Hotel-Area-north Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-parking Occur Num: 26</li><li>inform-hotel-internet Occur Num: 25</li><li>recommend-hotel-internet Occur Num: 2</li><li>inform-hotel-pricerange Occur Num: 2</li><li>inform-hotel-stars Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-address Occur Num: 19</li><li>request-hotel-internet Occur Num: 17</li><li>request-hotel-parking Occur Num: 17</li><li>request-hotel-postcode Occur Num: 13</li><li>request-hotel-phone Occur Num: 6</li><li>request-hotel-stars Occur Num: 3</li><li>request-hotel-type Occur Num: 3</li><li>request-hotel-pricerange Occur Num: 2</li><li>request-hotel-area Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-area Occur Num: 46</li><li>inform-hotel-type Occur Num: 46</li><li>inform-hotel-phone Occur Num: 24</li><li>inform-hotel-stars Occur Num: 9</li></ul></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 71.6 %</p><p> (Precision, Recall, F1) : (0.746, 0.802, 0.754) </p><p> Average Dialog Turn (Succ): 6.251 </p><p> Average Dialog Turn (All): 12.097 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Attraction-Fee-? Occur Num: 40</li><li>Request-Attraction-Addr-? Occur Num: 6</li><li>Request-Attraction-Post-? Occur Num: 5</li><li>Request-Attraction-Phone-? Occur Num: 2</li><li>Inform-Attraction-Name-saint john's college Occur Num: 1</li><li>Inform-Attraction-Name-the cambridge punter Occur Num: 1</li><li>Inform-Attraction-Area-centre Occur Num: 1</li><li>Request-Attraction-Area-? Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-entrance fee Occur Num: 49</li><li>request-attraction-address Occur Num: 32</li><li>request-attraction-postcode Occur Num: 20</li><li>request-attraction-phone Occur Num: 18</li><li>request-attraction-type Occur Num: 5</li><li>request-attraction-area Occur Num: 3</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-area Occur Num: 63</li><li>inform-attraction-phone Occur Num: 31</li><li>inform-attraction-type Occur Num: 18</li></ul></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 92.1 %</p><p> (Precision, Recall, F1) : (0.693, 0.832, 0.737) </p><p> Average Dialog Turn (Succ): 5.516 </p><p> Average Dialog Turn (All): 7.507 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Restaurant-Addr-? Occur Num: 6</li><li>Request-Restaurant-Post-? Occur Num: 5</li><li>Request-Restaurant-Phone-? Occur Num: 4</li><li>Inform-Restaurant-Food-dontcare Occur Num: 2</li><li>Request-Restaurant-Price-? Occur Num: 2</li><li>Inform-Restaurant-Day-sunday Occur Num: 1</li><li>Inform-Restaurant-Time-11:30 Occur Num: 1</li><li>Inform-Restaurant-Time-14:30 Occur Num: 1</li><li>Inform-Restaurant-Time-17:45 Occur Num: 1</li><li>Inform-Restaurant-Day-wednesday Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-restaurant-pricerange Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-address Occur Num: 13</li><li>request-restaurant-postcode Occur Num: 4</li><li>request-restaurant-pricerange Occur Num: 3</li><li>request-restaurant-phone Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-food Occur Num: 74</li><li>inform-restaurant-area Occur Num: 59</li><li>inform-restaurant-phone Occur Num: 20</li></ul></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 14.0 %</p><p> (Precision, Recall, F1) : (0.537, 0.341, 0.406) </p><p> Average Dialog Turn (Succ): 10.889 </p><p> Average Dialog Turn (All): 26.388 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Taxi-Car-? Occur Num: 108</li><li>Inform-Taxi-Dest-96 barton road Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-taxi-phone Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-taxi-car type Occur Num: 110</li><li>request-taxi-phone Occur Num: 60</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 21.4 %</p><p> (Precision, Recall, F1) : (0.293, 0.321, 0.272) </p><p> Average Dialog Turn (Succ): 18.150 </p><p> Average Dialog Turn (All): 32.048 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Train-Time-? Occur Num: 42</li><li>Request-Train-Ticket-? Occur Num: 26</li><li>Request-Train-Id-? Occur Num: 17</li><li>Request-Train-Arrive-? Occur Num: 6</li><li>Inform-Train-Arrive-09:15 Occur Num: 6</li><li>Inform-Train-Leave-12:15 Occur Num: 6</li><li>Inform-Train-Leave-08:15 Occur Num: 5</li><li>Request-Train-Leave-? Occur Num: 5</li><li>Inform-Train-Arrive-20:30 Occur Num: 4</li><li>Inform-Train-People-1 Occur Num: 4</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-train-arriveBy Occur Num: 95</li><li>inform-train-leaveAt Occur Num: 9</li><li>offerbook-train-arriveBy Occur Num: 1</li><li>offerbook-train-leaveAt Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-train-duration Occur Num: 130</li><li>request-train-price Occur Num: 118</li><li>request-train-trainID Occur Num: 39</li><li>request-train-leaveAt Occur Num: 21</li><li>request-train-arriveBy Occur Num: 17</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 78.3 %</p><p> (Precision, Recall, F1) : (0.783, 0.783, 0.783) </p><p> Average Dialog Turn (Succ): 3.556 </p><p> Average Dialog Turn (All): 11.478 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Police-Phone-? Occur Num: 5</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-police-phone Occur Num: 5</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 53.3 %</p><p> (Precision, Recall, F1) : (0.533, 0.533, 0.533) </p><p> Average Dialog Turn (Succ): 7.250 </p><p> Average Dialog Turn (All): 22.533 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Hospital-Phone-? Occur Num: 14</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-hospital-phone Occur Num: 14</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/DAMD/res.txt b/tests/results/DAMD/res.txt deleted file mode 100755 index e0d039d2e2d8ba5994ae6760181a2daacbae2107..0000000000000000000000000000000000000000 --- a/tests/results/DAMD/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.385 -success number of dialogs/tot: 0.336 -average precision: 0.6212610546826377 -average recall: 0.6068947897608333 -average f1: 0.5743577998956503 -average book rate: 0.5090811965811967 -average turn (succ): 10.44047619047619 -average turn (all): 28.206 diff --git a/tests/results/MDBT-RulePolicy-TemplateNLG/Frequency_of_domain.png b/tests/results/MDBT-RulePolicy-TemplateNLG/Frequency_of_domain.png deleted file mode 100755 index 484b5e916b74d64561b6598b53966389f6353466..0000000000000000000000000000000000000000 Binary files a/tests/results/MDBT-RulePolicy-TemplateNLG/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/MDBT-RulePolicy-TemplateNLG/Performance_for_each_domain.png b/tests/results/MDBT-RulePolicy-TemplateNLG/Performance_for_each_domain.png deleted file mode 100755 index 0d3d3e6a5e854bc4799dd3d58046f4c650a6f6e4..0000000000000000000000000000000000000000 Binary files a/tests/results/MDBT-RulePolicy-TemplateNLG/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/MDBT-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png b/tests/results/MDBT-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index aae3caa59b47b67d74a9239a592da8c6b03902a6..0000000000000000000000000000000000000000 Binary files a/tests/results/MDBT-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/MDBT-RulePolicy-TemplateNLG/report_multiwoz.html b/tests/results/MDBT-RulePolicy-TemplateNLG/report_multiwoz.html deleted file mode 100755 index b5051f0710145fdf8514a1fa14ecac5ea47882df..0000000000000000000000000000000000000000 --- a/tests/results/MDBT-RulePolicy-TemplateNLG/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: MDBT_RulePolicy_TemplateNLG</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-28 15:55:48</h3><h3>Overall Results</h3><p> Success Rate: 21.2 %</p><p> (Precision, Recall, F1) : (0.522, 0.410, 0.424) </p><p> Average Dialog Turn (Succ): 11.802 </p><p> Average Dialog Turn (All): 32.096 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">334</td><td align="center">0.225</td><td align="center">0.171</td><td align="center">0.173</td><td align="center">0.163</td><td align="center">0.599</td><td align="center">9.573</td><td align="center">24.581</td></tr><tr><td align="center">restaurant</td><td align="center">356</td><td align="center">0.528</td><td align="center">0.618</td><td align="center">0.581</td><td align="center">0.581</td><td align="center">0.388</td><td align="center">5.266</td><td align="center">17.938</td></tr><tr><td align="center">attraction</td><td align="center">298</td><td align="center">0.174</td><td align="center">0.499</td><td align="center">0.346</td><td align="center">0.388</td><td align="center">0.779</td><td align="center">8.577</td><td align="center">30.779</td></tr><tr><td align="center">train</td><td align="center">267</td><td align="center">0.828</td><td align="center">0.880</td><td align="center">0.901</td><td align="center">0.887</td><td align="center">0.105</td><td align="center">7.900</td><td align="center">8.172</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">1.000</td><td align="center">0.000</td><td align="center">40.000</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">0.067</td><td align="center">0.067</td><td align="center">0.067</td><td align="center">0.067</td><td align="center">0.933</td><td align="center">2.000</td><td align="center">37.467</td></tr><tr><td align="center">taxi</td><td align="center">51</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.882</td><td align="center">0.000</td><td align="center">35.294</td></tr></table><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 22.5 %</p><p> (Precision, Recall, F1) : (0.171, 0.173, 0.163) </p><p> Average Dialog Turn (Succ): 9.573 </p><p> Average Dialog Turn (All): 24.581 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 250</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 51</li><li>Inform-Train-Ticket-23.60 pounds Occur Num: 12</li><li>Inform-Hotel-Internet-none Occur Num: 10</li><li>Inform-Hotel-Type-guesthouse Occur Num: 8</li><li>Inform-Hotel-Choice-2 Occur Num: 7</li></ul></ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 168</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-none Occur Num: 41</li><li>Inform-Hotel-Parking-none Occur Num: 12</li><li>Inform-Hotel-Type-hotel Occur Num: 9</li><li>Inform-Hotel-Type-guesthouse Occur Num: 8</li><li>Inform-Hotel-Area-centre Occur Num: 6</li></ul></ul><li>NoOffer-Hotel-Internet-yes<ul><li>Occur Num: 139</li><li>NLU Output</li><ul><li>NoOffer-Hotel-Type-hotels Occur Num: 17</li><li>NoOffer-Hotel-Internet-none Occur Num: 14</li><li>NoOffer-Hotel-none-none Occur Num: 10</li><li>NoOffer-Hotel-Type-guesthouse Occur Num: 10</li><li>NoOffer-Hotel-Price-moderate Occur Num: 8</li></ul></ul><li>Inform-Hotel-Choice-2<ul><li>Occur Num: 136</li><li>NLU Output</li><ul><li>Inform-Train-Ticket-23.60 pounds Occur Num: 8</li><li>Inform-Hotel-Parking-none Occur Num: 7</li><li>Inform-Hotel-Choice-18 Occur Num: 5</li><li>Recommend-Hotel-Name-university arms hotel Occur Num: 4</li><li>Inform-Hotel-Choice-44 Occur Num: 4</li></ul></ul><li>NoOffer-Hotel-Price-moderate<ul><li>Occur Num: 126</li><li>NLU Output</li><ul><li>NoOffer-Hotel-Area-centre Occur Num: 13</li><li>NoOffer-Hotel-Type-hotels Occur Num: 12</li><li>NoOffer-Hotel-Type-guesthouse Occur Num: 7</li><li>NoOffer-Hotel-none-none Occur Num: 7</li><li>Inform-Train-Ticket-23.60 pounds Occur Num: 7</li></ul></ul><li>NoOffer-Hotel-Type-guesthouse<ul><li>Occur Num: 122</li><li>NLU Output</li><ul><li>NoOffer-Hotel-Type-hotels Occur Num: 12</li><li>NoOffer-Hotel-none-none Occur Num: 9</li><li>NoOffer-Hotel-Price-expensive Occur Num: 8</li><li>NoOffer-Hotel-Area-centre Occur Num: 7</li><li>NoOffer-Hotel-Price-moderate Occur Num: 7</li></ul></ul><li>NoOffer-Hotel-Parking-yes<ul><li>Occur Num: 118</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 11</li><li>NoOffer-Hotel-Parking-none Occur Num: 11</li><li>reqmore-general-none-none Occur Num: 8</li><li>NoOffer-Hotel-Type-hotels Occur Num: 8</li><li>NoOffer-Hotel-Price-moderate Occur Num: 7</li></ul></ul><li>NoOffer-Hotel-Area-centre<ul><li>Occur Num: 106</li><li>NLU Output</li><ul><li>NoOffer-Hotel-Price-moderate Occur Num: 13</li><li>NoOffer-Hotel-Type-hotels Occur Num: 11</li><li>NoOffer-Hotel-Type-guesthouse Occur Num: 7</li><li>NoOffer-Hotel-none-none Occur Num: 7</li><li>NoOffer-Hotel-Stars-4 Occur Num: 6</li></ul></ul><li>Inform-Hotel-Choice-5<ul><li>Occur Num: 104</li><li>NLU Output</li><ul><li>Inform-Train-Choice-5 Occur Num: 5</li><li>Inform-Hotel-Parking-none Occur Num: 4</li><li>Request-Train-Arrive-? Occur Num: 4</li><li>Inform-Restaurant-Choice-69 Occur Num: 4</li><li>Inform-Train-Ticket-23.60 pounds Occur Num: 4</li></ul></ul><li>NoOffer-Hotel-Price-expensive<ul><li>Occur Num: 101</li><li>NLU Output</li><ul><li>NoOffer-Hotel-Type-hotels Occur Num: 9</li><li>NoOffer-Hotel-Type-guesthouse Occur Num: 9</li><li>NoOffer-Hotel-none-none Occur Num: 8</li><li>NoOffer-Hotel-Internet-none Occur Num: 6</li><li>NoOffer-Hotel-Parking-none Occur Num: 5</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Hotel-Addr-? Occur Num: 72</li><li>Request-Hotel-Phone-? Occur Num: 49</li><li>Request-Hotel-Post-? Occur Num: 39</li><li>Request-Hotel-Price-? Occur Num: 15</li><li>Request-Hotel-Area-? Occur Num: 8</li><li>Inform-Hotel-Stay-4 Occur Num: 4</li><li>Inform-Hotel-Stay-2 Occur Num: 2</li><li>Inform-Hotel-Type-hotel Occur Num: 1</li><li>Inform-Hotel-Stay-3 Occur Num: 1</li><li>Inform-Hotel-Day-tuesday Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-parking Occur Num: 78</li><li>inform-hotel-internet Occur Num: 47</li><li>inform-hotel-stars Occur Num: 5</li><li>inform-hotel-area Occur Num: 4</li><li>recommend-hotel-internet Occur Num: 3</li><li>recommend-hotel-phone Occur Num: 3</li><li>inform-hotel-phone Occur Num: 3</li><li>recommend-hotel-parking Occur Num: 2</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-address Occur Num: 96</li><li>request-hotel-phone Occur Num: 67</li><li>request-hotel-postcode Occur Num: 52</li><li>request-hotel-internet Occur Num: 23</li><li>request-hotel-parking Occur Num: 19</li><li>request-hotel-pricerange Occur Num: 16</li><li>request-hotel-area Occur Num: 10</li><li>request-hotel-stars Occur Num: 3</li><li>request-hotel-type Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-phone Occur Num: 18</li><li>inform-hotel-stars Occur Num: 13</li><li>inform-hotel-area Occur Num: 11</li><li>inform-hotel-type Occur Num: 10</li></ul></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 52.8 %</p><p> (Precision, Recall, F1) : (0.618, 0.581, 0.581) </p><p> Average Dialog Turn (Succ): 5.266 </p><p> Average Dialog Turn (All): 17.938 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Restaurant-Choice-9<ul><li>Occur Num: 153</li><li>NLU Output</li><ul><li>Inform-Restaurant-Choice-13 Occur Num: 7</li><li>Request-Train-Arrive-? Occur Num: 6</li><li>Request-Taxi-Arrive-? Occur Num: 5</li><li>Inform-Restaurant-Choice-44 Occur Num: 5</li><li>Inform-Restaurant-Choice-10 Occur Num: 4</li></ul></ul><li>Inform-Restaurant-Choice-2<ul><li>Occur Num: 151</li><li>NLU Output</li><ul><li>Inform-Restaurant-Choice-4 Occur Num: 9</li><li>Inform-Restaurant-Choice-44 Occur Num: 4</li><li>Recommend-Attraction-Name-kings hedges learner pool Occur Num: 3</li><li>Inform-Restaurant-Choice-8 Occur Num: 3</li><li>Recommend-Restaurant-Name-Nandos Occur Num: 3</li></ul></ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 147</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 10</li><li>Recommend-Restaurant-Area-centre Occur Num: 9</li><li>Inform-Restaurant-Choice-44 Occur Num: 7</li><li>Inform-Booking-none-none Occur Num: 5</li><li>Inform-Restaurant-Choice-33 Occur Num: 4</li></ul></ul><li>Inform-Restaurant-Choice-5<ul><li>Occur Num: 131</li><li>NLU Output</li><ul><li>Inform-Restaurant-Choice-10 Occur Num: 7</li><li>Inform-Restaurant-Choice-8 Occur Num: 6</li><li>Recommend-Restaurant-Food-chinese Occur Num: 3</li><li>Inform-Restaurant-Choice-4 Occur Num: 3</li><li>Recommend-Restaurant-Name-restaurant two two Occur Num: 3</li></ul></ul><li>Inform-Restaurant-Choice-69<ul><li>Occur Num: 127</li><li>NLU Output</li><ul><li>Inform-Restaurant-Choice-44 Occur Num: 10</li><li>Request-Train-Arrive-? Occur Num: 6</li><li>Inform-Train-Choice-5 Occur Num: 5</li><li>Request-Taxi-Arrive-? Occur Num: 4</li><li>Recommend-Restaurant-Area-centre Occur Num: 4</li></ul></ul><li>Inform-Restaurant-Choice-33<ul><li>Occur Num: 127</li><li>NLU Output</li><ul><li>Inform-Restaurant-Choice-44 Occur Num: 18</li><li>Inform-Booking-none-none Occur Num: 6</li><li>Recommend-Restaurant-Name-The Occur Num: 3</li><li>Inform-Attraction-Choice-33 Occur Num: 3</li><li>Recommend-Attraction-Name-hughes hall Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Choice-3<ul><li>Occur Num: 125</li><li>NLU Output</li><ul><li>Inform-Restaurant-Choice-44 Occur Num: 5</li><li>Inform-Restaurant-Choice-10 Occur Num: 3</li><li>Request-Train-Arrive-? Occur Num: 3</li><li>Inform-Restaurant-Choice-13 Occur Num: 3</li><li>Recommend-Restaurant-Name-pizza hut fen ditton Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Food-indian<ul><li>Occur Num: 109</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Food-indian Occur Num: 10</li><li>Inform-Restaurant-Choice-4 Occur Num: 7</li><li>Inform-Restaurant-Choice-14 Occur Num: 5</li><li>Inform-Restaurant-Choice-44 Occur Num: 5</li><li>Inform-Restaurant-Choice-5 Occur Num: 3</li></ul></ul><li>Inform-Restaurant-Choice-21<ul><li>Occur Num: 109</li><li>NLU Output</li><ul><li>Inform-Restaurant-Choice-44 Occur Num: 17</li><li>Inform-Restaurant-Area-centre Occur Num: 4</li><li>Recommend-Restaurant-Area-centre Occur Num: 3</li><li>Recommend-Restaurant-Food-italian Occur Num: 3</li><li>Recommend-Restaurant-Name-Cotto Occur Num: 3</li></ul></ul><li>Inform-Restaurant-Choice-4<ul><li>Occur Num: 103</li><li>NLU Output</li><ul><li>Inform-Restaurant-Food-indian Occur Num: 5</li><li>Inform-Restaurant-Choice-44 Occur Num: 5</li><li>Inform-Restaurant-Area-centre Occur Num: 4</li><li>Inform-Restaurant-Food-chinese Occur Num: 3</li><li>Inform-Restaurant-Price-expensive Occur Num: 3</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Restaurant-Phone-? Occur Num: 119</li><li>Request-Restaurant-Post-? Occur Num: 4</li><li>Request-Restaurant-Addr-? Occur Num: 3</li><li>Request-Restaurant-Price-? Occur Num: 3</li><li>Inform-Restaurant-Time-11:00 Occur Num: 2</li><li>Request-Restaurant-Area-? Occur Num: 2</li><li>Inform-Restaurant-Time-13:45 Occur Num: 1</li><li>Inform-Restaurant-Time-17:00 Occur Num: 1</li><li>Inform-Restaurant-Time-14:30 Occur Num: 1</li><li>Inform-Restaurant-Time-17:45 Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>recommend-restaurant-area Occur Num: 2</li><li>inform-restaurant-area Occur Num: 1</li><li>recommend-restaurant-postcode Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-phone Occur Num: 123</li><li>request-restaurant-address Occur Num: 11</li><li>request-restaurant-postcode Occur Num: 8</li><li>request-restaurant-pricerange Occur Num: 6</li><li>request-restaurant-area Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-phone Occur Num: 26</li><li>inform-restaurant-area Occur Num: 11</li><li>inform-restaurant-food Occur Num: 4</li></ul></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 17.4 %</p><p> (Precision, Recall, F1) : (0.499, 0.346, 0.388) </p><p> Average Dialog Turn (Succ): 8.577 </p><p> Average Dialog Turn (All): 30.779 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Attraction-Choice-44<ul><li>Occur Num: 369</li><li>NLU Output</li><ul><li>Request-Train-Arrive-? Occur Num: 28</li><li>Inform-Train-Choice-5 Occur Num: 19</li><li>Inform-Restaurant-Choice-69 Occur Num: 16</li><li>Inform-Train-Choice-44 Occur Num: 13</li><li>Request-Train-Leave-? Occur Num: 12</li></ul></ul><li>Inform-Attraction-Choice-4<ul><li>Occur Num: 219</li><li>NLU Output</li><ul><li>Request-Train-Arrive-? Occur Num: 28</li><li>Inform-Train-Choice-4 Occur Num: 11</li><li>Request-Train-Leave-? Occur Num: 8</li><li>Inform-Restaurant-Choice-9 Occur Num: 6</li><li>Inform-Train-Choice-5 Occur Num: 5</li></ul></ul><li>Inform-Attraction-Choice-13<ul><li>Occur Num: 145</li><li>NLU Output</li><ul><li>Request-Train-Arrive-? Occur Num: 11</li><li>Inform-Attraction-Choice-14 Occur Num: 6</li><li>Recommend-Hotel-Name-huntingdon marriott hotel Occur Num: 4</li><li>Inform-Attraction-Choice-4 Occur Num: 4</li><li>Recommend-Attraction-Fee-free Occur Num: 4</li></ul></ul><li>Inform-Attraction-Fee-?<ul><li>Occur Num: 113</li><li>NLU Output</li><ul><li>Inform-Attraction-Post-cb21ab Occur Num: 23</li><li>Inform-Attraction-Addr-Regent Street City Centre Occur Num: 17</li><li>Request-Train-Leave-? Occur Num: 11</li><li>Request-Train-Arrive-? Occur Num: 10</li><li>Inform-Attraction-Type-park Occur Num: 8</li></ul></ul><li>Inform-Attraction-Choice-11<ul><li>Occur Num: 99</li><li>NLU Output</li><ul><li>Inform-Train-Choice-5 Occur Num: 8</li><li>Inform-Restaurant-Choice-69 Occur Num: 7</li><li>Request-Train-Leave-? Occur Num: 6</li><li>Request-Train-Arrive-? Occur Num: 6</li><li>Inform-Hotel-Type-guesthouse Occur Num: 4</li></ul></ul><li>Recommend-Attraction-Fee-free<ul><li>Occur Num: 95</li><li>NLU Output</li><ul><li>Request-Train-Arrive-? Occur Num: 8</li><li>Inform-Restaurant-Choice-69 Occur Num: 5</li><li>Inform-Attraction-Fee-free Occur Num: 4</li><li>Inform-Attraction-Choice-13 Occur Num: 3</li><li>Inform-Attraction-Choice-5 Occur Num: 3</li></ul></ul><li>Inform-Attraction-Choice-5<ul><li>Occur Num: 94</li><li>NLU Output</li><ul><li>Request-Train-Arrive-? Occur Num: 10</li><li>Inform-Train-Choice-5 Occur Num: 9</li><li>Inform-Attraction-Choice-69 Occur Num: 6</li><li>Inform-Hotel-Type-guesthouse Occur Num: 3</li><li>Request-Train-Leave-? Occur Num: 3</li></ul></ul><li>NoOffer-Attraction-Area-centre<ul><li>Occur Num: 89</li><li>NLU Output</li><ul><li>NoOffer-Attraction-Type-entertainment Occur Num: 8</li><li>Inform-Attraction-Choice-5 Occur Num: 8</li><li>Request-Train-Arrive-? Occur Num: 7</li><li>Inform-Restaurant-Choice-69 Occur Num: 7</li><li>Inform-Attraction-Choice-69 Occur Num: 5</li></ul></ul><li>Inform-Attraction-Choice-18<ul><li>Occur Num: 70</li><li>NLU Output</li><ul><li>Request-Train-Arrive-? Occur Num: 10</li><li>Inform-Train-Choice-18 Occur Num: 7</li><li>Request-Train-Day-? Occur Num: 4</li><li>Inform-Hotel-Type-guesthouse Occur Num: 3</li><li>Recommend-Attraction-Name-trinity college Occur Num: 3</li></ul></ul><li>Recommend-Attraction-Area-centre<ul><li>Occur Num: 63</li><li>NLU Output</li><ul><li>Inform-Train-Choice-5 Occur Num: 4</li><li>Inform-Attraction-Choice-5 Occur Num: 3</li><li>Inform-Attraction-Area-centre Occur Num: 3</li><li>Inform-Attraction-Choice-69 Occur Num: 3</li><li>Inform-Restaurant-Choice-69 Occur Num: 3</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Attraction-Fee-? Occur Num: 82</li><li>Request-Attraction-Addr-? Occur Num: 71</li><li>Request-Attraction-Phone-? Occur Num: 47</li><li>Request-Attraction-Post-? Occur Num: 23</li><li>Request-Attraction-Type-? Occur Num: 9</li></ul><b> Bad Inform Dialog Act</b><ul><li>recommend-attraction-phone Occur Num: 4</li><li>inform-attraction-phone Occur Num: 3</li><li>inform-attraction-area Occur Num: 2</li><li>recommend-attraction-postcode Occur Num: 1</li><li>inform-attraction-postcode Occur Num: 1</li><li>recommend-attraction-area Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-phone Occur Num: 134</li><li>request-attraction-address Occur Num: 126</li><li>request-attraction-entrance fee Occur Num: 78</li><li>request-attraction-postcode Occur Num: 66</li><li>request-attraction-type Occur Num: 17</li><li>request-attraction-area Occur Num: 6</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-phone Occur Num: 16</li><li>inform-attraction-type Occur Num: 12</li><li>inform-attraction-area Occur Num: 3</li></ul></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 82.8 %</p><p> (Precision, Recall, F1) : (0.880, 0.901, 0.887) </p><p> Average Dialog Turn (Succ): 7.900 </p><p> Average Dialog Turn (All): 8.172 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>Request-Train-Leave-?<ul><li>Occur Num: 231</li><li>NLU Output</li><ul><li>Request-Train-Arrive-? Occur Num: 44</li><li>Request-Taxi-Arrive-? Occur Num: 31</li><li>Request-Taxi-Leave-? Occur Num: 22</li><li>Inform-Train-Choice-5 Occur Num: 8</li><li>Request-Train-Day-? Occur Num: 7</li></ul></ul><li>Request-Train-Arrive-?<ul><li>Occur Num: 160</li><li>NLU Output</li><ul><li>Request-Taxi-Arrive-? Occur Num: 40</li><li>Request-Taxi-Leave-? Occur Num: 10</li><li>Inform-Train-Choice-5 Occur Num: 6</li><li>Request-Train-Leave-? Occur Num: 5</li><li>Inform-Hotel-Type-guesthouse Occur Num: 3</li></ul></ul><li>Inform-Train-Ticket-23.60 pounds<ul><li>Occur Num: 80</li><li>NLU Output</li><ul><li>Inform-Train-Time-51 minutes Occur Num: 4</li><li>Inform-Hotel-Parking-none Occur Num: 4</li><li>Inform-Train-Time-one Occur Num: 3</li><li>NoOffer-Hotel-Price-moderate Occur Num: 3</li><li>Inform-Hotel-Choice-18 Occur Num: 3</li></ul></ul><li>Inform-Train-Leave-05:00<ul><li>Occur Num: 42</li><li>NLU Output</li><ul><li>Select-Train-Leave-05:00 Occur Num: 11</li><li>Select-Train-none-none Occur Num: 10</li><li>Inform-Train-Arrive-05:51 Occur Num: 3</li><li>Inform-Train-Arrive-05:00 Occur Num: 2</li><li>NoOffer-Restaurant-Food-indian Occur Num: 1</li></ul></ul><li>Request-Train-Day-?<ul><li>Occur Num: 27</li><li>NLU Output</li><ul><li>Request-Train-Arrive-? Occur Num: 4</li><li>Request-Taxi-Arrive-? Occur Num: 2</li><li>Inform-Attraction-Choice-4 Occur Num: 1</li><li>Recommend-Attraction-Name-aylesbray Occur Num: 1</li><li>Recommend-Attraction-Phone-01223240089 Occur Num: 1</li></ul></ul><li>Inform-Train-Time-51 minutes<ul><li>Occur Num: 24</li><li>NLU Output</li><ul><li>Inform-Train-Time-one way Occur Num: 10</li><li>Inform-Train-Ticket-23.60 pounds Occur Num: 5</li><li>Inform-Train-Time-one Occur Num: 3</li><li>Inform-Train-Arrive-05:51 Occur Num: 2</li><li>Inform-Train-Depart-cambridge Occur Num: 1</li></ul></ul><li>Inform-Train-Depart-cambridge<ul><li>Occur Num: 9</li><li>NLU Output</li><ul><li>Inform-Train-Ticket-23.60 pounds Occur Num: 1</li><li>Inform-Train-Time-51 minutes Occur Num: 1</li><li>Inform-Train-Time-one Occur Num: 1</li><li>Request-Restaurant-Area-? Occur Num: 1</li><li>Request-Restaurant-Price-? Occur Num: 1</li></ul></ul><li>Inform-Train-Day-monday<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>OfferBooked-Train-Day-monday Occur Num: 6</li><li>Book-Booking-Day-monday Occur Num: 1</li></ul></ul><li>Inform-Train-Arrive-05:51<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Train-Time-51 minutes Occur Num: 2</li><li>Inform-Train-Arrive-05:00 Occur Num: 2</li><li>Inform-Train-Leave-05:00 Occur Num: 1</li><li>Inform-Train-Leave-would Occur Num: 1</li><li>Inform-Train-Time-one way Occur Num: 1</li></ul></ul><li>Request-Train-Depart-?<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Request-Train-Leave-? Occur Num: 3</li><li>Request-Train-Dest-? Occur Num: 1</li><li>Request-Train-Arrive-? Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Inform-Train-People-5 Occur Num: 5</li><li>Inform-Train-People-3 Occur Num: 5</li><li>Inform-Train-People-1 Occur Num: 4</li><li>Inform-Train-People-7 Occur Num: 4</li><li>Inform-Train-People-6 Occur Num: 3</li><li>Inform-Train-People-2 Occur Num: 2</li><li>Inform-Train-People-8 Occur Num: 2</li><li>Inform-Train-People-4 Occur Num: 1</li><li>Request-Train-Ticket-? Occur Num: 1</li><li>Inform-Train-Leave-09:45 Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-train-duration Occur Num: 13</li><li>inform-train-leaveAt Occur Num: 1</li><li>inform-train-price Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-train-duration Occur Num: 4</li><li>request-train-price Occur Num: 3</li><li>request-train-trainID Occur Num: 2</li><li>request-train-leaveAt Occur Num: 1</li><li>request-train-arriveBy Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 0.0 %</p><p> (Precision, Recall, F1) : (0.000, 0.000, 0.000) </p><p> Average Dialog Turn (Succ): 0.000 </p><p> Average Dialog Turn (All): 40.000 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Police-Addr-? Occur Num: 17</li><li>Request-Police-Phone-? Occur Num: 6</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-police-phone Occur Num: 20</li><li>request-police-address Occur Num: 17</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 6.7 %</p><p> (Precision, Recall, F1) : (0.067, 0.067, 0.067) </p><p> Average Dialog Turn (Succ): 2.000 </p><p> Average Dialog Turn (All): 37.467 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hospital-Department-neurosciences critical care unit<ul><li>Occur Num: 34</li><li>NLU Output</li><ul><li>Request-Train-Arrive-? Occur Num: 4</li><li>Request-Taxi-Leave-? Occur Num: 1</li><li>Inform-Restaurant-Choice-5 Occur Num: 1</li><li>Inform-Hotel-Name-gonville hotel Occur Num: 1</li><li>Inform-Police-Phone-01223366611 Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Hospital-Phone-? Occur Num: 28</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-hospital-phone Occur Num: 28</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 0.0 %</p><p> (Precision, Recall, F1) : (0.000, 0.000, 0.000) </p><p> Average Dialog Turn (Succ): 0.000 </p><p> Average Dialog Turn (All): 35.294 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Leave-?<ul><li>Occur Num: 173</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-guesthouse Occur Num: 44</li><li>Request-Taxi-Arrive-? Occur Num: 40</li><li>Request-Train-Arrive-? Occur Num: 9</li><li>Inform-Train-Time-51 minutes Occur Num: 9</li><li>Inform-Train-Leave-05:00 Occur Num: 9</li></ul></ul><li>Request-Taxi-Arrive-?<ul><li>Occur Num: 97</li><li>NLU Output</li><ul><li>Request-Taxi-Leave-? Occur Num: 22</li><li>Inform-Hotel-Type-guesthouse Occur Num: 20</li><li>Request-Train-Arrive-? Occur Num: 8</li><li>Inform-Train-Leave-05:00 Occur Num: 8</li><li>Inform-Train-Time-51 minutes Occur Num: 7</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Taxi-Car-? Occur Num: 30</li><li>Inform-Taxi-Arrive-dontcare Occur Num: 8</li><li>Inform-Taxi-Leave-dontcare Occur Num: 2</li><li>Inform-Taxi-Leave-05:15 Occur Num: 1</li><li>Inform-Taxi-Arrive-17:15 Occur Num: 1</li><li>Inform-Taxi-Arrive-19:30 Occur Num: 1</li><li>Inform-Taxi-Arrive-19:00 Occur Num: 1</li><li>Inform-Taxi-Arrive-13:15 Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-taxi-phone Occur Num: 51</li><li>request-taxi-car type Occur Num: 51</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/MDBT-RulePolicy-TemplateNLG/res.txt b/tests/results/MDBT-RulePolicy-TemplateNLG/res.txt deleted file mode 100755 index 942654faa93434735a073b46347891dcd315199e..0000000000000000000000000000000000000000 --- a/tests/results/MDBT-RulePolicy-TemplateNLG/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.277 -success number of dialogs/tot: 0.212 -average precision: 0.5221579389803689 -average recall: 0.4095551270006098 -average f1: 0.4241175318278122 -average book rate: 0.4543376068376068 -average turn (succ): 11.80188679245283 -average turn (all): 32.096 diff --git a/tests/results/MILU-RuleDST-RulePolicy-TemplateNLG/Frequency_of_domain.png b/tests/results/MILU-RuleDST-RulePolicy-TemplateNLG/Frequency_of_domain.png deleted file mode 100755 index 1ffd0bdb2c73ac3b3f8580921baea9ceb593f158..0000000000000000000000000000000000000000 Binary files a/tests/results/MILU-RuleDST-RulePolicy-TemplateNLG/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/MILU-RuleDST-RulePolicy-TemplateNLG/Performance_for_each_domain.png b/tests/results/MILU-RuleDST-RulePolicy-TemplateNLG/Performance_for_each_domain.png deleted file mode 100755 index 63d600729988b23e807a69107239acb6c179c97e..0000000000000000000000000000000000000000 Binary files a/tests/results/MILU-RuleDST-RulePolicy-TemplateNLG/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/MILU-RuleDST-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png b/tests/results/MILU-RuleDST-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index 9e3fcf80fd6e4cb367adf502d94130b7c144436b..0000000000000000000000000000000000000000 Binary files a/tests/results/MILU-RuleDST-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/MILU-RuleDST-RulePolicy-TemplateNLG/report_multiwoz.html b/tests/results/MILU-RuleDST-RulePolicy-TemplateNLG/report_multiwoz.html deleted file mode 100755 index 61a75df4f564bfd46d868ee20b179d833cbbd4fe..0000000000000000000000000000000000000000 --- a/tests/results/MILU-RuleDST-RulePolicy-TemplateNLG/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: MILU-RuleDST-RulePolicy-TemplateNLG</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-24 21:12:47</h3><h3>Overall Results</h3><p> Success Rate: 83.1 %</p><p> (Precision, Recall, F1) : (0.783, 0.917, 0.825) </p><p> Average Dialog Turn (Succ): 12.144 </p><p> Average Dialog Turn (All): 13.854 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">429</td><td align="center">0.751</td><td align="center">0.482</td><td align="center">0.649</td><td align="center">0.526</td><td align="center">0.058</td><td align="center">5.596</td><td align="center">6.410</td></tr><tr><td align="center">restaurant</td><td align="center">411</td><td align="center">0.954</td><td align="center">0.801</td><td align="center">0.850</td><td align="center">0.818</td><td align="center">0.032</td><td align="center">4.949</td><td align="center">5.781</td></tr><tr><td align="center">attraction</td><td align="center">331</td><td align="center">0.940</td><td align="center">0.878</td><td align="center">0.953</td><td align="center">0.905</td><td align="center">0.027</td><td align="center">5.633</td><td align="center">6.278</td></tr><tr><td align="center">taxi</td><td align="center">173</td><td align="center">0.879</td><td align="center">0.908</td><td align="center">0.893</td><td align="center">0.898</td><td align="center">0.121</td><td align="center">7.026</td><td align="center">9.399</td></tr><tr><td align="center">train</td><td align="center">389</td><td align="center">0.995</td><td align="center">0.840</td><td align="center">0.928</td><td align="center">0.867</td><td align="center">0.005</td><td align="center">5.767</td><td align="center">5.779</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">0.913</td><td align="center">0.913</td><td align="center">0.913</td><td align="center">0.913</td><td align="center">0.087</td><td align="center">5.524</td><td align="center">8.522</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">0.000</td><td align="center">4.067</td><td align="center">4.067</td></tr></table><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 75.1 %</p><p> (Precision, Recall, F1) : (0.482, 0.649, 0.526) </p><p> Average Dialog Turn (Succ): 5.596 </p><p> Average Dialog Turn (All): 6.410 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 381</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-hotel Occur Num: 85</li><li>Inform-Hotel-Stars-4 Occur Num: 45</li><li>Inform-Hotel-Type-guesthouse Occur Num: 44</li><li>Inform-Hotel-Internet-yes Occur Num: 41</li><li>Inform-Hotel-Price-moderate Occur Num: 34</li></ul></ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 369</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-hotel Occur Num: 50</li><li>Inform-Hotel-Type-guesthouse Occur Num: 49</li><li>Inform-Hotel-Stars-4 Occur Num: 42</li><li>Inform-Hotel-Price-moderate Occur Num: 40</li><li>Inform-Hotel-Parking-yes Occur Num: 39</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 259</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 40</li><li>Inform-Hotel-Internet-yes Occur Num: 33</li><li>Inform-Hotel-Stars-4 Occur Num: 30</li><li>Inform-Hotel-Price-moderate Occur Num: 28</li><li>Inform-Hotel-Type-hotel Occur Num: 24</li></ul></ul><li>Inform-Hotel-Stars-4<ul><li>Occur Num: 242</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-hotel Occur Num: 41</li><li>Inform-Hotel-Parking-yes Occur Num: 37</li><li>Inform-Hotel-Internet-yes Occur Num: 31</li><li>Inform-Hotel-Type-guesthouse Occur Num: 26</li><li>Inform-Hotel-Price-moderate Occur Num: 25</li></ul></ul><li>Inform-Hotel-Price-moderate<ul><li>Occur Num: 240</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-hotel Occur Num: 45</li><li>Inform-Hotel-Internet-yes Occur Num: 30</li><li>Inform-Hotel-Type-guesthouse Occur Num: 28</li><li>Inform-Hotel-Parking-yes Occur Num: 26</li><li>Inform-Hotel-Stars-4 Occur Num: 25</li></ul></ul><li>Inform-Hotel-Area-north<ul><li>Occur Num: 121</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 22</li><li>Inform-Hotel-Parking-yes Occur Num: 21</li><li>Inform-Hotel-Type-guesthouse Occur Num: 18</li><li>Inform-Hotel-Price-moderate Occur Num: 15</li><li>Inform-Hotel-Stars-4 Occur Num: 13</li></ul></ul><li>Inform-Hotel-Price-cheap<ul><li>Occur Num: 112</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-hotel Occur Num: 20</li><li>Inform-Hotel-Stars-4 Occur Num: 18</li><li>Inform-Hotel-Type-guesthouse Occur Num: 15</li><li>Inform-Hotel-Internet-yes Occur Num: 15</li><li>Inform-Hotel-Parking-yes Occur Num: 15</li></ul></ul><li>Request-Hotel-Addr-?<ul><li>Occur Num: 108</li><li>NLU Output</li><ul><li>Request-Hotel-Post-? Occur Num: 26</li><li>Request-Hotel-Phone-? Occur Num: 20</li><li>Request-Hotel-Internet-? Occur Num: 13</li><li>Request-Hotel-Price-? Occur Num: 7</li><li>Request-Hotel-Type-? Occur Num: 6</li></ul></ul><li>Inform-Hotel-Type-hotel<ul><li>Occur Num: 103</li><li>NLU Output</li><ul><li>Inform-Hotel-Stars-4 Occur Num: 15</li><li>Inform-Hotel-Parking-yes Occur Num: 13</li><li>Inform-Hotel-Price-moderate Occur Num: 13</li><li>Inform-Hotel-Area-east Occur Num: 10</li><li>Inform-Hotel-Internet-yes Occur Num: 10</li></ul></ul><li>Inform-Hotel-Parking-no<ul><li>Occur Num: 99</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 23</li><li>Inform-Hotel-Type-hotel Occur Num: 16</li><li>Inform-Hotel-Price-moderate Occur Num: 11</li><li>Inform-Hotel-Type-guesthouse Occur Num: 9</li><li>Inform-Hotel-Internet-yes Occur Num: 9</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 533</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-none Occur Num: 100</li><li>Inform-Hotel-Parking-none Occur Num: 66</li><li>Inform-Hotel-Stars-4 Occur Num: 32</li><li>Inform-Hotel-Type-guesthouse Occur Num: 30</li><li>Inform-Hotel-Type-hotel Occur Num: 22</li></ul></ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 438</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 80</li><li>Inform-Hotel-Internet-none Occur Num: 58</li><li>Inform-Hotel-Stars-4 Occur Num: 31</li><li>Inform-Hotel-Type-guesthouse Occur Num: 30</li><li>Inform-Hotel-Price-cheap Occur Num: 20</li></ul></ul><li>NoOffer-Hotel-Type-hotel<ul><li>Occur Num: 316</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 46</li><li>NoOffer-Hotel-Price-moderate Occur Num: 30</li><li>NoOffer-Hotel-Parking-none Occur Num: 27</li><li>Request-Hotel-Price-? Occur Num: 25</li><li>NoOffer-Hotel-Stars-4 Occur Num: 22</li></ul></ul><li>NoOffer-Hotel-Parking-yes<ul><li>Occur Num: 270</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 32</li><li>NoOffer-Hotel-Parking-none Occur Num: 31</li><li>NoOffer-Hotel-Type-hotel Occur Num: 23</li><li>reqmore-general-none-none Occur Num: 20</li><li>NoOffer-Hotel-Price-moderate Occur Num: 18</li></ul></ul><li>NoOffer-Hotel-Price-moderate<ul><li>Occur Num: 260</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 37</li><li>NoOffer-Hotel-Type-hotel Occur Num: 25</li><li>NoOffer-Hotel-Type-hotels Occur Num: 22</li><li>Request-Hotel-Price-? Occur Num: 20</li><li>NoOffer-Hotel-Parking-none Occur Num: 18</li></ul></ul><li>Inform-Hotel-Stars-4<ul><li>Occur Num: 243</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-none Occur Num: 37</li><li>Inform-Hotel-Parking-none Occur Num: 35</li><li>Inform-Hotel-Type-guesthouse Occur Num: 23</li><li>Inform-Hotel-Price-cheap Occur Num: 19</li><li>Inform-Hotel-Type-hotel Occur Num: 14</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 214</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 31</li><li>Inform-Hotel-Internet-none Occur Num: 30</li><li>Inform-Hotel-Stars-4 Occur Num: 18</li><li>Inform-Hotel-Price-cheap Occur Num: 13</li><li>Inform-Hotel-Price-moderate Occur Num: 13</li></ul></ul><li>NoOffer-Hotel-Internet-yes<ul><li>Occur Num: 201</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 24</li><li>NoOffer-Hotel-Internet-none Occur Num: 23</li><li>NoOffer-Hotel-Type-hotels Occur Num: 15</li><li>NoOffer-Hotel-Type-hotel Occur Num: 15</li><li>NoOffer-Hotel-Price-moderate Occur Num: 14</li></ul></ul><li>Inform-Hotel-Type-hotel<ul><li>Occur Num: 187</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-none Occur Num: 29</li><li>Inform-Hotel-Parking-none Occur Num: 26</li><li>Inform-Hotel-Stars-4 Occur Num: 11</li><li>Inform-Hotel-Area-west Occur Num: 10</li><li>Inform-Hotel-Price-expensive Occur Num: 9</li></ul></ul><li>Request-Hotel-Area-?<ul><li>Occur Num: 156</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 19</li><li>Request-Hotel-Price-? Occur Num: 18</li><li>NoOffer-Hotel-Price-moderate Occur Num: 12</li><li>NoOffer-Hotel-Area-centre Occur Num: 11</li><li>NoOffer-Hotel-Type-hotels Occur Num: 11</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Inform-Hotel-Price-moderate Occur Num: 4</li><li>Request-Hotel-Internet-? Occur Num: 3</li><li>Inform-Hotel-Parking-yes Occur Num: 3</li><li>Request-Hotel-Addr-? Occur Num: 2</li><li>Inform-Hotel-Internet-yes Occur Num: 2</li><li>Request-Hotel-Phone-? Occur Num: 2</li><li>Inform-Hotel-Area-centre Occur Num: 1</li><li>Inform-Hotel-Area-east Occur Num: 1</li><li>Inform-Hotel-Stay-4 Occur Num: 1</li><li>Inform-Hotel-Name-alexander bed and breakfast Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-internet Occur Num: 117</li><li>inform-hotel-parking Occur Num: 116</li><li>recommend-hotel-parking Occur Num: 20</li><li>inform-hotel-area Occur Num: 20</li><li>recommend-hotel-internet Occur Num: 14</li><li>inform-hotel-stars Occur Num: 13</li><li>inform-hotel-postcode Occur Num: 2</li><li>recommend-hotel-stars Occur Num: 2</li><li>recommend-hotel-area Occur Num: 1</li><li>recommend-hotel-phone Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-internet Occur Num: 29</li><li>request-hotel-parking Occur Num: 25</li><li>request-hotel-address Occur Num: 15</li><li>request-hotel-phone Occur Num: 14</li><li>request-hotel-postcode Occur Num: 11</li><li>request-hotel-area Occur Num: 4</li><li>request-hotel-stars Occur Num: 2</li><li>request-hotel-type Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-stars Occur Num: 46</li><li>inform-hotel-type Occur Num: 46</li><li>inform-hotel-area Occur Num: 40</li><li>inform-hotel-phone Occur Num: 26</li></ul></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 95.4 %</p><p> (Precision, Recall, F1) : (0.801, 0.850, 0.818) </p><p> Average Dialog Turn (Succ): 4.949 </p><p> Average Dialog Turn (All): 5.781 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 155</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 29</li><li>Inform-Restaurant-Food-indian Occur Num: 17</li><li>Inform-Restaurant-Price-expensively Occur Num: 13</li><li>Inform-Restaurant-Food-british Occur Num: 11</li><li>Inform-Restaurant-Food-italian Occur Num: 10</li></ul></ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 144</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 27</li><li>Inform-Restaurant-Price-moderate Occur Num: 23</li><li>Inform-Restaurant-Food-indian Occur Num: 13</li><li>Inform-Restaurant-Price-cheap Occur Num: 12</li><li>Inform-Restaurant-Food-chinese Occur Num: 9</li></ul></ul><li>Inform-Restaurant-Price-moderate<ul><li>Occur Num: 116</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 31</li><li>Inform-Restaurant-Price-moderately Occur Num: 15</li><li>Inform-Restaurant-Food-indian Occur Num: 11</li><li>Inform-Restaurant-Food-chinese Occur Num: 8</li><li>Inform-Hotel-Price-moderate Occur Num: 7</li></ul></ul><li>Inform-Restaurant-Price-cheap<ul><li>Occur Num: 80</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 14</li><li>Inform-Restaurant-Food-chinese Occur Num: 8</li><li>Inform-Restaurant-Food-indian Occur Num: 6</li><li>Inform-Restaurant-Area-east Occur Num: 6</li><li>Inform-Restaurant-Price-cheaply Occur Num: 6</li></ul></ul><li>Inform-Restaurant-Food-indian<ul><li>Occur Num: 63</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 15</li><li>Inform-Restaurant-Area-centre Occur Num: 12</li><li>Inform-Restaurant-Price-moderate Occur Num: 10</li><li>Inform-Restaurant-Price-cheap Occur Num: 6</li><li>Inform-Restaurant-Area-north Occur Num: 6</li></ul></ul><li>Request-Restaurant-Phone-?<ul><li>Occur Num: 58</li><li>NLU Output</li><ul><li>Request-Restaurant-Post-? Occur Num: 23</li><li>Request-Restaurant-Addr-? Occur Num: 13</li><li>Request-Attraction-Phone-? Occur Num: 8</li><li>Request-Hotel-Phone-? Occur Num: 5</li><li>Request-Restaurant-Price-? Occur Num: 3</li></ul></ul><li>Request-Restaurant-Addr-?<ul><li>Occur Num: 57</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 14</li><li>Request-Hotel-Addr-? Occur Num: 12</li><li>Request-Restaurant-Post-? Occur Num: 11</li><li>Request-Hospital-Addr-? Occur Num: 4</li><li>Request-Restaurant-Price-? Occur Num: 4</li></ul></ul><li>Inform-Restaurant-Day-thursday<ul><li>Occur Num: 48</li><li>NLU Output</li><ul><li>Inform-Restaurant-People-3 Occur Num: 5</li><li>Inform-Restaurant-People-6 Occur Num: 4</li><li>Inform-Restaurant-People-2 Occur Num: 4</li><li>Inform-Restaurant-People-5 Occur Num: 4</li><li>Inform-Restaurant-Time-18:30 Occur Num: 3</li></ul></ul><li>Inform-Restaurant-Food-chinese<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 9</li><li>Inform-Restaurant-Price-expensive Occur Num: 9</li><li>Inform-Restaurant-Price-cheap Occur Num: 8</li><li>Inform-Restaurant-Price-moderate Occur Num: 7</li><li>Inform-Restaurant-Area-east Occur Num: 6</li></ul></ul><li>Inform-Restaurant-Area-east<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 9</li><li>Inform-Restaurant-Food-chinese Occur Num: 6</li><li>Inform-Restaurant-Price-moderate Occur Num: 4</li><li>Inform-Restaurant-Price-cheap Occur Num: 4</li><li>Inform-Restaurant-Food-italian Occur Num: 3</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Request-Restaurant-Food-?<ul><li>Occur Num: 95</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 12</li><li>Request-Restaurant-Price-? Occur Num: 11</li><li>NoOffer-Restaurant-Area-centre Occur Num: 9</li><li>Request-Restaurant-Area-? Occur Num: 8</li><li>NoOffer-Restaurant-Price-cheap Occur Num: 7</li></ul></ul><li>NoOffer-Restaurant-Price-cheap<ul><li>Occur Num: 65</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 12</li><li>Request-Restaurant-Food-? Occur Num: 8</li><li>NoOffer-Restaurant-Area-centre Occur Num: 6</li><li>NoOffer-Restaurant-Area-east Occur Num: 5</li><li>NoOffer-Restaurant-Food-british Occur Num: 5</li></ul></ul><li>NoOffer-Restaurant-Area-centre<ul><li>Occur Num: 62</li><li>NLU Output</li><ul><li>Request-Restaurant-Food-? Occur Num: 9</li><li>NoOffer-Restaurant-none-none Occur Num: 7</li><li>NoOffer-Restaurant-Price-cheap Occur Num: 6</li><li>NoOffer-Restaurant-Food-indian Occur Num: 4</li><li>NoOffer-Restaurant-Price-moderate Occur Num: 4</li></ul></ul><li>Inform-Restaurant-Choice-4<ul><li>Occur Num: 58</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 3</li><li>Recommend-Restaurant-Name-hk fusion Occur Num: 2</li><li>Recommend-Restaurant-Name-The Occur Num: 2</li><li>Inform-Booking-none-none Occur Num: 2</li><li>Recommend-Restaurant-Name-Restaurant one seven Occur Num: 2</li></ul></ul><li>Request-Restaurant-Price-?<ul><li>Occur Num: 57</li><li>NLU Output</li><ul><li>Request-Restaurant-Food-? Occur Num: 12</li><li>NoOffer-Restaurant-none-none Occur Num: 6</li><li>Request-Restaurant-Area-? Occur Num: 6</li><li>NoOffer-Restaurant-Price-expensive Occur Num: 5</li><li>NoOffer-Restaurant-Area-east Occur Num: 4</li></ul></ul><li>NoOffer-Restaurant-Price-expensive<ul><li>Occur Num: 54</li><li>NLU Output</li><ul><li>Request-Restaurant-Food-? Occur Num: 7</li><li>NoOffer-Restaurant-none-none Occur Num: 7</li><li>Request-Restaurant-Price-? Occur Num: 6</li><li>NoOffer-Restaurant-Food-spanish Occur Num: 4</li><li>NoOffer-Restaurant-Area-in town Occur Num: 4</li></ul></ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 51</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 6</li><li>Recommend-Restaurant-Area-centre Occur Num: 4</li><li>Inform-Restaurant-Price-cheap Occur Num: 3</li><li>Inform-Attraction-Area-centre Occur Num: 3</li><li>Inform-Restaurant-Price-moderate Occur Num: 3</li></ul></ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 6</li><li>Recommend-Restaurant-Name-darrys cookhouse and wine shop Occur Num: 2</li><li>Inform-Restaurant-Food-modern european Occur Num: 2</li><li>Recommend-Restaurant-Price-expensive Occur Num: 2</li><li>Inform-Restaurant-Area-south Occur Num: 2</li></ul></ul><li>NoOffer-Restaurant-Price-moderate<ul><li>Occur Num: 44</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 7</li><li>NoOffer-Restaurant-Area-centre Occur Num: 7</li><li>Request-Restaurant-Food-? Occur Num: 6</li><li>NoOffer-Restaurant-Food-indian Occur Num: 4</li><li>NoOffer-Restaurant-Food-chinese Occur Num: 2</li></ul></ul><li>Request-Restaurant-Area-?<ul><li>Occur Num: 44</li><li>NLU Output</li><ul><li>Request-Restaurant-Food-? Occur Num: 9</li><li>Request-Restaurant-Price-? Occur Num: 7</li><li>NoOffer-Restaurant-Area-south Occur Num: 4</li><li>NoOffer-Restaurant-none-none Occur Num: 3</li><li>NoOffer-Restaurant-Price-expensive Occur Num: 3</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Restaurant-Phone-? Occur Num: 7</li><li>Request-Restaurant-Addr-? Occur Num: 5</li><li>Request-Restaurant-Post-? Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>recommend-restaurant-area Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-phone Occur Num: 8</li><li>request-restaurant-address Occur Num: 4</li><li>request-restaurant-postcode Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-area Occur Num: 26</li><li>inform-restaurant-phone Occur Num: 21</li><li>inform-restaurant-food Occur Num: 15</li></ul></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 94.0 %</p><p> (Precision, Recall, F1) : (0.878, 0.953, 0.905) </p><p> Average Dialog Turn (Succ): 5.633 </p><p> Average Dialog Turn (All): 6.278 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Attraction-Addr-?<ul><li>Occur Num: 171</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 55</li><li>Request-Attraction-Post-? Occur Num: 54</li><li>Request-Attraction-Fee-? Occur Num: 27</li><li>Request-Attraction-Area-? Occur Num: 14</li><li>Request-Attraction-Type-? Occur Num: 7</li></ul></ul><li>Request-Attraction-Phone-?<ul><li>Occur Num: 164</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 59</li><li>Request-Attraction-Addr-? Occur Num: 47</li><li>Request-Attraction-Fee-? Occur Num: 32</li><li>Request-Attraction-Area-? Occur Num: 12</li><li>Request-Attraction-Type-? Occur Num: 9</li></ul></ul><li>Request-Attraction-Post-?<ul><li>Occur Num: 161</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 60</li><li>Request-Attraction-Addr-? Occur Num: 50</li><li>Request-Attraction-Fee-? Occur Num: 34</li><li>Request-Attraction-Type-? Occur Num: 8</li><li>Request-Attraction-Area-? Occur Num: 5</li></ul></ul><li>Request-Attraction-Fee-?<ul><li>Occur Num: 110</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 36</li><li>Request-Attraction-Post-? Occur Num: 36</li><li>Request-Attraction-Addr-? Occur Num: 27</li><li>Request-Attraction-Type-? Occur Num: 7</li><li>Request-Attraction-Area-? Occur Num: 3</li></ul></ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 62</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-museum Occur Num: 15</li><li>Inform-Attraction-Type-entertainment Occur Num: 13</li><li>Inform-Attraction-Type-college Occur Num: 8</li><li>Inform-Restaurant-Area-centre Occur Num: 6</li><li>Inform-Attraction-Type-architecture Occur Num: 3</li></ul></ul><li>Inform-Attraction-Type-museum<ul><li>Occur Num: 44</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 22</li><li>Inform-Attraction-Type-museums Occur Num: 10</li><li>Inform-Attraction-Area-west Occur Num: 4</li><li>Inform-Attraction-Area-east Occur Num: 3</li><li>Inform-Attraction-Type-entertainment Occur Num: 3</li></ul></ul><li>Request-Attraction-Type-?<ul><li>Occur Num: 42</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 14</li><li>Request-Attraction-Post-? Occur Num: 11</li><li>Request-Attraction-Addr-? Occur Num: 8</li><li>Request-Attraction-Fee-? Occur Num: 7</li><li>Request-Attraction-Area-? Occur Num: 2</li></ul></ul><li>Request-Attraction-Area-?<ul><li>Occur Num: 30</li><li>NLU Output</li><ul><li>Request-Attraction-Addr-? Occur Num: 12</li><li>Request-Attraction-Phone-? Occur Num: 10</li><li>Request-Attraction-Post-? Occur Num: 5</li><li>Request-Attraction-Fee-? Occur Num: 3</li></ul></ul><li>Inform-Attraction-Type-college<ul><li>Occur Num: 25</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 12</li><li>Inform-Attraction-Type-colleges Occur Num: 5</li><li>Inform-Attraction-Area-west Occur Num: 3</li><li>Inform-Attraction-Area-south Occur Num: 2</li><li>Inform-Attraction-Area-north Occur Num: 2</li></ul></ul><li>Inform-Attraction-Area-west<ul><li>Occur Num: 21</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-entertainment Occur Num: 7</li><li>Inform-Attraction-Area-centre Occur Num: 4</li><li>Inform-Attraction-Type-museums Occur Num: 2</li><li>Inform-Attraction-Type-college Occur Num: 2</li><li>Inform-Attraction-Type-museum Occur Num: 2</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Attraction-Fee-?<ul><li>Occur Num: 212</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-park Occur Num: 22</li><li>Inform-Attraction-Fee-by our system Occur Num: 16</li><li>Inform-Attraction-Addr-whitehill road Occur Num: 15</li><li>Inform-Attraction-Addr-off newmarket road Occur Num: 15</li><li>Inform-Attraction-Phone-01223300085 Occur Num: 15</li></ul></ul><li>Inform-Attraction-Addr-pool way, whitehill road, off newmarket road<ul><li>Occur Num: 88</li><li>NLU Output</li><ul><li>Inform-Attraction-Addr-whitehill road Occur Num: 20</li><li>Inform-Attraction-Addr-off newmarket road Occur Num: 20</li><li>Inform-Attraction-Addr-pool way Occur Num: 15</li><li>Inform-Attraction-Post-cb58nt Occur Num: 11</li><li>Inform-Attraction-Phone-01223902088 Occur Num: 6</li></ul></ul><li>Inform-Attraction-Fee-free<ul><li>Occur Num: 72</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-park Occur Num: 12</li><li>Inform-Attraction-Addr-saint andrew 's street Occur Num: 11</li><li>Inform-Attraction-Post-cb23bu Occur Num: 7</li><li>Inform-Attraction-Post-cb21sj Occur Num: 4</li><li>Inform-Attraction-Phone-01223314960 Occur Num: 4</li></ul></ul><li>Inform-Attraction-Choice-44<ul><li>Occur Num: 68</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 5</li><li>Request-Attraction-Type-? Occur Num: 3</li><li>Recommend-Attraction-Name-king 's college Occur Num: 2</li><li>Recommend-Attraction-Name-the Occur Num: 2</li><li>Recommend-Attraction-Name-cambridge corn exchange Occur Num: 2</li></ul></ul><li>Inform-Attraction-Addr-saint andrew's street<ul><li>Occur Num: 60</li><li>NLU Output</li><ul><li>Inform-Attraction-Addr-saint andrew 's street Occur Num: 19</li><li>Inform-Attraction-Fee-free Occur Num: 11</li><li>Inform-Attraction-Post-cb23bu Occur Num: 10</li><li>Inform-Attraction-Phone-01223334900 Occur Num: 9</li><li>Inform-Attraction-Area-centre Occur Num: 2</li></ul></ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 60</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre of the city Occur Num: 6</li><li>Inform-Attraction-Area-within Occur Num: 6</li><li>Inform-Attraction-Addr-saint andrew 's street Occur Num: 4</li><li>Inform-Attraction-Type-park Occur Num: 4</li><li>Inform-Attraction-Phone-01223334900 Occur Num: 3</li></ul></ul><li>Inform-Attraction-Choice-5<ul><li>Occur Num: 58</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 6</li><li>Recommend-Attraction-Name-little saint mary 's church Occur Num: 4</li><li>Recommend-Attraction-Name-churchill college Occur Num: 3</li><li>Recommend-Attraction-Name-Old schools Occur Num: 2</li><li>Recommend-Attraction-Name-great saint mary 's church Occur Num: 2</li></ul></ul><li>Inform-Attraction-Choice-13<ul><li>Occur Num: 56</li><li>NLU Output</li><ul><li>Recommend-Attraction-Name-hughes hall Occur Num: 3</li><li>Recommend-Attraction-Name-christ 's college Occur Num: 2</li><li>reqmore-general-none-none Occur Num: 2</li><li>Recommend-Attraction-Name-churchill college Occur Num: 2</li><li>Inform-Attraction-Name-Christ 's college Occur Num: 2</li></ul></ul><li>Inform-Attraction-Post-cb58nt<ul><li>Occur Num: 52</li><li>NLU Output</li><ul><li>Inform-Attraction-Addr-whitehill road Occur Num: 11</li><li>Inform-Attraction-Addr-off newmarket road Occur Num: 11</li><li>Inform-Attraction-Addr-pool way Occur Num: 9</li><li>Inform-Attraction-Phone-01223902088 Occur Num: 7</li><li>Inform-Attraction-Fee-? Occur Num: 3</li></ul></ul><li>Request-Attraction-Type-?<ul><li>Occur Num: 50</li><li>NLU Output</li><ul><li>NoOffer-Attraction-Type-entertainment Occur Num: 11</li><li>Request-Attraction-Area-? Occur Num: 7</li><li>NoOffer-Attraction-Area-centre Occur Num: 5</li><li>NoOffer-Attraction-Area-north Occur Num: 4</li><li>NoOffer-Attraction-Area-south Occur Num: 3</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Attraction-Fee-? Occur Num: 4</li><li>Request-Attraction-Addr-? Occur Num: 2</li><li>Inform-Attraction-Type-boat Occur Num: 1</li><li>Request-Attraction-Phone-? Occur Num: 1</li><li>Inform-Attraction-Area-west Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-attraction-area Occur Num: 21</li><li>recommend-attraction-area Occur Num: 3</li><li>recommend-attraction-phone Occur Num: 2</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-entrance fee Occur Num: 10</li><li>request-attraction-address Occur Num: 7</li><li>request-attraction-phone Occur Num: 7</li><li>request-attraction-postcode Occur Num: 5</li><li>request-attraction-area Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-type Occur Num: 27</li><li>inform-attraction-phone Occur Num: 22</li><li>inform-attraction-area Occur Num: 17</li></ul></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 87.9 %</p><p> (Precision, Recall, F1) : (0.908, 0.893, 0.898) </p><p> Average Dialog Turn (Succ): 7.026 </p><p> Average Dialog Turn (All): 9.399 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Car-?<ul><li>Occur Num: 169</li><li>NLU Output</li><ul><li>Request-Taxi-Phone-? Occur Num: 56</li><li>Request-Police-Phone-? Occur Num: 38</li><li>Request-Hospital-Phone-? Occur Num: 24</li><li>Request-Hotel-Type-? Occur Num: 20</li><li>Request-Attraction-Phone-? Occur Num: 11</li></ul></ul><li>Request-Taxi-Phone-?<ul><li>Occur Num: 157</li><li>NLU Output</li><ul><li>Request-Taxi-Car-? Occur Num: 65</li><li>Request-Police-Phone-? Occur Num: 38</li><li>Request-Hospital-Phone-? Occur Num: 24</li><li>Request-Attraction-Phone-? Occur Num: 11</li><li>Request-Hotel-Phone-? Occur Num: 6</li></ul></ul><li>Inform-Taxi-Leave-dontcare<ul><li>Occur Num: 17</li><li>NLU Output</li><ul><li>Inform-Taxi-Arrive-16:45 Occur Num: 3</li><li>Inform-Taxi-Arrive-19:15 Occur Num: 2</li><li>Inform-Taxi-Arrive-16:30 Occur Num: 1</li><li>Inform-Taxi-Arrive-12:30 Occur Num: 1</li><li>Inform-Taxi-Arrive-17:15 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-dontcare<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Taxi-Leave-02:15 Occur Num: 2</li><li>Inform-Taxi-Leave-10:00 Occur Num: 1</li><li>Inform-Taxi-Leave-17:45 Occur Num: 1</li><li>Inform-Taxi-Leave-09:00 Occur Num: 1</li><li>Inform-Taxi-Leave-08:45 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-home from home<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-home Occur Num: 2</li><li>Inform-Hotel-Name-home Occur Num: 1</li><li>Inform-Taxi-Leave-02:45 Occur Num: 1</li><li>Inform-Taxi-Depart-arbury lodge guesthouse Occur Num: 1</li><li>Inform-Taxi-Arrive-02:30 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the fitzwilliam museum<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-golden house Occur Num: 1</li><li>Inform-Taxi-Depart-fitzwilliam museum Occur Num: 1</li><li>Inform-Taxi-Leave-13:45 Occur Num: 1</li><li>Inform-Taxi-Arrive-13:45 Occur Num: 1</li><li>Inform-Taxi-Depart-fitzwilliam Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-the lensfield hotel<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Depart-lensfield hotel Occur Num: 2</li><li>Inform-Taxi-Dest-thanh binh Occur Num: 1</li><li>Inform-Taxi-Arrive-02:00 Occur Num: 1</li><li>Inform-Taxi-Dest-williams art and antiques Occur Num: 1</li><li>Inform-Taxi-Leave-13:15 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-11:00<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-11:00 Occur Num: 3</li><li>Inform-Taxi-Dest-anatolia Occur Num: 1</li><li>Inform-Taxi-Depart-don pasquale pizzeria Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-12:15<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-12:15 Occur Num: 2</li><li>Inform-Taxi-Dest-lynne strover gallery Occur Num: 1</li><li>Inform-Taxi-Depart-tenpin Occur Num: 1</li><li>Inform-Taxi-Dest-sitar tandoori Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-18:15<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Inform-Taxi-Dest-churchill college Occur Num: 1</li><li>Inform-Taxi-Depart-city stop restaurant Occur Num: 1</li><li>Inform-Taxi-Depart-bedouin Occur Num: 1</li><li>Inform-Taxi-Dest-museum of classical archaeology Occur Num: 1</li><li>Inform-Train-Arrive-18:15 Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Leave-?<ul><li>Occur Num: 204</li><li>NLU Output</li><ul><li>Request-Taxi-Arrive-? Occur Num: 60</li><li>Request-Train-Arrive-? Occur Num: 42</li><li>Request-Train-Leave-? Occur Num: 36</li><li>Inform-Police-Phone-01223358966 Occur Num: 31</li><li>Inform-Attraction-Phone-01223216297 Occur Num: 6</li></ul></ul><li>Request-Taxi-Arrive-?<ul><li>Occur Num: 145</li><li>NLU Output</li><ul><li>Request-Taxi-Leave-? Occur Num: 44</li><li>Request-Train-Arrive-? Occur Num: 42</li><li>Inform-Police-Phone-01223358966 Occur Num: 29</li><li>Request-Train-Leave-? Occur Num: 14</li><li>Inform-Attraction-Phone-01223902088 Occur Num: 5</li></ul></ul><li>Inform-Taxi-Car-honda<ul><li>Occur Num: 28</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-52547806713 Occur Num: 1</li><li>Inform-Taxi-Phone-43863385242 Occur Num: 1</li><li>Inform-Taxi-Phone-01996214154 Occur Num: 1</li><li>Inform-Taxi-Phone-88297768151 Occur Num: 1</li><li>Inform-Taxi-Phone-69189867284 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-bmw<ul><li>Occur Num: 22</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-92208795533 Occur Num: 2</li><li>Inform-Hotel-Type-guesthouse Occur Num: 2</li><li>Inform-Taxi-Phone-78089461863 Occur Num: 1</li><li>Inform-Taxi-Phone-98052904604 Occur Num: 1</li><li>Inform-Taxi-Phone-79054617689 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-ford<ul><li>Occur Num: 21</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-guesthouse Occur Num: 2</li><li>Inform-Taxi-Phone-25725613015 Occur Num: 1</li><li>Inform-Taxi-Phone-69690993010 Occur Num: 1</li><li>Inform-Taxi-Phone-48521984046 Occur Num: 1</li><li>Inform-Taxi-Phone-94896745615 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-lexus<ul><li>Occur Num: 18</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-98682833553 Occur Num: 1</li><li>Inform-Taxi-Phone-83144288521 Occur Num: 1</li><li>Inform-Taxi-Phone-38074539777 Occur Num: 1</li><li>Inform-Taxi-Phone-76339421079 Occur Num: 1</li><li>Inform-Taxi-Phone-67771199099 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-audi<ul><li>Occur Num: 16</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-84045863603 Occur Num: 1</li><li>Inform-Taxi-Phone-74647122072 Occur Num: 1</li><li>Inform-Taxi-Phone-73297457818 Occur Num: 1</li><li>Inform-Taxi-Phone-47691148133 Occur Num: 1</li><li>Inform-Taxi-Phone-02411795305 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-toyota<ul><li>Occur Num: 16</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-83673902071 Occur Num: 1</li><li>Inform-Taxi-Phone-24219248656 Occur Num: 1</li><li>Inform-Taxi-Phone-87029902122 Occur Num: 1</li><li>Inform-Taxi-Phone-82310560899 Occur Num: 1</li><li>Inform-Taxi-Phone-78329616846 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-tesla<ul><li>Occur Num: 14</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-40662758012 Occur Num: 1</li><li>Inform-Taxi-Phone-88733073473 Occur Num: 1</li><li>Inform-Taxi-Phone-54087301478 Occur Num: 1</li><li>Inform-Hotel-Type-guesthouse Occur Num: 1</li><li>Inform-Taxi-Phone-87821614032 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-volkswagen<ul><li>Occur Num: 14</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-15214714504 Occur Num: 1</li><li>Inform-Taxi-Car-skoda Occur Num: 1</li><li>Inform-Taxi-Phone-35127718145 Occur Num: 1</li><li>Inform-Taxi-Phone-88416783908 Occur Num: 1</li><li>Inform-Taxi-Phone-47677872839 Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Taxi-Car-? Occur Num: 13</li><li>Inform-Taxi-Leave-dontcare Occur Num: 7</li><li>Inform-Taxi-Dest-gonville place Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-taxi-car type Occur Num: 21</li><li>request-taxi-phone Occur Num: 16</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 99.5 %</p><p> (Precision, Recall, F1) : (0.840, 0.928, 0.867) </p><p> Average Dialog Turn (Succ): 5.767 </p><p> Average Dialog Turn (All): 5.779 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Train-Dest-cambridge<ul><li>Occur Num: 452</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 34</li><li>Inform-Train-Day-friday Occur Num: 26</li><li>Inform-Train-Day-monday Occur Num: 21</li><li>Inform-Train-Day-saturday Occur Num: 20</li><li>Inform-Train-Day-wednesday Occur Num: 19</li></ul></ul><li>Inform-Train-Depart-cambridge<ul><li>Occur Num: 403</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 49</li><li>Inform-Train-Day-friday Occur Num: 24</li><li>Inform-Train-Day-wednesday Occur Num: 22</li><li>Inform-Train-Day-thursday Occur Num: 21</li><li>Inform-Train-Day-monday Occur Num: 18</li></ul></ul><li>Inform-Train-Day-friday<ul><li>Occur Num: 168</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 27</li><li>Inform-Train-Depart-cambridge Occur Num: 23</li><li>Inform-Train-Depart-leicester Occur Num: 8</li><li>Inform-Train-Dest-london liverpool street Occur Num: 6</li><li>Inform-Train-Depart-kings lynn Occur Num: 5</li></ul></ul><li>Inform-Train-Day-monday<ul><li>Occur Num: 154</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 24</li><li>Inform-Train-Depart-cambridge Occur Num: 14</li><li>Inform-Train-Dest-leicester Occur Num: 5</li><li>Inform-Train-Dest-ely Occur Num: 4</li><li>Inform-Train-Day-Saturday Occur Num: 4</li></ul></ul><li>Inform-Train-Day-wednesday<ul><li>Occur Num: 128</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 21</li><li>Inform-Train-Dest-cambridge Occur Num: 19</li><li>Inform-Train-Depart-london liverpool street Occur Num: 5</li><li>Inform-Train-Dest-ely Occur Num: 4</li><li>Inform-Train-Dest-london kings cross Occur Num: 4</li></ul></ul><li>Inform-Train-Day-thursday<ul><li>Occur Num: 120</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 20</li><li>Inform-Train-Depart-cambridge Occur Num: 20</li><li>Inform-Train-Depart-peterborough Occur Num: 4</li><li>Inform-Train-Dest-london kings cross Occur Num: 3</li><li>Inform-Train-Dest-leicester Occur Num: 3</li></ul></ul><li>Inform-Train-Day-tuesday<ul><li>Occur Num: 119</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 18</li><li>Inform-Train-Depart-cambridge Occur Num: 15</li><li>Inform-Train-Depart-peterborough Occur Num: 4</li><li>Inform-Train-Dest-stevenage Occur Num: 3</li><li>Inform-Train-Arrive-09:15 Occur Num: 3</li></ul></ul><li>Inform-Train-Day-saturday<ul><li>Occur Num: 100</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 21</li><li>Inform-Train-Depart-birmingham new street Occur Num: 6</li><li>Inform-Train-Depart-cambridge Occur Num: 5</li><li>Inform-Train-Depart-bishops stortford Occur Num: 3</li><li>Inform-Train-Depart-london liverpool street Occur Num: 3</li></ul></ul><li>Inform-Train-Day-sunday<ul><li>Occur Num: 87</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 9</li><li>Inform-Train-Depart-cambridge Occur Num: 8</li><li>Inform-Train-Arrive-08:00 Occur Num: 4</li><li>Inform-Train-Depart-london kings cross Occur Num: 4</li><li>Inform-Train-Depart-stansted airport Occur Num: 4</li></ul></ul><li>Request-Train-Id-?<ul><li>Occur Num: 79</li><li>NLU Output</li><ul><li>Request-Train-Time-? Occur Num: 40</li><li>Request-Train-Ticket-? Occur Num: 27</li><li>Request-Train-Leave-? Occur Num: 7</li><li>Request-Train-Arrive-? Occur Num: 5</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>OfferBook-Train-Dest-cambridge<ul><li>Occur Num: 620</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 109</li><li>OfferBook-Train-Day-thursday Occur Num: 27</li><li>OfferBook-Train-Arrive-that Occur Num: 25</li><li>OfferBook-Train-Day-friday Occur Num: 24</li><li>OfferBook-Train-Day-saturday Occur Num: 22</li></ul></ul><li>OfferBook-Train-Depart-cambridge<ul><li>Occur Num: 499</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 73</li><li>OfferBook-Train-Dest-cambridge Occur Num: 42</li><li>OfferBook-Train-Day-wednesday Occur Num: 22</li><li>OfferBook-Train-Day-friday Occur Num: 21</li><li>OfferBook-Train-Arrive-that Occur Num: 20</li></ul></ul><li>OfferBook-Train-Day-friday<ul><li>Occur Num: 201</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 29</li><li>OfferBook-Train-Dest-cambridge Occur Num: 24</li><li>OfferBook-Train-Depart-cambridge Occur Num: 16</li><li>OfferBook-Train-Arrive-that Occur Num: 9</li><li>OfferBook-Train-Arrive-, Occur Num: 8</li></ul></ul><li>Request-Train-Dest-?<ul><li>Occur Num: 198</li><li>NLU Output</li><ul><li>Request-Train-Depart-? Occur Num: 87</li><li>Request-Train-Arrive-? Occur Num: 58</li><li>Request-Train-Leave-? Occur Num: 40</li><li>Request-Taxi-Dest-? Occur Num: 6</li><li>Request-Taxi-Arrive-? Occur Num: 3</li></ul></ul><li>OfferBook-Train-Day-tuesday<ul><li>Occur Num: 194</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 32</li><li>OfferBook-Train-Dest-cambridge Occur Num: 20</li><li>OfferBook-Train-Depart-cambridge Occur Num: 15</li><li>OfferBook-Train-Arrive-that Occur Num: 12</li><li>OfferBook-Train-Arrive-, Occur Num: 9</li></ul></ul><li>OfferBook-Train-Day-thursday<ul><li>Occur Num: 183</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 32</li><li>OfferBook-Train-Dest-cambridge Occur Num: 25</li><li>OfferBook-Train-Depart-cambridge Occur Num: 12</li><li>OfferBook-Train-Depart-peterborough Occur Num: 5</li><li>OfferBook-Train-Arrive-, Occur Num: 5</li></ul></ul><li>OfferBook-Train-Day-saturday<ul><li>Occur Num: 181</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 32</li><li>OfferBook-Train-Dest-cambridge Occur Num: 24</li><li>OfferBook-Train-Depart-cambridge Occur Num: 12</li><li>Select-Train-Day-saturday Occur Num: 9</li><li>OfferBook-Train-Arrive-that Occur Num: 7</li></ul></ul><li>OfferBook-Train-Day-monday<ul><li>Occur Num: 174</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 28</li><li>OfferBook-Train-Dest-cambridge Occur Num: 18</li><li>OfferBook-Train-Depart-cambridge Occur Num: 10</li><li>OfferBook-Train-Arrive-, Occur Num: 9</li><li>OfferBook-Train-Arrive-that Occur Num: 9</li></ul></ul><li>Request-Train-Depart-?<ul><li>Occur Num: 168</li><li>NLU Output</li><ul><li>Request-Train-Dest-? Occur Num: 60</li><li>Request-Train-Arrive-? Occur Num: 56</li><li>Request-Train-Leave-? Occur Num: 39</li><li>Request-Taxi-Depart-? Occur Num: 5</li><li>Request-Taxi-Arrive-? Occur Num: 3</li></ul></ul><li>OfferBook-Train-Day-wednesday<ul><li>Occur Num: 166</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 29</li><li>OfferBook-Train-Dest-cambridge Occur Num: 21</li><li>OfferBook-Train-Depart-cambridge Occur Num: 16</li><li>OfferBook-Train-Dest-london Occur Num: 5</li><li>OfferBook-Train-Depart-london liverpool street Occur Num: 4</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Inform-Train-Day-thursday Occur Num: 1</li><li>Inform-Train-Day-sunday Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>offerbook-train-arriveBy Occur Num: 49</li><li>inform-train-duration Occur Num: 24</li><li>inform-train-price Occur Num: 2</li></ul><b> Request But Not Inform Dialog Act</b><p>Nothing</p><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 91.3 %</p><p> (Precision, Recall, F1) : (0.913, 0.913, 0.913) </p><p> Average Dialog Turn (Succ): 5.524 </p><p> Average Dialog Turn (All): 8.522 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Police-Addr-?<ul><li>Occur Num: 32</li><li>NLU Output</li><ul><li>Request-Police-Phone-? Occur Num: 12</li><li>Request-Police-Post-? Occur Num: 4</li><li>Request-Attraction-Addr-? Occur Num: 3</li><li>Request-Hospital-Addr-? Occur Num: 2</li><li>Request-Attraction-Post-? Occur Num: 2</li></ul></ul><li>Request-Police-Phone-?<ul><li>Occur Num: 28</li><li>NLU Output</li><ul><li>Request-Police-Addr-? Occur Num: 11</li><li>Request-Police-Post-? Occur Num: 4</li><li>Request-Attraction-Addr-? Occur Num: 2</li><li>Request-Restaurant-Addr-? Occur Num: 2</li><li>Request-Restaurant-Phone-? Occur Num: 2</li></ul></ul><li>Inform-Police-none-none<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Police-Name-Parkside Police Station Occur Num: 6</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Police-Addr-Parkside, Cambridge<ul><li>Occur Num: 29</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 15</li><li>Inform-Police-Phone-01223358966 Occur Num: 11</li><li>Inform-Attraction-Addr-pool way Occur Num: 1</li><li>Inform-Attraction-Addr-whitehill road Occur Num: 1</li><li>Inform-Attraction-Addr-off newmarket road Occur Num: 1</li></ul></ul><li>Inform-Police-Phone-01223358966<ul><li>Occur Num: 11</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 11</li></ul></ul><li>Inform-Police-Post-unknown<ul><li>Occur Num: 8</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 4</li><li>Inform-Police-Phone-01223358966 Occur Num: 4</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Police-Addr-? Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-police-phone Occur Num: 2</li><li>request-police-address Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 100.0 %</p><p> (Precision, Recall, F1) : (1.000, 1.000, 1.000) </p><p> Average Dialog Turn (Succ): 4.067 </p><p> Average Dialog Turn (All): 4.067 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Hospital-Phone-?<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Request-Hospital-Post-? Occur Num: 1</li><li>Request-Hospital-Addr-? Occur Num: 1</li><li>Request-Restaurant-Phone-? Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-acute medicine for the elderly<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-acute medicine Occur Num: 1</li><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-paediatric clinic<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-paediatric clinic department Occur Num: 2</li></ul></ul><li>Inform-Hospital-Department-plastic and vascular surgery plastics<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-and vascular surgery Occur Num: 1</li><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-neurology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurology neurosurgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-oncology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-hepatobillary and gastrointestinal surgery regional referral centre<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-hepatobillary and gastrointestinal surgery regional Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-oral and maxillofacial surgery and ent<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-oral and maxillofacial surgery and ent neurosurgery Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hospital-Choice-66<ul><li>Occur Num: 51</li><li>NLU Output</li><ul><li>Inform-Train-Ticket-66 Occur Num: 19</li><li>Inform-Train-Ref-66 Occur Num: 7</li><li>Inform-Hospital-Department-paediatric intensive care unit Occur Num: 2</li><li>Inform-Hospital-Department-66 Occur Num: 2</li><li>Inform-Hospital-Phone-01223217715 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Department-paediatric intensive care unit<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-paediatric intensive care unit Occur Num: 2</li><li>Inform-Hospital-Phone-01223217715 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223217715<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-paediatric intensive care unit Occur Num: 1</li><li>Inform-Hospital-Phone-01223217715 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Department-major trauma unit<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-66 Occur Num: 1</li><li>Inform-Hospital-Department-major trauma unit Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223217216<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Train-Ticket-66 Occur Num: 1</li><li>Inform-Hospital-Phone-01223217216 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223217231<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Train-Ticket-66 Occur Num: 1</li><li>Inform-Hospital-Phone-01223217231 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223596066<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Train-Ticket-66 Occur Num: 1</li><li>Inform-Hospital-Phone-01223596066 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223217297<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Phone-01223217297 Occur Num: 1</li><li>Inform-Hospital-Department-coronary care unit Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Department-coronary care unit<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Phone-01223217297 Occur Num: 1</li><li>Inform-Hospital-Department-coronary care unit Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223348336<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Train-Ticket-66 Occur Num: 1</li><li>Inform-Hospital-Phone-01223348336 Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><p>Nothing</p><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><p>Nothing</p><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/MILU-RuleDST-RulePolicy-TemplateNLG/res.txt b/tests/results/MILU-RuleDST-RulePolicy-TemplateNLG/res.txt deleted file mode 100755 index bf3d5584aaed67c14bb20889d5796dda339725b6..0000000000000000000000000000000000000000 --- a/tests/results/MILU-RuleDST-RulePolicy-TemplateNLG/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.899 -success number of dialogs/tot: 0.831 -average precision: 0.7828638161192679 -average recall: 0.9170754421533237 -average f1: 0.8247176199474817 -average book rate: 0.9085042735042735 -average turn (succ): 12.144404332129964 -average turn (all): 13.854 diff --git a/tests/results/SUMBT-LaRL/Frequency_of_domain.png b/tests/results/SUMBT-LaRL/Frequency_of_domain.png deleted file mode 100755 index 5641af7f137a2a2307e452f8b786e7443bc1afea..0000000000000000000000000000000000000000 Binary files a/tests/results/SUMBT-LaRL/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/SUMBT-LaRL/Performance_for_each_domain.png b/tests/results/SUMBT-LaRL/Performance_for_each_domain.png deleted file mode 100755 index 12d12f724965c4d449ae4ae4666b25e74f0fc682..0000000000000000000000000000000000000000 Binary files a/tests/results/SUMBT-LaRL/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/SUMBT-LaRL/Proportions_of_the_dialogue_loop.png b/tests/results/SUMBT-LaRL/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index c6c132238f0666209b728a9bd0fd9daa131e6f99..0000000000000000000000000000000000000000 Binary files a/tests/results/SUMBT-LaRL/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/SUMBT-LaRL/report_multiwoz.html b/tests/results/SUMBT-LaRL/report_multiwoz.html deleted file mode 100755 index d42e62f98f4760b216513b07ecc8053f9a96189c..0000000000000000000000000000000000000000 --- a/tests/results/SUMBT-LaRL/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: BERTNLU-RuleDST-LaRL</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-27 19:16:35</h3><h3>Overall Results</h3><p> Success Rate: 33.1 %</p><p> (Precision, Recall, F1) : (0.485, 0.560, 0.488) </p><p> Average Dialog Turn (Succ): 15.547 </p><p> Average Dialog Turn (All): 28.734 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">343</td><td align="center">0.364</td><td align="center">0.282</td><td align="center">0.471</td><td align="center">0.326</td><td align="center">0.458</td><td align="center">10.096</td><td align="center">16.711</td></tr><tr><td align="center">attraction</td><td align="center">294</td><td align="center">0.595</td><td align="center">0.619</td><td align="center">0.693</td><td align="center">0.631</td><td align="center">0.350</td><td align="center">9.337</td><td align="center">18.197</td></tr><tr><td align="center">restaurant</td><td align="center">377</td><td align="center">0.618</td><td align="center">0.521</td><td align="center">0.643</td><td align="center">0.554</td><td align="center">0.308</td><td align="center">10.876</td><td align="center">16.271</td></tr><tr><td align="center">train</td><td align="center">328</td><td align="center">0.634</td><td align="center">0.746</td><td align="center">0.676</td><td align="center">0.696</td><td align="center">0.354</td><td align="center">9.490</td><td align="center">18.470</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">0.783</td><td align="center">0.870</td><td align="center">0.848</td><td align="center">0.843</td><td align="center">0.217</td><td align="center">6.333</td><td align="center">13.652</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">0.900</td><td align="center">0.900</td><td align="center">0.900</td><td align="center">0.900</td><td align="center">0.100</td><td align="center">3.852</td><td align="center">7.467</td></tr><tr><td align="center">taxi</td><td align="center">94</td><td align="center">0.085</td><td align="center">0.138</td><td align="center">0.112</td><td align="center">0.121</td><td align="center">0.862</td><td align="center">8.750</td><td align="center">27.872</td></tr></table><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 36.4 %</p><p> (Precision, Recall, F1) : (0.282, 0.471, 0.326) </p><p> Average Dialog Turn (Succ): 10.096 </p><p> Average Dialog Turn (All): 16.711 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Hotel-Addr-? Occur Num: 24</li><li>Request-Hotel-Post-? Occur Num: 22</li><li>Request-Hotel-Phone-? Occur Num: 13</li><li>Inform-Hotel-Stay-4 Occur Num: 10</li><li>Inform-Hotel-Type-hotel Occur Num: 8</li><li>Inform-Hotel-Stay-2 Occur Num: 7</li><li>Request-Hotel-Area-? Occur Num: 6</li><li>Inform-Hotel-Stay-5 Occur Num: 6</li><li>Inform-Hotel-Stay-3 Occur Num: 5</li><li>Inform-Hotel-Parking-yes Occur Num: 5</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-pricerange Occur Num: 144</li><li>recommend-hotel-pricerange Occur Num: 55</li><li>inform-hotel-parking Occur Num: 43</li><li>inform-hotel-postcode Occur Num: 40</li><li>inform-hotel-internet Occur Num: 29</li><li>inform-hotel-stars Occur Num: 11</li><li>recommend-hotel-stars Occur Num: 7</li><li>recommend-hotel-internet Occur Num: 3</li><li>recommend-hotel-parking Occur Num: 2</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-postcode Occur Num: 43</li><li>request-hotel-phone Occur Num: 24</li><li>request-hotel-internet Occur Num: 21</li><li>request-hotel-address Occur Num: 21</li><li>request-hotel-parking Occur Num: 19</li><li>request-hotel-pricerange Occur Num: 19</li><li>request-hotel-area Occur Num: 8</li><li>request-hotel-stars Occur Num: 7</li><li>request-hotel-type Occur Num: 3</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-type Occur Num: 97</li><li>inform-hotel-area Occur Num: 53</li><li>inform-hotel-stars Occur Num: 16</li><li>inform-hotel-phone Occur Num: 15</li></ul></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 59.5 %</p><p> (Precision, Recall, F1) : (0.619, 0.693, 0.631) </p><p> Average Dialog Turn (Succ): 9.337 </p><p> Average Dialog Turn (All): 18.197 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Attraction-Addr-? Occur Num: 32</li><li>Request-Attraction-Fee-? Occur Num: 27</li><li>Request-Attraction-Post-? Occur Num: 23</li><li>Request-Attraction-Phone-? Occur Num: 14</li><li>Request-Attraction-Type-? Occur Num: 4</li><li>Request-Attraction-Area-? Occur Num: 2</li><li>Inform-Attraction-Type-museum Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-attraction-postcode Occur Num: 89</li><li>recommend-attraction-postcode Occur Num: 10</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-postcode Occur Num: 68</li><li>request-attraction-phone Occur Num: 55</li><li>request-attraction-address Occur Num: 37</li><li>request-attraction-entrance fee Occur Num: 27</li><li>request-attraction-area Occur Num: 8</li><li>request-attraction-type Occur Num: 5</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-phone Occur Num: 47</li><li>inform-attraction-area Occur Num: 12</li><li>inform-attraction-type Occur Num: 9</li></ul></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 61.8 %</p><p> (Precision, Recall, F1) : (0.521, 0.643, 0.554) </p><p> Average Dialog Turn (Succ): 10.876 </p><p> Average Dialog Turn (All): 16.271 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Restaurant-Addr-? Occur Num: 23</li><li>Request-Restaurant-Phone-? Occur Num: 15</li><li>Request-Restaurant-Post-? Occur Num: 8</li><li>Inform-Restaurant-Food-indian Occur Num: 6</li><li>Inform-Restaurant-Food-chinese Occur Num: 4</li><li>Inform-Restaurant-Day-friday Occur Num: 3</li><li>Request-Restaurant-Area-? Occur Num: 3</li><li>Inform-Restaurant-Food-international Occur Num: 3</li><li>Inform-Restaurant-Food-european Occur Num: 3</li><li>Inform-Restaurant-Food-british Occur Num: 3</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-restaurant-postcode Occur Num: 85</li><li>recommend-restaurant-postcode Occur Num: 30</li><li>inform-restaurant-pricerange Occur Num: 16</li><li>inform-restaurant-area Occur Num: 15</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-address Occur Num: 52</li><li>request-restaurant-postcode Occur Num: 50</li><li>request-restaurant-phone Occur Num: 13</li><li>request-restaurant-area Occur Num: 7</li><li>request-restaurant-pricerange Occur Num: 5</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-food Occur Num: 60</li><li>inform-restaurant-phone Occur Num: 20</li><li>inform-restaurant-area Occur Num: 16</li></ul></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 63.4 %</p><p> (Precision, Recall, F1) : (0.746, 0.676, 0.696) </p><p> Average Dialog Turn (Succ): 9.490 </p><p> Average Dialog Turn (All): 18.470 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Train-Ticket-? Occur Num: 75</li><li>Request-Train-Time-? Occur Num: 28</li><li>Request-Train-Leave-? Occur Num: 3</li><li>Inform-Train-People-1 Occur Num: 2</li><li>Inform-Train-People-7 Occur Num: 2</li><li>Request-Train-Arrive-? Occur Num: 2</li><li>Inform-Train-People-4 Occur Num: 1</li><li>Request-Train-Id-? Occur Num: 1</li><li>Inform-Train-Day-saturday Occur Num: 1</li><li>Inform-Train-People-2 Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>offerbooked-train-price Occur Num: 6</li><li>inform-train-duration Occur Num: 1</li><li>inform-train-price Occur Num: 1</li><li>offerbook-train-trainID Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-train-price Occur Num: 108</li><li>request-train-duration Occur Num: 3</li><li>request-train-leaveAt Occur Num: 3</li><li>request-train-arriveBy Occur Num: 3</li><li>request-train-trainID Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 78.3 %</p><p> (Precision, Recall, F1) : (0.870, 0.848, 0.843) </p><p> Average Dialog Turn (Succ): 6.333 </p><p> Average Dialog Turn (All): 13.652 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Police-Addr-? Occur Num: 3</li><li>Request-Police-Phone-? Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-police-postcode Occur Num: 3</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-police-phone Occur Num: 3</li><li>request-police-address Occur Num: 3</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 90.0 %</p><p> (Precision, Recall, F1) : (0.900, 0.900, 0.900) </p><p> Average Dialog Turn (Succ): 3.852 </p><p> Average Dialog Turn (All): 7.467 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Hospital-Phone-? Occur Num: 3</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-hospital-phone Occur Num: 3</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 8.5 %</p><p> (Precision, Recall, F1) : (0.138, 0.112, 0.121) </p><p> Average Dialog Turn (Succ): 8.750 </p><p> Average Dialog Turn (All): 27.872 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Taxi-Car-? Occur Num: 77</li><li>Request-Taxi-Phone-? Occur Num: 4</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-taxi-phone Occur Num: 86</li><li>request-taxi-car type Occur Num: 81</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/SUMBT-LaRL/res.txt b/tests/results/SUMBT-LaRL/res.txt deleted file mode 100755 index 568e965688d03c2f32ed6ca3cdaf31153cb441dc..0000000000000000000000000000000000000000 --- a/tests/results/SUMBT-LaRL/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.394 -success number of dialogs/tot: 0.331 -average precision: 0.48533434561013816 -average recall: 0.5600290098732467 -average f1: 0.4881878265335849 -average book rate: 0.39465811965811964 -average turn (succ): 15.546827794561933 -average turn (all): 28.734 diff --git a/tests/results/SUMBT-RulePolicy-TemplateNLG/Frequency_of_domain.png b/tests/results/SUMBT-RulePolicy-TemplateNLG/Frequency_of_domain.png deleted file mode 100755 index 039fd18d293bcfa7fa22a97202c36dff2c26515d..0000000000000000000000000000000000000000 Binary files a/tests/results/SUMBT-RulePolicy-TemplateNLG/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/SUMBT-RulePolicy-TemplateNLG/Performance_for_each_domain.png b/tests/results/SUMBT-RulePolicy-TemplateNLG/Performance_for_each_domain.png deleted file mode 100755 index 9b90bd7145b93fc84c790c7dbaa46302cff36161..0000000000000000000000000000000000000000 Binary files a/tests/results/SUMBT-RulePolicy-TemplateNLG/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/SUMBT-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png b/tests/results/SUMBT-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index e7a6f7d3fcb4c67574b31e221813193e6dec3bc0..0000000000000000000000000000000000000000 Binary files a/tests/results/SUMBT-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/SUMBT-RulePolicy-TemplateNLG/report_multiwoz.html b/tests/results/SUMBT-RulePolicy-TemplateNLG/report_multiwoz.html deleted file mode 100755 index f73c7b3207669857fb77e6f70e866f6d5d926665..0000000000000000000000000000000000000000 --- a/tests/results/SUMBT-RulePolicy-TemplateNLG/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: SUMBT-RulePolicy-TemplateNLG</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-27 11:53:17</h3><h3>Overall Results</h3><p> Success Rate: 33.8 %</p><p> (Precision, Recall, F1) : (0.523, 0.506, 0.473) </p><p> Average Dialog Turn (Succ): 12.089 </p><p> Average Dialog Turn (All): 26.552 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">338</td><td align="center">0.340</td><td align="center">0.174</td><td align="center">0.252</td><td align="center">0.184</td><td align="center">0.459</td><td align="center">10.104</td><td align="center">20.107</td></tr><tr><td align="center">restaurant</td><td align="center">368</td><td align="center">0.554</td><td align="center">0.550</td><td align="center">0.555</td><td align="center">0.531</td><td align="center">0.399</td><td align="center">5.000</td><td align="center">17.685</td></tr><tr><td align="center">attraction</td><td align="center">299</td><td align="center">0.311</td><td align="center">0.569</td><td align="center">0.516</td><td align="center">0.514</td><td align="center">0.545</td><td align="center">7.527</td><td align="center">21.993</td></tr><tr><td align="center">train</td><td align="center">278</td><td align="center">0.946</td><td align="center">0.809</td><td align="center">0.891</td><td align="center">0.833</td><td align="center">0.022</td><td align="center">7.202</td><td align="center">7.647</td></tr><tr><td align="center">taxi</td><td align="center">46</td><td align="center">0.891</td><td align="center">1.000</td><td align="center">0.946</td><td align="center">0.964</td><td align="center">0.109</td><td align="center">3.463</td><td align="center">7.435</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">1.000</td><td align="center">0.000</td><td align="center">40.000</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">0.367</td><td align="center">0.367</td><td align="center">0.367</td><td align="center">0.367</td><td align="center">0.633</td><td align="center">18.364</td><td align="center">32.067</td></tr></table><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 34.0 %</p><p> (Precision, Recall, F1) : (0.174, 0.252, 0.184) </p><p> Average Dialog Turn (Succ): 10.104 </p><p> Average Dialog Turn (All): 20.107 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 767</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 116</li><li>Inform-Hotel-Internet-none Occur Num: 73</li><li>Inform-Hotel-Stars-4 Occur Num: 41</li><li>Inform-Hotel-Type-guesthouse Occur Num: 37</li><li>Inform-Hotel-Type-hotel Occur Num: 36</li></ul></ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 642</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-none Occur Num: 105</li><li>Inform-Hotel-Parking-none Occur Num: 77</li><li>Inform-Hotel-Price-moderate Occur Num: 38</li><li>Inform-Hotel-Type-hotel Occur Num: 36</li><li>Inform-Hotel-Stars-4 Occur Num: 36</li></ul></ul><li>NoOffer-Hotel-Internet-yes<ul><li>Occur Num: 426</li><li>NLU Output</li><ul><li>NoOffer-Hotel-Internet-none Occur Num: 44</li><li>NoOffer-Hotel-none-none Occur Num: 41</li><li>NoOffer-Hotel-Type-hotels Occur Num: 36</li><li>NoOffer-Hotel-Type-hotel Occur Num: 33</li><li>NoOffer-Hotel-Stars-4 Occur Num: 24</li></ul></ul><li>NoOffer-Hotel-Type-hotel<ul><li>Occur Num: 399</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 43</li><li>NoOffer-Hotel-Stars-4 Occur Num: 36</li><li>NoOffer-Hotel-Price-moderate Occur Num: 27</li><li>NoOffer-Hotel-Internet-none Occur Num: 26</li><li>NoOffer-Hotel-Type-hotels Occur Num: 25</li></ul></ul><li>Inform-Hotel-Stars-4<ul><li>Occur Num: 337</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 48</li><li>Inform-Hotel-Internet-none Occur Num: 41</li><li>Inform-Hotel-Type-guesthouse Occur Num: 26</li><li>Inform-Hotel-Price-moderate Occur Num: 21</li><li>Inform-Hotel-Type-hotel Occur Num: 19</li></ul></ul><li>NoOffer-Hotel-Parking-yes<ul><li>Occur Num: 331</li><li>NLU Output</li><ul><li>NoOffer-Hotel-Parking-none Occur Num: 34</li><li>NoOffer-Hotel-Type-hotel Occur Num: 32</li><li>NoOffer-Hotel-none-none Occur Num: 30</li><li>NoOffer-Hotel-Stars-4 Occur Num: 20</li><li>NoOffer-Hotel-Price-moderate Occur Num: 19</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 298</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 38</li><li>Inform-Hotel-Internet-none Occur Num: 35</li><li>Inform-Hotel-Price-moderate Occur Num: 26</li><li>Inform-Hotel-Stars-4 Occur Num: 25</li><li>Inform-Hotel-Area-north Occur Num: 16</li></ul></ul><li>Inform-Hotel-Type-hotel<ul><li>Occur Num: 269</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 42</li><li>Inform-Hotel-Internet-none Occur Num: 36</li><li>Inform-Hotel-Price-expensive Occur Num: 18</li><li>Inform-Hotel-Stars-2 Occur Num: 15</li><li>Inform-Hotel-Price-moderate Occur Num: 12</li></ul></ul><li>Inform-Hotel-Price-moderate<ul><li>Occur Num: 266</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-none Occur Num: 37</li><li>Inform-Hotel-Parking-none Occur Num: 34</li><li>Inform-Hotel-Type-guesthouse Occur Num: 24</li><li>Inform-Hotel-Area-north Occur Num: 20</li><li>Inform-Hotel-Stars-4 Occur Num: 18</li></ul></ul><li>NoOffer-Hotel-Price-moderate<ul><li>Occur Num: 242</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 24</li><li>NoOffer-Hotel-Type-hotel Occur Num: 22</li><li>NoOffer-Hotel-Type-hotels Occur Num: 20</li><li>NoOffer-Hotel-Stars-4 Occur Num: 20</li><li>NoOffer-Hotel-Internet-none Occur Num: 16</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Hotel-Addr-? Occur Num: 59</li><li>Request-Hotel-Phone-? Occur Num: 41</li><li>Request-Hotel-Post-? Occur Num: 26</li><li>Request-Hotel-Price-? Occur Num: 11</li><li>Inform-Hotel-Parking-yes Occur Num: 6</li><li>Inform-Hotel-Internet-yes Occur Num: 2</li><li>Request-Hotel-Internet-? Occur Num: 2</li><li>Inform-Hotel-Stay-3 Occur Num: 2</li><li>Inform-Hotel-Day-wednesday Occur Num: 1</li><li>Inform-Hotel-Day-tuesday Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-parking Occur Num: 136</li><li>inform-hotel-internet Occur Num: 112</li><li>inform-hotel-stars Occur Num: 17</li><li>inform-hotel-area Occur Num: 14</li><li>recommend-hotel-parking Occur Num: 10</li><li>recommend-hotel-internet Occur Num: 6</li><li>recommend-hotel-stars Occur Num: 2</li><li>inform-hotel-phone Occur Num: 2</li><li>inform-hotel-pricerange Occur Num: 1</li><li>recommend-hotel-pricerange Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-address Occur Num: 89</li><li>request-hotel-phone Occur Num: 68</li><li>request-hotel-postcode Occur Num: 45</li><li>request-hotel-internet Occur Num: 25</li><li>request-hotel-parking Occur Num: 17</li><li>request-hotel-pricerange Occur Num: 13</li><li>request-hotel-area Occur Num: 4</li><li>request-hotel-stars Occur Num: 2</li><li>request-hotel-type Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-area Occur Num: 67</li><li>inform-hotel-type Occur Num: 60</li><li>inform-hotel-stars Occur Num: 44</li><li>inform-hotel-phone Occur Num: 11</li></ul></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 55.4 %</p><p> (Precision, Recall, F1) : (0.550, 0.555, 0.531) </p><p> Average Dialog Turn (Succ): 5.000 </p><p> Average Dialog Turn (All): 17.685 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 265</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 17</li><li>Request-Taxi-Arrive-? Occur Num: 12</li><li>Inform-Restaurant-Price-cheap Occur Num: 12</li><li>Inform-Attraction-Area-centre Occur Num: 12</li><li>Inform-Restaurant-Food-italian Occur Num: 8</li></ul></ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 167</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 16</li><li>Inform-Restaurant-Food-indian Occur Num: 12</li><li>Request-Taxi-Leave-? Occur Num: 7</li><li>Inform-Restaurant-Food-british Occur Num: 7</li><li>Request-Taxi-Arrive-? Occur Num: 6</li></ul></ul><li>Inform-Restaurant-Price-cheap<ul><li>Occur Num: 125</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 13</li><li>Recommend-Restaurant-Price-cheap Occur Num: 8</li><li>Inform-Restaurant-Food-italian Occur Num: 8</li><li>Inform-Restaurant-Food-international Occur Num: 3</li><li>Inform-Restaurant-Area-north Occur Num: 3</li></ul></ul><li>Inform-Restaurant-Food-indian<ul><li>Occur Num: 99</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 12</li><li>Inform-Restaurant-Area-centre Occur Num: 6</li><li>Inform-Restaurant-Choice-4 Occur Num: 5</li><li>Inform-Restaurant-Area-west Occur Num: 4</li><li>Inform-Restaurant-Price-moderate Occur Num: 4</li></ul></ul><li>Request-Restaurant-Food-?<ul><li>Occur Num: 99</li><li>NLU Output</li><ul><li>Request-Restaurant-Price-? Occur Num: 11</li><li>NoOffer-Restaurant-none-none Occur Num: 11</li><li>NoOffer-Restaurant-Price-expensive Occur Num: 8</li><li>NoOffer-Restaurant-Area-centre Occur Num: 8</li><li>NoOffer-Restaurant-Price-cheap Occur Num: 7</li></ul></ul><li>Inform-Restaurant-Food-italian<ul><li>Occur Num: 81</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-cheap Occur Num: 7</li><li>Inform-Restaurant-Area-centre Occur Num: 7</li><li>Recommend-Restaurant-Food-italian Occur Num: 4</li><li>Recommend-Restaurant-Name-da vinci pizzeria Occur Num: 3</li><li>Request-Taxi-Arrive-? Occur Num: 3</li></ul></ul><li>NoOffer-Restaurant-Area-centre<ul><li>Occur Num: 79</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 8</li><li>Request-Restaurant-Food-? Occur Num: 8</li><li>NoOffer-Restaurant-Price-expensive Occur Num: 7</li><li>NoOffer-Restaurant-Food-indian Occur Num: 6</li><li>NoOffer-Restaurant-Price-cheap Occur Num: 5</li></ul></ul><li>Inform-Restaurant-Food-chinese<ul><li>Occur Num: 77</li><li>NLU Output</li><ul><li>Inform-Restaurant-Choice-4 Occur Num: 7</li><li>Inform-Restaurant-Area-centre Occur Num: 4</li><li>Inform-Restaurant-Price-expensive Occur Num: 3</li><li>Inform-Restaurant-Area-east Occur Num: 3</li><li>Inform-Restaurant-Area-south Occur Num: 3</li></ul></ul><li>Inform-Restaurant-Choice-4<ul><li>Occur Num: 74</li><li>NLU Output</li><ul><li>Inform-Restaurant-Food-chinese Occur Num: 7</li><li>Inform-Restaurant-Food-indian Occur Num: 5</li><li>Inform-Restaurant-Area-centre Occur Num: 4</li><li>Recommend-Restaurant-Name-Jinling noodle bar Occur Num: 2</li><li>Recommend-Restaurant-Name-mahal of cambridge Occur Num: 2</li></ul></ul><li>NoOffer-Restaurant-Price-cheap<ul><li>Occur Num: 73</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 13</li><li>Request-Restaurant-Food-? Occur Num: 8</li><li>NoOffer-Restaurant-Area-centre Occur Num: 6</li><li>NoOffer-Restaurant-Area-east Occur Num: 5</li><li>NoOffer-Restaurant-Food-british Occur Num: 5</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Restaurant-Phone-? Occur Num: 130</li><li>Request-Restaurant-Addr-? Occur Num: 9</li><li>Request-Restaurant-Post-? Occur Num: 3</li><li>Request-Restaurant-Price-? Occur Num: 2</li><li>Inform-Restaurant-Time-13:30 Occur Num: 1</li><li>Inform-Restaurant-Time-11:45 Occur Num: 1</li><li>Inform-Restaurant-Food-portuguese Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-phone Occur Num: 133</li><li>request-restaurant-address Occur Num: 17</li><li>request-restaurant-postcode Occur Num: 7</li><li>request-restaurant-pricerange Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-food Occur Num: 55</li><li>inform-restaurant-area Occur Num: 54</li><li>inform-restaurant-phone Occur Num: 14</li></ul></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 31.1 %</p><p> (Precision, Recall, F1) : (0.569, 0.516, 0.514) </p><p> Average Dialog Turn (Succ): 7.527 </p><p> Average Dialog Turn (All): 21.993 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 229</li><li>NLU Output</li><ul><li>Inform-Attraction-Post-cb21ab Occur Num: 18</li><li>Inform-Attraction-Addr-Regent Street City Centre Occur Num: 14</li><li>Inform-Attraction-Area-within Occur Num: 14</li><li>Inform-Attraction-Type-museum Occur Num: 13</li><li>Inform-Attraction-Area-centre of the city Occur Num: 11</li></ul></ul><li>Inform-Attraction-Fee-free<ul><li>Occur Num: 124</li><li>NLU Output</li><ul><li>Inform-Attraction-Addr-Regent Street City Centre Occur Num: 20</li><li>Inform-Attraction-Post-cb21ab Occur Num: 17</li><li>Inform-Attraction-Type-park Occur Num: 13</li><li>Request-Train-Arrive-? Occur Num: 11</li><li>Inform-Attraction-Area-centre Occur Num: 10</li></ul></ul><li>Inform-Attraction-Type-museum<ul><li>Occur Num: 113</li><li>NLU Output</li><ul><li>Inform-Attraction-Addr-Regent Street City Centre Occur Num: 14</li><li>Inform-Attraction-Area-centre Occur Num: 12</li><li>Inform-Attraction-Post-cb21ab Occur Num: 11</li><li>Inform-Attraction-Fee-free Occur Num: 8</li><li>Inform-Attraction-Area-south Occur Num: 7</li></ul></ul><li>Request-Attraction-Type-?<ul><li>Occur Num: 110</li><li>NLU Output</li><ul><li>Request-Attraction-Area-? Occur Num: 10</li><li>NoOffer-Attraction-Area-centre Occur Num: 10</li><li>NoOffer-Attraction-Type-entertainment Occur Num: 9</li><li>NoOffer-Attraction-Area-east Occur Num: 7</li><li>NoOffer-Attraction-Type-college Occur Num: 7</li></ul></ul><li>NoOffer-Attraction-Area-centre<ul><li>Occur Num: 97</li><li>NLU Output</li><ul><li>Request-Attraction-Type-? Occur Num: 14</li><li>NoOffer-Attraction-Type-entertainment Occur Num: 14</li><li>NoOffer-Attraction-Area-that area of town Occur Num: 7</li><li>NoOffer-Attraction-Type-multiple sports Occur Num: 6</li><li>Request-Attraction-Area-? Occur Num: 5</li></ul></ul><li>Inform-Attraction-Fee-?<ul><li>Occur Num: 95</li><li>NLU Output</li><ul><li>Inform-Attraction-Post-cb21ab Occur Num: 16</li><li>Inform-Attraction-Addr-Regent Street City Centre Occur Num: 15</li><li>Request-Train-Arrive-? Occur Num: 7</li><li>Request-Taxi-Arrive-? Occur Num: 6</li><li>Inform-Attraction-Type-park Occur Num: 5</li></ul></ul><li>Inform-Attraction-Choice-44<ul><li>Occur Num: 71</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 5</li><li>Request-Attraction-Type-? Occur Num: 3</li><li>Recommend-Attraction-Type-architecturally Occur Num: 2</li><li>Inform-Attraction-Name-little saint mary 's church Occur Num: 1</li><li>Recommend-Attraction-Name-old schools Occur Num: 1</li></ul></ul><li>Inform-Attraction-Area-south<ul><li>Occur Num: 62</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-museum Occur Num: 6</li><li>Inform-Attraction-Addr-Regent Street City Centre Occur Num: 6</li><li>Inform-Attraction-Name-Byard art Occur Num: 5</li><li>Inform-Attraction-Post-cb21ab Occur Num: 5</li><li>Request-Train-Arrive-? Occur Num: 5</li></ul></ul><li>Inform-Attraction-Type-college<ul><li>Occur Num: 60</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 9</li><li>Inform-Attraction-Addr-Regent Street City Centre Occur Num: 8</li><li>Inform-Attraction-Post-cb21ab Occur Num: 7</li><li>Inform-Attraction-Area-west Occur Num: 5</li><li>Request-Taxi-Arrive-? Occur Num: 4</li></ul></ul><li>Inform-Attraction-Choice-13<ul><li>Occur Num: 57</li><li>NLU Output</li><ul><li>Recommend-Attraction-Name-cambridge and county folk museum Occur Num: 3</li><li>Recommend-Attraction-Name-saint john 's college Occur Num: 2</li><li>Book-Booking-Ref-00000058 Occur Num: 2</li><li>Recommend-Attraction-Name-christ 's college Occur Num: 2</li><li>Recommend-Attraction-Name-hughes hall Occur Num: 2</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Attraction-Phone-? Occur Num: 78</li><li>Request-Attraction-Fee-? Occur Num: 46</li><li>Request-Attraction-Addr-? Occur Num: 29</li><li>Request-Attraction-Post-? Occur Num: 7</li><li>Request-Attraction-Type-? Occur Num: 2</li><li>Inform-Attraction-Type-entertainment Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-attraction-area Occur Num: 37</li><li>recommend-attraction-area Occur Num: 2</li><li>recommend-attraction-phone Occur Num: 1</li><li>inform-attraction-phone Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-phone Occur Num: 141</li><li>request-attraction-address Occur Num: 71</li><li>request-attraction-entrance fee Occur Num: 55</li><li>request-attraction-postcode Occur Num: 44</li><li>request-attraction-type Occur Num: 8</li><li>request-attraction-area Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-type Occur Num: 42</li><li>inform-attraction-area Occur Num: 38</li><li>inform-attraction-phone Occur Num: 9</li></ul></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 94.6 %</p><p> (Precision, Recall, F1) : (0.809, 0.891, 0.833) </p><p> Average Dialog Turn (Succ): 7.202 </p><p> Average Dialog Turn (All): 7.647 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>OfferBook-Train-Dest-cambridge<ul><li>Occur Num: 533</li><li>NLU Output</li><ul><li>OfferBook-Train-Arrive-05:51 Occur Num: 35</li><li>OfferBook-Train-none-none Occur Num: 30</li><li>OfferBook-Train-Day-friday Occur Num: 20</li><li>OfferBook-Train-Day-thursday Occur Num: 19</li><li>OfferBook-Train-Day-saturday Occur Num: 18</li></ul></ul><li>OfferBook-Train-Depart-cambridge<ul><li>Occur Num: 476</li><li>NLU Output</li><ul><li>OfferBook-Train-Arrive-05:51 Occur Num: 32</li><li>OfferBook-Train-Dest-cambridge Occur Num: 24</li><li>OfferBook-Train-none-none Occur Num: 19</li><li>OfferBook-Train-Day-wednesday Occur Num: 19</li><li>OfferBook-Train-Day-thursday Occur Num: 17</li></ul></ul><li>Inform-Train-Arrive-05:51<ul><li>Occur Num: 467</li><li>NLU Output</li><ul><li>OfferBook-Train-Arrive-05:51 Occur Num: 70</li><li>OfferBook-Train-Depart-cambridge Occur Num: 37</li><li>OfferBook-Train-Dest-cambridge Occur Num: 36</li><li>OfferBook-Train-Leave-05:00 Occur Num: 21</li><li>OfferBook-Train-Day-friday Occur Num: 15</li></ul></ul><li>Inform-Train-Depart-cambridge<ul><li>Occur Num: 245</li><li>NLU Output</li><ul><li>OfferBook-Train-Depart-cambridge Occur Num: 33</li><li>OfferBook-Train-Dest-cambridge Occur Num: 14</li><li>OfferBook-Train-Arrive-05:51 Occur Num: 14</li><li>OfferBook-Train-Day-tuesday Occur Num: 9</li><li>OfferBook-Train-Day-friday Occur Num: 8</li></ul></ul><li>Inform-Train-Leave-05:00<ul><li>Occur Num: 229</li><li>NLU Output</li><ul><li>OfferBook-Train-Leave-05:00 Occur Num: 36</li><li>OfferBook-Train-Arrive-05:51 Occur Num: 17</li><li>OfferBook-Train-Depart-cambridge Occur Num: 13</li><li>OfferBook-Train-Dest-cambridge Occur Num: 11</li><li>OfferBook-Train-Day-thursday Occur Num: 9</li></ul></ul><li>OfferBook-Train-Day-friday<ul><li>Occur Num: 186</li><li>NLU Output</li><ul><li>OfferBook-Train-Dest-cambridge Occur Num: 20</li><li>OfferBook-Train-Arrive-05:51 Occur Num: 15</li><li>OfferBook-Train-Depart-cambridge Occur Num: 14</li><li>OfferBook-Train-none-none Occur Num: 8</li><li>OfferBook-Train-Arrive-, Occur Num: 6</li></ul></ul><li>OfferBook-Train-Day-thursday<ul><li>Occur Num: 183</li><li>NLU Output</li><ul><li>OfferBook-Train-Dest-cambridge Occur Num: 19</li><li>OfferBook-Train-Depart-cambridge Occur Num: 17</li><li>OfferBook-Train-Arrive-05:51 Occur Num: 11</li><li>OfferBook-Train-Leave-05:00 Occur Num: 9</li><li>OfferBook-Train-none-none Occur Num: 9</li></ul></ul><li>OfferBook-Train-Day-saturday<ul><li>Occur Num: 169</li><li>NLU Output</li><ul><li>OfferBook-Train-Dest-cambridge Occur Num: 16</li><li>OfferBook-Train-Depart-cambridge Occur Num: 12</li><li>OfferBook-Train-Arrive-05:51 Occur Num: 12</li><li>OfferBook-Train-Leave-05:00 Occur Num: 7</li><li>OfferBook-Train-none-none Occur Num: 6</li></ul></ul><li>OfferBook-Train-Day-tuesday<ul><li>Occur Num: 164</li><li>NLU Output</li><ul><li>OfferBook-Train-Depart-cambridge Occur Num: 19</li><li>OfferBook-Train-Dest-cambridge Occur Num: 14</li><li>OfferBook-Train-none-none Occur Num: 12</li><li>OfferBook-Train-Arrive-05:51 Occur Num: 10</li><li>OfferBook-Train-Arrive-that Occur Num: 5</li></ul></ul><li>OfferBook-Train-Day-wednesday<ul><li>Occur Num: 164</li><li>NLU Output</li><ul><li>OfferBook-Train-Depart-cambridge Occur Num: 19</li><li>OfferBook-Train-Dest-cambridge Occur Num: 13</li><li>OfferBook-Train-Arrive-05:51 Occur Num: 11</li><li>OfferBook-Train-none-none Occur Num: 6</li><li>OfferBook-Train-Leave-05:00 Occur Num: 5</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Train-Time-? Occur Num: 2</li><li>Request-Train-Ticket-? Occur Num: 2</li><li>Request-Train-Id-? Occur Num: 1</li><li>Inform-Train-Depart-norwich Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>offerbook-train-arriveBy Occur Num: 30</li><li>inform-train-duration Occur Num: 15</li><li>offerbook-train-leaveAt Occur Num: 8</li><li>inform-train-price Occur Num: 6</li><li>offerbook-train-price Occur Num: 3</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-train-duration Occur Num: 5</li><li>request-train-price Occur Num: 2</li><li>request-train-arriveBy Occur Num: 2</li><li>request-train-trainID Occur Num: 1</li><li>request-train-leaveAt Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 89.1 %</p><p> (Precision, Recall, F1) : (1.000, 0.946, 0.964) </p><p> Average Dialog Turn (Succ): 3.463 </p><p> Average Dialog Turn (All): 7.435 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Taxi-Car-skoda<ul><li>Occur Num: 15</li><li>NLU Output</li><ul><li>Inform-Taxi-Leave-05:00 Occur Num: 3</li><li>Inform-Taxi-Phone-62490184653 Occur Num: 1</li><li>Inform-Taxi-Arrive-05:51 Occur Num: 1</li><li>Inform-Taxi-Phone-56513788991 Occur Num: 1</li><li>Inform-Train-Time-51 minutes Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-toyota<ul><li>Occur Num: 14</li><li>NLU Output</li><ul><li>Inform-Taxi-Leave-05:00 Occur Num: 3</li><li>Inform-Taxi-Phone-41166927644 Occur Num: 1</li><li>Inform-Taxi-Phone-24219248656 Occur Num: 1</li><li>Inform-Taxi-Phone-87029902122 Occur Num: 1</li><li>Inform-Taxi-Phone-32149529100 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-lexus<ul><li>Occur Num: 14</li><li>NLU Output</li><ul><li>Inform-Taxi-Arrive-05:51 Occur Num: 2</li><li>Recommend-Restaurant-Name-the Occur Num: 1</li><li>Recommend-Restaurant-Name-river bar steakhouse and grill Occur Num: 1</li><li>Inform-Restaurant-Area-centre Occur Num: 1</li><li>Inform-Restaurant-Price-cheap Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-honda<ul><li>Occur Num: 12</li><li>NLU Output</li><ul><li>Inform-Taxi-Leave-05:00 Occur Num: 3</li><li>Inform-Taxi-Phone-52547806713 Occur Num: 1</li><li>Inform-Taxi-Phone-42223260114 Occur Num: 1</li><li>Inform-Taxi-Phone-84754151715 Occur Num: 1</li><li>Inform-Taxi-Arrive-05:51 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-bmw<ul><li>Occur Num: 12</li><li>NLU Output</li><ul><li>Inform-Taxi-Leave-05:00 Occur Num: 3</li><li>Inform-Taxi-Phone-79054617689 Occur Num: 1</li><li>Inform-Taxi-Phone-50382392404 Occur Num: 1</li><li>Inform-Taxi-Phone-31843013369 Occur Num: 1</li><li>Inform-Taxi-Phone-56545933342 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-ford<ul><li>Occur Num: 9</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-guesthouse Occur Num: 2</li><li>Inform-Taxi-Phone-94549609159 Occur Num: 1</li><li>Inform-Taxi-Phone-56629498460 Occur Num: 1</li><li>Inform-Taxi-Phone-47831724668 Occur Num: 1</li><li>Inform-Taxi-Phone-88614982646 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-audi<ul><li>Occur Num: 8</li><li>NLU Output</li><ul><li>Inform-Taxi-Leave-05:00 Occur Num: 2</li><li>Inform-Taxi-Phone-73297457818 Occur Num: 1</li><li>Inform-Taxi-Phone-02411795305 Occur Num: 1</li><li>OfferBooked-Train-Time-51 Occur Num: 1</li><li>Inform-Taxi-Phone-73390690110 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-tesla<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-guesthouse Occur Num: 3</li><li>Inform-Taxi-Phone-71931929499 Occur Num: 1</li><li>Inform-Taxi-Phone-40677505310 Occur Num: 1</li><li>Inform-Taxi-Car-lexus Occur Num: 1</li><li>Inform-Taxi-Phone-87623395811 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Phone-86986828335<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Name-the Occur Num: 1</li><li>Recommend-Restaurant-Name-river bar steakhouse and grill Occur Num: 1</li><li>Inform-Restaurant-Area-centre Occur Num: 1</li><li>Inform-Restaurant-Price-cheap Occur Num: 1</li><li>Inform-Hotel-Type-hotel Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-volvo<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-26438028145 Occur Num: 1</li><li>OfferBooked-Train-Time-51 Occur Num: 1</li><li>OfferBooked-Train-Time-one Occur Num: 1</li><li>Inform-Taxi-Phone-96368625434 Occur Num: 1</li><li>Inform-Train-Leave-05:00 Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Taxi-Car-? Occur Num: 5</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-taxi-car type Occur Num: 5</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 0.0 %</p><p> (Precision, Recall, F1) : (0.000, 0.000, 0.000) </p><p> Average Dialog Turn (Succ): 0.000 </p><p> Average Dialog Turn (All): 40.000 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Police-Addr-? Occur Num: 17</li><li>Request-Police-Phone-? Occur Num: 6</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-police-phone Occur Num: 20</li><li>request-police-address Occur Num: 17</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 36.7 %</p><p> (Precision, Recall, F1) : (0.367, 0.367, 0.367) </p><p> Average Dialog Turn (Succ): 18.364 </p><p> Average Dialog Turn (All): 32.067 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hospital-Choice-66<ul><li>Occur Num: 54</li><li>NLU Output</li><ul><li>Inform-Train-Ticket-66 Occur Num: 13</li><li>Inform-Hospital-Department-neurosciences critical care unit Occur Num: 8</li><li>Inform-Train-Ref-66 Occur Num: 5</li><li>Inform-Hospital-Department-paediatric intensive care unit Occur Num: 2</li><li>Inform-Hospital-Phone-01223274222 Occur Num: 2</li></ul></ul><li>Inform-Hospital-Department-neurosciences critical care unit<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-emergency department Occur Num: 1</li><li>Inform-Hospital-Phone-01223274222 Occur Num: 1</li><li>Inform-Hospital-Phone-01223348313 Occur Num: 1</li><li>Inform-Hospital-Department-clinical research facility Occur Num: 1</li><li>Inform-Hospital-Department-transplant unit Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223274222<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Inform-Hospital-Phone-01223274222 Occur Num: 2</li><li>Inform-Restaurant-Choice-66 Occur Num: 1</li><li>Inform-Hospital-Department-teenage cancer trust unit Occur Num: 1</li><li>Inform-Hospital-Department-neurosciences critical care unit Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223217484<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Request-Train-Arrive-? Occur Num: 1</li><li>Request-Taxi-Arrive-? Occur Num: 1</li><li>Inform-Hospital-Phone-01223217484 Occur Num: 1</li><li>Inform-Hospital-Department-medicine for the elderly Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Department-medicine for the elderly<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Request-Train-Arrive-? Occur Num: 1</li><li>Request-Taxi-Arrive-? Occur Num: 1</li><li>Inform-Hospital-Phone-01223217484 Occur Num: 1</li><li>Inform-Hospital-Department-medicine for the elderly Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Department-paediatric intensive care unit<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-paediatric intensive care unit Occur Num: 2</li><li>Inform-Hospital-Phone-01223217715 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223217231<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Restaurant-Name-66 Occur Num: 1</li><li>Inform-Hospital-Phone-01223217231 Occur Num: 1</li><li>Inform-Restaurant-Area-centre Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Department-teenage cancer trust unit<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Restaurant-Choice-66 Occur Num: 1</li><li>Inform-Hospital-Department-teenage cancer trust unit Occur Num: 1</li><li>Inform-Hospital-Phone-01223274222 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223217715<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-paediatric intensive care unit Occur Num: 1</li><li>Inform-Hospital-Phone-01223217715 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Department-emergency department<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurosciences critical care unit Occur Num: 1</li><li>Inform-Hospital-Department-emergency department Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Hospital-Phone-? Occur Num: 19</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-hospital-phone Occur Num: 19</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/SUMBT-RulePolicy-TemplateNLG/res.txt b/tests/results/SUMBT-RulePolicy-TemplateNLG/res.txt deleted file mode 100755 index 6c48c7eb03df6454ee9ceb36e4ea67e1d391aa4e..0000000000000000000000000000000000000000 --- a/tests/results/SUMBT-RulePolicy-TemplateNLG/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.347 -success number of dialogs/tot: 0.338 -average precision: 0.5234376186868399 -average recall: 0.5057916728477476 -average f1: 0.47292705156916265 -average book rate: 0.5775 -average turn (succ): 12.088757396449704 -average turn (all): 26.552 diff --git a/tests/results/SVMNLU-RuleDST-RulePolicy-TemplateNLG/Frequency_of_domain.png b/tests/results/SVMNLU-RuleDST-RulePolicy-TemplateNLG/Frequency_of_domain.png deleted file mode 100755 index 197f26e07d058306fc40f8b0d9d33cb0b678c9ac..0000000000000000000000000000000000000000 Binary files a/tests/results/SVMNLU-RuleDST-RulePolicy-TemplateNLG/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/SVMNLU-RuleDST-RulePolicy-TemplateNLG/Performance_for_each_domain.png b/tests/results/SVMNLU-RuleDST-RulePolicy-TemplateNLG/Performance_for_each_domain.png deleted file mode 100755 index 34c75e9fa1843be088754121916dccdf9f131269..0000000000000000000000000000000000000000 Binary files a/tests/results/SVMNLU-RuleDST-RulePolicy-TemplateNLG/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/SVMNLU-RuleDST-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png b/tests/results/SVMNLU-RuleDST-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index 73d242f2a3a7bead89909264917bd31525cb2627..0000000000000000000000000000000000000000 Binary files a/tests/results/SVMNLU-RuleDST-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/SVMNLU-RuleDST-RulePolicy-TemplateNLG/report_multiwoz.html b/tests/results/SVMNLU-RuleDST-RulePolicy-TemplateNLG/report_multiwoz.html deleted file mode 100755 index ba2750686482641f8e3d116d8c40cdf73e711f87..0000000000000000000000000000000000000000 --- a/tests/results/SVMNLU-RuleDST-RulePolicy-TemplateNLG/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: SVMNLU-RuleDST-RulePolicy-TemplateNLG</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-24 22:13:40</h3><h3>Overall Results</h3><p> Success Rate: 70.4 %</p><p> (Precision, Recall, F1) : (0.791, 0.888, 0.815) </p><p> Average Dialog Turn (Succ): 14.759 </p><p> Average Dialog Turn (All): 17.736 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">425</td><td align="center">0.605</td><td align="center">0.539</td><td align="center">0.647</td><td align="center">0.563</td><td align="center">0.122</td><td align="center">7.230</td><td align="center">8.551</td></tr><tr><td align="center">attraction</td><td align="center">333</td><td align="center">0.931</td><td align="center">0.891</td><td align="center">0.940</td><td align="center">0.907</td><td align="center">0.015</td><td align="center">6.361</td><td align="center">6.541</td></tr><tr><td align="center">taxi</td><td align="center">167</td><td align="center">0.844</td><td align="center">0.886</td><td align="center">0.865</td><td align="center">0.872</td><td align="center">0.156</td><td align="center">7.362</td><td align="center">10.060</td></tr><tr><td align="center">restaurant</td><td align="center">410</td><td align="center">0.849</td><td align="center">0.771</td><td align="center">0.807</td><td align="center">0.781</td><td align="center">0.054</td><td align="center">5.891</td><td align="center">6.941</td></tr><tr><td align="center">train</td><td align="center">389</td><td align="center">0.895</td><td align="center">0.825</td><td align="center">0.879</td><td align="center">0.842</td><td align="center">0.054</td><td align="center">11.247</td><td align="center">12.036</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">0.000</td><td align="center">5.652</td><td align="center">5.652</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">1.000</td><td align="center">0.000</td><td align="center">4.067</td><td align="center">4.067</td></tr></table><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 60.5 %</p><p> (Precision, Recall, F1) : (0.539, 0.647, 0.563) </p><p> Average Dialog Turn (Succ): 7.230 </p><p> Average Dialog Turn (All): 8.551 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 292</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 59</li><li>Inform-Hotel-Type-guesthouse Occur Num: 43</li><li>Inform-Hotel-Price-moderate Occur Num: 32</li><li>Inform-Hotel-Price-expensive Occur Num: 31</li><li>Inform-Hotel-Type-hotel Occur Num: 28</li></ul></ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 267</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-guesthouse Occur Num: 52</li><li>Inform-Hotel-Parking-yes Occur Num: 41</li><li>Inform-Hotel-Price-moderate Occur Num: 33</li><li>Inform-Hotel-Area-north Occur Num: 25</li><li>Inform-Hotel-Type-hotel Occur Num: 25</li></ul></ul><li>Inform-Hotel-Stars-4<ul><li>Occur Num: 224</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 36</li><li>Inform-Hotel-Parking-yes Occur Num: 30</li><li>Inform-Hotel-Type-guesthouse Occur Num: 25</li><li>Inform-Hotel-Price-moderate Occur Num: 20</li><li>Inform-Hotel-Price-expensive Occur Num: 18</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 194</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 34</li><li>Inform-Hotel-Internet-yes Occur Num: 33</li><li>Inform-Hotel-Area-north Occur Num: 26</li><li>Inform-Hotel-Price-expensive Occur Num: 23</li><li>Inform-Hotel-Price-moderate Occur Num: 21</li></ul></ul><li>Inform-Hotel-Price-moderate<ul><li>Occur Num: 156</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-yes Occur Num: 28</li><li>Inform-Hotel-Internet-yes Occur Num: 26</li><li>Inform-Hotel-Type-guesthouse Occur Num: 24</li><li>Inform-Hotel-Type-hotel Occur Num: 18</li><li>Inform-Hotel-Area-north Occur Num: 13</li></ul></ul><li>Request-Hotel-Post-?<ul><li>Occur Num: 140</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 52</li><li>Request-Restaurant-Post-? Occur Num: 26</li><li>Request-Hospital-Post-? Occur Num: 20</li><li>Request-Hotel-Addr-? Occur Num: 9</li><li>Request-Attraction-Phone-? Occur Num: 6</li></ul></ul><li>Request-Hotel-Addr-?<ul><li>Occur Num: 114</li><li>NLU Output</li><ul><li>Request-Attraction-Addr-? Occur Num: 24</li><li>Request-Hotel-Post-? Occur Num: 21</li><li>Request-Attraction-Phone-? Occur Num: 10</li><li>Request-Hotel-Phone-? Occur Num: 9</li><li>Request-Hotel-Type-? Occur Num: 8</li></ul></ul><li>Inform-Hotel-Area-north<ul><li>Occur Num: 96</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-yes Occur Num: 28</li><li>Inform-Hotel-Parking-yes Occur Num: 19</li><li>Inform-Hotel-Type-guesthouse Occur Num: 17</li><li>Inform-Hotel-Price-expensive Occur Num: 10</li><li>Inform-Hotel-Price-moderate Occur Num: 9</li></ul></ul><li>Request-Hotel-Phone-?<ul><li>Occur Num: 90</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 47</li><li>Request-Hotel-Addr-? Occur Num: 12</li><li>Request-Restaurant-Phone-? Occur Num: 9</li><li>Request-Attraction-Post-? Occur Num: 4</li><li>Request-Hotel-Parking-? Occur Num: 3</li></ul></ul><li>Inform-Hotel-Type-hotel<ul><li>Occur Num: 88</li><li>NLU Output</li><ul><li>Inform-Hotel-Price-moderate Occur Num: 13</li><li>Inform-Hotel-Price-expensive Occur Num: 13</li><li>Inform-Hotel-Internet-yes Occur Num: 12</li><li>Inform-Hotel-Parking-yes Occur Num: 11</li><li>Inform-Hotel-Area-east Occur Num: 6</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 300</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-none Occur Num: 61</li><li>Inform-Hotel-Parking-none Occur Num: 34</li><li>Inform-Hotel-Type-guesthouse Occur Num: 20</li><li>Inform-Hotel-Stars-4 Occur Num: 19</li><li>Inform-Hotel-Type-hotel Occur Num: 17</li></ul></ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 257</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 49</li><li>Inform-Hotel-Internet-none Occur Num: 30</li><li>Inform-Hotel-Stars-4 Occur Num: 20</li><li>Inform-Hotel-Type-guesthouse Occur Num: 20</li><li>Inform-Hotel-Type-hotel Occur Num: 14</li></ul></ul><li>Inform-Hotel-Stars-4<ul><li>Occur Num: 160</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 23</li><li>Inform-Hotel-Internet-none Occur Num: 21</li><li>Inform-Hotel-Type-guesthouse Occur Num: 16</li><li>Inform-Hotel-Price-cheap Occur Num: 12</li><li>Inform-Hotel-Type-hotel Occur Num: 10</li></ul></ul><li>Recommend-Hotel-Internet-yes<ul><li>Occur Num: 138</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-none Occur Num: 24</li><li>Recommend-Hotel-Internet-none Occur Num: 20</li><li>Inform-Hotel-Choice-9 Occur Num: 5</li><li>Inform-Hotel-Choice-2 Occur Num: 4</li><li>Inform-Hotel-Choice-5 Occur Num: 3</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 129</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 21</li><li>Inform-Hotel-Internet-none Occur Num: 19</li><li>Inform-Hotel-Stars-4 Occur Num: 13</li><li>Inform-Hotel-Price-cheap Occur Num: 9</li><li>Inform-Hotel-Area-north Occur Num: 9</li></ul></ul><li>NoOffer-Hotel-Internet-yes<ul><li>Occur Num: 128</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 15</li><li>NoOffer-Hotel-Type-hotels Occur Num: 12</li><li>NoOffer-Hotel-Internet-none Occur Num: 12</li><li>NoOffer-Hotel-Type-hotel Occur Num: 11</li><li>Request-Hotel-Price-? Occur Num: 10</li></ul></ul><li>NoOffer-Hotel-Type-hotel<ul><li>Occur Num: 112</li><li>NLU Output</li><ul><li>NoOffer-Hotel-none-none Occur Num: 13</li><li>NoOffer-Hotel-Type-hotels Occur Num: 10</li><li>reqmore-general-none-none Occur Num: 10</li><li>NoOffer-Hotel-Price-cheap Occur Num: 8</li><li>Request-Hotel-Area-? Occur Num: 7</li></ul></ul><li>Inform-Hotel-Price-moderate<ul><li>Occur Num: 105</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 15</li><li>Inform-Hotel-Internet-none Occur Num: 14</li><li>Inform-Hotel-Type-guesthouse Occur Num: 10</li><li>Inform-Hotel-Area-north Occur Num: 8</li><li>Inform-Hotel-Type-hotel Occur Num: 5</li></ul></ul><li>Inform-Hotel-Type-hotel<ul><li>Occur Num: 104</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-none Occur Num: 16</li><li>Inform-Hotel-Parking-none Occur Num: 15</li><li>Inform-Hotel-Stars-4 Occur Num: 9</li><li>Inform-Hotel-Area-west Occur Num: 7</li><li>Inform-Hotel-Price-expensive Occur Num: 6</li></ul></ul><li>Recommend-Hotel-Parking-yes<ul><li>Occur Num: 103</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 20</li><li>Recommend-Hotel-Parking-none Occur Num: 11</li><li>Inform-Hotel-Choice-9 Occur Num: 4</li><li>Inform-Hotel-Choice-2 Occur Num: 3</li><li>Inform-Hotel-Choice-11 Occur Num: 3</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Hotel-Post-? Occur Num: 16</li><li>Request-Hotel-Phone-? Occur Num: 11</li><li>Request-Hotel-Internet-? Occur Num: 3</li><li>Inform-Hotel-Internet-yes Occur Num: 2</li><li>Request-Hotel-Area-? Occur Num: 2</li><li>Inform-Hotel-Stay-2 Occur Num: 2</li><li>Inform-Hotel-Stay-5 Occur Num: 2</li><li>Inform-Hotel-Stay-3 Occur Num: 2</li><li>Inform-Hotel-Stars-0 Occur Num: 1</li><li>Inform-Hotel-Stars-2 Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-parking Occur Num: 84</li><li>inform-hotel-internet Occur Num: 82</li><li>recommend-hotel-internet Occur Num: 21</li><li>inform-hotel-stars Occur Num: 12</li><li>recommend-hotel-parking Occur Num: 12</li><li>inform-hotel-area Occur Num: 9</li><li>recommend-hotel-stars Occur Num: 3</li><li>recommend-hotel-area Occur Num: 1</li><li>recommend-hotel-phone Occur Num: 1</li><li>recommend-hotel-postcode Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-internet Occur Num: 29</li><li>request-hotel-parking Occur Num: 25</li><li>request-hotel-postcode Occur Num: 19</li><li>request-hotel-phone Occur Num: 14</li><li>request-hotel-address Occur Num: 12</li><li>request-hotel-type Occur Num: 4</li><li>request-hotel-area Occur Num: 3</li><li>request-hotel-stars Occur Num: 2</li><li>request-hotel-pricerange Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-type Occur Num: 38</li><li>inform-hotel-area Occur Num: 36</li><li>inform-hotel-stars Occur Num: 33</li><li>inform-hotel-phone Occur Num: 25</li></ul></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 93.1 %</p><p> (Precision, Recall, F1) : (0.891, 0.940, 0.907) </p><p> Average Dialog Turn (Succ): 6.361 </p><p> Average Dialog Turn (All): 6.541 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Attraction-Addr-?<ul><li>Occur Num: 260</li><li>NLU Output</li><ul><li>Request-Attraction-Fee-? Occur Num: 71</li><li>Request-Attraction-Phone-? Occur Num: 50</li><li>Request-Attraction-Post-? Occur Num: 50</li><li>Request-Restaurant-Addr-? Occur Num: 31</li><li>Request-Restaurant-Phone-? Occur Num: 17</li></ul></ul><li>Request-Attraction-Post-?<ul><li>Occur Num: 221</li><li>NLU Output</li><ul><li>Request-Attraction-Addr-? Occur Num: 64</li><li>Request-Attraction-Fee-? Occur Num: 56</li><li>Request-Attraction-Phone-? Occur Num: 37</li><li>Request-Restaurant-Addr-? Occur Num: 18</li><li>Request-Restaurant-Post-? Occur Num: 13</li></ul></ul><li>Request-Attraction-Phone-?<ul><li>Occur Num: 195</li><li>NLU Output</li><ul><li>Request-Attraction-Fee-? Occur Num: 55</li><li>Request-Attraction-Post-? Occur Num: 40</li><li>Request-Attraction-Addr-? Occur Num: 39</li><li>Request-Restaurant-Phone-? Occur Num: 24</li><li>Request-Attraction-Type-? Occur Num: 10</li></ul></ul><li>Request-Attraction-Fee-?<ul><li>Occur Num: 179</li><li>NLU Output</li><ul><li>Request-Attraction-Addr-? Occur Num: 63</li><li>Request-Attraction-Post-? Occur Num: 52</li><li>Request-Attraction-Phone-? Occur Num: 48</li><li>Request-Attraction-Type-? Occur Num: 7</li><li>Request-Attraction-Area-? Occur Num: 4</li></ul></ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 61</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 18</li><li>Inform-Attraction-Type-entertainment Occur Num: 16</li><li>Inform-Attraction-Type-college Occur Num: 7</li><li>Inform-Attraction-Type-museum Occur Num: 5</li><li>Inform-Hotel-Area-centre Occur Num: 4</li></ul></ul><li>Inform-Attraction-Type-museum<ul><li>Occur Num: 37</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 17</li><li>Inform-Train-Depart-cambridge Occur Num: 5</li><li>Inform-Attraction-Area-west Occur Num: 3</li><li>Inform-Attraction-Type-entertainment Occur Num: 3</li><li>Inform-Attraction-Area-east Occur Num: 3</li></ul></ul><li>Request-Attraction-Type-?<ul><li>Occur Num: 32</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 9</li><li>Request-Attraction-Fee-? Occur Num: 7</li><li>Request-Attraction-Post-? Occur Num: 7</li><li>Request-Attraction-Addr-? Occur Num: 6</li><li>Request-Restaurant-Phone-? Occur Num: 1</li></ul></ul><li>Request-Attraction-Area-?<ul><li>Occur Num: 28</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 7</li><li>Request-Attraction-Addr-? Occur Num: 6</li><li>Request-Attraction-Post-? Occur Num: 5</li><li>Request-Hotel-Addr-? Occur Num: 4</li><li>Request-Attraction-Fee-? Occur Num: 4</li></ul></ul><li>Inform-Attraction-Type-college<ul><li>Occur Num: 26</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 11</li><li>bye-general-none-none Occur Num: 4</li><li>Inform-Train-Depart-cambridge Occur Num: 3</li><li>Inform-Restaurant-Area-centre Occur Num: 2</li><li>Inform-Attraction-Area-west Occur Num: 2</li></ul></ul><li>Inform-Attraction-Area-west<ul><li>Occur Num: 19</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-entertainment Occur Num: 6</li><li>Inform-Attraction-Type-museum Occur Num: 3</li><li>Inform-Attraction-Type-college Occur Num: 3</li><li>Inform-Attraction-Area-centre Occur Num: 2</li><li>Request-Attraction-Type-? Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Attraction-Fee-?<ul><li>Occur Num: 198</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-park Occur Num: 22</li><li>Inform-Attraction-Phone-01223300085 Occur Num: 15</li><li>Inform-Attraction-Post-cb58as Occur Num: 15</li><li>Inform-Attraction-Fee-by Occur Num: 14</li><li>Inform-Attraction-Addr-whitehill road Occur Num: 12</li></ul></ul><li>Inform-Attraction-Choice-44<ul><li>Occur Num: 167</li><li>NLU Output</li><ul><li>Inform-Restaurant-Choice-69 Occur Num: 8</li><li>reqmore-general-none-none Occur Num: 7</li><li>Request-Attraction-Type-? Occur Num: 3</li><li>greet-general-none-none Occur Num: 3</li><li>Recommend-Attraction-Name-cambridge corn exchange Occur Num: 3</li></ul></ul><li>Inform-Attraction-Addr-pool way, whitehill road, off newmarket road<ul><li>Occur Num: 158</li><li>NLU Output</li><ul><li>Inform-Attraction-Addr-whitehill road Occur Num: 39</li><li>Inform-Attraction-Addr-off newmarket road Occur Num: 38</li><li>Inform-Attraction-Addr-pool way Occur Num: 34</li><li>Inform-Attraction-Phone-01223902088 Occur Num: 9</li><li>Inform-Attraction-Post-cb58nt Occur Num: 9</li></ul></ul><li>Inform-Attraction-Phone-01223902088<ul><li>Occur Num: 73</li><li>NLU Output</li><ul><li>Inform-Attraction-Addr-whitehill road Occur Num: 9</li><li>Inform-Attraction-Addr-off newmarket road Occur Num: 9</li><li>Inform-Attraction-Addr-pool way Occur Num: 8</li><li>Inform-Hotel-Phone-01223902088 Occur Num: 6</li><li>Inform-Attraction-Post-cb58nt Occur Num: 6</li></ul></ul><li>Inform-Attraction-Choice-5<ul><li>Occur Num: 72</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 4</li><li>Recommend-Attraction-Name-clare hall Occur Num: 3</li><li>Recommend-Attraction-Name-churchill college Occur Num: 2</li><li>Request-Attraction-Name-? Occur Num: 2</li><li>Inform-Attraction-Name-magdalene college Occur Num: 2</li></ul></ul><li>Inform-Attraction-Post-cb58nt<ul><li>Occur Num: 65</li><li>NLU Output</li><ul><li>Inform-Attraction-Addr-whitehill road Occur Num: 10</li><li>Inform-Attraction-Addr-off newmarket road Occur Num: 10</li><li>Inform-Attraction-Addr-pool way Occur Num: 9</li><li>Inform-Restaurant-Post-cb58nt Occur Num: 8</li><li>Inform-Attraction-Phone-01223902088 Occur Num: 6</li></ul></ul><li>Inform-Attraction-Fee-free<ul><li>Occur Num: 54</li><li>NLU Output</li><ul><li>Inform-Attraction-Type-park Occur Num: 12</li><li>Inform-Attraction-Addr-saint andrew 's street Occur Num: 6</li><li>Inform-Attraction-Addr-storey 's way Occur Num: 4</li><li>Inform-Attraction-Phone-01223336233 Occur Num: 3</li><li>Inform-Attraction-Post-cb23bu Occur Num: 3</li></ul></ul><li>Inform-Attraction-Choice-13<ul><li>Occur Num: 46</li><li>NLU Output</li><ul><li>Recommend-Attraction-Name-queens ' college Occur Num: 2</li><li>Inform-Attraction-Type-park Occur Num: 2</li><li>Recommend-Attraction-Name-Clare college Occur Num: 1</li><li>Recommend-Attraction-Addr-trinity lane Occur Num: 1</li><li>Inform-Attraction-Name-cafe jello gallery Occur Num: 1</li></ul></ul><li>Inform-Attraction-Phone-01223300085<ul><li>Occur Num: 41</li><li>NLU Output</li><ul><li>Inform-Attraction-Fee-? Occur Num: 6</li><li>Inform-Attraction-Post-cb58as Occur Num: 5</li><li>Inform-Hotel-Phone-01223300085 Occur Num: 4</li><li>Request-Taxi-Arrive-? Occur Num: 4</li><li>Inform-Attraction-Fee-Their Occur Num: 3</li></ul></ul><li>Recommend-Attraction-Type-museum<ul><li>Occur Num: 36</li><li>NLU Output</li><ul><li>Inform-Attraction-Choice-23 Occur Num: 5</li><li>Recommend-Attraction-Name-All Saints Church Occur Num: 3</li><li>Inform-Attraction-Choice-44 Occur Num: 3</li><li>Inform-Attraction-Choice-11 Occur Num: 2</li><li>Recommend-Attraction-Name-whipple museum of the history of science Occur Num: 2</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Attraction-Post-? Occur Num: 2</li><li>Request-Attraction-Phone-? Occur Num: 2</li><li>Inform-Attraction-Type-nightclub Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-attraction-area Occur Num: 9</li><li>recommend-attraction-area Occur Num: 1</li><li>recommend-attraction-phone Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-phone Occur Num: 12</li><li>request-attraction-postcode Occur Num: 11</li><li>request-attraction-address Occur Num: 11</li><li>request-attraction-entrance fee Occur Num: 10</li><li>request-attraction-type Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-type Occur Num: 26</li><li>inform-attraction-area Occur Num: 17</li><li>inform-attraction-phone Occur Num: 15</li></ul></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 84.4 %</p><p> (Precision, Recall, F1) : (0.886, 0.865, 0.872) </p><p> Average Dialog Turn (Succ): 7.362 </p><p> Average Dialog Turn (All): 10.060 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Phone-?<ul><li>Occur Num: 127</li><li>NLU Output</li><ul><li>Request-Taxi-Car-? Occur Num: 75</li><li>Request-Attraction-Phone-? Occur Num: 52</li></ul></ul><li>Request-Taxi-Car-?<ul><li>Occur Num: 115</li><li>NLU Output</li><ul><li>Request-Taxi-Phone-? Occur Num: 63</li><li>Request-Attraction-Phone-? Occur Num: 52</li></ul></ul><li>Inform-Taxi-Leave-dontcare<ul><li>Occur Num: 41</li><li>NLU Output</li><ul><li>bye-general-none-none Occur Num: 26</li><li>Inform-Train-Arrive-16:45 Occur Num: 3</li><li>Inform-Train-Arrive-18:00 Occur Num: 1</li><li>Inform-Train-Leave-12:15 Occur Num: 1</li><li>Inform-Train-Arrive-12:15 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-dontcare<ul><li>Occur Num: 15</li><li>NLU Output</li><ul><li>bye-general-none-none Occur Num: 7</li><li>Inform-Train-Leave-17:30 Occur Num: 2</li><li>Inform-Taxi-Leave-16:45 Occur Num: 1</li><li>Inform-Taxi-Leave-08:00 Occur Num: 1</li><li>Inform-Taxi-Leave-24:45 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-154 chesterton road<ul><li>Occur Num: 8</li><li>NLU Output</li><ul><li>Inform-Hotel-Addr-154 chesterton road Occur Num: 5</li><li>Inform-Taxi-none-none Occur Num: 1</li><li>Inform-Hotel-Addr-pool way, whitehill road, off newmarket road Occur Num: 1</li><li>greet-general-none-none Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-16:45<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-16:45 Occur Num: 4</li><li>Request-Train-Arrive-? Occur Num: 2</li><li>Inform-Restaurant-Time-16:45 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Arrive-17:45<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Inform-Train-Leave-17:45 Occur Num: 3</li><li>Inform-Train-Arrive-17:45 Occur Num: 2</li></ul></ul><li>Inform-Taxi-Arrive-15:15<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Request-Train-Arrive-? Occur Num: 2</li><li>Inform-Train-Arrive-15:15 Occur Num: 1</li><li>Inform-Train-Leave-15:15 Occur Num: 1</li><li>Inform-Restaurant-Food-chinese Occur Num: 1</li></ul></ul><li>Inform-Taxi-Dest-lynne strover gallery<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-12:15 Occur Num: 1</li><li>Recommend-Attraction-Name-Parkside Police Station Occur Num: 1</li><li>Recommend-Attraction-Name-lynne strover gallery Occur Num: 1</li><li>Inform-Taxi-Leave-21:30 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Depart-parkside police station<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Train-Arrive-12:15 Occur Num: 1</li><li>Recommend-Attraction-Name-Parkside Police Station Occur Num: 1</li><li>Inform-Taxi-Leave-12:00 Occur Num: 1</li><li>Inform-Taxi-Arrive-12:00 Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Request-Taxi-Leave-?<ul><li>Occur Num: 183</li><li>NLU Output</li><ul><li>Request-Taxi-Arrive-? Occur Num: 62</li><li>Request-Train-Arrive-? Occur Num: 41</li><li>Request-Train-Leave-? Occur Num: 29</li><li>Inform-Attraction-Phone-01223902088 Occur Num: 20</li><li>Inform-Hotel-Phone-01223902088 Occur Num: 6</li></ul></ul><li>Request-Taxi-Arrive-?<ul><li>Occur Num: 132</li><li>NLU Output</li><ul><li>Request-Taxi-Leave-? Occur Num: 45</li><li>Request-Train-Arrive-? Occur Num: 41</li><li>Inform-Attraction-Phone-01223902088 Occur Num: 16</li><li>Request-Train-Leave-? Occur Num: 10</li><li>Inform-Hotel-Phone-01223902088 Occur Num: 5</li></ul></ul><li>Inform-Taxi-Car-honda<ul><li>Occur Num: 28</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-52547806713 Occur Num: 1</li><li>Inform-Taxi-Phone-43863385242 Occur Num: 1</li><li>Inform-Taxi-Phone-01996214154 Occur Num: 1</li><li>Inform-Taxi-Phone-88297768151 Occur Num: 1</li><li>Inform-Taxi-Phone-68465805575 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-skoda<ul><li>Occur Num: 25</li><li>NLU Output</li><ul><li>Request-Train-Day-? Occur Num: 2</li><li>Inform-Taxi-Phone-81634767635 Occur Num: 1</li><li>Inform-Taxi-Phone-55611258127 Occur Num: 1</li><li>Inform-Taxi-Phone-28321970261 Occur Num: 1</li><li>Inform-Taxi-Phone-24924666316 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-bmw<ul><li>Occur Num: 25</li><li>NLU Output</li><ul><li>Request-Train-Day-? Occur Num: 4</li><li>Request-Train-Dest-? Occur Num: 3</li><li>Inform-Taxi-Phone-98052904604 Occur Num: 1</li><li>Inform-Taxi-Phone-79054617689 Occur Num: 1</li><li>Inform-Taxi-Phone-89940665962 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-volkswagen<ul><li>Occur Num: 19</li><li>NLU Output</li><ul><li>Request-Train-Depart-? Occur Num: 2</li><li>Request-Train-Day-? Occur Num: 2</li><li>Request-Train-Dest-? Occur Num: 2</li><li>Inform-Taxi-Phone-75085571961 Occur Num: 1</li><li>Inform-Taxi-Phone-27929115551 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-lexus<ul><li>Occur Num: 19</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-29828932965 Occur Num: 1</li><li>Inform-Taxi-Phone-83144288521 Occur Num: 1</li><li>Inform-Taxi-Phone-70392353736 Occur Num: 1</li><li>Inform-Taxi-Phone-76339421079 Occur Num: 1</li><li>Inform-Taxi-Phone-41740045662 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-toyota<ul><li>Occur Num: 15</li><li>NLU Output</li><ul><li>Request-Train-Day-? Occur Num: 1</li><li>Inform-Taxi-Phone-94438058116 Occur Num: 1</li><li>Inform-Taxi-Phone-10939629271 Occur Num: 1</li><li>Inform-Taxi-Phone-87029902122 Occur Num: 1</li><li>Inform-Taxi-Phone-75128605594 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-ford<ul><li>Occur Num: 12</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-91394454130 Occur Num: 1</li><li>Inform-Taxi-Phone-77993250755 Occur Num: 1</li><li>Inform-Taxi-Phone-46495898757 Occur Num: 1</li><li>Inform-Taxi-Phone-12327272764 Occur Num: 1</li><li>Inform-Taxi-Phone-00269515196 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-volvo<ul><li>Occur Num: 11</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-26438028145 Occur Num: 1</li><li>Inform-Taxi-Phone-27749757303 Occur Num: 1</li><li>Inform-Taxi-Phone-22565494549 Occur Num: 1</li><li>Inform-Taxi-Phone-22672239961 Occur Num: 1</li><li>Inform-Taxi-Phone-43235337410 Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Inform-Taxi-Leave-dontcare Occur Num: 14</li><li>Request-Taxi-Car-? Occur Num: 9</li><li>Inform-Taxi-Dest-5 greens road Occur Num: 1</li><li>Inform-Taxi-Arrive-12:15 Occur Num: 1</li><li>Inform-Taxi-Dest-154 chesterton road Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-taxi-car type Occur Num: 26</li><li>request-taxi-phone Occur Num: 19</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 84.9 %</p><p> (Precision, Recall, F1) : (0.771, 0.807, 0.781) </p><p> Average Dialog Turn (Succ): 5.891 </p><p> Average Dialog Turn (All): 6.941 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Restaurant-Phone-?<ul><li>Occur Num: 163</li><li>NLU Output</li><ul><li>Request-Attraction-Phone-? Occur Num: 68</li><li>Request-Restaurant-Addr-? Occur Num: 37</li><li>Inform-Hotel-Stars-one Occur Num: 24</li><li>Request-Restaurant-Post-? Occur Num: 12</li><li>Request-Hotel-Phone-? Occur Num: 7</li></ul></ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 157</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 33</li><li>Inform-Restaurant-Food-indian Occur Num: 15</li><li>Inform-Restaurant-Price-moderate Occur Num: 13</li><li>Inform-Restaurant-Food-british Occur Num: 12</li><li>Inform-Restaurant-Food-european Occur Num: 11</li></ul></ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 147</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderate Occur Num: 34</li><li>Inform-Restaurant-Price-expensive Occur Num: 22</li><li>Inform-Restaurant-Price-cheap Occur Num: 16</li><li>Inform-Restaurant-Food-indian Occur Num: 11</li><li>Inform-Restaurant-Food-british Occur Num: 9</li></ul></ul><li>Request-Restaurant-Addr-?<ul><li>Occur Num: 127</li><li>NLU Output</li><ul><li>Request-Police-Addr-? Occur Num: 34</li><li>Request-Restaurant-Phone-? Occur Num: 29</li><li>Request-Attraction-Addr-? Occur Num: 19</li><li>Request-Restaurant-Post-? Occur Num: 11</li><li>Request-Attraction-Phone-? Occur Num: 9</li></ul></ul><li>Request-Restaurant-Post-?<ul><li>Occur Num: 95</li><li>NLU Output</li><ul><li>Request-Attraction-Post-? Occur Num: 42</li><li>Request-Restaurant-Addr-? Occur Num: 22</li><li>Request-Hospital-Post-? Occur Num: 9</li><li>Request-Restaurant-Phone-? Occur Num: 8</li><li>Request-Attraction-Phone-? Occur Num: 5</li></ul></ul><li>Inform-Restaurant-Price-moderate<ul><li>Occur Num: 91</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 26</li><li>Inform-Restaurant-Food-indian Occur Num: 13</li><li>Inform-Restaurant-Food-italian Occur Num: 8</li><li>Inform-Restaurant-Area-east Occur Num: 6</li><li>Inform-Restaurant-Food-chinese Occur Num: 5</li></ul></ul><li>Inform-Restaurant-Price-cheap<ul><li>Occur Num: 77</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 16</li><li>Inform-Restaurant-Food-italian Occur Num: 6</li><li>Inform-Restaurant-Food-chinese Occur Num: 6</li><li>Inform-Restaurant-Area-east Occur Num: 5</li><li>Inform-Restaurant-Food-indian Occur Num: 4</li></ul></ul><li>Inform-Restaurant-Food-indian<ul><li>Occur Num: 58</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderate Occur Num: 15</li><li>Inform-Restaurant-Price-expensive Occur Num: 12</li><li>Inform-Restaurant-Area-centre Occur Num: 12</li><li>Inform-Restaurant-Price-cheap Occur Num: 5</li><li>Inform-Restaurant-Area-north Occur Num: 4</li></ul></ul><li>Inform-Restaurant-Area-east<ul><li>Occur Num: 56</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-moderate Occur Num: 13</li><li>Inform-Restaurant-Price-cheap Occur Num: 8</li><li>Inform-Restaurant-Price-expensive Occur Num: 7</li><li>Inform-Hotel-Area-east Occur Num: 5</li><li>Inform-Restaurant-Food-chinese Occur Num: 4</li></ul></ul><li>Inform-Restaurant-Area-north<ul><li>Occur Num: 41</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 8</li><li>Inform-Restaurant-Food-indian Occur Num: 8</li><li>Inform-Restaurant-Price-moderate Occur Num: 5</li><li>Inform-Hotel-Area-north Occur Num: 4</li><li>Inform-Restaurant-Price-cheap Occur Num: 4</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Request-Restaurant-Food-?<ul><li>Occur Num: 81</li><li>NLU Output</li><ul><li>Request-Restaurant-Price-? Occur Num: 10</li><li>NoOffer-Restaurant-Area-centre Occur Num: 8</li><li>NoOffer-Restaurant-none-none Occur Num: 7</li><li>NoOffer-Restaurant-Price-cheap Occur Num: 7</li><li>NoOffer-Restaurant-Price-expensive Occur Num: 6</li></ul></ul><li>Inform-Restaurant-Choice-4<ul><li>Occur Num: 74</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 3</li><li>Recommend-Restaurant-Name-meghna Occur Num: 2</li><li>Inform-Restaurant-Area-centre Occur Num: 2</li><li>Recommend-Restaurant-Name-Yippee noodle bar Occur Num: 2</li><li>Recommend-Restaurant-Name-Restaurant one seven Occur Num: 2</li></ul></ul><li>NoOffer-Restaurant-Price-cheap<ul><li>Occur Num: 62</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 9</li><li>Request-Restaurant-Food-? Occur Num: 8</li><li>NoOffer-Restaurant-Area-centre Occur Num: 6</li><li>Request-Restaurant-Price-? Occur Num: 6</li><li>NoOffer-Restaurant-Food-british Occur Num: 5</li></ul></ul><li>NoOffer-Restaurant-Area-centre<ul><li>Occur Num: 61</li><li>NLU Output</li><ul><li>Request-Restaurant-Food-? Occur Num: 9</li><li>NoOffer-Restaurant-none-none Occur Num: 8</li><li>NoOffer-Restaurant-Price-cheap Occur Num: 7</li><li>NoOffer-Restaurant-Food-indian Occur Num: 5</li><li>NoOffer-Restaurant-Area-in town Occur Num: 4</li></ul></ul><li>NoOffer-Restaurant-Price-moderate<ul><li>Occur Num: 57</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 10</li><li>NoOffer-Restaurant-Area-centre Occur Num: 6</li><li>NoOffer-Restaurant-Food-indian Occur Num: 5</li><li>Request-Restaurant-Food-? Occur Num: 5</li><li>NoOffer-Restaurant-Area-east Occur Num: 3</li></ul></ul><li>Inform-Restaurant-Choice-2<ul><li>Occur Num: 56</li><li>NLU Output</li><ul><li>Inform-Booking-none-none Occur Num: 3</li><li>Recommend-Restaurant-Name-galleria Occur Num: 3</li><li>Recommend-Restaurant-Name-Nandos city centre Occur Num: 2</li><li>Inform-Restaurant-Phone-01223362054 Occur Num: 2</li><li>Recommend-Restaurant-Name-anatolia Occur Num: 2</li></ul></ul><li>Request-Restaurant-Price-?<ul><li>Occur Num: 47</li><li>NLU Output</li><ul><li>Request-Restaurant-Food-? Occur Num: 10</li><li>NoOffer-Restaurant-Price-cheap Occur Num: 5</li><li>Request-Restaurant-Area-? Occur Num: 4</li><li>NoOffer-Restaurant-none-none Occur Num: 3</li><li>NoOffer-Restaurant-Price-expensive Occur Num: 3</li></ul></ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 39</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 5</li><li>Recommend-Restaurant-Area-centre Occur Num: 4</li><li>Inform-Restaurant-Price-moderate Occur Num: 4</li><li>Inform-Restaurant-Food-seafood Occur Num: 2</li><li>Recommend-Restaurant-Name-loch fyne Occur Num: 2</li></ul></ul><li>Inform-Restaurant-Choice-57<ul><li>Occur Num: 38</li><li>NLU Output</li><ul><li>Recommend-Restaurant-Name-cambridge chop house Occur Num: 3</li><li>Recommend-Restaurant-Name-The Occur Num: 2</li><li>Inform-Booking-none-none Occur Num: 2</li><li>Recommend-Restaurant-Name-Grafton hotel restaurant Occur Num: 2</li><li>Book-Booking-Ref-00000101 Occur Num: 2</li></ul></ul><li>Recommend-Restaurant-Area-centre<ul><li>Occur Num: 37</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 5</li><li>Inform-Restaurant-Choice-4 Occur Num: 3</li><li>Inform-Restaurant-Choice-69 Occur Num: 2</li><li>Inform-Restaurant-Choice-2 Occur Num: 2</li><li>Recommend-Restaurant-Name-Jinling noodle bar Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Restaurant-Phone-? Occur Num: 17</li><li>Request-Restaurant-Addr-? Occur Num: 2</li><li>Inform-Restaurant-Time-13:30 Occur Num: 1</li><li>Request-Restaurant-Post-? Occur Num: 1</li><li>Inform-Restaurant-Time-14:15 Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>recommend-restaurant-area Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-phone Occur Num: 23</li><li>request-restaurant-address Occur Num: 5</li><li>request-restaurant-postcode Occur Num: 2</li><li>request-restaurant-food Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-area Occur Num: 26</li><li>inform-restaurant-phone Occur Num: 17</li><li>inform-restaurant-food Occur Num: 17</li></ul></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 89.5 %</p><p> (Precision, Recall, F1) : (0.825, 0.879, 0.842) </p><p> Average Dialog Turn (Succ): 11.247 </p><p> Average Dialog Turn (All): 12.036 </p><b> System NLU Failed Dialog Act:</b><ul><li>Inform-Train-Dest-cambridge<ul><li>Occur Num: 526</li><li>NLU Output</li><ul><li>Inform-Train-Depart-cambridge Occur Num: 109</li><li>Inform-Train-Day-saturday Occur Num: 30</li><li>Inform-Train-Day-wednesday Occur Num: 30</li><li>Inform-Train-Day-tuesday Occur Num: 28</li><li>Inform-Train-Day-thursday Occur Num: 28</li></ul></ul><li>Inform-Train-Depart-cambridge<ul><li>Occur Num: 460</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 149</li><li>Inform-Train-Day-thursday Occur Num: 28</li><li>Inform-Train-Day-wednesday Occur Num: 21</li><li>Inform-Train-Day-friday Occur Num: 20</li><li>Inform-Train-Day-saturday Occur Num: 19</li></ul></ul><li>Inform-Train-Day-friday<ul><li>Occur Num: 162</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 32</li><li>Inform-Train-Depart-cambridge Occur Num: 16</li><li>Inform-Train-Day-thursday Occur Num: 4</li><li>Inform-Train-Day-tuesday Occur Num: 4</li><li>Inform-Hotel-Type-hotel Occur Num: 4</li></ul></ul><li>Inform-Train-Day-monday<ul><li>Occur Num: 146</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 18</li><li>Inform-Train-Depart-cambridge Occur Num: 11</li><li>Inform-Train-Dest-norwich Occur Num: 5</li><li>Inform-Train-Day-thursday Occur Num: 4</li><li>Inform-Train-Day-saturday Occur Num: 3</li></ul></ul><li>Inform-Train-Day-wednesday<ul><li>Occur Num: 143</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 25</li><li>Inform-Train-Depart-cambridge Occur Num: 23</li><li>Inform-Train-Day-tuesday Occur Num: 8</li><li>Inform-Train-Dest-london kings cross Occur Num: 4</li><li>Inform-Train-Day-thursday Occur Num: 4</li></ul></ul><li>Inform-Train-Day-tuesday<ul><li>Occur Num: 132</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 22</li><li>Inform-Train-Depart-cambridge Occur Num: 14</li><li>Inform-Train-Day-saturday Occur Num: 7</li><li>Inform-Train-Day-thursday Occur Num: 5</li><li>Inform-Hotel-Type-hotel Occur Num: 4</li></ul></ul><li>Inform-Train-Day-thursday<ul><li>Occur Num: 127</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 29</li><li>Inform-Train-Depart-cambridge Occur Num: 17</li><li>Inform-Train-Day-saturday Occur Num: 5</li><li>Inform-Train-Day-tuesday Occur Num: 5</li><li>Inform-Hotel-Type-hotel Occur Num: 4</li></ul></ul><li>Inform-Train-Day-saturday<ul><li>Occur Num: 112</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 19</li><li>Inform-Train-Depart-cambridge Occur Num: 11</li><li>Inform-Train-Day-thursday Occur Num: 5</li><li>Inform-Hotel-Type-hotel Occur Num: 4</li><li>Inform-Train-Day-tuesday Occur Num: 3</li></ul></ul><li>Inform-Train-Day-sunday<ul><li>Occur Num: 96</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 18</li><li>Inform-Train-Depart-cambridge Occur Num: 7</li><li>Inform-Hotel-Day-sunday Occur Num: 5</li><li>Inform-Train-Dest-norwich Occur Num: 3</li><li>Inform-Train-Depart-norwich Occur Num: 3</li></ul></ul><li>Inform-Train-Depart-bishops stortford<ul><li>Occur Num: 92</li><li>NLU Output</li><ul><li>Inform-Train-Dest-cambridge Occur Num: 15</li><li>Inform-Train-Depart-cambridge Occur Num: 14</li><li>Inform-Train-Dest-bishops stortford Occur Num: 9</li><li>Inform-Train-Day-saturday Occur Num: 4</li><li>Inform-Train-Day-monday Occur Num: 3</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>OfferBook-Train-Depart-cambridge<ul><li>Occur Num: 873</li><li>NLU Output</li><ul><li>OfferBook-Train-Dest-cambridge Occur Num: 132</li><li>OfferBook-Train-none-none Occur Num: 78</li><li>OfferBook-Train-Day-thursday Occur Num: 46</li><li>OfferBook-Train-Day-wednesday Occur Num: 35</li><li>OfferBook-Train-Day-friday Occur Num: 31</li></ul></ul><li>OfferBook-Train-Dest-cambridge<ul><li>Occur Num: 853</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 92</li><li>OfferBook-Train-Depart-cambridge Occur Num: 81</li><li>OfferBook-Train-Day-thursday Occur Num: 41</li><li>OfferBook-Train-Day-friday Occur Num: 36</li><li>OfferBook-Train-Day-wednesday Occur Num: 36</li></ul></ul><li>Request-Train-Depart-?<ul><li>Occur Num: 292</li><li>NLU Output</li><ul><li>Request-Train-Dest-? Occur Num: 100</li><li>Request-Train-Arrive-? Occur Num: 79</li><li>Request-Train-Leave-? Occur Num: 56</li><li>Request-Train-Day-? Occur Num: 45</li><li>Request-Taxi-Arrive-? Occur Num: 5</li></ul></ul><li>Request-Train-Dest-?<ul><li>Occur Num: 261</li><li>NLU Output</li><ul><li>Request-Train-Depart-? Occur Num: 119</li><li>Request-Train-Arrive-? Occur Num: 66</li><li>Request-Train-Leave-? Occur Num: 37</li><li>Request-Taxi-Dest-? Occur Num: 14</li><li>Request-Train-Day-? Occur Num: 14</li></ul></ul><li>OfferBook-Train-Day-thursday<ul><li>Occur Num: 257</li><li>NLU Output</li><ul><li>OfferBook-Train-Dest-cambridge Occur Num: 31</li><li>OfferBook-Train-Depart-cambridge Occur Num: 29</li><li>OfferBook-Train-none-none Occur Num: 26</li><li>Select-Train-Depart-cambridge Occur Num: 7</li><li>Inform-Train-none-none Occur Num: 4</li></ul></ul><li>OfferBook-Train-Day-friday<ul><li>Occur Num: 244</li><li>NLU Output</li><ul><li>OfferBook-Train-Dest-cambridge Occur Num: 32</li><li>OfferBook-Train-none-none Occur Num: 26</li><li>OfferBook-Train-Depart-cambridge Occur Num: 19</li><li>Select-Train-Day-friday Occur Num: 9</li><li>Select-Train-Dest-cambridge Occur Num: 6</li></ul></ul><li>OfferBook-Train-Day-monday<ul><li>Occur Num: 214</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 23</li><li>OfferBook-Train-Dest-cambridge Occur Num: 22</li><li>OfferBook-Train-Depart-cambridge Occur Num: 18</li><li>Inform-Train-Day-monday Occur Num: 5</li><li>OfferBook-Train-Arrive-that Occur Num: 5</li></ul></ul><li>OfferBook-Train-Day-wednesday<ul><li>Occur Num: 211</li><li>NLU Output</li><ul><li>OfferBook-Train-Dest-cambridge Occur Num: 33</li><li>OfferBook-Train-Depart-cambridge Occur Num: 23</li><li>OfferBook-Train-none-none Occur Num: 21</li><li>Select-Train-Depart-cambridge Occur Num: 7</li><li>Select-Train-Dest-cambridge Occur Num: 4</li></ul></ul><li>OfferBook-Train-Day-tuesday<ul><li>Occur Num: 208</li><li>NLU Output</li><ul><li>OfferBook-Train-Dest-cambridge Occur Num: 26</li><li>OfferBook-Train-none-none Occur Num: 21</li><li>OfferBook-Train-Depart-cambridge Occur Num: 19</li><li>OfferBook-Train-Leave-that Occur Num: 4</li><li>OfferBook-Train-Dest-peterborough Occur Num: 4</li></ul></ul><li>Request-Train-Leave-?<ul><li>Occur Num: 192</li><li>NLU Output</li><ul><li>Request-Train-Depart-? Occur Num: 70</li><li>Request-Train-Arrive-? Occur Num: 61</li><li>Request-Train-Dest-? Occur Num: 44</li><li>Request-Train-Day-? Occur Num: 8</li><li>Request-Taxi-Arrive-? Occur Num: 6</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Inform-Train-Depart-cambridge Occur Num: 8</li><li>Inform-Train-Dest-peterborough Occur Num: 2</li><li>Inform-Train-Depart-norwich Occur Num: 2</li><li>Inform-Train-Dest-broxbourne Occur Num: 2</li><li>Inform-Train-Dest-cambridge Occur Num: 2</li><li>Inform-Train-Dest-ely Occur Num: 1</li><li>Inform-Train-Depart-ely Occur Num: 1</li><li>Inform-Train-Dest-kings lynn Occur Num: 1</li><li>Inform-Train-People-1 Occur Num: 1</li><li>Inform-Train-Dest-leicester Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-train-duration Occur Num: 29</li><li>offerbook-train-arriveBy Occur Num: 16</li><li>offerbook-train-leaveAt Occur Num: 11</li><li>offerbook-train-trainID Occur Num: 2</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-train-duration Occur Num: 10</li><li>request-train-price Occur Num: 7</li><li>request-train-trainID Occur Num: 7</li><li>request-train-leaveAt Occur Num: 4</li><li>request-train-arriveBy Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 100.0 %</p><p> (Precision, Recall, F1) : (1.000, 1.000, 1.000) </p><p> Average Dialog Turn (Succ): 5.652 </p><p> Average Dialog Turn (All): 5.652 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Police-Addr-?<ul><li>Occur Num: 19</li><li>NLU Output</li><ul><li>Request-Police-Phone-? Occur Num: 10</li><li>Request-Attraction-Phone-? Occur Num: 4</li><li>Request-Restaurant-Phone-? Occur Num: 2</li><li>Inform-Police-none-none Occur Num: 1</li><li>Request-Attraction-Addr-? Occur Num: 1</li></ul></ul><li>Request-Police-Phone-?<ul><li>Occur Num: 14</li><li>NLU Output</li><ul><li>Request-Police-Addr-? Occur Num: 5</li><li>Request-Attraction-Phone-? Occur Num: 5</li><li>Request-Restaurant-Phone-? Occur Num: 2</li><li>Inform-Police-none-none Occur Num: 1</li><li>Request-Restaurant-Addr-? Occur Num: 1</li></ul></ul><li>Inform-Police-none-none<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Recommend-Hotel-Name-Parkside Police Station Occur Num: 5</li><li>Inform-Hotel-Name-Parkside Police Station Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Police-Addr-Parkside, Cambridge<ul><li>Occur Num: 20</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 17</li><li>Inform-Police-Phone-01223358966 Occur Num: 3</li></ul></ul><li>Inform-Police-Phone-01223358966<ul><li>Occur Num: 5</li><li>NLU Output</li><ul><li>Inform-Police-Addr-Parkside , Cambridge Occur Num: 3</li><li>Inform-Police-Phone-01223902088 Occur Num: 2</li></ul></ul></ul><b> Dialog Loop</b><p>Nothing</p><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><p>Nothing</p><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 100.0 %</p><p> (Precision, Recall, F1) : (1.000, 1.000, 1.000) </p><p> Average Dialog Turn (Succ): 4.067 </p><p> Average Dialog Turn (All): 4.067 </p><b> System NLU Failed Dialog Act:</b><ul><li>Request-Hospital-Phone-?<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Request-Restaurant-Phone-? Occur Num: 3</li></ul></ul><li>Inform-Hospital-Department-children's oncology and haematology<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-oncology Occur Num: 1</li><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-haematology and haematological oncology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-oncology Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-oncology<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-none-none Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-haematology day unit<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-haematology Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-hepatobillary and gastrointestinal surgery regional referral centre<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-surgery Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-neurology neurosurgery<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-neurology Occur Num: 1</li></ul></ul><li>Inform-Hospital-Department-oral and maxillofacial surgery and ent<ul><li>Occur Num: 1</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-surgery Occur Num: 1</li></ul></ul></ul><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hospital-Choice-66<ul><li>Occur Num: 57</li><li>NLU Output</li><ul><li>Inform-Train-Ticket-66 Occur Num: 21</li><li>Inform-Train-Ref-66 Occur Num: 7</li><li>Inform-Hospital-Department-paediatric intensive care unit Occur Num: 2</li><li>Inform-Hospital-Department-66 Occur Num: 2</li><li>Inform-Hospital-Department-clinical decisions unit Occur Num: 2</li></ul></ul><li>Recommend-Hospital-Phone-01223217667<ul><li>Occur Num: 4</li><li>NLU Output</li><ul><li>Inform-Hospital-Phone-01223217667 Occur Num: 2</li><li>Inform-Train-Ticket-66 Occur Num: 1</li><li>Inform-Hospital-Department-postnatal Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Department-paediatric intensive care unit<ul><li>Occur Num: 3</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-paediatric intensive care unit Occur Num: 2</li><li>Inform-Hospital-Phone-01223217715 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223217715<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-paediatric intensive care unit Occur Num: 1</li><li>Inform-Hospital-Phone-01223217715 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Department-major trauma unit<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-66 Occur Num: 1</li><li>Inform-Hospital-Department-major trauma unit Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Department-clinical decisions unit<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Department-clinical decisions unit Occur Num: 2</li></ul></ul><li>Recommend-Hospital-Phone-01223217216<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Train-Ticket-66 Occur Num: 1</li><li>Inform-Hospital-Phone-01223217216 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223217231<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Train-Ticket-66 Occur Num: 1</li><li>Inform-Hospital-Phone-01223217231 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223596066<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Train-Ticket-66 Occur Num: 1</li><li>Inform-Hospital-Phone-01223596066 Occur Num: 1</li></ul></ul><li>Recommend-Hospital-Phone-01223217297<ul><li>Occur Num: 2</li><li>NLU Output</li><ul><li>Inform-Hospital-Phone-01223217297 Occur Num: 1</li><li>Inform-Hospital-Department-coronary care unit Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><p>Nothing</p><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><p>Nothing</p><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/SVMNLU-RuleDST-RulePolicy-TemplateNLG/res.txt b/tests/results/SVMNLU-RuleDST-RulePolicy-TemplateNLG/res.txt deleted file mode 100755 index 98a4e4ecabd6ee4482869f6af29d9da747ed4cb4..0000000000000000000000000000000000000000 --- a/tests/results/SVMNLU-RuleDST-RulePolicy-TemplateNLG/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.842 -success number of dialogs/tot: 0.704 -average precision: 0.7909328570932932 -average recall: 0.8878809606223936 -average f1: 0.8148240530602635 -average book rate: 0.7613247863247864 -average turn (succ): 14.758522727272727 -average turn (all): 17.736 diff --git a/tests/results/Sequicity/Frequency_of_domain.png b/tests/results/Sequicity/Frequency_of_domain.png deleted file mode 100755 index 2f577a9fa27e5747b8cf187db9ba642ef192754b..0000000000000000000000000000000000000000 Binary files a/tests/results/Sequicity/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/Sequicity/Performance_for_each_domain.png b/tests/results/Sequicity/Performance_for_each_domain.png deleted file mode 100755 index 6ca95b7ed4abac0a723733ed35fe0a0e4a5374b7..0000000000000000000000000000000000000000 Binary files a/tests/results/Sequicity/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/Sequicity/Proportions_of_the_dialogue_loop.png b/tests/results/Sequicity/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index 2670f54b6f8c301154a4ce66be0324270ca88fac..0000000000000000000000000000000000000000 Binary files a/tests/results/Sequicity/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/Sequicity/report_multiwoz.html b/tests/results/Sequicity/report_multiwoz.html deleted file mode 100755 index 988b66cad4da3eefe9aeb9101aa51bf1bacd7667..0000000000000000000000000000000000000000 --- a/tests/results/Sequicity/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: Sequicity</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-26 00:56:51</h3><h3>Overall Results</h3><p> Success Rate: 10.5 %</p><p> (Precision, Recall, F1) : (0.414, 0.308, 0.313) </p><p> Average Dialog Turn (Succ): 12.876 </p><p> Average Dialog Turn (All): 38.320 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">244</td><td align="center">0.164</td><td align="center">0.272</td><td align="center">0.328</td><td align="center">0.275</td><td align="center">0.803</td><td align="center">10.550</td><td align="center">31.336</td></tr><tr><td align="center">restaurant</td><td align="center">332</td><td align="center">0.265</td><td align="center">0.494</td><td align="center">0.464</td><td align="center">0.456</td><td align="center">0.720</td><td align="center">6.409</td><td align="center">29.229</td></tr><tr><td align="center">train</td><td align="center">251</td><td align="center">0.191</td><td align="center">0.345</td><td align="center">0.272</td><td align="center">0.295</td><td align="center">0.789</td><td align="center">15.500</td><td align="center">32.430</td></tr><tr><td align="center">attraction</td><td align="center">234</td><td align="center">0.162</td><td align="center">0.635</td><td align="center">0.430</td><td align="center">0.488</td><td align="center">0.833</td><td align="center">5.842</td><td align="center">33.103</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">0.087</td><td align="center">0.087</td><td align="center">0.087</td><td align="center">0.087</td><td align="center">0.913</td><td align="center">16.000</td><td align="center">37.913</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">0.867</td><td align="center">0.867</td><td align="center">0.867</td><td align="center">0.867</td><td align="center">0.133</td><td align="center">7.692</td><td align="center">12.000</td></tr><tr><td align="center">taxi</td><td align="center">44</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">1.000</td><td align="center">0.000</td><td align="center">40.000</td></tr></table><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 16.4 %</p><p> (Precision, Recall, F1) : (0.272, 0.328, 0.275) </p><p> Average Dialog Turn (Succ): 10.550 </p><p> Average Dialog Turn (All): 31.336 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Hotel-Addr-? Occur Num: 75</li><li>Inform-Hotel-Stay-4 Occur Num: 14</li><li>Request-Hotel-Post-? Occur Num: 12</li><li>Inform-Hotel-Stay-2 Occur Num: 10</li><li>Inform-Hotel-Stay-3 Occur Num: 8</li><li>Request-Hotel-Stars-? Occur Num: 8</li><li>Inform-Hotel-Stay-5 Occur Num: 6</li><li>Request-Hotel-Price-? Occur Num: 6</li><li>Inform-Hotel-Type-guesthouse Occur Num: 5</li><li>Request-Hotel-Phone-? Occur Num: 5</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-pricerange Occur Num: 34</li><li>inform-hotel-area Occur Num: 19</li><li>inform-hotel-postcode Occur Num: 7</li><li>inform-hotel-internet Occur Num: 6</li><li>inform-hotel-parking Occur Num: 5</li><li>recommend-hotel-area Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-address Occur Num: 92</li><li>request-hotel-internet Occur Num: 11</li><li>request-hotel-stars Occur Num: 11</li><li>request-hotel-postcode Occur Num: 11</li><li>request-hotel-pricerange Occur Num: 11</li><li>request-hotel-parking Occur Num: 9</li><li>request-hotel-area Occur Num: 8</li><li>request-hotel-phone Occur Num: 5</li><li>request-hotel-type Occur Num: 3</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-type Occur Num: 86</li><li>inform-hotel-area Occur Num: 43</li><li>inform-hotel-phone Occur Num: 15</li><li>inform-hotel-stars Occur Num: 5</li></ul></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 26.5 %</p><p> (Precision, Recall, F1) : (0.494, 0.464, 0.456) </p><p> Average Dialog Turn (Succ): 6.409 </p><p> Average Dialog Turn (All): 29.229 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Restaurant-Addr-? Occur Num: 129</li><li>Request-Restaurant-Phone-? Occur Num: 38</li><li>Request-Restaurant-Post-? Occur Num: 8</li><li>Inform-Restaurant-Day-sunday Occur Num: 4</li><li>Request-Restaurant-Price-? Occur Num: 4</li><li>Inform-Restaurant-Time-16:45 Occur Num: 3</li><li>Inform-Restaurant-Day-saturday Occur Num: 3</li><li>Inform-Restaurant-Time-18:30 Occur Num: 3</li><li>Inform-Restaurant-People-1 Occur Num: 2</li><li>Inform-Restaurant-Day-wednesday Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-restaurant-area Occur Num: 33</li><li>inform-restaurant-pricerange Occur Num: 19</li><li>inform-restaurant-postcode Occur Num: 3</li><li>recommend-restaurant-pricerange Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-address Occur Num: 166</li><li>request-restaurant-phone Occur Num: 11</li><li>request-restaurant-pricerange Occur Num: 6</li><li>request-restaurant-postcode Occur Num: 5</li><li>request-restaurant-area Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-area Occur Num: 54</li><li>inform-restaurant-phone Occur Num: 11</li><li>inform-restaurant-food Occur Num: 11</li></ul></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 19.1 %</p><p> (Precision, Recall, F1) : (0.345, 0.272, 0.295) </p><p> Average Dialog Turn (Succ): 15.500 </p><p> Average Dialog Turn (All): 32.430 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Train-Time-? Occur Num: 62</li><li>Request-Train-Id-? Occur Num: 52</li><li>Request-Train-Ticket-? Occur Num: 42</li><li>Request-Train-Leave-? Occur Num: 13</li><li>Request-Train-Arrive-? Occur Num: 9</li><li>Inform-Train-People-6 Occur Num: 3</li><li>Inform-Train-People-5 Occur Num: 3</li><li>Inform-Train-People-3 Occur Num: 3</li><li>Inform-Train-People-1 Occur Num: 3</li><li>Inform-Train-People-8 Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-train-trainID Occur Num: 2</li><li>offerbooked-train-price Occur Num: 2</li><li>inform-train-day Occur Num: 2</li><li>inform-train-arriveBy Occur Num: 1</li><li>inform-train-leaveAt Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-train-trainID Occur Num: 100</li><li>request-train-price Occur Num: 57</li><li>request-train-duration Occur Num: 46</li><li>request-train-arriveBy Occur Num: 20</li><li>request-train-leaveAt Occur Num: 15</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 16.2 %</p><p> (Precision, Recall, F1) : (0.635, 0.430, 0.488) </p><p> Average Dialog Turn (Succ): 5.842 </p><p> Average Dialog Turn (All): 33.103 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Attraction-Fee-? Occur Num: 109</li><li>Request-Attraction-Addr-? Occur Num: 71</li><li>Request-Attraction-Post-? Occur Num: 6</li><li>Request-Attraction-Type-? Occur Num: 6</li><li>Request-Attraction-Phone-? Occur Num: 2</li><li>Request-Attraction-Area-? Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-attraction-area Occur Num: 13</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-address Occur Num: 132</li><li>request-attraction-entrance fee Occur Num: 109</li><li>request-attraction-phone Occur Num: 17</li><li>request-attraction-postcode Occur Num: 16</li><li>request-attraction-area Occur Num: 11</li><li>request-attraction-type Occur Num: 8</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-phone Occur Num: 24</li><li>inform-attraction-type Occur Num: 15</li><li>inform-attraction-area Occur Num: 5</li></ul></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 8.7 %</p><p> (Precision, Recall, F1) : (0.087, 0.087, 0.087) </p><p> Average Dialog Turn (Succ): 16.000 </p><p> Average Dialog Turn (All): 37.913 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Police-Addr-? Occur Num: 17</li><li>Request-Police-Phone-? Occur Num: 4</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-police-phone Occur Num: 18</li><li>request-police-address Occur Num: 17</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 86.7 %</p><p> (Precision, Recall, F1) : (0.867, 0.867, 0.867) </p><p> Average Dialog Turn (Succ): 7.692 </p><p> Average Dialog Turn (All): 12.000 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Hospital-Phone-? Occur Num: 4</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-hospital-phone Occur Num: 4</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 0.0 %</p><p> (Precision, Recall, F1) : (0.000, 0.000, 0.000) </p><p> Average Dialog Turn (Succ): 0.000 </p><p> Average Dialog Turn (All): 40.000 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Taxi-Car-? Occur Num: 44</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-taxi-phone Occur Num: 44</li><li>request-taxi-car type Occur Num: 44</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/Sequicity/res.txt b/tests/results/Sequicity/res.txt deleted file mode 100755 index 705fbd6082518c4ac5d3d550cb0bcec9b9ed00b0..0000000000000000000000000000000000000000 --- a/tests/results/Sequicity/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.131 -success number of dialogs/tot: 0.105 -average precision: 0.4144451861741581 -average recall: 0.30776920667886404 -average f1: 0.31279910524458815 -average book rate: 0.05128205128205128 -average turn (succ): 12.876190476190477 -average turn (all): 38.32 diff --git a/tests/results/TRADE-RulePolicy-TemplateNLG/Frequency_of_domain.png b/tests/results/TRADE-RulePolicy-TemplateNLG/Frequency_of_domain.png deleted file mode 100755 index 7496d3c9ef25427e62cc9fc2c41b19939e6201b9..0000000000000000000000000000000000000000 Binary files a/tests/results/TRADE-RulePolicy-TemplateNLG/Frequency_of_domain.png and /dev/null differ diff --git a/tests/results/TRADE-RulePolicy-TemplateNLG/Performance_for_each_domain.png b/tests/results/TRADE-RulePolicy-TemplateNLG/Performance_for_each_domain.png deleted file mode 100755 index 1498110a9e2cf71ed76f82dd312bf60c3631546d..0000000000000000000000000000000000000000 Binary files a/tests/results/TRADE-RulePolicy-TemplateNLG/Performance_for_each_domain.png and /dev/null differ diff --git a/tests/results/TRADE-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png b/tests/results/TRADE-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png deleted file mode 100755 index 3005cde540687c5c82925a3b8d0bc9df7e71ad96..0000000000000000000000000000000000000000 Binary files a/tests/results/TRADE-RulePolicy-TemplateNLG/Proportions_of_the_dialogue_loop.png and /dev/null differ diff --git a/tests/results/TRADE-RulePolicy-TemplateNLG/report_multiwoz.html b/tests/results/TRADE-RulePolicy-TemplateNLG/report_multiwoz.html deleted file mode 100755 index 903248e21900900cecc8e9d5adf80985694ef250..0000000000000000000000000000000000000000 --- a/tests/results/TRADE-RulePolicy-TemplateNLG/report_multiwoz.html +++ /dev/null @@ -1 +0,0 @@ -<html><body><h1>Test Report</h1><h3>Model Name: TRADE-RulePolicy-TemplateNLG</h3><h3>Dataset: multiwoz</h3><h3>Time: 2020-04-29 21:10:04</h3><h3>Overall Results</h3><p> Success Rate: 25.3 %</p><p> (Precision, Recall, F1) : (0.493, 0.481, 0.444) </p><p> Average Dialog Turn (Succ): 12.735 </p><p> Average Dialog Turn (All): 24.684 </p><table><tr><td><img src="Frequency_of_domain.png" width=600 border=0/></td><td><img src="Performance_for_each_domain.png" width=600 border=0/></td></tr></table><h3>Metric</h3><table width="900" border="1"><tr><th> </th><th>Total Num</th><th>Succ Rate</th><th>Precision</th><th>Recall</th><th>F1</th><th>Dialog Loop Failed Rate</th><th>Dialog Turn (Succ)</th><th>Dialog Turn (All)</th></tr><tr><td align="center">hotel</td><td align="center">330</td><td align="center">0.285</td><td align="center">0.167</td><td align="center">0.286</td><td align="center">0.188</td><td align="center">0.473</td><td align="center">9.617</td><td align="center">16.448</td></tr><tr><td align="center">attraction</td><td align="center">291</td><td align="center">0.299</td><td align="center">0.614</td><td align="center">0.541</td><td align="center">0.547</td><td align="center">0.533</td><td align="center">6.368</td><td align="center">20.192</td></tr><tr><td align="center">restaurant</td><td align="center">361</td><td align="center">0.335</td><td align="center">0.479</td><td align="center">0.490</td><td align="center">0.465</td><td align="center">0.427</td><td align="center">4.744</td><td align="center">14.964</td></tr><tr><td align="center">train</td><td align="center">273</td><td align="center">0.897</td><td align="center">0.795</td><td align="center">0.865</td><td align="center">0.815</td><td align="center">0.084</td><td align="center">9.902</td><td align="center">11.773</td></tr><tr><td align="center">police</td><td align="center">23</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">1.000</td><td align="center">0.000</td><td align="center">40.000</td></tr><tr><td align="center">hospital</td><td align="center">30</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">0.000</td><td align="center">1.000</td><td align="center">0.000</td><td align="center">40.000</td></tr><tr><td align="center">taxi</td><td align="center">46</td><td align="center">0.935</td><td align="center">1.000</td><td align="center">0.967</td><td align="center">0.978</td><td align="center">0.065</td><td align="center">5.209</td><td align="center">7.478</td></tr></table><h3>Dialogue Loop</h3><img src="Proportions_of_the_dialogue_loop.png" width=800 /><h3>Domain hotel</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 28.5 %</p><p> (Precision, Recall, F1) : (0.167, 0.286, 0.188) </p><p> Average Dialog Turn (Succ): 9.617 </p><p> Average Dialog Turn (All): 16.448 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Hotel-Parking-yes<ul><li>Occur Num: 886</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 139</li><li>Inform-Hotel-Internet-none Occur Num: 72</li><li>Inform-Hotel-Stars-4 Occur Num: 51</li><li>Inform-Hotel-Type-guesthouse Occur Num: 47</li><li>Inform-Hotel-Type-hotel Occur Num: 37</li></ul></ul><li>Inform-Hotel-Internet-yes<ul><li>Occur Num: 732</li><li>NLU Output</li><ul><li>Inform-Hotel-Internet-none Occur Num: 120</li><li>Inform-Hotel-Parking-none Occur Num: 82</li><li>Inform-Hotel-Stars-4 Occur Num: 44</li><li>Inform-Hotel-Price-moderate Occur Num: 37</li><li>Inform-Hotel-Type-guesthouse Occur Num: 36</li></ul></ul><li>NoOffer-Hotel-Internet-yes<ul><li>Occur Num: 462</li><li>NLU Output</li><ul><li>NoOffer-Hotel-Internet-none Occur Num: 39</li><li>NoOffer-Hotel-Type-hotels Occur Num: 36</li><li>NoOffer-Hotel-Type-hotel Occur Num: 30</li><li>NoOffer-Hotel-none-none Occur Num: 28</li><li>NoOffer-Hotel-Price-moderate Occur Num: 28</li></ul></ul><li>Inform-Hotel-Stars-4<ul><li>Occur Num: 394</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 53</li><li>Inform-Hotel-Internet-none Occur Num: 48</li><li>Inform-Hotel-Type-guesthouse Occur Num: 32</li><li>Inform-Hotel-Price-moderate Occur Num: 24</li><li>Inform-Hotel-Area-north Occur Num: 18</li></ul></ul><li>Inform-Hotel-Type-guesthouse<ul><li>Occur Num: 365</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 47</li><li>Inform-Hotel-Internet-none Occur Num: 36</li><li>Inform-Hotel-Stars-4 Occur Num: 30</li><li>Inform-Hotel-Price-moderate Occur Num: 29</li><li>Inform-Hotel-Area-north Occur Num: 23</li></ul></ul><li>NoOffer-Hotel-Type-hotel<ul><li>Occur Num: 341</li><li>NLU Output</li><ul><li>NoOffer-Hotel-Stars-4 Occur Num: 24</li><li>NoOffer-Hotel-Price-moderate Occur Num: 23</li><li>NoOffer-Hotel-Type-hotels Occur Num: 22</li><li>NoOffer-Hotel-none-none Occur Num: 22</li><li>NoOffer-Hotel-Internet-none Occur Num: 18</li></ul></ul><li>Inform-Hotel-Price-moderate<ul><li>Occur Num: 302</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 39</li><li>Inform-Hotel-Internet-none Occur Num: 39</li><li>Inform-Hotel-Type-guesthouse Occur Num: 28</li><li>Inform-Hotel-Area-north Occur Num: 25</li><li>Inform-Hotel-Stars-4 Occur Num: 18</li></ul></ul><li>NoOffer-Hotel-Parking-yes<ul><li>Occur Num: 295</li><li>NLU Output</li><ul><li>NoOffer-Hotel-Type-hotel Occur Num: 28</li><li>NoOffer-Hotel-none-none Occur Num: 21</li><li>NoOffer-Hotel-Parking-none Occur Num: 21</li><li>reqmore-general-none-none Occur Num: 17</li><li>Request-Hotel-Area-? Occur Num: 16</li></ul></ul><li>NoOffer-Hotel-Price-moderate<ul><li>Occur Num: 273</li><li>NLU Output</li><ul><li>NoOffer-Hotel-Type-hotels Occur Num: 21</li><li>NoOffer-Hotel-none-none Occur Num: 21</li><li>NoOffer-Hotel-Type-hotel Occur Num: 17</li><li>NoOffer-Hotel-Stars-4 Occur Num: 15</li><li>Request-Hotel-Price-? Occur Num: 15</li></ul></ul><li>Inform-Hotel-Type-hotel<ul><li>Occur Num: 251</li><li>NLU Output</li><ul><li>Inform-Hotel-Parking-none Occur Num: 43</li><li>Inform-Hotel-Internet-none Occur Num: 35</li><li>Inform-Hotel-Price-expensive Occur Num: 14</li><li>Inform-Hotel-Stars-4 Occur Num: 11</li><li>Inform-Hotel-Price-moderate Occur Num: 10</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Hotel-Addr-? Occur Num: 50</li><li>Request-Hotel-Phone-? Occur Num: 39</li><li>Request-Hotel-Post-? Occur Num: 19</li><li>Inform-Hotel-Stay-4 Occur Num: 7</li><li>Request-Hotel-Internet-? Occur Num: 6</li><li>Inform-Hotel-Parking-yes Occur Num: 6</li><li>Request-Hotel-Price-? Occur Num: 5</li><li>Inform-Hotel-Internet-yes Occur Num: 3</li><li>Inform-Hotel-Stay-5 Occur Num: 3</li><li>Inform-Hotel-Stay-2 Occur Num: 2</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-hotel-parking Occur Num: 152</li><li>inform-hotel-internet Occur Num: 126</li><li>inform-hotel-stars Occur Num: 25</li><li>inform-hotel-area Occur Num: 18</li><li>recommend-hotel-parking Occur Num: 11</li><li>recommend-hotel-internet Occur Num: 9</li><li>recommend-hotel-stars Occur Num: 4</li><li>inform-hotel-phone Occur Num: 2</li><li>inform-hotel-pricerange Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-hotel-address Occur Num: 82</li><li>request-hotel-phone Occur Num: 68</li><li>request-hotel-postcode Occur Num: 35</li><li>request-hotel-internet Occur Num: 25</li><li>request-hotel-parking Occur Num: 17</li><li>request-hotel-pricerange Occur Num: 9</li><li>request-hotel-type Occur Num: 4</li><li>request-hotel-stars Occur Num: 3</li><li>request-hotel-area Occur Num: 2</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-hotel-area Occur Num: 70</li><li>inform-hotel-type Occur Num: 65</li><li>inform-hotel-stars Occur Num: 60</li><li>inform-hotel-phone Occur Num: 8</li></ul></div><h3>Domain attraction</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 29.9 %</p><p> (Precision, Recall, F1) : (0.614, 0.541, 0.547) </p><p> Average Dialog Turn (Succ): 6.368 </p><p> Average Dialog Turn (All): 20.192 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Attraction-Area-centre<ul><li>Occur Num: 292</li><li>NLU Output</li><ul><li>Inform-Attraction-Post-cb21ab Occur Num: 23</li><li>Inform-Attraction-Type-museum Occur Num: 18</li><li>Inform-Attraction-Addr-Regent Street City Centre Occur Num: 18</li><li>Inform-Attraction-Area-within Occur Num: 12</li><li>Inform-Attraction-Type-park Occur Num: 11</li></ul></ul><li>Inform-Attraction-Type-museum<ul><li>Occur Num: 156</li><li>NLU Output</li><ul><li>Inform-Attraction-Area-centre Occur Num: 20</li><li>Inform-Attraction-Post-cb21ab Occur Num: 16</li><li>Inform-Attraction-Addr-Regent Street City Centre Occur Num: 15</li><li>Inform-Attraction-Fee-free Occur Num: 7</li><li>Recommend-Attraction-Type-museum Occur Num: 6</li></ul></ul><li>Request-Attraction-Type-?<ul><li>Occur Num: 138</li><li>NLU Output</li><ul><li>NoOffer-Attraction-Area-centre Occur Num: 13</li><li>NoOffer-Attraction-Type-entertainment Occur Num: 11</li><li>Request-Attraction-Name-? Occur Num: 7</li><li>NoOffer-Attraction-Type-college Occur Num: 6</li><li>Request-Attraction-Area-? Occur Num: 6</li></ul></ul><li>NoOffer-Attraction-Area-centre<ul><li>Occur Num: 130</li><li>NLU Output</li><ul><li>NoOffer-Attraction-Type-entertainment Occur Num: 16</li><li>Request-Attraction-Type-? Occur Num: 12</li><li>NoOffer-Attraction-Area-that area of town Occur Num: 7</li><li>NoOffer-Attraction-Type-college Occur Num: 6</li><li>Inform-Hotel-Type-guesthouse Occur Num: 5</li></ul></ul><li>Inform-Attraction-Fee-?<ul><li>Occur Num: 117</li><li>NLU Output</li><ul><li>Inform-Attraction-Addr-Regent Street City Centre Occur Num: 17</li><li>Inform-Attraction-Post-cb21ab Occur Num: 17</li><li>Inform-Attraction-Area-centre Occur Num: 7</li><li>Request-Taxi-Leave-? Occur Num: 6</li><li>Inform-Attraction-Area-north Occur Num: 4</li></ul></ul><li>Inform-Attraction-Fee-free<ul><li>Occur Num: 102</li><li>NLU Output</li><ul><li>Inform-Attraction-Post-cb21ab Occur Num: 14</li><li>Inform-Attraction-Addr-Regent Street City Centre Occur Num: 12</li><li>Inform-Attraction-Area-centre Occur Num: 11</li><li>Inform-Attraction-Type-park Occur Num: 8</li><li>Request-Train-Arrive-? Occur Num: 4</li></ul></ul><li>Inform-Attraction-Choice-44<ul><li>Occur Num: 94</li><li>NLU Output</li><ul><li>reqmore-general-none-none Occur Num: 3</li><li>Request-Attraction-Type-? Occur Num: 2</li><li>Recommend-Attraction-Name-primavera Occur Num: 2</li><li>Recommend-Attraction-Name-scudamores punting co Occur Num: 2</li><li>Recommend-Attraction-Post-cb21qy Occur Num: 2</li></ul></ul><li>NoOffer-Attraction-Type-college<ul><li>Occur Num: 93</li><li>NLU Output</li><ul><li>NoOffer-Attraction-Area-west Occur Num: 9</li><li>Request-Attraction-Type-? Occur Num: 8</li><li>NoOffer-Attraction-Type-entertainment Occur Num: 7</li><li>Inform-Attraction-Post-cb21ab Occur Num: 5</li><li>NoOffer-Attraction-Area-centre Occur Num: 5</li></ul></ul><li>Request-Attraction-Name-?<ul><li>Occur Num: 84</li><li>NLU Output</li><ul><li>Request-Attraction-Type-? Occur Num: 7</li><li>NoOffer-Attraction-Type-college Occur Num: 5</li><li>Inform-Attraction-Addr-Regent Street City Centre Occur Num: 5</li><li>NoOffer-Attraction-Area-centre Occur Num: 5</li><li>NoOffer-Attraction-Type-entertainment Occur Num: 4</li></ul></ul><li>NoOffer-Attraction-Area-west<ul><li>Occur Num: 72</li><li>NLU Output</li><ul><li>NoOffer-Attraction-Type-college Occur Num: 7</li><li>NoOffer-Attraction-Type-entertainment Occur Num: 5</li><li>Inform-Attraction-Post-cb21ab Occur Num: 5</li><li>Request-Attraction-Type-? Occur Num: 4</li><li>NoOffer-Attraction-Type-museum Occur Num: 4</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Attraction-Phone-? Occur Num: 78</li><li>Request-Attraction-Fee-? Occur Num: 42</li><li>Request-Attraction-Addr-? Occur Num: 23</li><li>Request-Attraction-Post-? Occur Num: 9</li><li>Inform-Attraction-Type-college Occur Num: 1</li><li>Inform-Attraction-Type-nightclub Occur Num: 1</li><li>Request-Attraction-Type-? Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>inform-attraction-area Occur Num: 33</li><li>recommend-attraction-area Occur Num: 2</li><li>inform-attraction-phone Occur Num: 1</li><li>recommend-attraction-phone Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-attraction-phone Occur Num: 140</li><li>request-attraction-entrance fee Occur Num: 59</li><li>request-attraction-address Occur Num: 54</li><li>request-attraction-postcode Occur Num: 28</li><li>request-attraction-type Occur Num: 5</li><li>request-attraction-area Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-attraction-area Occur Num: 47</li><li>inform-attraction-type Occur Num: 43</li><li>inform-attraction-phone Occur Num: 7</li></ul></div><h3>Domain restaurant</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 33.5 %</p><p> (Precision, Recall, F1) : (0.479, 0.490, 0.465) </p><p> Average Dialog Turn (Succ): 4.744 </p><p> Average Dialog Turn (All): 14.964 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Restaurant-Area-centre<ul><li>Occur Num: 269</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 20</li><li>Inform-Attraction-Area-centre Occur Num: 14</li><li>Request-Taxi-Arrive-? Occur Num: 14</li><li>Inform-Restaurant-Price-moderate Occur Num: 13</li><li>Inform-Restaurant-Price-cheap Occur Num: 11</li></ul></ul><li>Inform-Restaurant-Price-expensive<ul><li>Occur Num: 187</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 19</li><li>Request-Taxi-Arrive-? Occur Num: 8</li><li>Inform-Restaurant-Food-indian Occur Num: 8</li><li>Request-Taxi-Leave-? Occur Num: 6</li><li>Recommend-Restaurant-Name-bangkok city Occur Num: 5</li></ul></ul><li>Request-Restaurant-Food-?<ul><li>Occur Num: 173</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 20</li><li>Request-Restaurant-Price-? Occur Num: 17</li><li>NoOffer-Restaurant-Area-centre Occur Num: 16</li><li>NoOffer-Restaurant-Price-expensive Occur Num: 15</li><li>Request-Restaurant-Area-? Occur Num: 11</li></ul></ul><li>NoOffer-Restaurant-Price-expensive<ul><li>Occur Num: 163</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 24</li><li>NoOffer-Restaurant-Area-centre Occur Num: 15</li><li>Request-Restaurant-Food-? Occur Num: 14</li><li>Request-Restaurant-Price-? Occur Num: 9</li><li>NoOffer-Restaurant-Food-indian Occur Num: 6</li></ul></ul><li>NoOffer-Restaurant-Area-centre<ul><li>Occur Num: 151</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 21</li><li>Request-Restaurant-Food-? Occur Num: 16</li><li>NoOffer-Restaurant-Price-expensive Occur Num: 14</li><li>NoOffer-Restaurant-Price-cheap Occur Num: 8</li><li>NoOffer-Restaurant-Food-indian Occur Num: 8</li></ul></ul><li>Request-Restaurant-Price-?<ul><li>Occur Num: 119</li><li>NLU Output</li><ul><li>Request-Restaurant-Food-? Occur Num: 19</li><li>NoOffer-Restaurant-none-none Occur Num: 10</li><li>NoOffer-Restaurant-Price-expensive Occur Num: 10</li><li>NoOffer-Restaurant-Area-centre Occur Num: 8</li><li>Request-Restaurant-Area-? Occur Num: 7</li></ul></ul><li>Inform-Restaurant-Food-indian<ul><li>Occur Num: 118</li><li>NLU Output</li><ul><li>Inform-Restaurant-Price-expensive Occur Num: 8</li><li>Inform-Restaurant-Choice-4 Occur Num: 6</li><li>Recommend-Restaurant-Food-indian Occur Num: 5</li><li>Inform-Restaurant-Area-east Occur Num: 5</li><li>Inform-Restaurant-Area-centre Occur Num: 4</li></ul></ul><li>NoOffer-Restaurant-Price-moderate<ul><li>Occur Num: 99</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 14</li><li>Request-Restaurant-Food-? Occur Num: 8</li><li>NoOffer-Restaurant-Food-indian Occur Num: 7</li><li>NoOffer-Restaurant-Area-centre Occur Num: 7</li><li>NoOffer-Restaurant-Area-east Occur Num: 5</li></ul></ul><li>NoOffer-Restaurant-Price-cheap<ul><li>Occur Num: 95</li><li>NLU Output</li><ul><li>NoOffer-Restaurant-none-none Occur Num: 15</li><li>NoOffer-Restaurant-Area-centre Occur Num: 10</li><li>Request-Restaurant-Food-? Occur Num: 10</li><li>Request-Restaurant-Price-? Occur Num: 7</li><li>NoOffer-Restaurant-Area-east Occur Num: 4</li></ul></ul><li>Inform-Restaurant-Price-cheap<ul><li>Occur Num: 94</li><li>NLU Output</li><ul><li>Inform-Restaurant-Area-centre Occur Num: 10</li><li>Inform-Restaurant-Food-italian Occur Num: 6</li><li>Recommend-Restaurant-Price-cheap Occur Num: 5</li><li>Request-Taxi-Arrive-? Occur Num: 3</li><li>Recommend-Restaurant-Name-la raza Occur Num: 3</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Restaurant-Phone-? Occur Num: 108</li><li>Request-Restaurant-Addr-? Occur Num: 7</li><li>Request-Restaurant-Post-? Occur Num: 4</li><li>Inform-Restaurant-Day-saturday Occur Num: 3</li><li>Inform-Restaurant-People-4 Occur Num: 2</li><li>Inform-Restaurant-Time-14:15 Occur Num: 2</li><li>Inform-Restaurant-Day-sunday Occur Num: 2</li><li>Request-Restaurant-Price-? Occur Num: 2</li><li>Inform-Restaurant-Time-16:45 Occur Num: 1</li><li>Inform-Restaurant-Time-13:30 Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-restaurant-phone Occur Num: 127</li><li>request-restaurant-address Occur Num: 41</li><li>request-restaurant-postcode Occur Num: 19</li><li>request-restaurant-pricerange Occur Num: 3</li></ul><b> Inform But Not Request Dialog Act</b><ul><li>inform-restaurant-food Occur Num: 56</li><li>inform-restaurant-area Occur Num: 50</li><li>inform-restaurant-phone Occur Num: 18</li></ul></div><h3>Domain train</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 89.7 %</p><p> (Precision, Recall, F1) : (0.795, 0.865, 0.815) </p><p> Average Dialog Turn (Succ): 9.902 </p><p> Average Dialog Turn (All): 11.773 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>OfferBook-Train-Depart-cambridge<ul><li>Occur Num: 712</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 43</li><li>OfferBook-Train-Dest-cambridge Occur Num: 41</li><li>OfferBook-Train-Day-tuesday Occur Num: 24</li><li>OfferBook-Train-Arrive-05:51 Occur Num: 24</li><li>OfferBook-Train-Day-thursday Occur Num: 22</li></ul></ul><li>OfferBook-Train-Dest-cambridge<ul><li>Occur Num: 641</li><li>NLU Output</li><ul><li>OfferBook-Train-none-none Occur Num: 50</li><li>OfferBook-Train-Arrive-05:51 Occur Num: 33</li><li>OfferBook-Train-Depart-cambridge Occur Num: 27</li><li>OfferBook-Train-Day-saturday Occur Num: 26</li><li>OfferBook-Train-Day-tuesday Occur Num: 23</li></ul></ul><li>Inform-Train-Arrive-05:51<ul><li>Occur Num: 475</li><li>NLU Output</li><ul><li>OfferBook-Train-Arrive-05:51 Occur Num: 67</li><li>OfferBook-Train-Dest-cambridge Occur Num: 37</li><li>OfferBook-Train-Depart-cambridge Occur Num: 31</li><li>OfferBook-Train-Leave-05:00 Occur Num: 21</li><li>OfferBook-Train-none-none Occur Num: 17</li></ul></ul><li>OfferBook-Train-Day-tuesday<ul><li>Occur Num: 256</li><li>NLU Output</li><ul><li>OfferBook-Train-Dest-cambridge Occur Num: 21</li><li>OfferBook-Train-Depart-cambridge Occur Num: 21</li><li>OfferBook-Train-none-none Occur Num: 14</li><li>OfferBook-Train-Arrive-05:51 Occur Num: 12</li><li>OfferBook-Train-Leave-05:00 Occur Num: 7</li></ul></ul><li>OfferBook-Train-Day-saturday<ul><li>Occur Num: 254</li><li>NLU Output</li><ul><li>OfferBook-Train-Dest-cambridge Occur Num: 26</li><li>OfferBook-Train-Depart-cambridge Occur Num: 18</li><li>OfferBook-Train-none-none Occur Num: 13</li><li>OfferBook-Train-Arrive-05:51 Occur Num: 12</li><li>OfferBook-Train-Leave-05:00 Occur Num: 8</li></ul></ul><li>Inform-Train-Depart-cambridge<ul><li>Occur Num: 253</li><li>NLU Output</li><ul><li>OfferBook-Train-Depart-cambridge Occur Num: 32</li><li>OfferBook-Train-Dest-cambridge Occur Num: 14</li><li>OfferBook-Train-Arrive-05:51 Occur Num: 13</li><li>OfferBook-Train-Day-tuesday Occur Num: 10</li><li>OfferBook-Train-Day-thursday Occur Num: 7</li></ul></ul><li>Inform-Train-Leave-05:00<ul><li>Occur Num: 242</li><li>NLU Output</li><ul><li>OfferBook-Train-Leave-05:00 Occur Num: 37</li><li>OfferBook-Train-Arrive-05:51 Occur Num: 16</li><li>OfferBook-Train-Dest-cambridge Occur Num: 13</li><li>OfferBook-Train-Depart-cambridge Occur Num: 10</li><li>OfferBook-Train-Day-saturday Occur Num: 8</li></ul></ul><li>OfferBook-Train-Day-friday<ul><li>Occur Num: 237</li><li>NLU Output</li><ul><li>OfferBook-Train-Depart-cambridge Occur Num: 20</li><li>OfferBook-Train-Dest-cambridge Occur Num: 19</li><li>OfferBook-Train-none-none Occur Num: 13</li><li>OfferBook-Train-Arrive-05:51 Occur Num: 11</li><li>OfferBook-Train-Leave-05:00 Occur Num: 7</li></ul></ul><li>OfferBook-Train-Day-thursday<ul><li>Occur Num: 233</li><li>NLU Output</li><ul><li>OfferBook-Train-Depart-cambridge Occur Num: 23</li><li>OfferBook-Train-Dest-cambridge Occur Num: 17</li><li>OfferBook-Train-none-none Occur Num: 16</li><li>OfferBook-Train-Arrive-05:51 Occur Num: 9</li><li>OfferBook-Train-Leave-05:00 Occur Num: 5</li></ul></ul><li>Inform-Train-Time-51 minutes<ul><li>Occur Num: 194</li><li>NLU Output</li><ul><li>OfferBook-Train-Depart-cambridge Occur Num: 13</li><li>OfferBook-Train-Dest-cambridge Occur Num: 13</li><li>Inform-Train-Time-one way Occur Num: 10</li><li>OfferBook-Train-Arrive-05:51 Occur Num: 9</li><li>OfferBook-Train-People-51 Occur Num: 8</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Train-Time-? Occur Num: 9</li><li>Inform-Train-People-6 Occur Num: 2</li><li>Inform-Train-People-4 Occur Num: 2</li><li>Inform-Train-People-3 Occur Num: 2</li><li>Inform-Train-People-5 Occur Num: 1</li><li>Inform-Train-People-7 Occur Num: 1</li><li>Inform-Train-Depart-london liverpool street Occur Num: 1</li><li>Inform-Train-Dest-cambridge Occur Num: 1</li><li>Inform-Train-Dest-birmingham new street Occur Num: 1</li><li>Request-Train-Leave-? Occur Num: 1</li></ul><b> Bad Inform Dialog Act</b><ul><li>offerbook-train-arriveBy Occur Num: 30</li><li>inform-train-duration Occur Num: 14</li><li>offerbook-train-leaveAt Occur Num: 8</li><li>offerbook-train-price Occur Num: 2</li><li>inform-train-price Occur Num: 1</li><li>offerbook-train-trainID Occur Num: 1</li></ul><b> Request But Not Inform Dialog Act</b><ul><li>request-train-duration Occur Num: 13</li><li>request-train-price Occur Num: 2</li><li>request-train-leaveAt Occur Num: 2</li><li>request-train-arriveBy Occur Num: 1</li><li>request-train-trainID Occur Num: 1</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain police</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 0.0 %</p><p> (Precision, Recall, F1) : (0.000, 0.000, 0.000) </p><p> Average Dialog Turn (Succ): 0.000 </p><p> Average Dialog Turn (All): 40.000 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Police-Addr-? Occur Num: 17</li><li>Request-Police-Phone-? Occur Num: 6</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-police-phone Occur Num: 20</li><li>request-police-address Occur Num: 17</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain hospital</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 0.0 %</p><p> (Precision, Recall, F1) : (0.000, 0.000, 0.000) </p><p> Average Dialog Turn (Succ): 0.000 </p><p> Average Dialog Turn (All): 40.000 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><p>Nothing</p><b> Dialog Loop</b><ul><li>Request-Hospital-Phone-? Occur Num: 30</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-hospital-phone Occur Num: 30</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div><h3>Domain taxi</h3><div style="margin-left:50"><h3>Overall Results</h3><p> Success Rate: 93.5 %</p><p> (Precision, Recall, F1) : (1.000, 0.967, 0.978) </p><p> Average Dialog Turn (Succ): 5.209 </p><p> Average Dialog Turn (All): 7.478 </p><b> System NLU Failed Dialog Act:</b><p>Nothing</p><b> User NLU Failed Dialog Act:</b><ul><li>Inform-Taxi-Car-skoda<ul><li>Occur Num: 27</li><li>NLU Output</li><ul><li>Inform-Taxi-Leave-05:00 Occur Num: 3</li><li>Inform-Hotel-Type-guesthouse Occur Num: 2</li><li>Inform-Train-Time-51 minutes Occur Num: 2</li><li>Inform-Taxi-Phone-62698234072 Occur Num: 1</li><li>Inform-Taxi-Car-toyota Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-toyota<ul><li>Occur Num: 17</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-guesthouse Occur Num: 3</li><li>Inform-Taxi-Leave-05:00 Occur Num: 2</li><li>Inform-Taxi-Phone-62698234072 Occur Num: 1</li><li>Inform-Taxi-Phone-83153889561 Occur Num: 1</li><li>Inform-Taxi-Phone-69714552379 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-audi<ul><li>Occur Num: 16</li><li>NLU Output</li><ul><li>Inform-Taxi-Leave-05:00 Occur Num: 2</li><li>Inform-Taxi-Phone-73297457818 Occur Num: 1</li><li>Inform-Taxi-Phone-02411795305 Occur Num: 1</li><li>OfferBooked-Train-Time-51 Occur Num: 1</li><li>Inform-Taxi-Phone-17927153736 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-ford<ul><li>Occur Num: 14</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-guesthouse Occur Num: 3</li><li>Inform-Taxi-Phone-94549609159 Occur Num: 1</li><li>Inform-Taxi-Phone-56629498460 Occur Num: 1</li><li>Inform-Taxi-Phone-44947488614 Occur Num: 1</li><li>Inform-Train-Time-51 minutes Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-honda<ul><li>Occur Num: 13</li><li>NLU Output</li><ul><li>Inform-Taxi-Leave-05:00 Occur Num: 3</li><li>Inform-Taxi-Phone-52547806713 Occur Num: 1</li><li>Inform-Taxi-Phone-42223260114 Occur Num: 1</li><li>Inform-Taxi-Phone-84754151715 Occur Num: 1</li><li>Inform-Taxi-Arrive-05:51 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-lexus<ul><li>Occur Num: 13</li><li>NLU Output</li><ul><li>Inform-Taxi-Arrive-05:51 Occur Num: 2</li><li>Inform-Hotel-Type-guesthouse Occur Num: 2</li><li>Inform-Taxi-Phone-83144288521 Occur Num: 1</li><li>Inform-Taxi-Phone-69714552379 Occur Num: 1</li><li>Inform-Taxi-Phone-19626794559 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-bmw<ul><li>Occur Num: 13</li><li>NLU Output</li><ul><li>Inform-Taxi-Leave-05:00 Occur Num: 3</li><li>Inform-Taxi-Phone-79054617689 Occur Num: 1</li><li>Inform-Taxi-Phone-73609361703 Occur Num: 1</li><li>Inform-Taxi-Phone-49920327578 Occur Num: 1</li><li>Inform-Hotel-Type-guesthouse Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-tesla<ul><li>Occur Num: 13</li><li>NLU Output</li><ul><li>Inform-Hotel-Type-guesthouse Occur Num: 3</li><li>Inform-Taxi-Phone-00253869902 Occur Num: 1</li><li>Inform-Taxi-Phone-97488259243 Occur Num: 1</li><li>Inform-Taxi-Leave-05:00 Occur Num: 1</li><li>Inform-Taxi-Phone-36951803842 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-volkswagen<ul><li>Occur Num: 7</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-15214714504 Occur Num: 1</li><li>Inform-Taxi-Arrive-05:51 Occur Num: 1</li><li>Inform-Taxi-Phone-53827293134 Occur Num: 1</li><li>Inform-Hotel-Type-guesthouse Occur Num: 1</li><li>Inform-Taxi-Phone-29737767268 Occur Num: 1</li></ul></ul><li>Inform-Taxi-Car-volvo<ul><li>Occur Num: 6</li><li>NLU Output</li><ul><li>Inform-Taxi-Phone-26438028145 Occur Num: 1</li><li>OfferBooked-Train-Time-51 Occur Num: 1</li><li>Inform-Taxi-Phone-96368625434 Occur Num: 1</li><li>Inform-Train-Leave-05:00 Occur Num: 1</li><li>Inform-Taxi-Phone-10921657762 Occur Num: 1</li></ul></ul></ul><b> Dialog Loop</b><ul><li>Request-Taxi-Car-? Occur Num: 3</li></ul><b> Bad Inform Dialog Act</b><p>Nothing</p><b> Request But Not Inform Dialog Act</b><ul><li>request-taxi-car type Occur Num: 3</li></ul><b> Inform But Not Request Dialog Act</b><p>Nothing</p></div></body></html> \ No newline at end of file diff --git a/tests/results/TRADE-RulePolicy-TemplateNLG/res.txt b/tests/results/TRADE-RulePolicy-TemplateNLG/res.txt deleted file mode 100755 index ac4b617beae4af29ab206483eb88d9991d7dd720..0000000000000000000000000000000000000000 --- a/tests/results/TRADE-RulePolicy-TemplateNLG/res.txt +++ /dev/null @@ -1,8 +0,0 @@ -complete number of dialogs/tot: 0.299 -success number of dialogs/tot: 0.253 -average precision: 0.49259671331322424 -average recall: 0.48095032059206516 -average f1: 0.4437872685208314 -average book rate: 0.3691452991452991 -average turn (succ): 12.735177865612648 -average turn (all): 24.684