diff --git a/.gitignore b/.gitignore
index 39548e53ce43c63e08a25e379b22cf6bd4ac402a..6012819da84535f1108916153f3116b3a609ba07 100644
--- a/.gitignore
+++ b/.gitignore
@@ -66,7 +66,8 @@ convlab/nlu/jointBERT_new/**/output/
 convlab/nlu/milu/09*
 convlab/nlu/jointBERT/multiwoz/configs/multiwoz_new_usr_context.json
 convlab/nlu/milu/multiwoz/configs/system_without_context.jsonnet
-convlab/nlu/milu/multiwoz/configs/user_without_context.jsonnet
+convlab/nlu/milu/multiwoz/configs/user_without_context.jsonnet\
+*.pkl
 
 # test script
 *_test.py
@@ -87,7 +88,6 @@ dist
 convlab.egg-info
 
 # configs
-
 *experiment*
 *pretrained_models*
 .ipynb_checkpoints
@@ -102,8 +102,8 @@ convlab/dst/trade/multiwoz_config/
 convlab/deploy/bert_multiwoz_all.zip
 convlab/deploy/templates/dialog_eg.html
 test.py
-
 *convlab/policy/vector/action_dicts
+
 *.egg-info
 pre-trained-models/
 venv
diff --git a/README.md b/README.md
index 38ff5f214a8e243e9b665559fc4dd5ffe8db4434..4f20e80a64732f56211872f43bf31653ce761de5 100755
--- a/README.md
+++ b/README.md
@@ -1,283 +1,126 @@
-# ConvLab-2
-[![Build Status](https://travis-ci.com/thu-coai/ConvLab-2.svg?branch=master)](https://travis-ci.com/thu-coai/ConvLab-2)
+# ConvLab-3
 
-**ConvLab-2** is an open-source toolkit that enables researchers to build task-oriented dialogue systems with state-of-the-art models, perform an end-to-end evaluation, and diagnose the weakness of systems. As the successor of [ConvLab](https://github.com/ConvLab/ConvLab), ConvLab-2 inherits ConvLab's framework but integrates more powerful dialogue models and supports more datasets. Besides, we have developed an analysis tool and an interactive tool to assist researchers in diagnosing dialogue systems. [[paper]](https://arxiv.org/abs/2002.04793)
+![PyPI](https://img.shields.io/pypi/v/convlab) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/convlab) ![GitHub](https://img.shields.io/github/license/ConvLab/ConvLab-3)
+
+**ConvLab-3** is a flexible dialog system platform based on a **unified data format** for task-oriented dialog (TOD) datasets. The unified format serves as the adapter between TOD datasets and models: datasets are first transformed to the unified format and then loaded by models. In this way, the cost of adapting $M$ models to $N$ datasets is reduced from $M\times N$ to $M+N$. While retaining all features of [ConvLab-2](https://github.com/thu-coai/ConvLab-2),  ConvLab-3 greatly enlarges supported datasets and models thanks to the unified format, and enhances the utility of reinforcement learning (RL) toolkit for dialog policy module. For typical usage, see our [paper](). Datasets and Trained models are also available on [Hugging Face Hub](https://huggingface.co/ConvLab).
 
 - [Installation](#installation)
 - [Tutorials](#tutorials)
-- [Documents](#documents)
+- [Unified Datasets](#Unified-Datasets)
 - [Models](#models)
-- [Supported Datasets](#Supported-Datasets)
-- [End-to-end Performance on MultiWOZ](#End-to-end-Performance-on-MultiWOZ)
-- [Module Performance on MultiWOZ](#Module-Performance-on-MultiWOZ)
-- [Issues](#issues)
-- [Contributions](#contributions)
+- [Code Structure]($Code-Structure)
+- [Contributing](#contributing)
+- [Team](#Team)
 - [Citing](#citing)
 - [License](#license)
 
 ## Updates
 
-2021.9.13:
+- **2022.11.30**: ConvLab-3 release.
 
-- Add [MultiWOZ 2.3](https://github.com/lexmen318/MultiWOZ-coref) dataset in `data` dir. The dataset adds co-reference annotations in addition to corrections of dialogue acts and dialogue states. [[paper]](https://arxiv.org/abs/2010.05594)
+## Installation
 
-2021.6.18:
+You can install ConvLab-3 in one of the following ways according to your need. Higher versions of `torch` and `transformers` may also work.
 
-- Add [LAUG](https://github.com/thu-coai/LAUG), an open-source toolkit for Language understanding AUGmentation. It is an automatic method to approximate the natural perturbations to existing data. Augmented data could be used to conduct black-box robustness testing or enhancing training. [[paper]](https://arxiv.org/abs/2012.15262)
-- Add [SC-GPT](https://github.com/pengbaolin/SC-GPT) for NLG. [[paper]](https://arxiv.org/abs/2002.12328)
+### Git clone and pip install in development mode (Recommend)
 
-## Installation
+For the latest and most configurable version, we recommend installing ConvLab-3 in development mode.
 
-Require python >= 3.6.
+Clone the newest repository:
 
-Clone this repository:
 ```bash
-git clone https://github.com/thu-coai/ConvLab-2.git
+git clone --depth 1 https://github.com/ConvLab/ConvLab-3.git
 ```
 
-Install ConvLab-2 via pip:
+Install ConvLab-3 via pip:
 
 ```bash
-cd ConvLab-2
+cd ConvLab-3
 pip install -e .
 ```
 
-## Tutorials
-
-- [Getting Started](https://github.com/thu-coai/ConvLab-2/blob/master/tutorials/Getting_Started.ipynb) (Have a try on [Colab](https://colab.research.google.com/github/thu-coai/ConvLab-2/blob/master/tutorials/Getting_Started.ipynb)!)
-- [Add New Model](https://github.com/thu-coai/ConvLab-2/blob/master/tutorials/Add_New_Model.md)
-- [Train RL Policies](https://github.com/thu-coai/ConvLab-2/blob/master/tutorials/Train_RL_Policies)
-- [Interactive Tool](https://github.com/thu-coai/ConvLab-2/blob/master/deploy) [[demo video]](https://youtu.be/00VWzbcx26E)
-
-## Documents
-Our documents are on https://thu-coai.github.io/ConvLab-2_docs/convlab.html.
-
-## Models
-
-We provide following models:
-
-- NLU: SVMNLU, MILU, BERTNLU
-- DST: rule, TRADE, SUMBT
-- Policy: rule, Imitation, REINFORCE, PPO, GDPL, MDRG, HDSA, LaRL
-- Simulator policy: Agenda, VHUS
-- NLG: Template, SCLSTM
-- End2End: Sequicity, DAMD, RNN_rollout
-
-For  more details about these models, You can refer to `README.md` under `convlab/$module/$model/$dataset` dir such as `convlab/nlu/jointBERT/multiwoz/README.md`.
-
-## Supported Datasets
-
-- [Multiwoz 2.1](https://github.com/budzianowski/multiwoz)
-  - We add user dialogue act (*inform*, *request*, *bye*, *greet*, *thank*), remove 5 sessions that have incomplete dialogue act annotation and place it under `data/multiwoz` dir.
-  - Train/val/test size: 8434/999/1000. Split as original data.
-  - LICENSE: Attribution 4.0 International, url: http://creativecommons.org/licenses/by/4.0/
-- [CrossWOZ](https://github.com/thu-coai/CrossWOZ)
-  - We offers a rule-based user simulator and a complete set of models for building a pipeline system on the CrossWOZ dataset. We correct few state annotation and place it under `data/crosswoz` dir.
-  - Train/val/test size: 5012/500/500. Split as original data.
-  - LICENSE: Attribution 4.0 International, url: http://creativecommons.org/licenses/by/4.0/
-- [Camrest](https://www.repository.cam.ac.uk/handle/1810/260970)
-  - We add system dialogue act (*inform*, *request*, *nooffer*) and place it under `data/camrest` dir.
-  - Train/val/test size: 406/135/135. Split as original data.
-  - LICENSE: Attribution 4.0 International, url: http://creativecommons.org/licenses/by/4.0/
-- [Dealornot](https://github.com/facebookresearch/end-to-end-negotiator/tree/master/src/data/negotiate)
-  - Placed under `data/dealornot` dir.
-  - Train/val/test size: 5048/234/526. Split as original data.
-  - LICENSE: Attribution-NonCommercial 4.0 International, url: https://creativecommons.org/licenses/by-nc/4.0/
-
-## End-to-end Performance on MultiWOZ
-
-*Notice*: The results are for commits before [`bdc9dba`](https://github.com/thu-coai/ConvLab-2/commit/bdc9dba72c957d97788e533f9458ed03a4b0137b) (inclusive). We will update the results after improving user policy.
-
-We perform end-to-end evaluation (1000 dialogues) on MultiWOZ using the user simulator below (a full example on `tests/test_end2end.py`) :
-
-```python
-# BERT nlu trained on sys utterance
-user_nlu = BERTNLU(mode='sys', config_file='multiwoz_sys_context.json', model_file='https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_sys_context.zip')
-user_dst = None
-user_policy = RulePolicy(character='usr')
-user_nlg = TemplateNLG(is_user=True)
-user_agent = PipelineAgent(user_nlu, user_dst, user_policy, user_nlg, name='user')
-
-analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz')
-
-set_seed(20200202)
-analyzer.comprehensive_analyze(sys_agent=sys_agent, model_name='sys_agent', total_dialog=1000)
-```
-
-Main metrics (refer to `convlab/evaluator/multiwoz_eval.py` for more details):
-
-- Complete: whether complete the goal. Judged by the Agenda policy instead of external evaluator.
-- Success: whether all user requests have been informed and the booked entities satisfy the constraints.
-- Book: how many the booked entities satisfy the user constraints.
-- Inform Precision/Recall/F1: how many user requests have been informed.
-- Turn(succ/all): average turn number for successful/all dialogues.
-
-Performance (the first row is the default config for each module. Empty entries are set to default config.):
-
-| NLU         | DST       | Policy         | NLG         | Complete rate | Success rate | Book rate | Inform P/R/F1 | Turn(succ/all) |
-| ----------- | --------- | -------------- | ----------- | ------------- | ------------ | --------- | --------- | -------------- |
-| **BERTNLU** | RuleDST   | RulePolicy     | TemplateNLG |   90.5       |     81.3    |   91.1 | 79.7/92.6/83.5 | 11.6/12.3      |
-| **MILU**    | RuleDST | RulePolicy | TemplateNLG |    93.3       |   81.8      |   93.0    | 80.4/94.7/84.8 | 11.3/12.1      |
-| BERTNLU | RuleDST | RulePolicy | **SCLSTM**  |   48.5    | 40.2 | 56.9   | 62.3/62.5/58.7 |  11.9/27.1         |
-| BERTNLU     | RuleDST | **MLEPolicy**  | TemplateNLG |     42.7          |    35.9      |  17.6   | 62.8/69.8/62.9  |  12.1/24.1    |
-| BERTNLU | RuleDST | **PGPolicy**   | TemplateNLG |     37.4         |    31.7     |   17.4  |  57.4/63.7/56.9  |   11.0/25.3    |
-| BERTNLU | RuleDST | **PPOPolicy**  | TemplateNLG |     75.5         |    71.7    |   86.6    | 69.4/85.8/74.1  |  13.1/17.8   |
-| BERTNLU | RuleDST | **GDPLPolicy** | TemplateNLG |     49.4         |     38.4    |  20.1     |  64.5/73.8/65.6 |  11.5/21.3    |
-| None        | **TRADE** | RulePolicy | TemplateNLG |    32.4      |    20.1     |    34.7      |  46.9/48.5/44.0 |  11.4/23.9      |
-| None        | **SUMBT** | RulePolicy | TemplateNLG |   34.5       |   29.4     |   62.4    |  54.1/50.3/48.3  |   11.0/28.1     |
-| BERTNLU | RuleDST | **MDRG**       | None        | 21.6 | 17.8 | 31.2 | 39.9/36.3/34.8 | 15.6/30.5|
-| BERTNLU | RuleDST | **LaRL**       | None        | 34.8 | 27.0 | 29.6 | 49.1/53.6/47.8 |13.2/24.4|
-| None | **SUMBT** | **LaRL** | None |  32.9 | 23.7  |  25.9 | 48.6/52.0/46.7 | 12.5/24.3|
-| None | None | **DAMD***      | None | 39.5| 34.3 | 51.4 | 60.4/59.8/56.3 | 15.8/29.8 |
-
-*: end-to-end models used as sys_agent directly.
-
-## Module Performance on MultiWOZ
-
-### NLU
-
-By running `convlab/nlu/evaluate.py MultiWOZ $model all`:
-
-|         | Precision | Recall | F1    |
-| ------- | --------- | ------ | ----- |
-| BERTNLU | 82.48     | 85.59  | 84.01 |
-| MILU    | 80.29     | 83.63  | 81.92 |
-| SVMNLU  | 74.96     | 50.74  | 60.52 |
-
-### DST 
-
-By running `convlab/dst/evaluate.py MultiWOZ $model`:
-
-|             |  Joint accuracy  | Slot accuracy | Joint F1  |
-| --------    |   -------------   | -------------  | --------|
-|  MDBT       |   0.06           |      0.89       | 0.43    |
-|  SUMBT      |    0.30         |       0.96       | 0.83    |
-|   TRADE     |    0.40         |       0.96       | 0.84    |
-
-### Policy
+### Pip install from PyPI
 
-*Notice*: The results are for commits before [`bdc9dba`](https://github.com/thu-coai/ConvLab-2/commit/bdc9dba72c957d97788e533f9458ed03a4b0137b) (inclusive). We will update the results after improving user policy.
+To use ConvLab-3 as an off-the-shelf tool, you can install via:
 
-By running `convlab/policy/evalutate.py --model_name $model`
-
-|           | Task Success Rate |
-| --------- | ----------------- |
-| MLE       | 0.56              |
-| PG        | 0.54              |
-| PPO       | 0.89              |
-| GDPL      | 0.58              |
-
-### NLG
-
-By running `convlab/nlg/evaluate.py MultiWOZ $model sys`
-
-|          | corpus BLEU-4 |
-| -------- | ------------- |
-| Template | 0.3309        |
-| SCLSTM   | 0.4884        |
-
-## Translation-train SUMBT for cross-lingual DST
-
-### Train
-
-With Convlab-2, you can train SUMBT on a machine-translated dataset like this:
-
-```python
-# train.py
-import os
-from sys import argv
-
-if __name__ == "__main__":
-    if len(argv) != 2:
-        print('usage: python3 train.py [dataset]')
-        exit(1)
-    assert argv[1] in ['multiwoz', 'crosswoz']
-
-    from convlab.dst.sumbt.multiwoz_zh.sumbt import SUMBT_PATH
-    if argv[1] == 'multiwoz':
-        from convlab.dst.sumbt.multiwoz_zh.sumbt import SUMBTTracker as SUMBT
-    elif argv[1] == 'crosswoz':
-        from convlab.dst.sumbt.crosswoz_en.sumbt import SUMBTTracker as SUMBT
-
-    sumbt = SUMBT()
-    sumbt.train(True)
+```bash
+pip install convlab
 ```
+Note that the `data` directory will not be included due to the package size limitation.
 
-### Evaluate
+### Using Docker
 
-Execute `evaluate.py` (under `convlab/dst/`) with following command:
+We also provide [Dockerfile](https://github.com/ConvLab/ConvLab-3/blob/master/Dockerfile) for building docker. Basically it uses the `requirement.txt` and then installs ConvLab-3 in development mode.
 
 ```bash
-python3 evaluate.py [CrossWOZ-en|MultiWOZ-zh] [val|test|human_val]
-```
+# create image
+docker build -t convlab .
 
-evaluation of our pre-trained models are: (joint acc.)
+# run container
+docker run -dit convlab
 
-| type  | CrossWOZ-en | MultiWOZ-zh |
-| ----- | ----------- | ----------- |
-| val   | 12.4%       | 48.5%       |
-| test  | 12.4%       | 46.0%       |
-| human_val | 10.6%       | 47.4%       |
+# open bash in container
+docker exec -it CONTAINER_ID bash
+```
 
-`human_val` option will make the model evaluate on the validation set translated by human. 
+## Tutorials
 
-Note: You may want to download pre-traiend BERT models and translation-train SUMBT models provided by us.
+- [Introduction to Unified Data Format](https://github.com/ConvLab/ConvLab-3/tree/master/data/unified_datasets)
+- [Utility functions for unified datasets](https://github.com/ConvLab/ConvLab-3/blob/master/convlab/util/unified_datasets_util.py)
+- [RL Toolkit](https://github.com/ConvLab/ConvLab-3/tree/master/convlab/policy)
+- [Interactive Tool](https://github.com/ConvLab/ConvLab-3/blob/master/deploy) [[demo video]](https://youtu.be/00VWzbcx26E)
 
-Without modifying any code, you could:
+## Unified Datasets
 
-- download pre-trained BERT models from:
+Current datasets in unified data format: (DA-U/DA-S stands for user/system dialog acts)
 
-  - [bert-base-uncased](https://huggingface.co/bert-base-uncased)  for CrossWOZ-en
-  - [chinese-bert-wwm-ext](https://huggingface.co/hfl/chinese-bert-wwm-ext)  for MultiWOZ-zh
+| Dataset       | Dialogs | Goal               | DA-U               | DA-S               | State              | API result         | DataBase           |
+| ------------- | ------- | ------------------ | ------------------ | ------------------ | ------------------ | ------------------ | ------------------ |
+| Camrest       | 676     | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |                    | :white_check_mark: |
+| WOZ 2.0       | 1200    |                    | :white_check_mark: |                    | :white_check_mark: |                    |                    |
+| KVRET         | 3030    |                    | :white_check_mark: |                    | :white_check_mark: | :white_check_mark: |                    |
+| DailyDialog   | 13118   |                    | :white_check_mark: |                    |                    |                    |                    |
+| Taskmaster-1  | 13175   |                    | :white_check_mark: | :white_check_mark: | :white_check_mark: |                    |                    |
+| Taskmaster-2  | 17303   |                    | :white_check_mark: | :white_check_mark: | :white_check_mark: |                    |                    |
+| MultiWOZ 2.1  | 10438   | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |                    | :white_check_mark: |
+| Schema-Guided | 22825   |                    | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |                    |
+| MetaLWOZ      | 40203   | :white_check_mark: |                    |                    |                    |                    |                    |
+| CrossWOZ (zh) | 6012    | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
+| Taskmaster-3  | 23757   |                    | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |                    |
 
-  extract it to `./pre-trained-models`.
+Unified datasets are available under `data/unified_datasets` directory as well as [Hugging Face Hub](https://huggingface.co/ConvLab). We will continue adding more datasets listed in [this issue](https://github.com/ConvLab/ConvLab-3/issues/11). If you want to add a listed/custom dataset to ConvLab-3, you can create an issue for discussion and then create pull-request. We will list you as the [contributors](#Team) and highly appreciate your contributions!
 
-- for translation-train SUMBT model:
+## Models
 
-  - [trained on CrossWOZ-en](https://convlab.blob.core.windows.net/convlab-2/crosswoz_en-pytorch_model.bin.zip)
-  - [trained on MultiWOZ-zh](https://convlab.blob.core.windows.net/convlab-2/multiwoz_zh-pytorch_model.bin.zip)
-  - Say the data set is CrossWOZ (English), (after extraction) just save the pre-trained model under `./convlab/dst/sumbt/crosswoz_en/pre-trained` and name it with `pytorch_model.bin`. 
+We list newly integrated models in ConvLab-3 that support unified data format and obtain strong performance. You can follow the link for more details about these models. Other models can be used in the same way as in ConvLab-2.
 
-## Issues
+| Task                           | Models                                                       | Input           | Output           |
+| ------------------------------ | ------------------------------------------------------------ | --------------- | ---------------- |
+| Response Generation            | [T5](https://github.com/ConvLab/ConvLab-3/tree/master/convlab/base_models/t5) | Context         | Response         |
+| Goal-to-Dialogue                 | [T5](https://github.com/ConvLab/ConvLab-3/tree/master/convlab/base_models/t5) | Goal            | Dialog           |
+| Natural Language Understanding | [T5](https://github.com/ConvLab/ConvLab-3/tree/master/convlab/base_models/t5), [BERTNLU](https://github.com/ConvLab/ConvLab-3/tree/master/convlab/nlu/jointBERT), [MILU](https://github.com/ConvLab/ConvLab-3/tree/master/convlab/nlu/milu) | Context         | DA-U             |
+| Dialog State Tracking          | [T5](https://github.com/ConvLab/ConvLab-3/tree/master/convlab/base_models/t5), [SUMBT](https://github.com/ConvLab/ConvLab-3/tree/master/convlab/dst/sumbt), [SetSUMBT](https://github.com/ConvLab/ConvLab-3/tree/master/convlab/dst/setsumbt), TripPy | Context         | State            |
+| RL Policy                      | [DDPT](https://github.com/ConvLab/ConvLab-3/tree/master/convlab/policy/vtrace_DPT), [PPO](https://github.com/ConvLab/ConvLab-3/tree/master/convlab/policy/ppo), [PG](https://github.com/ConvLab/ConvLab-3/tree/master/convlab/policy/pg) | State, DA-U, DB | DA-S             |
+| Natural Language Generation    | [T5](https://github.com/ConvLab/ConvLab-3/tree/master/convlab/base_models/t5), SC-GPT | DA-S            | Response         |
+| End-to-End                     | SOLOIST                                                      | Context, DB     | State, Response  |
+| User simulator                 | [TUS](https://github.com/ConvLab/ConvLab-3/tree/master/convlab/policy/tus), [GenTUS](https://github.com/ConvLab/ConvLab-3/tree/master/convlab/policy/genTUS) | Goal, DA-S      | DA-U, (Response) |
 
-You are welcome to create an issue if you want to request a feature, report a bug or ask a general question.
+Trained models are available on [Hugging Face Hub](https://huggingface.co/ConvLab).
 
-## Contributions
+## Contributing
 
-We welcome contributions from community.
+We welcome contributions from community. Please see issues to find what we need.
 
-- If you want to make a big change, we recommend first creating an issue with your design.
-- Small contributions can be directly made by a pull request.
-- If you like make contributions to our library, see issues to find what we need.
+- If you want to add a new dataset, model, or other feature, please describe the dataset/model/feature in an issue before creating pull-request.
+- Small change like fixing a bug can be directly made by a pull-request.
 
 ## Team
 
-**ConvLab-3** is maintained and developed by Tsinghua University Conversational AI group (THU-coai), the [Dialogue Systems and Machine Learning Group](https://www.cs.hhu.de/en/research-groups/dialog-systems-and-machine-learning.html) at Heinrich Heine University, Düsseldorf, Germany and Microsoft Research (MSR).
-
-We would like to thank:
+**ConvLab-3** is maintained and developed by [Tsinghua University Conversational AI](http://coai.cs.tsinghua.edu.cn/) group (THU-COAI), the [Dialogue Systems and Machine Learning Group](https://www.cs.hhu.de/en/research-groups/dialog-systems-and-machine-learning.html) at Heinrich Heine University, Düsseldorf, Germany and Microsoft Research (MSR).
 
-Yan Fang, Zhuoer Feng, Jianfeng Gao, Qihan Guo, Kaili Huang, Minlie Huang, Sungjin Lee, Bing Li, Jinchao Li, Xiang Li, Xiujun Li, Jiexi Liu, Lingxiao Luo, Wenchang Ma, Mehrad Moradshahi, Baolin Peng, Runze Liang, Ryuichi Takanobu, Hongru Wang, Jiaxin Wen, Yaoqin Zhang, Zheng Zhang, Qi Zhu, Xiaoyan Zhu, Carel van Niekerk, Christian Geishauser, Hsien-chin Lin, Nurul Lubis, Xiaochen Zhu, Michael Heck, Shutong Feng, Milica Gašić.
+We would like to thank all contributors of ConvLab:
 
-
-## Citing
-
-If you use ConvLab-2 in your research, please cite:
-
-```
-@inproceedings{zhu2020convlab,
-    title={ConvLab-2: An Open-Source Toolkit for Building, Evaluating, and Diagnosing Dialogue Systems},
-    author={Qi Zhu and Zheng Zhang and Yan Fang and Xiang Li and Ryuichi Takanobu and Jinchao Li and Baolin Peng and Jianfeng Gao and Xiaoyan Zhu and Minlie Huang},
-    year={2020},
-    booktitle={Proceedings of the 58th Annual Meeting of the Association for Computational Linguistics},
-}
-
-@inproceedings{liu2021robustness,
-    title={Robustness Testing of Language Understanding in Task-Oriented Dialog},
-    author={Liu, Jiexi and Takanobu, Ryuichi and Wen, Jiaxin and Wan, Dazhen and Li, Hongguang and Nie, Weiran and Li, Cheng and Peng, Wei and Huang, Minlie},
-    year={2021},
-    booktitle={Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics},
-}
-```
+Yan Fang, Zhuoer Feng, Jianfeng Gao, Qihan Guo, Kaili Huang, Minlie Huang, Sungjin Lee, Bing Li, Jinchao Li, Xiang Li, Xiujun Li, Jiexi Liu, Lingxiao Luo, Wenchang Ma, Mehrad Moradshahi, Baolin Peng, Runze Liang, Ryuichi Takanobu, Dazhen Wan, Hongru Wang, Jiaxin Wen, Yaoqin Zhang, Zheng Zhang, Qi Zhu, Xiaoyan Zhu, Carel van Niekerk, Christian Geishauser, Hsien-chin Lin, Nurul Lubis, Xiaochen Zhu, Michael Heck, Shutong Feng, Milica Gašić.
 
 ## License
 
diff --git a/convlab/base_models/gpt/create_data.py b/convlab/base_models/gpt/create_data.py
deleted file mode 100644
index e6c4d67bb4babd509feb026d256264c6ff1c0051..0000000000000000000000000000000000000000
--- a/convlab/base_models/gpt/create_data.py
+++ /dev/null
@@ -1,39 +0,0 @@
-import os
-import json
-from tqdm import tqdm
-import re
-from convlab.util import load_dataset
-
-
-def create_lm_data(dataset, data_dir, args):
-    data_by_split = dataset
-    os.makedirs(data_dir, exist_ok=True)
-
-    data_splits = data_by_split.keys()
-    for data_split in data_splits:
-        data = []
-        for sample in tqdm(data_by_split[data_split], desc=f'{data_split} sample', leave=False):
-            if args.model_type == 'dialogpt':
-                dialogue = ' <|endoftext|> '.join([turn['utterance'] for turn in sample['turns']]) + ' <|endoftext|>'
-            else:
-                dialogue = '\n'.join([f"{turn['speaker']}: {turn['utterance']}" for turn in sample['turns']])
-            data.append(json.dumps({'dialogue': dialogue}, ensure_ascii=False)+'\n')
-
-        file_name = os.path.join(data_dir, f"{data_split}.json")
-        with open(file_name, "w", encoding='utf-8') as f:
-            f.writelines(data)
-
-
-if __name__ == '__main__':
-    from argparse import ArgumentParser
-    parser = ArgumentParser(description="create data for seq2seq training")
-    parser.add_argument('--tasks', '-t', metavar='task_name', nargs='*', choices=['lm'], help='names of tasks')
-    parser.add_argument('--datasets', '-d', metavar='dataset_name', nargs='*', help='names of unified datasets')
-    parser.add_argument('--model_type', '-m', metavar='model_type', help='type of the language model: gpt, dialogpt, ..')
-    args = parser.parse_args()
-    print(args)
-    for dataset_name in tqdm(args.datasets, desc='datasets'):
-        dataset = load_dataset(dataset_name)
-        for task_name in tqdm(args.tasks, desc='tasks', leave=False):
-            data_dir = os.path.join('data', task_name, args.model_type, dataset_name)
-            eval(f"create_{task_name}_data")(dataset, data_dir, args)
diff --git a/convlab/base_models/gpt/keyword_extraction/eval_key2gen.py b/convlab/base_models/gpt/keyword_extraction/eval_key2gen.py
deleted file mode 100644
index 6b1068cef045550f57621fe0ab4aad8a4047cfbb..0000000000000000000000000000000000000000
--- a/convlab/base_models/gpt/keyword_extraction/eval_key2gen.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import json
-import datasets
-from tabulate import tabulate
-
-def main(predict_result):
-    data = {
-        "grounded keywords": {
-            "positive_keywords": [], "negative_keywords": None,
-            "predictions": [], "references": []
-        },
-        "all keywords": {
-            "positive_keywords": [], "negative_keywords": [],
-            "predictions": [], "references": []
-        },
-        "no keywords": {
-            "positive_keywords": None, "negative_keywords": None,
-            "predictions": [], "references": []
-        }
-    }
-    with open(predict_result) as f:
-        for line in f:
-            item = json.loads(line)
-            prediction = item['predictions'].strip()
-            reference = item['target'].strip()
-            if 'all_keywords' in item and item['all_keywords']:
-                sample_type = 'all keywords'
-
-                positive_keywords = [k for g in item['keywords'] for k in g]
-                data[sample_type]["positive_keywords"].append(positive_keywords)
-
-                all_keywords = [k for g in item['all_keywords'] for k in g]
-                for keyword in positive_keywords:
-                    all_keywords.remove(keyword)
-                data[sample_type]["negative_keywords"].append(all_keywords)
-
-            elif 'keywords' in item and item['keywords']:
-                sample_type = 'grounded keywords'
-
-                positive_keywords = [k for g in item['keywords'] for k in g]
-                data[sample_type]["positive_keywords"].append(positive_keywords)
-            
-            else:
-                sample_type = 'no keywords'
-
-            data[sample_type]["predictions"].append(prediction)
-            data[sample_type]["references"].append(reference)
-
-    metric = datasets.load_metric('./key2gen_metric.py')
-    table = []
-    for sample_type in data:
-        table.append({'sample_type': sample_type, **metric.compute(**data[sample_type])})
-    print(tabulate(table, headers='keys', tablefmt='github'))
-
-
-if __name__ == '__main__':
-    from argparse import ArgumentParser
-    parser = ArgumentParser(description="evaluate keywords to response generation performance")
-    parser.add_argument('--predict_result', '-p', type=str, required=True, help='path to the output file generated_predictions.json')
-    args = parser.parse_args()
-    print(args)
-    main(args.predict_result)
diff --git a/convlab/base_models/gpt/keyword_extraction/gen_pretraining_data.py b/convlab/base_models/gpt/keyword_extraction/gen_pretraining_data.py
deleted file mode 100644
index b6ef65db298378b744a45130fd71c072243bcfca..0000000000000000000000000000000000000000
--- a/convlab/base_models/gpt/keyword_extraction/gen_pretraining_data.py
+++ /dev/null
@@ -1,88 +0,0 @@
-import json
-import json_lines
-import os
-import random
-from tqdm import tqdm
-from nltk import sent_tokenize
-
-def main(args):
-    random.seed(42)
-    os.makedirs(args.output_dir, exist_ok=True)
-    filenames = [os.path.join(args.input_dir, f) for (_, _, fs) in os.walk(args.input_dir) for f in fs if 'keywords' in f]
-    for filename in filenames:
-        dataset_name = filename.split('/')[-2]
-        data_split = filename.split('/')[-1].split('_')[-1].split('.')[0]
-        output_file = os.path.join(args.output_dir, f"{filename.split('/')[-1].split('_')[-1]}")
-        print(f'processing {dataset_name}: {filename} => {output_file}')
-        with open(filename, 'rb') as fin, open(output_file, 'w', encoding='utf-8') as fout:
-            for dial in tqdm(json_lines.reader(fin)):
-                context = []
-                turns_keywords = [turn['keywords'] for turn in dial]
-                for i, turn in enumerate(dial):
-                    if dataset_name == 'wikidialog':
-                        # skip user turns that generated by T5 in wikidialog 
-                        speaker = 'user' if i % 2 == 1 else 'system'
-                    else:
-                        speaker = 'user' if i % 2 == 0 else 'system'
-                    utt = turn['utterance']
-                    context_seq = '\n'.join([f"{turn['speaker']}: {turn['utt']}" for turn in context]+[f'{speaker}: '])
-                    context.append({'speaker': speaker, 'utt': utt})
-                    if i == 0 or (dataset_name == 'wikidialog' and speaker == 'user'):
-                        continue
-                    
-                    if args.mode == 'rg':
-                        input_seq = f'generate a response: all knowledge: | | context:\n\n{context_seq}'
-                        fout.write(json.dumps({
-                            'dataset': dataset_name,
-                            'source': input_seq, 
-                            'target': utt
-                            }, ensure_ascii=False)+'\n')
-                        continue
-
-                    if args.mode == 'key2gen':
-                        random.shuffle(turn['keywords'])
-                        for j in range(len(turn['keywords'])):
-                            random.shuffle(turn['keywords'][j])
-                        keywords = ' | '.join([' : '.join(sent_keywords) for sent_keywords in turn['keywords']])
-                        input_seq = f'generate a response: grounded knowledge: | {keywords} | context:\n\n{context_seq}'
-                        json2dump = {
-                            'dataset': dataset_name,
-                            'source': input_seq, 
-                            'target': utt
-                            }
-                        if data_split == 'validation':
-                            json2dump.update({'keywords': turn['keywords']})
-                        fout.write(json.dumps(json2dump, ensure_ascii=False)+'\n')
-                        continue
-
-                    if args.mode == 'key2gen_noisy':
-                        if random.random() < 0.8:
-                            possible_keywords_sents = turn['keywords'][:]
-                        else:
-                            possible_keywords_sents = []
-                        num_possible_keywords_turns = min(random.randint(1, 5), len(turns_keywords) - 1)
-                        for turn_keywords in random.sample(turns_keywords[:i] + turns_keywords[i+1:], num_possible_keywords_turns):
-                            possible_keywords_sents.extend(turn_keywords)
-                        random.shuffle(possible_keywords_sents)
-                        possible_keywords = ' | '.join([' : '.join(sent_keywords) for sent_keywords in possible_keywords_sents])
-                        input_seq = f'generate a response: all knowledge: | {possible_keywords} | context:\n\n{context_seq}'
-                        json2dump = {
-                            'dataset': dataset_name,
-                            'source': input_seq, 
-                            'target': utt
-                            }
-                        if data_split == 'validation':
-                            json2dump.update({'keywords': turn['keywords'], 'all_keywords': possible_keywords_sents})
-                        fout.write(json.dumps(json2dump, ensure_ascii=False)+'\n')
-                        continue
-    
-
-if __name__ == '__main__':
-    from argparse import ArgumentParser
-    parser = ArgumentParser(description="calculate NLU metrics for unified datasets")
-    parser.add_argument('--input_dir', '-i', type=str, help='path to the input files')
-    parser.add_argument('--output_dir', '-o', type=str, help='path to the output files')
-    parser.add_argument('--mode', '-m', type=str, choices=['rg', 'key2gen', 'key2gen_noisy'], help='which task to perform')
-    args = parser.parse_args()
-    print(args)
-    main(args)
diff --git a/convlab/base_models/gpt/keyword_extraction/gen_pretraining_data.sh b/convlab/base_models/gpt/keyword_extraction/gen_pretraining_data.sh
deleted file mode 100644
index eb67a18b22e0480323f132abbf42a2d1508755b9..0000000000000000000000000000000000000000
--- a/convlab/base_models/gpt/keyword_extraction/gen_pretraining_data.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-# generate data for response generation, key2gen, key2gen_noisy
-for task_name in rg key2gen key2gen_noisy
-do
-    dataset_name="dailydialog+metalwoz+tm1+tm2+tm3+sgd+reddit+wikidialog"
-    names=$(echo ${dataset_name} | tr "+" "\n")
-    model_type="gpt"
-    data_dir=data/${task_name}/${model_type}/${dataset_name}
-    mkdir -p ${data_dir}
-    train_file="${data_dir}/train.json"
-    validation_file="${data_dir}/validation.json"
-    rm ${train_file} ${validation_file}
-    for name in ${names}
-    do
-        echo "preprocessing ${name}"
-        python gen_pretraining_data.py -i data/lm/${model_type}/${name} -o data/${task_name}/${model_type}/${name} -m ${task_name}
-        if [ "${name}" != "${dataset_name}" ]; then
-            cat "data/${task_name}/${model_type}/${name}/train.json" >> ${train_file}
-            cat "data/${task_name}/${model_type}/${name}/validation.json" >> ${validation_file}
-        fi
-    done
-done
-
-# merge key2gen+key2gen_noisy data
-task_name="key2gen+key2gen_noisy"
-dataset_name="dailydialog+metalwoz+tm1+tm2+tm3+sgd+reddit+wikidialog"
-names=$(echo ${task_name} | tr "+" "\n")
-model_type="gpt"
-data_dir=data/${task_name}/${model_type}/${dataset_name}
-mkdir -p ${data_dir}
-train_file="${data_dir}/train.json"
-validation_file="${data_dir}/validation.json"
-rm ${train_file} ${validation_file}
-for name in ${names}
-do
-    echo "preprocessing ${name}"
-    if [ "${name}" != "${task_name}" ]; then
-        cat "data/${name}/${model_type}/${dataset_name}/train.json" >> ${train_file}
-        cat "data/${name}/${model_type}/${dataset_name}/validation.json" >> ${validation_file}
-    fi
-done
\ No newline at end of file
diff --git a/convlab/base_models/gpt/keyword_extraction/get_keywords.sh b/convlab/base_models/gpt/keyword_extraction/get_keywords.sh
deleted file mode 100644
index d3051ba65b2458bfad7af288746a28456f4936e5..0000000000000000000000000000000000000000
--- a/convlab/base_models/gpt/keyword_extraction/get_keywords.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-task_name="lm"
-model_type="gpt"
-model_name_or_path="gpt2-large"
-keywords_num=100
-keywords_ratio=0.3
-keywords_loss_th=0
-stopwords=True
-for dataset_name in dailydialog metalwoz tm1 tm2 tm3 sgd reddit wikidialog
-do
-    data_dir="data/${task_name}/${model_type}/${dataset_name}"
-    for data_split in validation train
-    do
-        token_loss_file="${data_dir}/token_loss_${data_split}.json"
-        output_file="${data_dir}/keywords_${data_split}.json"
-        python lmloss2keywords.py \
-            --model_type ${model_type} \
-            --model_name_or_path ${model_name_or_path} \
-            --token_loss_file ${token_loss_file} \
-            --keywords_num ${keywords_num} \
-            --keywords_ratio ${keywords_ratio} \
-            --keywords_loss_th ${keywords_loss_th} \
-            --stopwords ${stopwords} \
-            --output_file ${output_file}
-    done
-done
\ No newline at end of file
diff --git a/convlab/base_models/gpt/keyword_extraction/get_token_loss.sh b/convlab/base_models/gpt/keyword_extraction/get_token_loss.sh
deleted file mode 100644
index 7c2b57dafa89d0bd711c927a0e97994d43d15bfc..0000000000000000000000000000000000000000
--- a/convlab/base_models/gpt/keyword_extraction/get_token_loss.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-n_gpus=4
-master_port=23456
-task_name="lm"
-model_type="gpt"
-cache_dir="../cache"
-source_column="dialogue"
-max_length=512
-model_name_or_path="gpt2-large"
-per_device_eval_batch_size=16
-
-for dataset_name in dailydialog metalwoz tm1 tm2 tm3 sgd reddit wikidialog
-do
-    data_dir="data/${task_name}/${model_type}/${dataset_name}"
-    output_dir="output/${task_name}/${model_type}/${dataset_name}"
-
-    python ../create_data.py --tasks ${task_name} --datasets ${dataset_name} --model_type ${model_type}
-    for data_split in validation train
-    do
-        validation_file="${data_dir}/${data_split}.json"
-        dump_eval_loss_to="${data_dir}/token_loss_${data_split}.json"
-        rm ${dump_eval_loss_to}
-        python -m torch.distributed.launch --master_port ${master_port} \
-            --nproc_per_node ${n_gpus} ../run_clm.py \
-            --dump_eval_loss_to ${dump_eval_loss_to}\
-            --model_name_or_path ${model_name_or_path} \
-            --output_dir ${data_dir} \
-            --validation_file ${validation_file} \
-            --source_column ${source_column} \
-            --max_length ${max_length} \
-            --do_eval \
-            --cache_dir ${cache_dir} \
-            --preprocessing_num_workers 4 \
-            --per_device_eval_batch_size ${per_device_eval_batch_size}
-    done
-done
diff --git a/convlab/base_models/gpt/keyword_extraction/key2gen_metric.py b/convlab/base_models/gpt/keyword_extraction/key2gen_metric.py
deleted file mode 100644
index d9722d96ca71a961dc7ad837191fa202848111f3..0000000000000000000000000000000000000000
--- a/convlab/base_models/gpt/keyword_extraction/key2gen_metric.py
+++ /dev/null
@@ -1,96 +0,0 @@
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""key2gen Metric"""
-
-import datasets
-import sacrebleu
-
-# TODO: Add BibTeX citation
-_CITATION = """\
-@inproceedings{post-2018-call,
-    title = "A Call for Clarity in Reporting {BLEU} Scores",
-    author = "Post, Matt",
-    booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers",
-    month = oct,
-    year = "2018",
-    address = "Belgium, Brussels",
-    publisher = "Association for Computational Linguistics",
-    url = "https://www.aclweb.org/anthology/W18-6319",
-    pages = "186--191",
-}
-"""
-
-_DESCRIPTION = """\
-Metric to evaluate text-to-text models on the keywords grounded generation task.
-"""
-
-_KWARGS_DESCRIPTION = """
-Calculates corpus-bleu4, positive keywords recall, negative keywords recall 
-Args:
-    positive_keywords: list of keywords (list of string) in the ground truth references
-    negative_keywords: list of keywords (list of string) in the random sampled references
-    predictions: list of predictions to score. Each predictions
-        should be a string.
-    references: list of reference for each prediction. Each
-        reference should be a string.
-Returns:
-    bleu: corpus-bleu score
-    positive_keywords_recall: how many keywords in the ground truth response are generated, micro-averaged
-    negative_keywords_recall: how many keywords in the random sampled response are generated, micro-averaged
-"""
-
-
-@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class Key2GenMetrics(datasets.Metric):
-    """Metric to evaluate text-to-text models on the keywords grounded generation task."""
-
-    def _info(self):
-        return datasets.MetricInfo(
-            description=_DESCRIPTION,
-            citation=_CITATION,
-            inputs_description=_KWARGS_DESCRIPTION,
-            # This defines the format of each prediction and reference
-            features=datasets.Features({
-                'predictions': datasets.Value('string'),
-                'references': datasets.Value('string'),
-            })
-        )
-
-    def _compute(self, predictions, references, positive_keywords, negative_keywords=None):
-        """Returns the scores: bleu, positive_keywords_recall, negative_keywords_recall"""
-        bleu = sacrebleu.corpus_bleu(predictions, [references], lowercase=True).score
-        cnt = {'pos': 0, 'neg': 0, 'pos_recall': 0, 'neg_recall': 0}
-        if positive_keywords:
-            if not negative_keywords:
-                negative_keywords = [[]] * len(positive_keywords)
-            for poskeys, negkeys, prediction in zip(positive_keywords, negative_keywords, predictions):
-                cnt['pos'] += len(poskeys)
-                cnt['neg'] += len(negkeys)
-
-                prediction = prediction.lower()
-                for key in poskeys:
-                    key = key.lower()
-                    if key in prediction:
-                        cnt['pos_recall'] += 1
-                
-                for key in negkeys:
-                    key = key.lower()
-                    if key in prediction:
-                        cnt['neg_recall'] += 1
-            
-        return {
-            "bleu": bleu,
-            "positive_keywords_recall": cnt['pos_recall']/cnt['pos'] if cnt['pos'] > 0 else 0,
-            "negative_keywords_recall": cnt['neg_recall']/cnt['neg'] if cnt['neg'] > 0 else 0,
-        }
diff --git a/convlab/base_models/gpt/keyword_extraction/lmloss2keywords.py b/convlab/base_models/gpt/keyword_extraction/lmloss2keywords.py
deleted file mode 100644
index bb221f6d78b026c61f10846c385b5fa903c64e7f..0000000000000000000000000000000000000000
--- a/convlab/base_models/gpt/keyword_extraction/lmloss2keywords.py
+++ /dev/null
@@ -1,174 +0,0 @@
-import json
-import json_lines
-from pprint import pprint
-import os
-from tqdm import tqdm
-import numpy as np
-from nltk.corpus import stopwords
-from nltk.tokenize import word_tokenize, PunktSentenceTokenizer
-from transformers import GPT2Tokenizer
-from string import punctuation
-
-
-def merge_tokens(tokens, losses):
-    """Merge tokens into words"""
-    res = []
-    i = 0
-    while i < len(tokens):
-        token = tokens[i]
-        loss = losses[i]
-        if token in ['Ġ', 'Ċ']:
-            # "Ġ" means " ", "Ċ" means "\n"
-            if token == 'Ċ' and i < len(tokens) - 1 and not tokens[i+1].startswith('Ġ'):
-                tokens[i+1] = 'Ġ'+tokens[i+1]
-            i += 1
-            continue
-        if token in ['user', 'system', 'Ġuser', 'Ġsystem'] and i < len(tokens)-1 and tokens[i+1] == ':':
-            if i > 0:
-                tokens[i+1] = '<|endoftext|>'
-                i += 1
-            else:
-                i += 2
-            continue
-        if token.startswith('Ġ'):
-            # token = token.replace("Ġ", "")
-            res.append([[token], [loss]])
-        elif token == '<|endoftext|>':
-            res.append([[token], [0.]])
-        else:
-            assert 'Ġ' not in token
-            if len(res) > 0:
-                res[-1][0].append(token)
-                res[-1][1].append(loss)
-            else:
-                res.append([[token], [loss]])
-        i += 1
-    return res
-
-
-def convert_token_loss2word_loss(token_loss_file):
-    """generate a word loss file according to the token loss file"""
-    word_loss_file = os.path.join(os.path.dirname(token_loss_file), token_loss_file.split('/')[-1].replace('token', 'word'))
-    fin = open(token_loss_file, 'rb')
-    fout = open(word_loss_file, 'w', encoding='utf-8')
-
-    for item in tqdm(json_lines.reader(fin)):
-        tokens, losses = item['tokens'], item['losses']
-        assert len(tokens) == len(losses)
-        word2losses = merge_tokens(tokens, losses)
-        fout.write(json.dumps({"words": [x[0] for x in word2losses], "losses": [x[1] for x in word2losses]}, ensure_ascii=False)+'\n')
-
-    fin.close()
-    fout.close()
-    return word_loss_file
-
-def main(args):
-    if not args.word_loss_file:
-        word_loss_file = convert_token_loss2word_loss(args.token_loss_file)
-    else:
-        word_loss_file = args.word_loss_file
-
-    if not args.output_file:
-        return
-
-    stop_words = set(stopwords.words('english'))
-    tokenizer = GPT2Tokenizer.from_pretrained(args.model_name_or_path)
-    sent_tokenizer = PunktSentenceTokenizer()
-
-    def keywords_filter(words, losses):
-        word_loss_pairs = list(zip(words, losses))
-        index2keyword = {}
-        index2turn_sent = {}
-        num_turns = 0
-        turns_sent_spans = [list(sent_tokenizer.span_tokenize(utt)) for utt in ''.join(words).strip().split('<|endoftext|>')]
-        utt = ''
-        for i, word_loss_pair in enumerate(word_loss_pairs):
-            if word_loss_pair[0].startswith('<|endoftext|>'):
-                num_turns += 1
-                utt = ''
-                continue
-            utt += word_loss_pair[0]
-            words = word_tokenize(word_loss_pair[0])
-            if args.stopwords and any([w.lower() in stop_words for w in words]):
-                # skip stopwords
-                continue
-            if word_loss_pair[1] <= args.keywords_loss_th:
-                # skip if loss is too small
-                continue
-            # strip punctuation
-            strip_punctuation = word_loss_pair[0].strip(punctuation).strip()
-            if len(strip_punctuation) == 0:
-                # skip punctuation
-                continue
-            index2keyword[i] = strip_punctuation
-            for sent_idx, (sent_start, sent_end) in enumerate(turns_sent_spans[num_turns]):
-                if len(utt.strip()) <= sent_end:
-                    index2turn_sent[i] = (num_turns, sent_idx)
-                    break
-        candidate_indexes = list(index2keyword.keys())
-        topk = min(round(args.keywords_ratio*(len(word_loss_pairs)-num_turns)), args.keywords_num)
-        topk_indexes = sorted(candidate_indexes, key=lambda x: word_loss_pairs[x][1], reverse=True)[:topk]
-        topk_indexes = sorted(topk_indexes)
-        keywords = []
-        keywords_turn_sent2idx = {}
-        for i, index in enumerate(topk_indexes):
-            if i > 0 and index == topk_indexes[i-1] + 1 and \
-                word_loss_pairs[index][0].strip().startswith(index2keyword[index]) and \
-                word_loss_pairs[topk_indexes[i-1]][0].strip().endswith(index2keyword[topk_indexes[i-1]]):
-                keywords[-1]+= ' '+index2keyword[index]
-            else:
-                keywords_turn_sent2idx.setdefault(index2turn_sent[index][0], {})
-                keywords_turn_sent2idx[index2turn_sent[index][0]].setdefault(index2turn_sent[index][1], [])
-                keywords_turn_sent2idx[index2turn_sent[index][0]][index2turn_sent[index][1]].append(len(keywords))
-                keywords.append(index2keyword[index])
-
-        return keywords, keywords_turn_sent2idx
-
-    fin = open(word_loss_file, 'rb')
-    fout = open(args.output_file, 'w', encoding='utf-8')
-
-    for item in tqdm(json_lines.reader(fin)):
-        words = [tokenizer.convert_tokens_to_string(tokens) for tokens in item['words']]
-        losses = [np.mean(loss) for loss in item['losses']]
-        dialog_keywords, keywords_turn_sent2idx = keywords_filter(words, losses)
-        # print(keywords_turn_sent2idx)
-        turns = []
-        turn = {'words': [], 'losses': []}
-        for i, (word, loss) in enumerate(zip(words, losses)):
-            if word != '<|endoftext|>':
-                turn['words'].append(word)
-                turn['losses'].append(loss)
-            if word == '<|endoftext|>' or i == len(words) - 1:
-                # switch turn
-                turn['utterance'] = ''.join(turn['words']).strip()
-                # 1) extract keywords according to LM loss within the turn
-                # keywords, _ = keywords_filter(turn['words'], turn['losses'])
-                # turn['turn-level_keywords'] = keywords
-                # 1) extract keywords according to LM loss over the dialog, and group them by sentence
-                turn['keywords'] = [[dialog_keywords[idx] for idx in k_idxes] for sent_idx, k_idxes in keywords_turn_sent2idx.get(len(turns), {}).items()]
-                turn.pop('words')
-                turn.pop('losses')
-                turns.append(turn)
-                turn = {'words': [], 'losses': []}
-                
-        fout.write(json.dumps(turns, ensure_ascii=False)+'\n')
-    
-    fin.close()
-    fout.close()
-
-
-if __name__ == '__main__':
-    from argparse import ArgumentParser
-    parser = ArgumentParser(description="extract keywords according to lm loss")
-    parser.add_argument('--model_type', '-m', type=str, help='gpt or dialogpt')
-    parser.add_argument('--model_name_or_path', type=str, help='model name or path')
-    parser.add_argument('--token_loss_file', '-t', type=str, help='path to the token loss file that contains two columns: [tokens, losses]')
-    parser.add_argument('--word_loss_file', '-w', type=str, help='path to the token loss file that contains two columns: [tokens, losses]')
-    parser.add_argument('--output_file', '-o', type=str, help='path to the output file')
-    parser.add_argument('--keywords_num', '-n', type=int, default=100, help='how many words in an utterance serve as keywords')
-    parser.add_argument('--keywords_ratio', '-r', type=float, default=1.0, help='how many words (in ratio) in an utterance serve as keywords')
-    parser.add_argument('--keywords_loss_th', '-th', type=float, default=0., help='loss threshold for the keywords')
-    parser.add_argument('--stopwords', '-s', type=lambda x: bool(eval(x)), default=True, help='filter out stopwords')
-    args = parser.parse_args()
-    print(args)
-    main(args)
diff --git a/convlab/base_models/gpt/keyword_extraction/merge_keywords_res.py b/convlab/base_models/gpt/keyword_extraction/merge_keywords_res.py
deleted file mode 100644
index 94af288a38845f1cf72470da4af916be5a6f0dda..0000000000000000000000000000000000000000
--- a/convlab/base_models/gpt/keyword_extraction/merge_keywords_res.py
+++ /dev/null
@@ -1,29 +0,0 @@
-import json
-
-def main(args):
-    filename2data = {f.split('/')[-1]: json.load(open(f)) for f in args.keywords_files}
-    first_filename = args.keywords_files[0].split('/')[-1]
-    dialogs = []
-    for i in range(len(filename2data[first_filename])):
-        turns = []
-        for j in range(min([len(filename2data[filename][i]) for filename in filename2data])):
-            utt = filename2data[first_filename][i][j]['utterance']
-            keywords = {filename.split('_')[3]+'_nonstopword'+filename.split('_')[-1]: ' | '.join(filename2data[filename][i][j]['keywords']) for filename in filename2data}
-            turns.append({
-                "utterance": utt,
-                **keywords
-            })
-        dialogs.append(turns)
-    json.dump(dialogs, open(args.output_file, "w", encoding='utf-8'), indent=2, ensure_ascii=False)
-
-
-    
-
-if __name__ == '__main__':
-    from argparse import ArgumentParser
-    parser = ArgumentParser(description="calculate NLU metrics for unified datasets")
-    parser.add_argument('--keywords_files', '-f', metavar='keywords_files', nargs='*', help='keywords files')
-    parser.add_argument('--output_file', '-o', type=str, help='path to the output file')
-    args = parser.parse_args()
-    print(args)
-    main(args)
diff --git a/convlab/base_models/gpt/keyword_extraction/train_lm_dialogpt.sh b/convlab/base_models/gpt/keyword_extraction/train_lm_dialogpt.sh
deleted file mode 100644
index f260f7071529e6837f9c7807d6d5ecf2469494a2..0000000000000000000000000000000000000000
--- a/convlab/base_models/gpt/keyword_extraction/train_lm_dialogpt.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-set -e
-n_gpus=1
-task_name="lm"
-dataset_name="multiwoz21"
-model_type="dialogpt"
-data_dir="data/${task_name}/${dataset_name}/${model_type}"
-output_dir="output/${task_name}/${dataset_name}/${model_type}"
-cache_dir="../cache"
-logging_dir="${output_dir}/runs"
-train_file="${data_dir}/train.json"
-validation_file="${data_dir}/validation.json"
-test_file="${data_dir}/test.json"
-source_column="dialogue"
-max_length=512
-model_name_or_path="microsoft/DialoGPT-large"
-per_device_train_batch_size=16
-per_device_eval_batch_size=16
-gradient_accumulation_steps=4
-lr=5e-5
-num_train_epochs=3
-
-python ../create_data.py --tasks ${task_name} --datasets ${dataset_name} --model_type ${model_type}
-
-python ../run_clm.py \
-    --model_name_or_path ${model_name_or_path} \
-    --train_file ${train_file} \
-    --validation_file ${validation_file} \
-    --source_column ${source_column} \
-    --max_length ${max_length} \
-    --do_train \
-    --do_eval \
-    --save_strategy epoch \
-    --evaluation_strategy epoch \
-    --load_best_model_at_end \
-    --prediction_loss_only \
-    --cache_dir ${cache_dir} \
-    --output_dir ${output_dir} \
-    --logging_dir ${logging_dir} \
-    --overwrite_output_dir \
-    --preprocessing_num_workers 4 \
-    --per_device_train_batch_size ${per_device_train_batch_size} \
-    --per_device_eval_batch_size ${per_device_eval_batch_size} \
-    --gradient_accumulation_steps ${gradient_accumulation_steps} \
-    --learning_rate ${lr} \
-    --num_train_epochs ${num_train_epochs} \
-    --gradient_checkpointing
diff --git a/convlab/base_models/gpt/keyword_extraction/train_lm_gpt.sh b/convlab/base_models/gpt/keyword_extraction/train_lm_gpt.sh
deleted file mode 100644
index 82c63a1f4c4a1633ad5e7d4a721a3bbac558cefb..0000000000000000000000000000000000000000
--- a/convlab/base_models/gpt/keyword_extraction/train_lm_gpt.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-set -e
-n_gpus=1
-task_name="lm"
-dataset_name="multiwoz21"
-model_type="gpt"
-data_dir="data/${task_name}/${dataset_name}/${model_type}"
-output_dir="output/${task_name}/${dataset_name}/${model_type}"
-cache_dir="../cache"
-logging_dir="${output_dir}/runs"
-train_file="${data_dir}/train.json"
-validation_file="${data_dir}/validation.json"
-test_file="${data_dir}/test.json"
-source_column="dialogue"
-max_length=512
-model_name_or_path="gpt2-large"
-per_device_train_batch_size=16
-per_device_eval_batch_size=16
-gradient_accumulation_steps=4
-lr=5e-5
-num_train_epochs=3
-
-python ../create_data.py --tasks ${task_name} --datasets ${dataset_name} --model_type ${model_type}
-
-python ../run_clm.py \
-    --model_name_or_path ${model_name_or_path} \
-    --train_file ${train_file} \
-    --validation_file ${validation_file} \
-    --source_column ${source_column} \
-    --max_length ${max_length} \
-    --do_train \
-    --do_eval \
-    --save_strategy epoch \
-    --evaluation_strategy epoch \
-    --load_best_model_at_end \
-    --prediction_loss_only \
-    --cache_dir ${cache_dir} \
-    --output_dir ${output_dir} \
-    --logging_dir ${logging_dir} \
-    --overwrite_output_dir \
-    --preprocessing_num_workers 4 \
-    --per_device_train_batch_size ${per_device_train_batch_size} \
-    --per_device_eval_batch_size ${per_device_eval_batch_size} \
-    --gradient_accumulation_steps ${gradient_accumulation_steps} \
-    --learning_rate ${lr} \
-    --num_train_epochs ${num_train_epochs} \
-    --gradient_checkpointing
diff --git a/convlab/base_models/gpt/keyword_extraction/train_t5_key2gen+key2gen_noisy.sh b/convlab/base_models/gpt/keyword_extraction/train_t5_key2gen+key2gen_noisy.sh
deleted file mode 100644
index 8e0b3617210408d3226bd7da9f675534c9458398..0000000000000000000000000000000000000000
--- a/convlab/base_models/gpt/keyword_extraction/train_t5_key2gen+key2gen_noisy.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-set -e
-n_gpus=8
-master_port=23456
-task_name="key2gen+key2gen_noisy"
-dataset_name="dailydialog+metalwoz+tm1+tm2+tm3+sgd+reddit+wikidialog"
-model_type="gpt"
-model_name="t5-small"
-data_dir="data/${task_name}/${model_type}/${dataset_name}"
-output_dir="output/${task_name}/${model_name}/${dataset_name}"
-cache_dir="../cache"
-logging_dir="${output_dir}/runs"
-train_file="${data_dir}/train.json"
-source_column="source"
-target_column="target"
-truncation_side="left"
-max_source_length=512
-max_target_length=128
-model_name_or_path="${model_name}"
-per_device_train_batch_size=64
-per_device_eval_batch_size=128
-gradient_accumulation_steps=1
-num_workers=16
-lr=1e-3
-num_train_epochs=1
-
-python -m torch.distributed.launch --master_port ${master_port} \
-    --nproc_per_node ${n_gpus} ../../t5/run_seq2seq.py \
-    --task_name ${task_name} \
-    --train_file ${train_file} \
-    --source_column ${source_column} \
-    --target_column ${target_column} \
-    --max_source_length ${max_source_length} \
-    --max_target_length ${max_target_length} \
-    --truncation_side ${truncation_side} \
-    --model_name_or_path ${model_name_or_path} \
-    --do_train \
-    --save_steps 5000 \
-    --save_total_limit 1 \
-    --cache_dir ${cache_dir} \
-    --output_dir ${output_dir} \
-    --logging_dir ${logging_dir} \
-    --preprocessing_num_workers ${num_workers} \
-    --dataloader_num_workers ${num_workers} \
-    --per_device_train_batch_size ${per_device_train_batch_size} \
-    --per_device_eval_batch_size ${per_device_eval_batch_size} \
-    --gradient_accumulation_steps ${gradient_accumulation_steps} \
-    --learning_rate ${lr} \
-    --num_train_epochs ${num_train_epochs} \
-    --optim adafactor \
-    --lr_scheduler_type constant \
-    --gradient_checkpointing
diff --git a/convlab/base_models/gpt/keyword_extraction/train_t5_rg.sh b/convlab/base_models/gpt/keyword_extraction/train_t5_rg.sh
deleted file mode 100644
index 8d9a019bd0fa10d63586c023705807a3eafd5ff0..0000000000000000000000000000000000000000
--- a/convlab/base_models/gpt/keyword_extraction/train_t5_rg.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-set -e
-n_gpus=8
-master_port=23456
-task_name="rg"
-dataset_name="dailydialog+metalwoz+tm1+tm2+tm3+sgd+reddit+wikidialog"
-model_type="gpt"
-model_name="t5-small"
-data_dir="data/${task_name}/${model_type}/${dataset_name}"
-output_dir="output/${task_name}/${model_name}/${dataset_name}"
-cache_dir="../cache"
-logging_dir="${output_dir}/runs"
-train_file="${data_dir}/train.json"
-source_column="source"
-target_column="target"
-truncation_side="left"
-max_source_length=512
-max_target_length=128
-model_name_or_path="${model_name}"
-per_device_train_batch_size=64
-per_device_eval_batch_size=128
-gradient_accumulation_steps=1
-num_workers=16
-lr=1e-3
-num_train_epochs=1
-
-python -m torch.distributed.launch --master_port ${master_port} \
-    --nproc_per_node ${n_gpus} ../../t5/run_seq2seq.py \
-    --task_name ${task_name} \
-    --train_file ${train_file} \
-    --source_column ${source_column} \
-    --target_column ${target_column} \
-    --max_source_length ${max_source_length} \
-    --max_target_length ${max_target_length} \
-    --truncation_side ${truncation_side} \
-    --model_name_or_path ${model_name_or_path} \
-    --do_train \
-    --save_steps 5000 \
-    --save_total_limit 1 \
-    --cache_dir ${cache_dir} \
-    --output_dir ${output_dir} \
-    --logging_dir ${logging_dir} \
-    --preprocessing_num_workers ${num_workers} \
-    --dataloader_num_workers ${num_workers} \
-    --per_device_train_batch_size ${per_device_train_batch_size} \
-    --per_device_eval_batch_size ${per_device_eval_batch_size} \
-    --gradient_accumulation_steps ${gradient_accumulation_steps} \
-    --learning_rate ${lr} \
-    --num_train_epochs ${num_train_epochs} \
-    --optim adafactor \
-    --lr_scheduler_type constant \
-    --gradient_checkpointing
diff --git a/convlab/base_models/gpt/keyword_extraction/train_t5_rg_key2gen+key2gen_noisy.sh b/convlab/base_models/gpt/keyword_extraction/train_t5_rg_key2gen+key2gen_noisy.sh
deleted file mode 100644
index 75b79932bb94b0699d2e2349a4c8cb8846915cb3..0000000000000000000000000000000000000000
--- a/convlab/base_models/gpt/keyword_extraction/train_t5_rg_key2gen+key2gen_noisy.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-set -e
-n_gpus=8
-master_port=23456
-task_name="key2gen+key2gen_noisy"
-dataset_name="dailydialog+metalwoz+tm1+tm2+tm3+sgd+reddit+wikidialog"
-model_type="gpt"
-model_name="t5-small"
-data_dir="data/${task_name}/${model_type}/${dataset_name}"
-output_dir="output/${task_name}/${model_name}/${dataset_name}"
-cache_dir="../cache"
-logging_dir="${output_dir}/runs"
-train_file="${data_dir}/train.json"
-source_column="source"
-target_column="target"
-truncation_side="left"
-max_source_length=512
-max_target_length=128
-model_name_or_path="output/rg/${model_name}/${dataset_name}"
-per_device_train_batch_size=64
-per_device_eval_batch_size=128
-gradient_accumulation_steps=1
-num_workers=16
-lr=1e-3
-num_train_epochs=1
-
-python -m torch.distributed.launch --master_port ${master_port} \
-    --nproc_per_node ${n_gpus} ../../t5/run_seq2seq.py \
-    --task_name ${task_name} \
-    --train_file ${train_file} \
-    --source_column ${source_column} \
-    --target_column ${target_column} \
-    --max_source_length ${max_source_length} \
-    --max_target_length ${max_target_length} \
-    --truncation_side ${truncation_side} \
-    --model_name_or_path ${model_name_or_path} \
-    --do_train \
-    --save_steps 5000 \
-    --save_total_limit 1 \
-    --cache_dir ${cache_dir} \
-    --output_dir ${output_dir} \
-    --logging_dir ${logging_dir} \
-    --preprocessing_num_workers ${num_workers} \
-    --dataloader_num_workers ${num_workers} \
-    --per_device_train_batch_size ${per_device_train_batch_size} \
-    --per_device_eval_batch_size ${per_device_eval_batch_size} \
-    --gradient_accumulation_steps ${gradient_accumulation_steps} \
-    --learning_rate ${lr} \
-    --num_train_epochs ${num_train_epochs} \
-    --optim adafactor \
-    --lr_scheduler_type constant \
-    --gradient_checkpointing
diff --git a/convlab/base_models/gpt/run_clm.py b/convlab/base_models/gpt/run_clm.py
deleted file mode 100644
index ace68609af00bb6a05d3b6c45378719a98732414..0000000000000000000000000000000000000000
--- a/convlab/base_models/gpt/run_clm.py
+++ /dev/null
@@ -1,564 +0,0 @@
-#!/usr/bin/env python
-# coding=utf-8
-# Copyright 2020 The HuggingFace Inc. team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""
-Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset.
-Modified from https://github.com/huggingface/transformers/blob/main/examples/pytorch/language-modeling/run_clm.py
-Here is the full list of checkpoints on the hub that can be fine-tuned by this script:
-https://huggingface.co/models?filter=text-generation
-"""
-# You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments.
-
-import logging
-import math
-import os
-import sys
-from dataclasses import dataclass, field
-from itertools import chain
-from typing import Optional
-
-import datasets
-from datasets import load_dataset
-from tqdm import tqdm
-from torch.utils.data import DataLoader
-import torch
-import json
-
-import transformers
-from transformers import (
-    CONFIG_MAPPING,
-    MODEL_FOR_CAUSAL_LM_MAPPING,
-    AutoConfig,
-    AutoModelForCausalLM,
-    AutoTokenizer,
-    HfArgumentParser,
-    TrainingArguments,
-    DataCollatorForTokenClassification,
-    is_torch_tpu_available,
-    set_seed,
-)
-from transformers.trainer_utils import get_last_checkpoint
-from transformers.utils import check_min_version
-from transformers.utils.versions import require_version
-from convlab.base_models.gpt.trainer import DumpTokenLossTrainer
-
-
-# Will error if the minimal version of Transformers is not installed. Remove at your own risks.
-check_min_version("4.17.0")
-
-require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt")
-
-logger = logging.getLogger(__name__)
-
-
-MODEL_CONFIG_CLASSES = list(MODEL_FOR_CAUSAL_LM_MAPPING.keys())
-MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
-
-
-@dataclass
-class ModelArguments:
-    """
-    Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
-    """
-
-    model_name_or_path: Optional[str] = field(
-        default=None,
-        metadata={
-            "help": "The model checkpoint for weights initialization."
-            "Don't set if you want to train a model from scratch."
-        },
-    )
-    model_type: Optional[str] = field(
-        default=None,
-        metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)},
-    )
-    config_overrides: Optional[str] = field(
-        default=None,
-        metadata={
-            "help": "Override some existing default config settings when a model is trained from scratch. Example: "
-            "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index"
-        },
-    )
-    config_name: Optional[str] = field(
-        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
-    )
-    tokenizer_name: Optional[str] = field(
-        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
-    )
-    cache_dir: Optional[str] = field(
-        default=None,
-        metadata={"help": "Where to store the pretrained models downloaded from huggingface.co"},
-    )
-    use_fast_tokenizer: bool = field(
-        default=True,
-        metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."},
-    )
-    truncation_side: Optional[str] = field(
-        default="right",
-        metadata={"help": "Which side to truncate, left or right."}
-    )
-    model_revision: str = field(
-        default="main",
-        metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."},
-    )
-    use_auth_token: bool = field(
-        default=False,
-        metadata={
-            "help": "Will use the token generated when running `transformers-cli login` (necessary to use this script "
-            "with private models)."
-        },
-    )
-    resize_position_embeddings: Optional[bool] = field(
-        default=None,
-        metadata={
-            "help": "Whether to automatically resize the position embeddings if `max_source_length` exceeds "
-                    "the model's position embeddings."
-        },
-    )
-
-    def __post_init__(self):
-        if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None):
-            raise ValueError(
-                "--config_overrides can't be used in combination with --config_name or --model_name_or_path"
-            )
-
-
-@dataclass
-class DataTrainingArguments:
-    """
-    Arguments pertaining to what data we are going to input our model for training and eval.
-    """
-
-    dataset_name: Optional[str] = field(
-        default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."}
-    )
-    dataset_config_name: Optional[str] = field(
-        default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}
-    )
-    source_column: Optional[str] = field(
-        default=None,
-        metadata={"help": "The name of the column in the datasets containing the texts."},
-    )
-    train_file: Optional[str] = field(
-        default=None, metadata={"help": "The input training data file (a text, jsonlines or csv file)."}
-    )
-    validation_file: Optional[str] = field(
-        default=None,
-        metadata={
-            "help": "An optional input evaluation data file to evaluate the metrics on (a text, jsonlines or csv file)."
-        },
-    )
-    dump_eval_loss_to: Optional[str] = field(
-        default=None, metadata={"help": "Where to dump the tokens' losses in the evaluation data, default not to"}
-    )
-    overwrite_cache: bool = field(
-        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
-    )
-    preprocessing_num_workers: Optional[int] = field(
-        default=None,
-        metadata={"help": "The number of processes to use for the preprocessing."},
-    )
-    max_length: Optional[int] = field(
-        default=1024,
-        metadata={
-            "help": "The maximum total input sequence length after tokenization. Sequences longer "
-                    "than this will be truncated, sequences shorter will be padded."
-        },
-    )
-    pad_to_max_length: bool = field(
-        default=False,
-        metadata={
-            "help": "Whether to pad all samples to model maximum sentence length. "
-                    "If False, will pad the samples dynamically when batching to the maximum length in the batch. More "
-                    "efficient on GPU but very bad for TPU."
-        },
-    )
-    max_train_samples: Optional[int] = field(
-        default=None,
-        metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of training examples to this "
-            "value if set."
-        },
-    )
-    max_eval_samples: Optional[int] = field(
-        default=None,
-        metadata={
-            "help": "For debugging purposes or quicker training, truncate the number of evaluation examples to this "
-            "value if set."
-        },
-    )
-    ignore_pad_token_for_loss: bool = field(
-        default=True,
-        metadata={
-            "help": "Whether to ignore the tokens corresponding to padded labels in the loss computation or not."
-        },
-    )
-    validation_split_percentage: Optional[int] = field(
-        default=5,
-        metadata={
-            "help": "The percentage of the train set used as validation set in case there's no validation split"
-        },
-    )
-    keep_linebreaks: bool = field(
-        default=True, metadata={"help": "Whether to keep line breaks when using TXT files or not."}
-    )
-
-    def __post_init__(self):
-        if self.dataset_name is None and self.train_file is None and self.validation_file is None:
-            raise ValueError("Need either a dataset name or a training/validation file.")
-        else:
-            if self.train_file is not None:
-                extension = self.train_file.split(".")[-1]
-                assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, a json or a txt file."
-            if self.validation_file is not None:
-                extension = self.validation_file.split(".")[-1]
-                assert extension in ["csv", "json", "txt"], "`validation_file` should be a csv, a json or a txt file."
-
-
-def main():
-    # See all possible arguments in src/transformers/training_args.py
-    # or by passing the --help flag to this script.
-    # We now keep distinct sets of args, for a cleaner separation of concerns.
-
-    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
-    if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
-        # If we pass only one argument to the script and it's the path to a json file,
-        # let's parse it to get our arguments.
-        model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]))
-    else:
-        model_args, data_args, training_args = parser.parse_args_into_dataclasses()
-
-    # Setup logging
-    logging.basicConfig(
-        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
-        datefmt="%m/%d/%Y %H:%M:%S",
-        handlers=[logging.StreamHandler(sys.stdout)],
-    )
-    log_level = training_args.get_process_log_level()
-    logger.setLevel(log_level)
-    datasets.utils.logging.set_verbosity(log_level)
-    transformers.utils.logging.set_verbosity(log_level)
-    transformers.utils.logging.enable_default_handler()
-    transformers.utils.logging.enable_explicit_format()
-
-    # Log on each process the small summary:
-    logger.warning(
-        f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}"
-        + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}"
-    )
-    logger.info(f"Training/evaluation parameters {training_args}")
-
-    # Detecting last checkpoint.
-    last_checkpoint = None
-    if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir:
-        last_checkpoint = get_last_checkpoint(training_args.output_dir)
-        if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0:
-            raise ValueError(
-                f"Output directory ({training_args.output_dir}) already exists and is not empty. "
-                "Use --overwrite_output_dir to overcome."
-            )
-        elif last_checkpoint is not None and training_args.resume_from_checkpoint is None:
-            logger.info(
-                f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change "
-                "the `--output_dir` or add `--overwrite_output_dir` to train from scratch."
-            )
-
-    # Set seed before initializing model.
-    set_seed(training_args.seed)
-
-    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
-    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
-    # (the dataset will be downloaded automatically from the datasets Hub).
-    #
-    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
-    # 'text' is found. You can easily tweak this behavior (see below).
-    #
-    # In distributed training, the load_dataset function guarantee that only one local process can concurrently
-    # download the dataset.
-    if data_args.dataset_name is not None:
-        # Downloading and loading a dataset from the hub.
-        raw_datasets = load_dataset(
-            data_args.dataset_name,
-            data_args.dataset_config_name,
-            cache_dir=model_args.cache_dir,
-            use_auth_token=True if model_args.use_auth_token else None,
-        )
-        if "validation" not in raw_datasets.keys():
-            raw_datasets["validation"] = load_dataset(
-                data_args.dataset_name,
-                data_args.dataset_config_name,
-                split=f"train[:{data_args.validation_split_percentage}%]",
-                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
-            )
-            raw_datasets["train"] = load_dataset(
-                data_args.dataset_name,
-                data_args.dataset_config_name,
-                split=f"train[{data_args.validation_split_percentage}%:]",
-                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
-            )
-    else:
-        data_files = {}
-        dataset_args = {}
-        if data_args.train_file is not None:
-            data_files["train"] = data_args.train_file
-        if data_args.validation_file is not None:
-            data_files["validation"] = data_args.validation_file
-        extension = (
-            data_args.train_file.split(".")[-1]
-            if data_args.train_file is not None
-            else data_args.validation_file.split(".")[-1]
-        )
-        if extension == "txt":
-            extension = "text"
-            dataset_args["keep_linebreaks"] = data_args.keep_linebreaks
-        raw_datasets = load_dataset(
-            extension,
-            data_files=data_files,
-            cache_dir=model_args.cache_dir,
-            use_auth_token=True if model_args.use_auth_token else None,
-            **dataset_args,
-        )
-        # If no validation data is there, validation_split_percentage will be used to divide the dataset.
-        if "validation" not in raw_datasets.keys():
-            raw_datasets["validation"] = load_dataset(
-                extension,
-                data_files=data_files,
-                split=f"train[:{data_args.validation_split_percentage}%]",
-                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
-                **dataset_args,
-            )
-            raw_datasets["train"] = load_dataset(
-                extension,
-                data_files=data_files,
-                split=f"train[{data_args.validation_split_percentage}%:]",
-                cache_dir=model_args.cache_dir,
-                use_auth_token=True if model_args.use_auth_token else None,
-                **dataset_args,
-            )
-
-    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
-    # https://huggingface.co/docs/datasets/loading_datasets.html.
-
-    # Load pretrained model and tokenizer
-    #
-    # Distributed training:
-    # The .from_pretrained methods guarantee that only one local process can concurrently
-    # download model & vocab.
-    config_kwargs = {
-        "cache_dir": model_args.cache_dir,
-        "revision": model_args.model_revision,
-        "use_auth_token": True if model_args.use_auth_token else None,
-    }
-    if model_args.config_name:
-        config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs)
-    elif model_args.model_name_or_path:
-        config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs)
-    else:
-        config = CONFIG_MAPPING[model_args.model_type]()
-        logger.warning("You are instantiating a new config instance from scratch.")
-        if model_args.config_overrides is not None:
-            logger.info(f"Overriding config: {model_args.config_overrides}")
-            config.update_from_string(model_args.config_overrides)
-            logger.info(f"New config: {config}")
-
-    tokenizer_kwargs = {
-        "cache_dir": model_args.cache_dir,
-        "use_fast": model_args.use_fast_tokenizer,
-        "truncation_side": model_args.truncation_side,
-        "revision": model_args.model_revision,
-        "use_auth_token": True if model_args.use_auth_token else None,
-    }
-    if model_args.tokenizer_name:
-        tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, **tokenizer_kwargs)
-    elif model_args.model_name_or_path:
-        tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, **tokenizer_kwargs)
-    else:
-        raise ValueError(
-            "You are instantiating a new tokenizer from scratch. This is not supported by this script."
-            "You can do it from another script, save it, and load it from here, using --tokenizer_name."
-        )
-
-    if not tokenizer.pad_token:
-        tokenizer.pad_token = tokenizer.eos_token
-
-    if model_args.model_name_or_path:
-        model = AutoModelForCausalLM.from_pretrained(
-            model_args.model_name_or_path,
-            from_tf=bool(".ckpt" in model_args.model_name_or_path),
-            config=config,
-            cache_dir=model_args.cache_dir,
-            revision=model_args.model_revision,
-            use_auth_token=True if model_args.use_auth_token else None,
-        )
-    else:
-        model = AutoModelForCausalLM.from_config(config)
-        n_params = sum(dict((p.data_ptr(), p.numel()) for p in model.parameters()).values())
-        logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params")
-
-    model.resize_token_embeddings(len(tokenizer))
-
-    if training_args.gradient_checkpointing:
-        # use_cache=True is incompatible with gradient checkpointing.
-        config.use_cache = False
-
-    # Preprocessing the datasets.
-    # First we tokenize all the texts.
-    if training_args.do_train:
-        column_names = raw_datasets["train"].column_names
-    elif training_args.do_eval:
-        column_names = raw_datasets["validation"].column_names
-    else:
-        logger.info("There is nothing to do. Please pass `do_train` and/or `do_eval`.")
-        return
-    if data_args.source_column is None:
-        source_column = column_names[0]
-    else:
-        source_column = data_args.source_column
-        if source_column not in column_names:
-            raise ValueError(
-                f"--source_column' value '{data_args.source_column}' needs to be one of: {', '.join(column_names)}"
-            )
-
-    def preprocess_function(examples):
-
-        inputs = []
-        for i in range(len(examples[source_column])):
-            if len(examples[source_column][i]) > 0:
-                inputs.append(examples[source_column][i])
-        
-        padding = "max_length" if data_args.pad_to_max_length else False
-        model_inputs = tokenizer(inputs, max_length=data_args.max_length, padding=padding, truncation=True)
-
-        # If we are padding here, replace all tokenizer.pad_token_id in the labels by -100 when we want to ignore
-        # padding in the loss. Else pad in data_collator.
-        if padding == "max_length" and data_args.ignore_pad_token_for_loss:
-            model_inputs["labels"] = [
-                [(l if l != tokenizer.pad_token_id else -100) for l in label] for label in model_inputs["input_ids"]
-            ]
-        else:
-            model_inputs["labels"] = model_inputs["input_ids"].copy()
-
-        return model_inputs
-
-    with training_args.main_process_first(desc="dataset map tokenization"):
-        tokenized_datasets = raw_datasets.map(
-            preprocess_function,
-            batched=True,
-            num_proc=data_args.preprocessing_num_workers,
-            remove_columns=column_names,
-            load_from_cache_file=not data_args.overwrite_cache,
-            desc="Running tokenizer on dataset",
-        )
-    
-    lm_datasets = tokenized_datasets
-
-    if training_args.do_train:
-        if "train" not in tokenized_datasets:
-            raise ValueError("--do_train requires a train dataset")
-        train_dataset = lm_datasets["train"]
-        if data_args.max_train_samples is not None:
-            max_train_samples = min(len(train_dataset), data_args.max_train_samples)
-            train_dataset = train_dataset.select(range(max_train_samples))
-
-    if training_args.do_eval:
-        if "validation" not in tokenized_datasets:
-            raise ValueError("--do_eval requires a validation dataset")
-        eval_dataset = lm_datasets["validation"]
-        if data_args.max_eval_samples is not None:
-            max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples)
-            eval_dataset = eval_dataset.select(range(max_eval_samples))
-
-    # Data collator
-    label_pad_token_id = -100 if data_args.ignore_pad_token_for_loss else tokenizer.pad_token_id
-    data_collator = DataCollatorForTokenClassification(
-        tokenizer,
-        label_pad_token_id=label_pad_token_id,
-        pad_to_multiple_of=8 if training_args.fp16 else None,
-    )
-
-    training_args.dump_eval_loss_to = data_args.dump_eval_loss_to
-    
-    # Initialize our Trainer
-    trainer = DumpTokenLossTrainer(
-        model=model,
-        args=training_args,
-        train_dataset=train_dataset if training_args.do_train else None,
-        eval_dataset=eval_dataset if training_args.do_eval else None,
-        tokenizer=tokenizer,
-        # Data collator will default to DataCollatorWithPadding, so we change it.
-        data_collator=data_collator,
-    )
-
-    # Training
-    if training_args.do_train:
-        checkpoint = None
-        if training_args.resume_from_checkpoint is not None:
-            checkpoint = training_args.resume_from_checkpoint
-        elif last_checkpoint is not None:
-            checkpoint = last_checkpoint
-        train_result = trainer.train(resume_from_checkpoint=checkpoint)
-        trainer.save_model()  # Saves the tokenizer too for easy upload
-
-        metrics = train_result.metrics
-        max_train_samples = (
-            data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset)
-        )
-        metrics["train_samples"] = min(max_train_samples, len(train_dataset))
-
-        trainer.log_metrics("train", metrics)
-        trainer.save_metrics("train", metrics)
-        trainer.save_state()
-
-    # Evaluation
-    if training_args.do_eval:
-        logger.info("*** Evaluate ***")
-        metrics = trainer.evaluate(metric_key_prefix="eval")
-        max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset)
-        metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset))
-        try:
-            perplexity = math.exp(metrics["eval_loss"])
-        except OverflowError:
-            perplexity = float("inf")
-        metrics["eval_perplexity"] = perplexity
-        logger.info(f"eval_perplexity: {perplexity}")
-
-        trainer.log_metrics("eval", metrics)
-        trainer.save_metrics("eval", metrics)
-        
-    kwargs = {"finetuned_from": model_args.model_name_or_path, "tasks": "text-generation"}
-    if data_args.dataset_name is not None:
-        kwargs["dataset_tags"] = data_args.dataset_name
-        if data_args.dataset_config_name is not None:
-            kwargs["dataset_args"] = data_args.dataset_config_name
-            kwargs["dataset"] = f"{data_args.dataset_name} {data_args.dataset_config_name}"
-        else:
-            kwargs["dataset"] = data_args.dataset_name
-
-    if training_args.push_to_hub:
-        trainer.push_to_hub(**kwargs)
-    else:
-        trainer.create_model_card(**kwargs)
-
-
-def _mp_fn(index):
-    # For xla_spawn (TPUs)
-    main()
-
-
-if __name__ == "__main__":
-    main()
diff --git a/convlab/base_models/gpt/trainer.py b/convlab/base_models/gpt/trainer.py
deleted file mode 100644
index 5a8ed11c6566e897a5d3ef7a0d16a130968cd6aa..0000000000000000000000000000000000000000
--- a/convlab/base_models/gpt/trainer.py
+++ /dev/null
@@ -1,243 +0,0 @@
-from transformers import Trainer
-from transformers.trainer_utils import EvalLoopOutput, has_length
-from transformers.deepspeed import deepspeed_init
-from transformers.utils import logging
-from transformers.trainer_pt_utils import find_batch_size, nested_concat, nested_numpify, IterableDatasetShard, nested_truncate
-from transformers.trainer_utils import EvalPrediction, denumpify_detensorize
-import torch
-from torch.utils.data import DataLoader
-import numpy as np
-from typing import List, Optional
-import json
-
-
-logger = logging.get_logger(__name__)
-
-class DumpTokenLossTrainer(Trainer):
-    def evaluation_loop(
-        self,
-        dataloader: DataLoader,
-        description: str,
-        prediction_loss_only: Optional[bool] = None,
-        ignore_keys: Optional[List[str]] = None,
-        metric_key_prefix: str = "eval",
-    ) -> EvalLoopOutput:
-        """
-        Prediction/evaluation loop, shared by `Trainer.evaluate()` and `Trainer.predict()`.
-        Works both with or without labels.
-        """
-        args = self.args
-
-        prediction_loss_only = args.prediction_loss_only
-
-        # if eval is called w/o train init deepspeed here
-        if args.deepspeed and not self.deepspeed:
-
-            # XXX: eval doesn't have `resume_from_checkpoint` arg but we should be able to do eval
-            # from the checkpoint eventually
-            deepspeed_engine, _, _ = deepspeed_init(
-                self, num_training_steps=0, resume_from_checkpoint=None, inference=True
-            )
-            self.model = deepspeed_engine.module
-            self.model_wrapped = deepspeed_engine
-            self.deepspeed = deepspeed_engine
-
-        model = self._wrap_model(self.model, training=False, dataloader=dataloader)
-
-        # if full fp16 or bf16 eval is wanted and this ``evaluation`` or ``predict`` isn't called
-        # while ``train`` is running, cast it to the right dtype first and then put on device
-        if not self.is_in_train:
-            if args.fp16_full_eval:
-                model = model.to(dtype=torch.float16, device=args.device)
-            elif args.bf16_full_eval:
-                model = model.to(dtype=torch.bfloat16, device=args.device)
-
-        batch_size = self.args.eval_batch_size
-
-        logger.info(f"***** Running {description} *****")
-        if has_length(dataloader):
-            logger.info(f"  Num examples = {self.num_examples(dataloader)}")
-        else:
-            logger.info("  Num examples: Unknown")
-        logger.info(f"  Batch size = {batch_size}")
-
-        model.eval()
-
-        self.callback_handler.eval_dataloader = dataloader
-        # Do this before wrapping.
-        eval_dataset = getattr(dataloader, "dataset", None)
-
-        if args.past_index >= 0:
-            self._past = None
-
-        # Initialize containers
-        # losses/preds/labels on GPU/TPU (accumulated for eval_accumulation_steps)
-        losses_host = None
-        preds_host = None
-        labels_host = None
-        inputs_host = None
-
-        # losses/preds/labels on CPU (final containers)
-        all_losses = None
-        all_preds = None
-        all_labels = None
-        all_inputs = None
-        # Will be useful when we have an iterable dataset so don't know its length.
-
-        if args.dump_eval_loss_to:
-            writer = open(args.dump_eval_loss_to, "a", encoding='utf-8')
-            loss_fct = torch.nn.CrossEntropyLoss(reduction='none')
-            num_sample_to_write = len(eval_dataset)
-
-        observed_num_examples = 0
-        # Main evaluation loop
-        for step, inputs in enumerate(dataloader):
-            # Update the observed num examples
-            observed_batch_size = find_batch_size(inputs)
-            if observed_batch_size is not None:
-                observed_num_examples += observed_batch_size
-                # For batch samplers, batch_size is not known by the dataloader in advance.
-                if batch_size is None:
-                    batch_size = observed_batch_size
-
-            # Prediction step
-            loss, logits, labels = self.prediction_step(model, inputs, prediction_loss_only, ignore_keys=ignore_keys)
-            inputs_decode = self._prepare_input(inputs["input_ids"]) if args.include_inputs_for_metrics else None
-
-            # Update containers on host
-            if loss is not None:
-                losses = self._nested_gather(loss.repeat(batch_size))
-                losses_host = losses if losses_host is None else torch.cat((losses_host, losses), dim=0)
-            if labels is not None:
-                labels = self._pad_across_processes(labels)
-                labels = self._nested_gather(labels)
-                # labels_host = labels if labels_host is None else nested_concat(labels_host, labels, padding_index=-100)
-            if inputs_decode is not None:
-                inputs_decode = self._pad_across_processes(inputs_decode)
-                inputs_decode = self._nested_gather(inputs_decode)
-                inputs_host = (
-                    inputs_decode
-                    if inputs_host is None
-                    else nested_concat(inputs_host, inputs_decode, padding_index=-100)
-                )
-            if logits is not None:
-                logits = self._pad_across_processes(logits)
-                logits = self._nested_gather(logits)
-                if self.preprocess_logits_for_metrics is not None:
-                    logits = self.preprocess_logits_for_metrics(logits, labels)
-                # preds_host = logits if preds_host is None else nested_concat(preds_host, logits, padding_index=-100)
-
-            if args.dump_eval_loss_to:
-                if self.is_world_process_zero() and num_sample_to_write > 0:
-                    assert logits is not None and labels is not None, print('prediction_loss_only', prediction_loss_only)
-                    shift_logits = logits[..., :-1, :].contiguous()
-                    shift_labels = labels[..., 1:].contiguous()
-                    batch_token_loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
-                    batch_token_loss = batch_token_loss.view(shift_labels.size()).tolist()
-                    labels = labels.tolist()
-                    for i in range(len(labels)):
-                        if num_sample_to_write > 0:
-                            num_sample_to_write -= 1
-                        else:
-                            break
-                        token_ids = [x for x in labels[i] if x != -100]
-                        tokens = self.tokenizer.convert_ids_to_tokens(token_ids)
-                        token_losses = [0] + batch_token_loss[i][:len(token_ids)-1]
-                        writer.write(json.dumps({"tokens": tokens, "losses": token_losses}, ensure_ascii=False)+'\n')
-
-            self.control = self.callback_handler.on_prediction_step(args, self.state, self.control)
-
-            # Gather all tensors and put them back on the CPU if we have done enough accumulation steps.
-            if args.eval_accumulation_steps is not None and (step + 1) % args.eval_accumulation_steps == 0:
-                if losses_host is not None:
-                    losses = nested_numpify(losses_host)
-                    all_losses = losses if all_losses is None else np.concatenate((all_losses, losses), axis=0)
-                if preds_host is not None:
-                    logits = nested_numpify(preds_host)
-                    all_preds = logits if all_preds is None else nested_concat(all_preds, logits, padding_index=-100)
-                if inputs_host is not None:
-                    inputs_decode = nested_numpify(inputs_host)
-                    all_inputs = (
-                        inputs_decode
-                        if all_inputs is None
-                        else nested_concat(all_inputs, inputs_decode, padding_index=-100)
-                    )
-                if labels_host is not None:
-                    labels = nested_numpify(labels_host)
-                    all_labels = (
-                        labels if all_labels is None else nested_concat(all_labels, labels, padding_index=-100)
-                    )
-
-                # Set back to None to begin a new accumulation
-                losses_host, preds_host, inputs_host, labels_host = None, None, None, None
-
-        if args.dump_eval_loss_to:
-            writer.close()
-        
-        if args.past_index and hasattr(self, "_past"):
-            # Clean the state at the end of the evaluation loop
-            delattr(self, "_past")
-
-        # Gather all remaining tensors and put them back on the CPU
-        if losses_host is not None:
-            losses = nested_numpify(losses_host)
-            all_losses = losses if all_losses is None else np.concatenate((all_losses, losses), axis=0)
-        if preds_host is not None:
-            logits = nested_numpify(preds_host)
-            all_preds = logits if all_preds is None else nested_concat(all_preds, logits, padding_index=-100)
-        if inputs_host is not None:
-            inputs_decode = nested_numpify(inputs_host)
-            all_inputs = (
-                inputs_decode if all_inputs is None else nested_concat(all_inputs, inputs_decode, padding_index=-100)
-            )
-        if labels_host is not None:
-            labels = nested_numpify(labels_host)
-            all_labels = labels if all_labels is None else nested_concat(all_labels, labels, padding_index=-100)
-
-        # Number of samples
-        if has_length(eval_dataset):
-            num_samples = len(eval_dataset)
-        # The instance check is weird and does not actually check for the type, but whether the dataset has the right
-        # methods. Therefore we need to make sure it also has the attribute.
-        elif isinstance(eval_dataset, IterableDatasetShard) and hasattr(eval_dataset, "num_examples"):
-            num_samples = eval_dataset.num_examples
-        else:
-            if has_length(dataloader):
-                num_samples = self.num_examples(dataloader)
-            else:  # both len(dataloader.dataset) and len(dataloader) fail
-                num_samples = observed_num_examples
-
-        # Number of losses has been rounded to a multiple of batch_size and in a distributed training, the number of
-        # samplers has been rounded to a multiple of batch_size, so we truncate.
-        if all_losses is not None:
-            all_losses = all_losses[:num_samples]
-        if all_preds is not None:
-            all_preds = nested_truncate(all_preds, num_samples)
-        if all_labels is not None:
-            all_labels = nested_truncate(all_labels, num_samples)
-        if all_inputs is not None:
-            all_inputs = nested_truncate(all_inputs, num_samples)
-
-        # Metrics!
-        if self.compute_metrics is not None and all_preds is not None and all_labels is not None:
-            if args.include_inputs_for_metrics:
-                metrics = self.compute_metrics(
-                    EvalPrediction(predictions=all_preds, label_ids=all_labels, inputs=all_inputs)
-                )
-            else:
-                metrics = self.compute_metrics(EvalPrediction(predictions=all_preds, label_ids=all_labels))
-        else:
-            metrics = {}
-
-        # To be JSON-serializable, we need to remove numpy types or zero-d tensors
-        metrics = denumpify_detensorize(metrics)
-
-        if all_losses is not None:
-            metrics[f"{metric_key_prefix}_loss"] = all_losses.mean().item()
-
-        # Prefix all keys with metric_key_prefix + '_'
-        for key in list(metrics.keys()):
-            if not key.startswith(f"{metric_key_prefix}_"):
-                metrics[f"{metric_key_prefix}_{key}"] = metrics.pop(key)
-
-        return EvalLoopOutput(predictions=all_preds, label_ids=all_labels, metrics=metrics, num_samples=num_samples)
diff --git a/convlab/base_models/t5/README.md b/convlab/base_models/t5/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..c0894e6c5cd38b5738ef85f7d900ecdf304d3fa6
--- /dev/null
+++ b/convlab/base_models/t5/README.md
@@ -0,0 +1,80 @@
+# T5 models
+
+By converting NLP tasks into a text-to-text format, we can use one single model to solve various tasks. Here we use T5 as backbone model and provide a unified training script `run_seq2seq.py` for many tasks. **See `*.sh` under each task directory for usage.**
+
+## Create Data
+Currently we support natural language understanding (**NLU**), dialog state tracking (**DST**), natural language generation (**NLG**), response generation (**RG**), and generating a dialog from a user goal (**Goal2Dialogue**). We provide serialization and deserialization methods for dialog acts and state in the unified data format (user goals are already natural language instruction). An example of serialized dialog acts and state:
+
+```
+User: I am looking for a cheap restaurant.
+System: Is there a particular area of town you prefer?
+User: In the centre of town.
+
+User dialog acts: [inform][restaurant]([area][centre])
+State: [restaurant]([area][centre],[price range][cheap])
+System dialog acts: [recommend][restaurant]([name][Zizzi Cambridge])
+
+System: I would recommend Zizzi Cambridge.
+```
+
+Dialogue acts are in the form of `[intent][domain]([slot][value],...);...`. State is in the form of `[domain]([slot][value],...);...`. Multiple items will be concatenated by a semicolon `;`.
+
+To create data for a specific task, run `create_data.py` with corresponding arguments. For example, create data for single turn NLU on MultiWOZ 2.1:
+
+```bash
+python create_data.py --tasks nlu --datasets multiwoz21 --speaker user
+```
+
+Note that the script only supported **datasets in the unified format**.
+
+## Training
+
+To train the model, specify the arguments like data path, learning rate, epochs, etc., and then run `run_seq2seq.py`. See `nlu/run_nlu.sh` for an example.
+
+## Evaluation
+
+The standard evaluation scripts of NLU, DST, and NLG task are located under `../../$task/evaluate_unified_datasets.py` directories. See `nlu/run_nlu.sh` for an example.
+
+## Trained Models
+
+Trained models and their performance are available in [Hugging Face Hub](https://huggingface.co/ConvLab). You can try some example with hosted inference API.
+
+| Name                                                         | Task          | Training Dataset             |
+| ------------------------------------------------------------ | ------------- | ---------------------------- |
+| [t5-small-goal2dialogue-multiwoz21](https://huggingface.co/ConvLab/t5-small-goal2dialogue-multiwoz21) | Goal2Dialogue | MultiWOZ 2.1                 |
+| [t5-small-nlu-multiwoz21](https://huggingface.co/ConvLab/t5-small-nlu-multiwoz21) | NLU           | MultiWOZ 2.1                 |
+| [t5-small-nlu-sgd](https://huggingface.co/ConvLab/t5-small-nlu-sgd) | NLU           | SGD                          |
+| [t5-small-nlu-tm1_tm2_tm3](https://huggingface.co/ConvLab/t5-small-nlu-tm1_tm2_tm3) | NLU           | TM1+TM2+TM3                  |
+| [t5-small-nlu-multiwoz21_sgd_tm1_tm2_tm3](https://huggingface.co/ConvLab/t5-small-nlu-multiwoz21_sgd_tm1_tm2_tm3) | NLU           | MultiWOZ 2.1+SGD+TM1+TM2+TM3 |
+| [t5-small-dst-multiwoz21](https://huggingface.co/ConvLab/t5-small-dst-multiwoz21) | DST           | MultiWOZ 2.1                 |
+| [t5-small-dst-sgd](https://huggingface.co/ConvLab/t5-small-dst-sgd) | DST           | SGD                          |
+| [t5-small-dst-tm1_tm2_tm3](https://huggingface.co/ConvLab/t5-small-dst-tm1_tm2_tm3) | DST           | TM1+TM2+TM3                  |
+| [t5-small-dst-multiwoz21_sgd_tm1_tm2_tm3](https://huggingface.co/ConvLab/t5-small-dst-multiwoz21_sgd_tm1_tm2_tm3) | DST           | MultiWOZ 2.1+SGD+TM1+TM2+TM3 |
+| [t5-small-nlg-multiwoz21](https://huggingface.co/ConvLab/t5-small-nlg-multiwoz21) | NLG           | MultiWOZ 2.1                 |
+| [t5-small-nlg-sgd](https://huggingface.co/ConvLab/t5-small-nlg-sgd) | NLG           | SGD                          |
+| [t5-small-nlg-tm1_tm2_tm3](https://huggingface.co/ConvLab/t5-small-nlg-tm1_tm2_tm3) | NLG           | TM1+TM2+TM3                  |
+| [t5-small-nlg-multiwoz21_sgd_tm1_tm2_tm3](https://huggingface.co/ConvLab/t5-small-nlg-multiwoz21_sgd_tm1_tm2_tm3) | NLG           | MultiWOZ 2.1+SGD+TM1+TM2+TM3 |
+
+## Interface
+
+To use trained models in a dialog system, import them through:
+
+```python
+from convlab.base_models.t5.nlu import T5NLU
+from convlab.base_models.t5.dst import T5DST
+from convlab.base_models.t5.nlg import T5NLG
+
+# example instantiation
+# model_name_or_path could be model in hugging face hub or local path
+nlu = T5NLU(speaker='user', context_window_size=0, model_name_or_path='ConvLab/t5-small-nlu-multiwoz21')
+```
+
+See `nlu/nlu.py`, `dst/dst.py`, `nlg/nlg.py` for example usage.
+
+## Support a New Task
+
+To support a new task, you can first serialize model input and output like `create_data.py`, and then train the model with `run_seq2seq.py`. Finally, write a evaluation script for the task or pass the `metric_name_or_path` for an existing metric to `run_seq2seq.py`.
+
+## Author
+
+Qi Zhu(zhuq96 at gmail dot com)
\ No newline at end of file
diff --git a/convlab/base_models/t5/dst/__init__.py b/convlab/base_models/t5/dst/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e4e6903b1093e645eed5ae6ba6b9b88d6150fe39
--- /dev/null
+++ b/convlab/base_models/t5/dst/__init__.py
@@ -0,0 +1 @@
+from convlab.base_models.t5.dst.dst import T5DST
\ No newline at end of file
diff --git a/convlab/base_models/t5/dst/dst.py b/convlab/base_models/t5/dst/dst.py
index c34395c02fd188568b9f4c4bc1956240fbbc88a9..3c5ec2525091cfdcb423460ed1b01871087deb21 100755
--- a/convlab/base_models/t5/dst/dst.py
+++ b/convlab/base_models/t5/dst/dst.py
@@ -8,16 +8,12 @@ from convlab.util.custom_util import model_downloader
 
 
 class T5DST(DST):
-    def __init__(self, speaker, context_window_size, model_name_or_path, model_file=None, device='cuda'):
+    def __init__(self, speaker, context_window_size, model_name_or_path, device='cuda'):
         assert speaker in ['user', 'system']
         assert context_window_size > 0
         self.speaker = speaker
         self.opponent = 'system' if speaker == 'user' else 'user'
         self.context_window_size = context_window_size
-
-        model_dir = os.path.dirname(os.path.abspath(__file__))
-        if not os.path.exists(model_name_or_path):
-            model_downloader(model_dir, model_file)
         
         self.config = AutoConfig.from_pretrained(model_name_or_path)
         self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
diff --git a/convlab/base_models/t5/key2gen/create_data.py b/convlab/base_models/t5/key2gen/create_data.py
deleted file mode 100644
index 138808f2a13d4b3fad71b57a2aa7977917f8143c..0000000000000000000000000000000000000000
--- a/convlab/base_models/t5/key2gen/create_data.py
+++ /dev/null
@@ -1,162 +0,0 @@
-import os
-import json
-from tqdm import tqdm
-from convlab.util import load_dataset, load_unified_data, load_nlu_data
-
-def create_nlg_data(dataset, data_dir, args):
-    data_by_split = load_nlu_data(dataset, speaker='system', use_context=True, context_window_size=3)
-    os.makedirs(data_dir, exist_ok=True)
-
-    data_splits = data_by_split.keys()
-    for data_split in data_splits:
-        data = []
-        for sample in tqdm(data_by_split[data_split], desc=f'{data_split} sample', leave=False):
-            context = [(turn['speaker'], turn['utterance']) for turn in sample['context']]
-            response = sample['utterance']
-            if len(context) > 0 and len(response) > 0:
-                knowledge = sample['dialogue_acts']
-                data.append(json.dumps({'context': context, 'knowledge': knowledge, 'response': response}, ensure_ascii=False)+'\n')
-
-        if 'test' in data_split:
-            file_name = os.path.join(os.path.dirname(data_dir), f"{data_split}.json")
-        else:
-            file_name = os.path.join(data_dir, f"{data_split}.json")
-        with open(file_name, "w", encoding='utf-8') as f:
-            f.writelines(data)
-        data_by_split[data_split] = data
-    return data_by_split
-
-def create_kvret_data(dataset, data_dir, args):
-    data_by_split = load_unified_data(dataset, speaker='system', utterance=True, db_results=True, use_context=True, context_window_size=100)
-    os.makedirs(data_dir, exist_ok=True)
-
-    domain2entity_col = {'schedule': 'event' ,'navigate': 'poi', 'weather': 'location'}
-    data_splits = data_by_split.keys()
-    for data_split in data_splits:
-        data = []
-        for sample in tqdm(data_by_split[data_split], desc=f'{data_split} sample', leave=False):
-            context = [(turn['speaker'], turn['utterance']) for turn in sample['context']]
-            response = sample['utterance']
-            if len(context) > 0 and len(response) > 0:
-                knowledge = sample['db_results']
-                for domain, db_items in knowledge.items():
-                    entity_col = domain2entity_col[domain]
-                    for db_item in db_items:
-                        db_item['entity'] = db_item.pop(entity_col)
-
-                data.append(json.dumps({'context': context, 'knowledge': knowledge, 'response': response}, ensure_ascii=False)+'\n')
-
-        if 'test' in data_split:
-            file_name = os.path.join(os.path.dirname(data_dir), f"{data_split}.json")
-        else:
-            file_name = os.path.join(data_dir, f"{data_split}.json")
-        with open(file_name, "w", encoding='utf-8') as f:
-            f.writelines(data)
-        data_by_split[data_split] = data
-    return data_by_split
-
-def create_personachat_data(dataset, data_dir, args):
-    data_by_split = dataset
-    os.makedirs(data_dir, exist_ok=True)
-
-    data_splits = data_by_split.keys()
-    for data_split in data_splits:
-        data = []
-        for dial in tqdm(data_by_split[data_split], desc=f'{data_split} sample', leave=False):
-            knowledge = dial['persona']['system']
-            context = []
-            for turn in dial['turns']:
-                response = turn['utterance']
-                if turn['speaker'] == 'system' and len(context) > 0 and len(response) > 0:
-                    data.append(json.dumps({'context': context, 'knowledge': knowledge, 'response': response}, ensure_ascii=False)+'\n')
-                context.append((turn['speaker'], turn['utterance']))
-
-        if 'test' in data_split:
-            file_name = os.path.join(os.path.dirname(data_dir), f"{data_split}.json")
-        else:
-            file_name = os.path.join(data_dir, f"{data_split}.json")
-        with open(file_name, "w", encoding='utf-8') as f:
-            f.writelines(data)
-        data_by_split[data_split] = data
-    return data_by_split
-
-def create_wow_data(dataset, data_dir, args):
-    data_by_split = dataset
-    os.makedirs(data_dir, exist_ok=True)
-    data_by_split['test'] = data_by_split['test_seen'] + data_by_split['test_unseen']
-    data_by_split.pop('test_seen')
-    data_by_split.pop('test_unseen')
-
-    data_splits = data_by_split.keys()
-    for data_split in data_splits:
-        data = []
-        for dial in tqdm(data_by_split[data_split], desc=f'{data_split} sample', leave=False):
-            context = []
-            for turn in dial['turns']:
-                response = turn['utterance']
-                if turn['speaker'] == 'system' and len(context) > 0 and len(response) > 0:
-                    knowledge = turn['checked_passage']
-                    if knowledge is None:
-                        knowledge = []
-                    elif isinstance(knowledge, str):
-                        knowledge = [knowledge]
-                    data.append(json.dumps({'context': context, 'knowledge': knowledge, 'response': response}, ensure_ascii=False)+'\n')
-                context.append((turn['speaker'], turn['utterance']))
-
-        if 'test' in data_split:
-            file_name = os.path.join(os.path.dirname(data_dir), f"{data_split}.json")
-        else:
-            file_name = os.path.join(data_dir, f"{data_split}.json")
-        with open(file_name, "w", encoding='utf-8') as f:
-            f.writelines(data)
-        data_by_split[data_split] = data
-    return data_by_split
-
-def create_opendialkg_data(dataset, data_dir, args):
-    data_by_split = dataset
-    os.makedirs(data_dir, exist_ok=True)
-
-    data_splits = data_by_split.keys()
-    for data_split in data_splits:
-        data = []
-        for dial in tqdm(data_by_split[data_split], desc=f'{data_split} sample', leave=False):
-            context = []
-            for turn in dial['turns']:
-                response = turn['utterance']
-                if turn['speaker'] == 'system' and 'kg_path' in turn and len(context) > 0 and len(response) > 0:
-                    knowledge = turn['kg_path']['triples']
-                    data.append(json.dumps({'context': context, 'knowledge': knowledge, 'response': response}, ensure_ascii=False)+'\n')
-                context.append((turn['speaker'], turn['utterance']))
-
-        if 'test' in data_split:
-            file_name = os.path.join(os.path.dirname(data_dir), f"{data_split}.json")
-        else:
-            file_name = os.path.join(data_dir, f"{data_split}.json")
-        with open(file_name, "w", encoding='utf-8') as f:
-            f.writelines(data)
-        data_by_split[data_split] = data
-    return data_by_split
-
-
-if __name__ == '__main__':
-    from argparse import ArgumentParser
-    parser = ArgumentParser(description="create data for seq2seq training")
-    parser.add_argument('--tasks', '-t', metavar='task_name', nargs='*', choices=['nlg', 'kvret', 'opendialkg', 'personachat', 'wow'], help='names of tasks')
-    parser.add_argument('--datasets', '-d', metavar='dataset_name', nargs='*', help='names of unified datasets')
-    parser.add_argument('--shot', '-s', type=float, default=None, help='how many data is used for training and evaluation, ratio if < 1 else absolute number')
-    parser.add_argument('--dial_ids_order', '-o', type=int, default=None, help='which data order is used for experiments')
-    args = parser.parse_args()
-    print(args)
-    for dataset_name in tqdm(args.datasets, desc='datasets'):
-        dataset = load_dataset(dataset_name, dial_ids_order=args.dial_ids_order)
-        if args.shot:
-            if args.shot < 1:
-                dataset['train'] = dataset['train'][:round(len(dataset['train'])*args.shot)]
-                dataset['validation'] = dataset['validation'][:round(len(dataset['validation'])*args.shot)]
-            else:
-                args.shot = int(args.shot)
-                dataset['train'] = dataset['train'][:args.shot]
-                dataset['validation'] = dataset['validation'][:args.shot]
-        for task_name in tqdm(args.tasks, desc='tasks', leave=False):
-            data_dir = os.path.join('data', task_name, (dataset_name if not args.shot else f'{dataset_name}_{args.shot}shot_order{args.dial_ids_order}'))
-            data_by_split = eval(f"create_{task_name}_data")(dataset, data_dir, args)
diff --git a/convlab/base_models/t5/key2gen/dataset_godel.py b/convlab/base_models/t5/key2gen/dataset_godel.py
deleted file mode 100644
index caf7b8ab7b1fb10b8de03c01dac9a147f5540af1..0000000000000000000000000000000000000000
--- a/convlab/base_models/t5/key2gen/dataset_godel.py
+++ /dev/null
@@ -1,123 +0,0 @@
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Data processing for vanilla generator"""
-
-import json
-import datasets
-from convlab.base_models.t5.key2gen.features import FEATURES
-from copy import deepcopy
-
-
-class GodelDataset(datasets.GeneratorBasedBuilder):
-    """Dataset for vanilla generator (e.g., t5)"""
-
-    VERSION = datasets.Version("1.18.0")
-
-    BUILDER_CONFIGS = [
-        datasets.BuilderConfig(name="nlg", version=VERSION, description="DA grounded generation task"),
-        datasets.BuilderConfig(name="kvret", version=VERSION, description="KB grounded generation task"),
-        datasets.BuilderConfig(name="opendialkg", version=VERSION, description="KG grounded generation task"),
-        datasets.BuilderConfig(name="wow", version=VERSION, description="Passage grounded generation task"),
-        datasets.BuilderConfig(name="personachat", version=VERSION, description="Persona grounded generation task"),
-    ]
-
-    def _info(self):
-        return datasets.DatasetInfo(
-            description=f"Vanilla Dataset for {self.config.description}",
-            features=datasets.Features(deepcopy(FEATURES[self.config.name]))
-        )
-
-    def _split_generators(self, dl_manager):
-        generators = []
-        if "train" in self.config.data_files:
-            generators.append(datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                gen_kwargs={
-                    "filepath": self.config.data_files["train"][0],
-                    "split": "train",
-                },
-            ))
-        if "validation" in self.config.data_files:
-            generators.append(datasets.SplitGenerator(
-                name=datasets.Split.VALIDATION,
-                gen_kwargs={
-                    "filepath": self.config.data_files["validation"][0],
-                    "split": "validation",
-                },
-            ))
-        if "test" in self.config.data_files:
-            generators.append(datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={
-                    "filepath": self.config.data_files["test"][0],
-                    "split": "test",
-                },
-            ))
-            
-        return generators
-
-    def _generate_examples(self, filepath, split):
-        with open(filepath, encoding="utf-8") as f:
-            for key, row in enumerate(f):
-                item = json.loads(row)
-                if self.config.name == "nlg":
-                    knowledge = item["knowledge"]
-                    triples = []
-                    for da_type in knowledge:
-                        for da in knowledge[da_type]:
-                            intent, domain, slot, value = da["intent"], da["domain"], da["slot"], da.get("value", "")
-                            if 'start' in da:
-                                da.pop('start')
-                                da.pop('end')
-                            intent_domain = f"{intent}-{domain}"
-                            triples.append([intent_domain])
-                            if len(slot) > 0:
-                                triples[-1].append(slot)
-                            if len(value) > 0:
-                                triples[-1].append(value)
-                    knowledge_seq = "| {} |".format(" | ".join([" : ".join(da_keywords) for da_keywords in triples]))
-                    
-                elif self.config.name == "kvret":
-                    knowledge = {"schedule": [], "weather": [], "navigate": []}
-                    triples = []
-                    for domain, db_items in item["knowledge"].items():
-                        knowledge[domain] = db_items
-                        for db_item in db_items:
-                            entity = db_item["entity"]
-                            for db_key, db_value in db_item.items():
-                                if db_key == "entity":
-                                    continue
-                                triples.append([entity, db_key, db_value])
-                    knowledge_seq = "| {} |".format(" | ".join([" : ".join(triple) for triple in triples]))
-
-                elif self.config.name == "opendialkg":
-                    knowledge = item["knowledge"]
-                    knowledge_seq = "| {} |".format(" | ".join([" : ".join(triple) for triple in item["knowledge"]]))
-                
-                elif self.config.name in ["wow", "personachat"]:
-                    knowledge = item["knowledge"]
-                    try:
-                        knowledge_seq = "| {} |".format(" | ".join(item["knowledge"]))
-                    except:
-                        print([knowledge])
-                        raise
-                
-                context = " EOS ".join([turn[1] for turn in item["context"]])
-                context_knowledge = context + ' <|Knowledge|> \n\n' + knowledge_seq + ' => '
-                
-                yield key, {
-                    "context+knowledge": context_knowledge,
-                    "response": item["response"],
-                    "knowledge": knowledge,
-                }
diff --git a/convlab/base_models/t5/key2gen/dataset_vanilla.py b/convlab/base_models/t5/key2gen/dataset_vanilla.py
deleted file mode 100644
index 15a8c7b4ac8cfbf1057e090f675a3fc7a4051f2c..0000000000000000000000000000000000000000
--- a/convlab/base_models/t5/key2gen/dataset_vanilla.py
+++ /dev/null
@@ -1,126 +0,0 @@
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Data processing for vanilla generator"""
-
-import json
-import datasets
-from convlab.base_models.t5.key2gen.features import FEATURES
-from copy import deepcopy
-
-
-class VanillaDataset(datasets.GeneratorBasedBuilder):
-    """Dataset for vanilla generator (e.g., t5)"""
-
-    VERSION = datasets.Version("1.18.0")
-
-    BUILDER_CONFIGS = [
-        datasets.BuilderConfig(name="nlg", version=VERSION, description="DA grounded generation task"),
-        datasets.BuilderConfig(name="kvret", version=VERSION, description="KB grounded generation task"),
-        datasets.BuilderConfig(name="opendialkg", version=VERSION, description="KG grounded generation task"),
-        datasets.BuilderConfig(name="wow", version=VERSION, description="Passage grounded generation task"),
-        datasets.BuilderConfig(name="personachat", version=VERSION, description="Persona grounded generation task"),
-    ]
-
-    def _info(self):
-        return datasets.DatasetInfo(
-            description=f"Vanilla Dataset for {self.config.description}",
-            features=datasets.Features(deepcopy(FEATURES[self.config.name]))
-        )
-
-    def _split_generators(self, dl_manager):
-        generators = []
-        if "train" in self.config.data_files:
-            generators.append(datasets.SplitGenerator(
-                name=datasets.Split.TRAIN,
-                gen_kwargs={
-                    "filepath": self.config.data_files["train"][0],
-                    "split": "train",
-                },
-            ))
-        if "validation" in self.config.data_files:
-            generators.append(datasets.SplitGenerator(
-                name=datasets.Split.VALIDATION,
-                gen_kwargs={
-                    "filepath": self.config.data_files["validation"][0],
-                    "split": "validation",
-                },
-            ))
-        if "test" in self.config.data_files:
-            generators.append(datasets.SplitGenerator(
-                name=datasets.Split.TEST,
-                gen_kwargs={
-                    "filepath": self.config.data_files["test"][0],
-                    "split": "test",
-                },
-            ))
-            
-        return generators
-
-    def _generate_examples(self, filepath, split):
-        with open(filepath, encoding="utf-8") as f:
-            for key, row in enumerate(f):
-                item = json.loads(row)
-                if self.config.name == "nlg":
-                    knowledge = item["knowledge"]
-                    triples = []
-                    for da_type in knowledge:
-                        for da in knowledge[da_type]:
-                            intent, domain, slot, value = da["intent"], da["domain"], da["slot"], da.get("value", "")
-                            if 'start' in da:
-                                da.pop('start')
-                                da.pop('end')
-                            intent_domain = f"{intent}-{domain}"
-                            triples.append([intent_domain])
-                            if len(slot) > 0:
-                                triples[-1].append(slot)
-                            if len(value) > 0:
-                                triples[-1].append(value)
-                    knowledge_seq = "| {} |".format(" | ".join([" : ".join(da_keywords) for da_keywords in triples]))
-                    
-                elif self.config.name == "kvret":
-                    knowledge = {"schedule": [], "weather": [], "navigate": []}
-                    triples = []
-                    for domain, db_items in item["knowledge"].items():
-                        knowledge[domain] = db_items
-                        for db_item in db_items:
-                            entity = db_item["entity"]
-                            for db_key, db_value in db_item.items():
-                                if db_key == "entity":
-                                    continue
-                                triples.append([entity, db_key, db_value])
-                    knowledge_seq = "| {} |".format(" | ".join([" : ".join(triple) for triple in triples]))
-
-                elif self.config.name == "opendialkg":
-                    knowledge = item["knowledge"]
-                    knowledge_seq = "| {} |".format(" | ".join([" : ".join(triple) for triple in item["knowledge"]]))
-                
-                elif self.config.name in ["wow", "personachat"]:
-                    knowledge = item["knowledge"]
-                    try:
-                        knowledge_seq = "| {} |".format(" | ".join(item["knowledge"]))
-                    except:
-                        print([knowledge])
-                        raise
-                
-                context = "\n".join([f"{turn[0]}: {turn[1]}" for turn in item["context"]]+["system: "])
-                if self.config.name in ["kvret", "wow", "personachat"]:
-                    context_knowledge = f"generate a response: all knowledge: \n\n{knowledge_seq} context:\n\n{context}"
-                else:
-                    context_knowledge = f"generate a response: grounded knowledge: \n\n{knowledge_seq} context:\n\n{context}"
-                
-                yield key, {
-                    "context+knowledge": context_knowledge,
-                    "response": item["response"],
-                    "knowledge": knowledge,
-                }
diff --git a/convlab/base_models/t5/key2gen/eval.ipynb b/convlab/base_models/t5/key2gen/eval.ipynb
deleted file mode 100644
index 51fcc5e0da1321ef740084d0a8b0241b5721a2fc..0000000000000000000000000000000000000000
--- a/convlab/base_models/t5/key2gen/eval.ipynb
+++ /dev/null
@@ -1 +0,0 @@
-{"cells":[{"cell_type":"code","execution_count":1,"metadata":{},"outputs":[],"source":["import json\n","import re"]},{"cell_type":"code","execution_count":2,"metadata":{},"outputs":[],"source":["def read_jsonline(path):\n","    return [json.loads(line) for line in open(path)]"]},{"cell_type":"code","execution_count":3,"metadata":{},"outputs":[],"source":["origin = read_jsonline('output/wow/wow/test_unseen.json')"]},{"cell_type":"code","execution_count":22,"metadata":{},"outputs":[],"source":["key2gen = read_jsonline('output/wow/key2gen_wow/test_unseen.json')"]},{"cell_type":"code","execution_count":23,"metadata":{},"outputs":[],"source":["with open('tmp_wow.txt', 'w') as f:\n","    for d1, d2 in zip(origin, key2gen):\n","        print(re.split('context:|grounded knowledge:', d1['context+knowledge'])[1].strip(), file=f)\n","        print(re.split('context:|grounded knowledge:', d2['context+knowledge'])[1].strip(), file=f)\n","        print(d1['context+knowledge'].split('context:')[1].replace('\\n\\n', '\\n'), file=f)\n","        print(file=f)\n","        print('target', d1['response'], file=f)\n","        print('origin', d1['predictions'], file=f)\n","        print('key2gen', d2['predictions'], file=f)\n","        print('='*100, file=f)"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["for ratio in [0.1, 0.01]:\n","    for order in [0, 1, 2]:\n","        origin = read_jsonline(f'output/personachat/key2gen_personachat_{ratio}_order{order}/generated_predictions.json')\n","        score = metric.compute(predictions=[d['predictions'] for d in origin], references=[d['response'] for d in origin])\n","        print(ratio, order)\n","        print(score)\n","        "]},{"cell_type":"code","execution_count":51,"metadata":{},"outputs":[{"name":"stdout","output_type":"stream","text":["0.01 1\n","{'bleu-1': 24.322560358946276, 'bleu-2': 13.03630111937752, 'bleu-3': 7.43647978674912, 'bleu-4': 4.450365738541082, 'unigram f1': 0.20101056184593705, 'unigram f1 (non-stop words)': 0.09881569367818614, 'rouge1': 21.359332522961864, 'rouge2': 6.532120354812852, 'rougeL': 19.76437990594138}\n"]}],"source":["for ratio in [0.01]:\n","    for order in [1]:\n","        origin = read_jsonline(f'output/personachat/personachat/generated_predictions.json')\n","        score = metric.compute(predictions=[d['predictions'] for d in origin], references=[d['response'] for d in origin])\n","        print(ratio, order)\n","        print(score)\n","        "]},{"cell_type":"code","execution_count":4,"metadata":{},"outputs":[],"source":["from datasets import load_metric"]},{"cell_type":"code","execution_count":7,"metadata":{},"outputs":[],"source":["metric = load_metric('metric.py')"]},{"cell_type":"code","execution_count":58,"metadata":{},"outputs":[{"data":{"text/plain":["{'bleu-1': 47.9848465486215,\n"," 'bleu-2': 37.18000679532912,\n"," 'bleu-3': 29.346646172092814,\n"," 'bleu-4': 23.410526740211363,\n"," 'unigram f1': 0.4999850046010773,\n"," 'unigram f1 (non-stop words)': 0.5150265227462978,\n"," 'rouge1': 50.536642578692195,\n"," 'rouge2': 33.10681789367832,\n"," 'rougeL': 46.84702913163778,\n"," 'meteor': 0.4641962079490068}"]},"execution_count":58,"metadata":{},"output_type":"execute_result"}],"source":["metric.compute(predictions=[d['predictions'] for d in key2gen], references=[d['response'] for d in key2gen])"]},{"cell_type":"code","execution_count":8,"metadata":{},"outputs":[{"data":{"text/plain":["{'bleu-1': 37.570099942714585,\n"," 'bleu-2': 26.77393964962893,\n"," 'bleu-3': 21.115954644820572,\n"," 'bleu-4': 17.513316671216046,\n"," 'unigram f1': 0.3656930567072274,\n"," 'unigram f1 (non-stop words)': 0.36456219281235724,\n"," 'rouge1': 39.1982724920493,\n"," 'rouge2': 20.825159884632743,\n"," 'rougeL': 34.98278542180112,\n"," 'meteor': 0.3405671227693821,\n"," 'distinct-1': 0.07838670580160921,\n"," 'distinct-2': 0.29689084413659694}"]},"execution_count":8,"metadata":{},"output_type":"execute_result"}],"source":["metric.compute(predictions=[d['predictions'] for d in origin], references=[d['response'] for d in origin])"]},{"cell_type":"code","execution_count":34,"metadata":{},"outputs":[{"data":{"text/plain":["{'bleu-1': 47.9848465486215,\n"," 'bleu-2': 37.18000679532912,\n"," 'bleu-3': 29.346646172092814,\n"," 'bleu-4': 23.410526740211363,\n"," 'unigram f1': 0.4999850046010773,\n"," 'unigram f1 (non-stop words)': 0.5150265227462978,\n"," 'rouge1': AggregateScore(low=Score(precision=0.5301926525013549, recall=0.4821419251082986, fmeasure=0.48565655175230005), mid=Score(precision=0.5513392693168799, recall=0.50235850981064, fmeasure=0.5053664257869219), high=Score(precision=0.5760132731228504, recall=0.5268580272115051, fmeasure=0.5279111393835526)),\n"," 'rouge2': AggregateScore(low=Score(precision=0.34772127155901306, recall=0.30411953889228, fmeasure=0.31029658993105447), mid=Score(precision=0.3696898381097765, recall=0.32612705034192035, fmeasure=0.3310681789367832), high=Score(precision=0.3947745596965405, recall=0.34880792116864995, fmeasure=0.35356317521641434)),\n"," 'rougeL': AggregateScore(low=Score(precision=0.4874189522136045, recall=0.4413343070361347, fmeasure=0.4464463084888409), mid=Score(precision=0.5108530997712726, recall=0.4642203560120527, fmeasure=0.46847029131637785), high=Score(precision=0.5350154077389535, recall=0.4855131911095939, fmeasure=0.4899950876629784)),\n"," 'rougeLsum': AggregateScore(low=Score(precision=0.4871840444049138, recall=0.44081531444183386, fmeasure=0.44514075751478493), mid=Score(precision=0.5105975305923949, recall=0.4639265647317744, fmeasure=0.46779186414456864), high=Score(precision=0.5348015149575474, recall=0.48693312722760357, fmeasure=0.4918651382986408))}"]},"execution_count":34,"metadata":{},"output_type":"execute_result"}],"source":["metric.compute(predictions=[d['predictions'] for d in key2gen], references=[d['response'] for d in key2gen])"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]}],"metadata":{"interpreter":{"hash":"0f9333403d680bc010aa5ce5a2f27ba398c9e47e92ba3724506306aa234cd07d"},"kernelspec":{"display_name":"Python 3.8.12 ('py38')","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.12"},"orig_nbformat":4},"nbformat":4,"nbformat_minor":2}
diff --git a/convlab/base_models/t5/key2gen/evaluate.py b/convlab/base_models/t5/key2gen/evaluate.py
deleted file mode 100644
index 769fdfcf3d1c899aad1b5389dad2c8d9465c05c6..0000000000000000000000000000000000000000
--- a/convlab/base_models/t5/key2gen/evaluate.py
+++ /dev/null
@@ -1,91 +0,0 @@
-from tabulate import tabulate
-import os
-import json
-from tqdm import tqdm
-from datasets import load_metric
-import numpy as np
-import csv
-
-def evaluate(filename, metric):
-    """
-    It reads the predictions, references, and knowledge from a file, and then computes the metric
-    
-    :param filename: the path to the file containing the predictions
-    :param metric: the metric to use for evaluation
-    :return: The result of the evaluation.
-    """
-    predictions, references, knowledge = [], [], []
-    with open(filename, 'r') as f:
-        for line in f:
-            item = json.loads(line)
-            predictions.append(item['predictions'])
-            references.append(item['response'])
-            knowledge.append(item['knowledge'])
-    result = metric.compute(predictions=predictions, references=references, knowledge=knowledge)
-    return result
-
-
-def avg_result(results):
-    """
-    It takes a list of dictionaries, and returns a dictionary with the same keys, but the values are the
-    mean and standard deviation of the values in the input dictionaries
-    
-    :param results: a list of dictionaries, each dictionary is the result of a single run of the model
-    :return: The average and standard deviation of the results.
-    """
-    ret = {}
-    for k in results[0]:
-        m = round(np.mean([result[k] for result in results]), 2)
-        v = round(np.std([result[k] for result in results], ddof=1), 2) if len(results) > 1 else None
-        ret[k] = f"{m}({v})"
-    return ret
-
-
-if __name__ == '__main__':
-    from argparse import ArgumentParser
-    parser = ArgumentParser(description="create data for seq2seq training")
-    parser.add_argument("--output_dirs", type=str, nargs='*', required=True)
-    parser.add_argument('--tasks', '-t', type=str, nargs='*', choices=['nlg', 'kvret', 'opendialkg', 'personachat', 'wow'], help='names of tasks')
-    parser.add_argument('--shots', '-s', type=int, nargs='*', help='how many data is used for training and evaluation, ratio if < 1 else absolute number')
-    parser.add_argument('--dial_ids_orders', '-o', type=int, nargs='*', help='which data order is used for experiments')
-    args = parser.parse_args()
-    print(args)
-    
-    table = []
-    fieldnames = []
-    for task_name in tqdm(args.tasks, desc='tasks'):
-        metric = load_metric("metric.py", task_name)
-        dataset_name = task_name if task_name != "nlg" else "multiwoz21"
-        for shot in tqdm(args.shots, desc='shots', leave=False):
-            for output_dir in tqdm(args.output_dirs, desc='models', leave=False):
-                model_name = output_dir.split('/')[-1]
-                results = []
-                for dial_ids_order in tqdm(args.dial_ids_orders, desc='dial_ids_orders', leave=False):
-                    result_dir = os.path.join(output_dir, task_name, f"{dataset_name}_{shot}shot_order{dial_ids_order}/gen")
-                    result_file = os.path.join(result_dir, "result.json")
-                    if not os.path.exists(result_file):
-                        filename = os.path.join(output_dir, task_name, f"{dataset_name}_{shot}shot_order{dial_ids_order}/gen/generated_predictions.json")
-                        result = evaluate(filename, metric)
-                        json.dump(result, open(result_file, 'w', encoding='utf-8'), indent=2, ensure_ascii=False)
-                    else:
-                        result = json.load(open(result_file))
-                    results.append(result)
-                res = {
-                    "dataset": f"{task_name}-{shot}shot",
-                    "model": f"{model_name}",
-                    **avg_result(results)
-                }
-                table.append(res)
-                for k in res:
-                    if k not in fieldnames:
-                        fieldnames.append(k)
-                    
-    res = tabulate(table, headers='keys', tablefmt='github')
-    with open(f'eval_results.txt', 'w', encoding='utf-8') as f:
-        print(res, file=f)
-    with open('eval_results.csv', 'w', newline='') as csvfile:
-        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
-
-        writer.writeheader()
-        for res in table:
-            writer.writerow(res)
diff --git a/convlab/base_models/t5/key2gen/features.py b/convlab/base_models/t5/key2gen/features.py
deleted file mode 100644
index 0ac768b5cbe61d46e430580b025182e515db93ef..0000000000000000000000000000000000000000
--- a/convlab/base_models/t5/key2gen/features.py
+++ /dev/null
@@ -1,72 +0,0 @@
-import datasets
-
-FEATURES = {
-    "nlg": {
-        "context+knowledge": datasets.Value("string"),
-        "response": datasets.Value("string"),
-        "knowledge": {
-            "categorical": datasets.Sequence({
-                "intent": datasets.Value("string"),
-                "domain": datasets.Value("string"),
-                "slot": datasets.Value("string"),
-                "value": datasets.Value("string"),
-            }), 
-            "non-categorical": datasets.Sequence({
-                "intent": datasets.Value("string"),
-                "domain": datasets.Value("string"),
-                "slot": datasets.Value("string"),
-                "value": datasets.Value("string"),
-            }), 
-            "binary": datasets.Sequence({
-                "intent": datasets.Value("string"),
-                "domain": datasets.Value("string"),
-                "slot": datasets.Value("string"),
-            })
-        }},
-    "kvret": {
-        "context+knowledge": datasets.Value("string"),
-        "response": datasets.Value("string"),
-        "knowledge": {
-            "schedule": datasets.Sequence({
-                "entity": datasets.Value("string"),
-                "time": datasets.Value("string"),
-                "date": datasets.Value("string"),
-                "party": datasets.Value("string"),
-                "room": datasets.Value("string"),
-                "agenda": datasets.Value("string")
-            }),
-            "weather": datasets.Sequence({
-                "entity": datasets.Value("string"),
-                "today": datasets.Value("string"),
-                "monday": datasets.Value("string"),
-                "tuesday": datasets.Value("string"),
-                "wednesday": datasets.Value("string"),
-                "thursday": datasets.Value("string"),
-                "friday": datasets.Value("string"),
-                "saturday": datasets.Value("string"),
-                "sunday": datasets.Value("string"),
-            }),
-            "navigate": datasets.Sequence({
-                "entity": datasets.Value("string"),
-                "traffic_info": datasets.Value("string"),
-                "poi_type": datasets.Value("string"),
-                "address": datasets.Value("string"),
-                "distance": datasets.Value("string")
-            })
-        }},
-    "opendialkg": {
-        "context+knowledge": datasets.Value("string"),
-        "response": datasets.Value("string"),
-        "knowledge": datasets.Sequence(datasets.Sequence(datasets.Value("string"))),
-        },
-    "wow": {
-        "context+knowledge": datasets.Value("string"),
-        "response": datasets.Value("string"),
-        "knowledge": datasets.Sequence(datasets.Value("string")),
-        },
-    "personachat": {
-        "context+knowledge": datasets.Value("string"),
-        "response": datasets.Value("string"),
-        "knowledge": datasets.Sequence(datasets.Value("string")),
-    }
-}
\ No newline at end of file
diff --git a/convlab/base_models/t5/key2gen/finetune.sh b/convlab/base_models/t5/key2gen/finetune.sh
deleted file mode 100644
index 8b2eb8d208966ed8f8056f01ece1b1a373033014..0000000000000000000000000000000000000000
--- a/convlab/base_models/t5/key2gen/finetune.sh
+++ /dev/null
@@ -1,116 +0,0 @@
-set -e
-dataset_path=$1
-model_name=$2
-model_name_or_path=$3
-dataset_name=$4
-if [ "${dataset_name}" == "multiwoz21" ]
-then
-    task_name="nlg"
-else
-    task_name=${dataset_name}
-fi
-master_port=$5
-
-n_gpus=2
-cache_dir="../cache"
-metric_name_or_path="metric.py"
-source_column="context+knowledge"
-target_column="response"
-truncation_side="left"
-max_source_length=512
-max_target_length=512
-per_device_train_batch_size=64
-per_device_eval_batch_size=64
-gradient_accumulation_steps=1
-num_workers=16
-lr=1e-3
-num_train_epochs=100
-
-for shot in 50 100 200
-do
-    for dial_ids_order in 0 1 2 3 4
-    do
-        python create_data.py -t ${task_name} -d ${dataset_name} -o ${dial_ids_order} -s ${shot}
-
-        data_dir="data/${task_name}/${dataset_name}_${shot}shot_order${dial_ids_order}"
-        output_dir="output/${model_name}/${task_name}/${dataset_name}_${shot}shot_order${dial_ids_order}"
-        logging_dir="${output_dir}/runs"
-        train_file="${data_dir}/train.json"
-        validation_file="${data_dir}/validation.json"
-
-        # training
-        python -m torch.distributed.launch --master_port ${master_port} \
-            --nproc_per_node ${n_gpus} ../run_seq2seq.py \
-            --task_name ${task_name} \
-            --dataset_name ${dataset_path} \
-            --dataset_config_name ${task_name} \
-            --train_file ${train_file} \
-            --validation_file ${validation_file} \
-            --source_column ${source_column} \
-            --target_column ${target_column} \
-            --max_source_length ${max_source_length} \
-            --max_target_length ${max_target_length} \
-            --truncation_side ${truncation_side} \
-            --model_name_or_path ${model_name_or_path} \
-            --do_train \
-            --do_eval \
-            --save_strategy epoch \
-            --evaluation_strategy epoch \
-            --save_total_limit 1 \
-            --prediction_loss_only \
-            --load_best_model_at_end \
-            --overwrite_output_dir \
-            --cache_dir ${cache_dir} \
-            --output_dir ${output_dir} \
-            --logging_dir ${logging_dir} \
-            --preprocessing_num_workers ${num_workers} \
-            --dataloader_num_workers ${num_workers} \
-            --per_device_train_batch_size ${per_device_train_batch_size} \
-            --per_device_eval_batch_size ${per_device_eval_batch_size} \
-            --gradient_accumulation_steps ${gradient_accumulation_steps} \
-            --learning_rate ${lr} \
-            --num_train_epochs ${num_train_epochs} \
-            --optim adafactor \
-            --lr_scheduler_type constant \
-            --gradient_checkpointing
-
-        # inference
-        test_file="data/${task_name}/test.json"
-        gen_output_dir="${output_dir}/gen"
-
-        python -m torch.distributed.launch --master_port ${master_port} \
-            --nproc_per_node ${n_gpus} ../run_seq2seq.py \
-            --task_name ${task_name} \
-            --dataset_name ${dataset_path} \
-            --dataset_config_name ${task_name} \
-            --metric_name_or_path ${metric_name_or_path} \
-            --metric_config_name ${task_name} \
-            --test_file ${test_file} \
-            --source_column ${source_column} \
-            --target_column ${target_column} \
-            --max_source_length ${max_source_length} \
-            --max_target_length ${max_target_length} \
-            --truncation_side ${truncation_side} \
-            --model_name_or_path ${output_dir} \
-            --do_predict \
-            --predict_with_generate \
-            --cache_dir ${cache_dir} \
-            --output_dir ${gen_output_dir} \
-            --logging_dir ${logging_dir} \
-            --overwrite_output_dir \
-            --preprocessing_num_workers ${num_workers} \
-            --dataloader_num_workers ${num_workers} \
-            --per_device_train_batch_size ${per_device_train_batch_size} \
-            --per_device_eval_batch_size ${per_device_eval_batch_size} \
-            --gradient_accumulation_steps ${gradient_accumulation_steps} \
-            --learning_rate ${lr} \
-            --num_train_epochs ${num_train_epochs} \
-            --optim adafactor \
-            --lr_scheduler_type constant \
-            --gradient_checkpointing
-        
-    done
-done
-
-# evaluation
-python evaluate.py --output_dirs output/${model_name} -t ${task_name} -s 50 100 200 -o 0 1 2 3 4
diff --git a/convlab/base_models/t5/key2gen/metric.py b/convlab/base_models/t5/key2gen/metric.py
deleted file mode 100644
index 808934b65268ab2ae4180b9bbe64457fb5ca1b68..0000000000000000000000000000000000000000
--- a/convlab/base_models/t5/key2gen/metric.py
+++ /dev/null
@@ -1,434 +0,0 @@
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Grounded Dialog Generation Metric"""
-
-from weakref import ref
-import datasets
-from sacrebleu.metrics import BLEU
-from sacrebleu.utils import sum_of_lists
-import re
-from collections import Counter
-import numpy as np
-from nltk.corpus import stopwords
-from rouge_score import rouge_scorer, scoring
-from nltk.translate import meteor_score
-from datasets.config import importlib_metadata, version
-from convlab.base_models.t5.key2gen.features import FEATURES
-from convlab.util import load_ontology
-from copy import deepcopy
-
-
-NLTK_VERSION = version.parse(importlib_metadata.version("nltk"))
-if NLTK_VERSION >= version.Version("3.6.5"):
-    from nltk import word_tokenize
-
-# Uncomment to download nltk_data for the first time running.
-# import nltk
-# nltk.download("wordnet")
-# if NLTK_VERSION >= version.Version("3.6.5"):
-#     nltk.download("punkt")
-# if NLTK_VERSION >= version.Version("3.6.6"):
-#     nltk.download("omw-1.4")
-
-
-_CITATION = """
-"""
-
-_DESCRIPTION = """\
-Metric to evaluate text generation models on the grounded dialog generation task.
-"""
-
-# TODO
-_KWARGS_DESCRIPTION = """
-Args:
-    predictions: list of predictions to score. Each predictions
-        should be a string.
-    references: list of reference for each prediction. Each
-        reference should be a string.
-    knowledge: task-specific grounded knowledge
-
-Returns:
-    bleu-1/2/3/4: corpus-bleu score, from sacrebleu
-    rouge-1/2/L: ROUGE-F1, from rouge_score
-    meteor: METEOR, from nltk
-    unigram f1: unigram overlap, from parlai
-    distinct-1/2: from parlai
-    other knowledge utility score: task-specific knowledge utility metrics
-"""
-
-re_art = re.compile(r'\b(a|an|the)\b')
-re_punc = re.compile(r'[!"#$%&()*+,-./:;<=>?@\[\]\\^`{|}~_\']')
-stop_words = set(stopwords.words("english"))
-def utt2words(s):
-    """Lower text and remove punctuation, articles and extra whitespace.
-    from parlai https://github.com/facebookresearch/ParlAI/blob/9daae69320c07104493486e022c0e46a7871b253/parlai/core/metrics.py#L810"""
-    s = s.lower()
-    s = re_punc.sub(' ', s)
-    s = re_art.sub(' ', s)
-    return s.split()
-
-
-def get_bleu(predictions, references):
-    """bleu-1/2/3/4 from sacrebleu"""
-    references = [" " if ref=="" else ref for ref in references]
-    metrics = {}
-    bleu = BLEU(lowercase=True, force=False, tokenize=BLEU.TOKENIZER_DEFAULT, smooth_method="exp", smooth_value=None, effective_order=False)
-    stats = sum_of_lists(bleu._extract_corpus_statistics(predictions, [references]))
-    for n in range(1,5):
-        metrics[f"bleu-{n}"] = bleu.compute_bleu(
-            correct=stats[2: 2 + bleu.max_ngram_order],
-            total=stats[2 + bleu.max_ngram_order:],
-            sys_len=int(stats[0]), ref_len=int(stats[1]),
-            smooth_method=bleu.smooth_method, smooth_value=bleu.smooth_value,
-            effective_order=bleu.effective_order,
-            max_ngram_order=n).score
-    return metrics
-
-
-def get_unigram_f1(predictions, references):
-    """unigram f1 between prediction and reference, from parlai"""
-    metrics = {}
-    metrics["unigram f1"] = []
-    metrics["unigram f1 (non-stop words)"] = []
-    for prediction, reference in zip(predictions, references):
-        pred_items = utt2words(prediction)
-        gold_items = utt2words(reference)
-        for remove_stopwords in [False, True]:
-            if remove_stopwords:
-                pred_items = [w for w in pred_items if w not in stop_words]
-                gold_items = [w for w in gold_items if w not in stop_words]
-            common = Counter(pred_items) & Counter(gold_items)
-            num_same = sum(common.values())
-            if num_same == 0:
-                f1 = 0
-            else:
-                precision = 1.0 * num_same / len(pred_items)
-                recall = 1.0 * num_same / len(gold_items)
-                f1 = (2 * precision * recall) / (precision + recall)
-            if not remove_stopwords:
-                metrics["unigram f1"].append(f1)
-            else:
-                metrics["unigram f1 (non-stop words)"].append(f1)
-    metrics["unigram f1"] = np.mean(metrics["unigram f1"]) * 100
-    metrics["unigram f1 (non-stop words)"] = np.mean(metrics["unigram f1 (non-stop words)"]) * 100
-    return metrics
-
-
-def get_rouge(predictions, references):
-    """rouge-1/2/L from rouge-score"""
-    rouge_types=["rouge1", "rouge2", "rougeL"]
-    scorer = rouge_scorer.RougeScorer(rouge_types=rouge_types, use_stemmer=True)
-    aggregator = scoring.BootstrapAggregator()
-
-    for prediction, reference in zip(predictions, references):
-        score = scorer.score(reference, prediction)
-        aggregator.add_scores(score)
-
-    return {key: 100 * (value.mid.fmeasure if key == "rougeL" else value.mid.recall) for key, value in aggregator.aggregate().items()}
-
-
-def get_meteor(predictions, references):
-    """meteor from nltk"""
-    alpha=0.9
-    beta=3
-    gamma=0.5
-    if NLTK_VERSION >= version.Version("3.6.5"):
-        scores = [
-            meteor_score.single_meteor_score(
-                word_tokenize(ref), word_tokenize(pred), alpha=alpha, beta=beta, gamma=gamma
-            )
-            for ref, pred in zip(references, predictions)
-        ]
-    else:
-        scores = [
-            meteor_score.single_meteor_score(ref, pred, alpha=alpha, beta=beta, gamma=gamma)
-            for ref, pred in zip(references, predictions)
-        ]
-    return {"meteor": np.mean(scores) * 100}
-
-
-def get_distinct(predictions):
-    """distinct-1/2 
-    from parlai https://github.com/facebookresearch/ParlAI/blob/9daae69320c07104493486e022c0e46a7871b253/parlai/core/metrics.py#L781"""
-    def _ngram(seq, n):
-        for i in range(len(seq) - n + 1):
-            yield tuple(seq[i : i + n])
-    
-    metrics = {}
-    for k in [1, 2]:
-        inter_cnt = Counter()
-        for prediction in predictions:
-            ngram = Counter(_ngram(utt2words(prediction), k))
-            inter_cnt += ngram
-        metrics[f"distinct-{k}"] = max(len(inter_cnt), 1e-12) / max(sum(inter_cnt.values()), 1e-5) * 100
-    return metrics
-
-
-def get_nlg_slot_err(predictions, knowledge):
-    """slot error rate: (missing_count + redundant_count) / all_count for value in dialog acts"""
-    val2ds_dict = {}
-    ontology = load_ontology("multiwoz21")
-    for domain_name in ontology["domains"]:
-        domain = ontology["domains"][domain_name]
-        for slot_name in domain["slots"]:
-            slot = domain["slots"][slot_name]
-            if "possible_values" not in slot:
-                continue
-            possible_vals = slot["possible_values"]
-            if len(possible_vals) > 0:
-                for val in possible_vals:
-                    val2ds_dict[val] = f"{domain_name}-{slot_name}"
-    score_list = []
-    for utterance, da in zip(predictions, knowledge):
-        missing_count = 0
-        redundant_count = 0
-        all_count = 0
-        all_values = set()
-        ## missing values
-        # print(da)
-        # print(utterance)
-        for key in ['categorical', 'non-categorical']:
-            for value in da[key]['value']:
-                if len(value) > 0:
-                    # print(value)
-                    all_values.add(value)
-                    if value.strip().lower() not in utterance.lower():
-                        missing_count += 1
-                        # print(f"\tmissing: {value}")
-                    all_count += 1
-        if all_count == 0:
-            continue
-        ## redundant values
-        for val in val2ds_dict:
-            if f" {val.strip().lower()} " in f" {utterance.strip().lower()} " and val.strip().lower() not in all_values:
-                wlist = val2ds_dict[val].split("-")
-                domain, slot = wlist[0], wlist[1]
-                if f" {slot.strip().lower()}" in f" {utterance.strip().lower()} ":
-                    redundant_count += 1
-                    # print(f"redundant: {val}/{val2ds_dict[val]}")
-        item_score = float(missing_count + redundant_count) / all_count
-        # print(f"\tredundant: {redundant_count} | missing_count: {missing_count} |all_count: {all_count}")
-        # print('-'*100)
-        score_list.append(item_score)
-    return {"err": np.mean(score_list) * 100}
-
-
-def load_entities():
-    """modified (load from unified ontology) from UnifiedSKG
-    https://github.com/HKUNLP/UnifiedSKG/blob/49a2ff950bb312b980c22ad72b11520db72ab6a3/metrics/kvret/evaluator.py#L8"""
-
-    ontology = load_ontology("kvret")
-    all_entities = set()
-    for domain in ontology["domains"]:
-        for slot in ontology["domains"][domain]["slots"]:
-            all_entities |= set(ontology["domains"][domain]["slots"][slot]["possible_values"])
-    missed_entities = ["yoga", "tennis", "swimming", "football", " lab ", "doctor", "optometrist", "dentist", "1st",
-                        "2nd", "3rd", "4th", "5th", "6th", "7th", "8th", "9th", "10th",
-                        "11th", "12th", "13th", "14th", "15th", "16th", "17th", "18th", "19th", "20th", "Jill",
-                        "Jack"]
-    all_entities |= set(missed_entities)
-    all_entities.remove("HR")
-    all_entities.add(" HR ")
-    all_entities = sorted(list(all_entities), key=lambda i: len(i), reverse=True)
-    return all_entities
-
-
-def check_sub_str(str_list: list, sub_str: str):
-    """
-    It takes a list of strings and a substring as input, and returns True if the substring is found
-    in any of the strings in the list, and False otherwise
-    """
-    for str_item in str_list:
-        if sub_str in str_item or sub_str.lower() in str_item.lower():
-            return True
-    return False
-
-
-def extract_entities_from_utterance(utterance, sorted_entities):
-    """modified (remove underscore) from UnifiedSKG
-    https://github.com/HKUNLP/UnifiedSKG/blob/49a2ff950bb312b980c22ad72b11520db72ab6a3/metrics/kvret/response_entity_hit.py#L45"""
-
-    utterance = " {} ".format(utterance)  # for entity matching
-    for h in range(0, 13): # for formulating am & pm
-        utterance = utterance.replace("{} am".format(h), "{}am".format(h))
-        utterance = utterance.replace("{} pm".format(h), "{}pm".format(h))
-    for entity_item_a in [20, 30, 40, 50, 60, 70, 80, 90, 100]:
-        for entity_item_b in [20, 30, 40, 50, 60, 70, 80, 90, 100]:
-            utterance = utterance.replace("{}-{}f".format(str(entity_item_a), str(entity_item_b)), "{}f-{}f".format(str(entity_item_a), str(entity_item_b)))
-    entities_in_this_utterance = []
-    for entity in sorted_entities:
-        # len(entity) decreases
-        if (entity in utterance) or (entity.lower() in utterance.lower()):
-            if not check_sub_str(entities_in_this_utterance, entity):
-                # in case of "week & weekend", "week & next_week" etc
-                entities_in_this_utterance.append(entity)
-    return entities_in_this_utterance
-
-
-def f1_score(y_pred, y_true, average="micro"):
-    """micro/marco-F1 score, modified from UnifiedSKG
-    https://github.com/HKUNLP/UnifiedSKG/blob/49a2ff950bb312b980c22ad72b11520db72ab6a3/metrics/kvret/response_entity_hit.py#L76"""
-
-    assert len(y_pred) == len(y_true)
-
-    def _compute_F1(precision, recall):
-        return 2 * precision * recall / float(precision + recall) if (precision + recall) != 0 else 0
-
-    def _compute_prf(gold, pred):
-        TP, FP, FN = 0, 0, 0
-        if len(gold) != 0:
-            count = 1
-            for g in gold:
-                if g in pred:
-                    TP += 1
-                else:
-                    FN += 1
-            for p in set(pred):
-                if p not in gold:
-                    FP += 1
-            precision = TP / float(TP + FP) if (TP + FP) != 0 else 0
-            recall = TP / float(TP + FN) if (TP + FN) != 0 else 0
-            F1 = _compute_F1(precision, recall)
-        else:
-            precision, recall, F1, count = 0, 0, 0, 0
-        return TP, FP, FN, F1, count
-
-    F1_pred, F1_count, TP_all, FP_all, FN_all = 0, 0, 0, 0, 0
-
-    for y_true_item, y_pred_item in zip(y_true, y_pred):
-        single_tp, single_fp, single_fn, single_f1, count = _compute_prf(y_true_item, y_pred_item)
-        F1_pred += single_f1
-        F1_count += count
-        TP_all += single_tp
-        FP_all += single_fp
-        FN_all += single_fn
-
-    if average == "macro":
-        F1_macro_score = F1_pred / float(F1_count) if F1_count != 0 else 0
-        return F1_macro_score * 100
-    elif average == "micro":
-        P_score = TP_all / float(TP_all + FP_all) if (TP_all + FP_all) != 0 else 0
-        R_score = TP_all / float(TP_all + FN_all) if (TP_all + FN_all) != 0 else 0
-        F1_micro_score = _compute_F1(P_score, R_score)
-        return F1_micro_score * 100
-    else:
-        raise ValueError("Options other than micro/macro are not supported.")
-
-
-def get_kvret_entity_f1(predictions, references, knowledge):
-    """entity f1 for kvret, modified from
-    https://github.com/HKUNLP/UnifiedSKG/blob/49a2ff950bb312b980c22ad72b11520db72ab6a3/metrics/kvret/response_entity_hit.py#L178"""
-
-    global_entities = load_entities()
-    F1_scores = {}
-    entities_from_predictions_and_references = {
-        d: {"predictions_entities": [], "references_entities": []} for d in ["all", "schedule", "weather", "navigate"]
-    }
-    for prediction, reference, kb in zip(predictions, references, knowledge):
-        prediction_entities = extract_entities_from_utterance(utterance=prediction, sorted_entities=global_entities)
-        reference_entities = extract_entities_from_utterance(utterance=reference, sorted_entities=global_entities)
-        entities_from_predictions_and_references["all"]["predictions_entities"].append(prediction_entities)
-        entities_from_predictions_and_references["all"]["references_entities"].append(reference_entities)
-        domain = "schedule"
-        for d in kb:
-            if len(kb[d]["entity"]) > 0:
-                domain = d
-                break
-        entities_from_predictions_and_references[domain]["predictions_entities"].append(prediction_entities)
-        entities_from_predictions_and_references[domain]["references_entities"].append(reference_entities)
-    
-    for category in entities_from_predictions_and_references.keys():
-        predictions_entities = entities_from_predictions_and_references[category]["predictions_entities"]
-        references_entities = entities_from_predictions_and_references[category]["references_entities"]
-        F1_scores["{} micro entity F1".format(category)] = f1_score(y_pred=predictions_entities, y_true=references_entities, average="micro")
-        F1_scores["{} macro entity F1".format(category)] = f1_score(y_pred=predictions_entities, y_true=references_entities, average="macro")
-
-    return {**F1_scores}
-
-
-def get_opendialkg_entity_f1(predictions, references, knowledge):
-    predictions_entities, references_entities = [], []
-    for prediction, reference, kg_path in zip(predictions, references, knowledge):
-        kg_entities = set()
-        for kg_triple in kg_path:
-            # add head and tail entities
-            kg_entities.add(kg_triple[0])
-            kg_entities.add(kg_triple[-1])
-        kg_entities = sorted(list(kg_entities), key=lambda i: len(i), reverse=True)
-        
-        for utterance, entities in zip([prediction, reference], [predictions_entities, references_entities]):
-            entities_in_this_utterance = []
-            for entity in kg_entities:
-                if (entity in utterance) or (entity.lower() in utterance.lower()):
-                    if not check_sub_str(entities_in_this_utterance, entity):
-                        # in case of "week & weekend", "week & next_week" etc
-                        entities_in_this_utterance.append(entity)
-            entities.append(entities_in_this_utterance)
-
-    return {
-        "micro entity f1": f1_score(y_pred=predictions_entities, y_true=references_entities, average="micro"),
-        "macro entity f1": f1_score(y_pred=predictions_entities, y_true=references_entities, average="macro")
-    }
-
-def get_knowledge_sentences_f1(predictions, knowledge):
-    knowledge_reference = [' '.join(k_sens) for k_sens in knowledge]
-    f1_score = get_unigram_f1(predictions, knowledge_reference)
-    return {f"knowledge {k}": v for k, v in f1_score.items()}
-
-
-@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class GroundedDialogGenerationMetrics(datasets.Metric):
-    """Metric to evaluate text generation models on the grounded dialog generation task."""
-    def _info(self):
-        return datasets.MetricInfo(
-            description=_DESCRIPTION,
-            citation=_CITATION,
-            inputs_description=_KWARGS_DESCRIPTION,
-            features=datasets.Features({
-                "predictions": datasets.Value("string"),
-                "references": datasets.Value("string"),
-                "knowledge": deepcopy(FEATURES[self.config_name]["knowledge"])
-            })
-        )
-
-    def compute(self, predictions, references, knowledge=None):
-        """Returns the scores: bleu"""
-        metrics = {}
-
-        # bleu
-        metrics.update(get_bleu(predictions, references))
-                
-        # unigram f1
-        metrics.update(get_unigram_f1(predictions, references))
-        
-        # rouge-1/2/L-fmeasure
-        metrics.update(get_rouge(predictions, references))
-
-        # meteor
-        metrics.update(get_meteor(predictions, references))
-
-        # inter-distinct-1/2
-        metrics.update(get_distinct(predictions))
-        
-        if knowledge is not None:
-            if self.config_name == "nlg":
-                metrics.update(get_nlg_slot_err(predictions, knowledge))
-            elif self.config_name == "kvret":
-                metrics.update(get_kvret_entity_f1(predictions, references, knowledge))
-            elif self.config_name == "opendialkg":
-                metrics.update(get_opendialkg_entity_f1(predictions, references, knowledge))
-            elif self.config_name in ["wow", "personachat"]:
-                metrics.update(get_knowledge_sentences_f1(predictions, knowledge))
-
-        return metrics
diff --git a/convlab/base_models/t5/nlg/__init__.py b/convlab/base_models/t5/nlg/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..32275a5755248f5cfb67192b0a0f530cee0a276e
--- /dev/null
+++ b/convlab/base_models/t5/nlg/__init__.py
@@ -0,0 +1 @@
+from convlab.base_models.t5.nlg.nlg import T5NLG
\ No newline at end of file
diff --git a/convlab/base_models/t5/nlg/nlg.py b/convlab/base_models/t5/nlg/nlg.py
index 2781fded74c3b02a9c46a6c289d6f0e5fb850f2b..214dc01eed75cfbb85a740e8f5fee8a759d813b0 100755
--- a/convlab/base_models/t5/nlg/nlg.py
+++ b/convlab/base_models/t5/nlg/nlg.py
@@ -8,17 +8,13 @@ from convlab.util.custom_util import model_downloader
 
 
 class T5NLG(NLG):
-    def __init__(self, speaker, context_window_size, model_name_or_path, model_file=None, device='cuda'):
+    def __init__(self, speaker, context_window_size, model_name_or_path, device='cuda'):
         assert speaker in ['user', 'system']
         self.speaker = speaker
         self.opponent = 'system' if speaker == 'user' else 'user'
         self.context_window_size = context_window_size
         self.use_context = context_window_size > 0
 
-        model_dir = os.path.dirname(os.path.abspath(__file__))
-        if not os.path.exists(model_name_or_path):
-            model_downloader(model_dir, model_file)
-        
         self.config = AutoConfig.from_pretrained(model_name_or_path)
         self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
         self.model = AutoModelForSeq2SeqLM.from_pretrained(model_name_or_path, config=self.config)
diff --git a/convlab/base_models/t5/nlu/__init__.py b/convlab/base_models/t5/nlu/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..ed4bbd0306fe68dfbbaa69f7e93cdb2b68c23e6d
--- /dev/null
+++ b/convlab/base_models/t5/nlu/__init__.py
@@ -0,0 +1 @@
+from convlab.base_models.t5.nlu.nlu import T5NLU
\ No newline at end of file
diff --git a/convlab/base_models/t5/nlu/nlu.py b/convlab/base_models/t5/nlu/nlu.py
index 2862cea7aa74c8c365a047dc74aa41dc79ead405..a5a6e6a23ec184b15fc073d88fa1a6b3fece34d8 100755
--- a/convlab/base_models/t5/nlu/nlu.py
+++ b/convlab/base_models/t5/nlu/nlu.py
@@ -8,16 +8,12 @@ from convlab.util.custom_util import model_downloader
 
 
 class T5NLU(NLU):
-    def __init__(self, speaker, context_window_size, model_name_or_path, model_file=None, device='cuda'):
+    def __init__(self, speaker, context_window_size, model_name_or_path, device='cuda'):
         assert speaker in ['user', 'system']
         self.speaker = speaker
         self.opponent = 'system' if speaker == 'user' else 'user'
         self.context_window_size = context_window_size
         self.use_context = context_window_size > 0
-
-        model_dir = os.path.dirname(os.path.abspath(__file__))
-        if not os.path.exists(model_name_or_path):
-            model_downloader(model_dir, model_file)
         
         self.config = AutoConfig.from_pretrained(model_name_or_path)
         self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
diff --git a/convlab/dialog_agent/agent.py b/convlab/dialog_agent/agent.py
index 0815fad0abe59fb72c0c0a3bfb59b0f6ac140d67..79f61e2b18f04e702c690a33cdf313656b2615c6 100755
--- a/convlab/dialog_agent/agent.py
+++ b/convlab/dialog_agent/agent.py
@@ -7,6 +7,7 @@ from convlab.policy import Policy
 from convlab.nlg import NLG
 from copy import deepcopy
 import time
+import pdb
 from pprint import pprint
 
 
@@ -63,7 +64,7 @@ class PipelineAgent(Agent):
            =====   =====    ======  ===     ==      ===
     """
 
-    def __init__(self, nlu: NLU, dst: DST, policy: Policy, nlg: NLG, name: str, return_semantic_acts=False):
+    def __init__(self, nlu: NLU, dst: DST, policy: Policy, nlg: NLG, name: str):
         """The constructor of PipelineAgent class.
 
         Here are some special combination cases:
@@ -94,7 +95,7 @@ class PipelineAgent(Agent):
         self.dst = dst
         self.policy = policy
         self.nlg = nlg
-        self.return_semantic_acts = return_semantic_acts
+
         self.init_session()
         self.agent_saves = []
         self.history = []
@@ -151,6 +152,7 @@ class PipelineAgent(Agent):
 
                 self.input_action = self.nlu.predict(
                     observation, context=[x[1] for x in self.history[:-1]])
+                # print("system semantic action: ", self.input_action)
             else:
                 self.input_action = observation
                 self.input_action_eval = observation
@@ -186,7 +188,7 @@ class PipelineAgent(Agent):
 
                 if type(self.output_action) == list:
                     for intent, domain, slot, value in self.output_action:
-                        if intent == "book":
+                        if intent.lower() == "book":
                             self.dst.state['booked'][domain] = [{slot: value}]
             else:
                 self.dst.state['user_action'] = self.output_action
@@ -196,8 +198,6 @@ class PipelineAgent(Agent):
         self.history.append([self.name, model_response])
 
         self.turn += 1
-        if self.return_semantic_acts:
-            return self.output_action
         self.agent_saves.append(self.save_info())
         return model_response
 
diff --git a/convlab/dialog_agent/env.py b/convlab/dialog_agent/env.py
index 6216eaaac9fb81615f903f048d6d85766ce663c5..a8915301153f8e824e0a2ed91cd6eb9cd34e2605 100755
--- a/convlab/dialog_agent/env.py
+++ b/convlab/dialog_agent/env.py
@@ -49,11 +49,14 @@ class Environment():
         dialog_act = self.sys_nlu.predict(
             observation) if self.sys_nlu else observation
         self.sys_dst.state['user_action'] = dialog_act
+        self.sys_dst.state['history'].append(["sys", model_response])
+        self.sys_dst.state['history'].append(["user", observation])
+
         state = self.sys_dst.update(dialog_act)
-        state = deepcopy(state)
+        self.sys_dst.state['history'].append(["sys", model_response])
+        self.sys_dst.state['history'].append(["usr", observation])
 
-        state['history'].append(["sys", model_response])
-        state['history'].append(["usr", observation])
+        state = deepcopy(state)
 
         terminated = self.usr.is_terminated()
 
diff --git a/convlab/dst/evaluate_unified_datasets.py b/convlab/dst/evaluate_unified_datasets.py
index d4e0720dc02bf90efcfebb1780630211f0722f7f..d4b3ad2464be6c8de0d6f1f51ecdf1bf6bfdbd0e 100644
--- a/convlab/dst/evaluate_unified_datasets.py
+++ b/convlab/dst/evaluate_unified_datasets.py
@@ -7,7 +7,6 @@ def evaluate(predict_result):
 
     metrics = {'TP':0, 'FP':0, 'FN':0}
     acc = []
-
     for sample in predict_result:
         pred_state = sample['predictions']['state']
         gold_state = sample['state']
@@ -37,7 +36,7 @@ def evaluate(predict_result):
                         flag = False
 
         acc.append(flag)
-    
+
     TP = metrics.pop('TP')
     FP = metrics.pop('FP')
     FN = metrics.pop('FN')
diff --git a/convlab/dst/setsumbt/__init__.py b/convlab/dst/setsumbt/__init__.py
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..9492faa9c9a20d1c476819bb995900ca71d56607 100644
--- a/convlab/dst/setsumbt/__init__.py
+++ b/convlab/dst/setsumbt/__init__.py
@@ -0,0 +1 @@
+from convlab.dst.setsumbt.tracker import SetSUMBTTracker
\ No newline at end of file
diff --git a/convlab/dst/setsumbt/calibration_plots.py b/convlab/dst/setsumbt/calibration_plots.py
index 379057e6411082d466b10a81027bb57e4131bb9b..a41f280d3349164a2a67333d0ab176a37cbe50ea 100644
--- a/convlab/dst/setsumbt/calibration_plots.py
+++ b/convlab/dst/setsumbt/calibration_plots.py
@@ -35,7 +35,7 @@ def main():
     path = args.data_dir
 
     models = os.listdir(path)
-    models = [os.path.join(path, model, 'test.belief') for model in models]
+    models = [os.path.join(path, model, 'test.predictions') for model in models]
 
     fig = plt.figure(figsize=(14,8))
     font=20
@@ -56,16 +56,16 @@ def main():
 
 
 def get_calibration(path, device, n_bins=10, temperature=1.00):
-    logits = torch.load(path, map_location=device)
-    y_true = logits['labels']
-    logits = logits['belief_states']
+    probs = torch.load(path, map_location=device)
+    y_true = probs['state_labels']
+    probs = probs['belief_states']
 
-    y_pred = {slot: logits[slot].reshape(-1, logits[slot].size(-1)).argmax(-1) for slot in logits}
+    y_pred = {slot: probs[slot].reshape(-1, probs[slot].size(-1)).argmax(-1) for slot in probs}
     goal_acc = {slot: (y_pred[slot] == y_true[slot].reshape(-1)).int() for slot in y_pred}
     goal_acc = sum([goal_acc[slot] for slot in goal_acc])
     goal_acc = (goal_acc == len(y_true)).int()
 
-    scores = [logits[slot].reshape(-1, logits[slot].size(-1)).max(-1)[0].unsqueeze(0) for slot in logits]
+    scores = [probs[slot].reshape(-1, probs[slot].size(-1)).max(-1)[0].unsqueeze(0) for slot in probs]
     scores = torch.cat(scores, 0).min(0)[0]
 
     step = 1.0 / float(n_bins)
diff --git a/convlab/dst/setsumbt/configs/setsumbt_multitask.json b/convlab/dst/setsumbt/configs/setsumbt_multitask.json
new file mode 100644
index 0000000000000000000000000000000000000000..c076a557cb3e1d567784c70559fb1922fe05c545
--- /dev/null
+++ b/convlab/dst/setsumbt/configs/setsumbt_multitask.json
@@ -0,0 +1,11 @@
+{
+  "model_type": "SetSUMBT",
+  "dataset": "multiwoz21+sgd+tm1+tm2+tm3",
+  "no_action_prediction": true,
+  "model_name_or_path": "/gpfs/project/niekerk/models/transformers/roberta-base",
+  "transformers_local_files_only": true,
+  "train_batch_size": 3,
+  "dev_batch_size": 8,
+  "test_batch_size": 8,
+  "run_nbt": true
+}
\ No newline at end of file
diff --git a/convlab/dst/setsumbt/configs/setsumbt_multiwoz21.json b/convlab/dst/setsumbt/configs/setsumbt_multiwoz21.json
new file mode 100644
index 0000000000000000000000000000000000000000..0bff751c16f0bdcdf61f04ce33d616370c0d32d8
--- /dev/null
+++ b/convlab/dst/setsumbt/configs/setsumbt_multiwoz21.json
@@ -0,0 +1,11 @@
+{
+  "model_type": "SetSUMBT",
+  "dataset": "multiwoz21",
+  "no_action_prediction": true,
+  "model_name_or_path": "/gpfs/project/niekerk/models/transformers/roberta-base",
+  "transformers_local_files_only": true,
+  "train_batch_size": 3,
+  "dev_batch_size": 16,
+  "test_batch_size": 16,
+  "run_nbt": true
+}
\ No newline at end of file
diff --git a/convlab/dst/setsumbt/configs/setsumbt_pretrain.json b/convlab/dst/setsumbt/configs/setsumbt_pretrain.json
new file mode 100644
index 0000000000000000000000000000000000000000..fdc22d157840e7494b0266d0bd99f8a99d242969
--- /dev/null
+++ b/convlab/dst/setsumbt/configs/setsumbt_pretrain.json
@@ -0,0 +1,11 @@
+{
+  "model_type": "SetSUMBT",
+  "dataset": "sgd+tm1+tm2+tm3",
+  "no_action_prediction": true,
+  "model_name_or_path": "/gpfs/project/niekerk/models/transformers/roberta-base",
+  "transformers_local_files_only": true,
+  "train_batch_size": 3,
+  "dev_batch_size": 12,
+  "test_batch_size": 12,
+  "run_nbt": true
+}
\ No newline at end of file
diff --git a/convlab/dst/setsumbt/configs/setsumbt_sgd.json b/convlab/dst/setsumbt/configs/setsumbt_sgd.json
new file mode 100644
index 0000000000000000000000000000000000000000..97f5818334af4c7984ec24448861b627315820e3
--- /dev/null
+++ b/convlab/dst/setsumbt/configs/setsumbt_sgd.json
@@ -0,0 +1,11 @@
+{
+  "model_type": "SetSUMBT",
+  "dataset": "sgd",
+  "no_action_prediction": true,
+  "model_name_or_path": "/gpfs/project/niekerk/models/transformers/roberta-base",
+  "transformers_local_files_only": true,
+  "train_batch_size": 3,
+  "dev_batch_size": 6,
+  "test_batch_size": 3,
+  "run_nbt": true
+}
\ No newline at end of file
diff --git a/convlab/dst/setsumbt/configs/setsumbt_tm.json b/convlab/dst/setsumbt/configs/setsumbt_tm.json
new file mode 100644
index 0000000000000000000000000000000000000000..138f84c358067389d5f7b478ae94c3eb2aa90ea3
--- /dev/null
+++ b/convlab/dst/setsumbt/configs/setsumbt_tm.json
@@ -0,0 +1,11 @@
+{
+  "model_type": "SetSUMBT",
+  "dataset": "tm1+tm2+tm3",
+  "no_action_prediction": true,
+  "model_name_or_path": "/gpfs/project/niekerk/models/transformers/roberta-base",
+  "transformers_local_files_only": true,
+  "train_batch_size": 3,
+  "dev_batch_size": 8,
+  "test_batch_size": 8,
+  "run_nbt": true
+}
\ No newline at end of file
diff --git a/convlab/dst/setsumbt/dataset/__init__.py b/convlab/dst/setsumbt/dataset/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..17b1f93b3b39f95827cf6c09e8826383cd00b805
--- /dev/null
+++ b/convlab/dst/setsumbt/dataset/__init__.py
@@ -0,0 +1,2 @@
+from convlab.dst.setsumbt.dataset.unified_format import get_dataloader, change_batch_size
+from convlab.dst.setsumbt.dataset.ontology import get_slot_candidate_embeddings
diff --git a/convlab/dst/setsumbt/dataset/ontology.py b/convlab/dst/setsumbt/dataset/ontology.py
new file mode 100644
index 0000000000000000000000000000000000000000..ce150a61077ad61ab9d7af2ae3537971ae925f55
--- /dev/null
+++ b/convlab/dst/setsumbt/dataset/ontology.py
@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
+# Authors: Carel van Niekerk (niekerk@hhu.de)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Create Ontology Embeddings"""
+
+import json
+import os
+import random
+from copy import deepcopy
+
+import torch
+import numpy as np
+from tqdm import tqdm
+
+
+def set_seed(args):
+    """
+    Set random seeds
+
+    Args:
+        args (Arguments class): Arguments class containing seed and number of gpus to use
+    """
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    if args.n_gpu > 0:
+        torch.cuda.manual_seed_all(args.seed)
+
+
+def encode_candidates(candidates: list, args, tokenizer, embedding_model) -> torch.tensor:
+    """
+    Embed candidates
+
+    Args:
+        candidates (list): List of candidate descriptions
+        args (argument class): Runtime arguments
+        tokenizer (transformers Tokenizer): Tokenizer for the embedding_model
+        embedding_model (transformer Model): Transformer model for embedding candidate descriptions
+
+    Returns:
+        feats (torch.tensor): Embeddings of the candidate descriptions
+    """
+    # Tokenize candidate descriptions
+    feats = [tokenizer.encode_plus(val, add_special_tokens=True,max_length=args.max_candidate_len,
+                                   padding='max_length', truncation='longest_first')
+             for val in candidates]
+
+    # Encode tokenized descriptions
+    with torch.no_grad():
+        feats = {key: torch.tensor([f[key] for f in feats]).to(embedding_model.device) for key in feats[0]}
+        embedded_feats = embedding_model(**feats)  # [num_candidates, max_candidate_len, hidden_dim]
+
+    # Reduce/pool descriptions embeddings if required
+    if args.set_similarity:
+        feats = embedded_feats.last_hidden_state.detach().cpu()  # [num_candidates, max_candidate_len, hidden_dim]
+    elif args.candidate_pooling == 'cls':
+        feats = embedded_feats.pooler_output.detach().cpu()  # [num_candidates, hidden_dim]
+    elif args.candidate_pooling == "mean":
+        feats = embedded_feats.last_hidden_state.detach().cpu()
+        feats = feats.sum(1)
+        feats = torch.nn.functional.layer_norm(feats, feats.size())
+        feats = feats.detach().cpu()  # [num_candidates, hidden_dim]
+
+    return feats
+
+
+def get_slot_candidate_embeddings(ontology: dict, set_type: str, args, tokenizer, embedding_model, save_to_file=True):
+    """
+    Get embeddings for slots and candidates
+
+    Args:
+        ontology (dict): Dictionary of domain-slot pair descriptions and possible value sets
+        set_type (str): Subset of the dataset being used (train/validation/test)
+        args (argument class): Runtime arguments
+        tokenizer (transformers Tokenizer): Tokenizer for the embedding_model
+        embedding_model (transformer Model): Transormer model for embedding candidate descriptions
+        save_to_file (bool): Indication of whether to save information to file
+
+    Returns:
+        slots (dict): domain-slot description embeddings, candidate embeddings and requestable flag for each domain-slot
+    """
+    # Set model to eval mode
+    embedding_model.eval()
+
+    slots = dict()
+    for domain, subset in tqdm(ontology.items(), desc='Domains'):
+        for slot, slot_info in tqdm(subset.items(), desc='Slots'):
+            # Get description or use "domain-slot"
+            if args.use_descriptions:
+                desc = slot_info['description']
+            else:
+                desc = f"{domain}-{slot}"
+
+            # Encode domain-slot pair description
+            slot_emb = encode_candidates([desc], args, tokenizer, embedding_model)[0]
+
+            # Obtain possible value set and discard requestable value
+            values = deepcopy(slot_info['possible_values'])
+            is_requestable = False
+            if '?' in values:
+                is_requestable = True
+                values.remove('?')
+
+            # Encode value candidates
+            if values:
+                feats = encode_candidates(values, args, tokenizer, embedding_model)
+            else:
+                feats = None
+
+            # Store domain-slot description embeddings, candidate embeddings and requestabke flag for each domain-slot
+            slots[f"{domain}-{slot}"] = (slot_emb, feats, is_requestable)
+
+    # Dump tensors and ontology for use in training and evaluation
+    if save_to_file:
+        writer = os.path.join(args.output_dir, 'database', '%s.db' % set_type)
+        torch.save(slots, writer)
+
+        writer = open(os.path.join(args.output_dir, 'database', '%s.json' % set_type), 'w')
+        json.dump(ontology, writer, indent=2)
+        writer.close()
+    
+    return slots
diff --git a/convlab/dst/setsumbt/dataset/unified_format.py b/convlab/dst/setsumbt/dataset/unified_format.py
new file mode 100644
index 0000000000000000000000000000000000000000..1c3a68c3b2e627ac60f555a642dfa837734249b6
--- /dev/null
+++ b/convlab/dst/setsumbt/dataset/unified_format.py
@@ -0,0 +1,429 @@
+# -*- coding: utf-8 -*-
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
+# Authors: Carel van Niekerk (niekerk@hhu.de)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Convlab3 Unified Format Dialogue Datasets"""
+
+from copy import deepcopy
+
+import torch
+import transformers
+from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler
+from transformers.tokenization_utils import PreTrainedTokenizer
+from tqdm import tqdm
+
+from convlab.util import load_dataset
+from convlab.dst.setsumbt.dataset.utils import (get_ontology_slots, ontology_add_values,
+                                                get_values_from_data, ontology_add_requestable_slots,
+                                                get_requestable_slots, load_dst_data, extract_dialogues,
+                                                combine_value_sets, IdTensor)
+
+transformers.logging.set_verbosity_error()
+
+
+def convert_examples_to_features(data: list,
+                                 ontology: dict,
+                                 tokenizer: PreTrainedTokenizer,
+                                 max_turns: int = 12,
+                                 max_seq_len: int = 64) -> dict:
+    """
+    Convert dialogue examples to model input features and labels
+
+    Args:
+        data (list): List of all extracted dialogues
+        ontology (dict): Ontology dictionary containing slots, slot descriptions and
+        possible value sets including requests
+        tokenizer (PreTrainedTokenizer): Tokenizer for the encoder model used
+        max_turns (int): Maximum numbers of turns in a dialogue
+        max_seq_len (int): Maximum number of tokens in a dialogue turn
+
+    Returns:
+        features (dict): All inputs and labels required to train the model
+    """
+    features = dict()
+    ontology = deepcopy(ontology)
+
+    # Get encoder input for system, user utterance pairs
+    input_feats = []
+    for dial in tqdm(data):
+        dial_feats = []
+        for turn in dial:
+            if len(turn['system_utterance']) == 0:
+                usr = turn['user_utterance']
+                dial_feats.append(tokenizer.encode_plus(usr, add_special_tokens=True,
+                                                        max_length=max_seq_len, padding='max_length',
+                                                        truncation='longest_first'))
+            else:
+                usr = turn['user_utterance']
+                sys = turn['system_utterance']
+                dial_feats.append(tokenizer.encode_plus(usr, sys, add_special_tokens=True,
+                                                        max_length=max_seq_len, padding='max_length',
+                                                        truncation='longest_first'))
+            # Truncate
+            if len(dial_feats) >= max_turns:
+                break
+        input_feats.append(dial_feats)
+    del dial_feats
+
+    # Perform turn level padding
+    dial_ids = list()
+    for dial in data:
+        _ids = [turn['dialogue_id'] for turn in dial][:max_turns]
+        _ids += [''] * (max_turns - len(_ids))
+        dial_ids.append(_ids)
+    input_ids = [[turn['input_ids'] for turn in dial] + [[0] * max_seq_len] * (max_turns - len(dial))
+                 for dial in input_feats]
+    if 'token_type_ids' in input_feats[0][0]:
+        token_type_ids = [[turn['token_type_ids'] for turn in dial] + [[0] * max_seq_len] * (max_turns - len(dial))
+                          for dial in input_feats]
+    else:
+        token_type_ids = None
+    if 'attention_mask' in input_feats[0][0]:
+        attention_mask = [[turn['attention_mask'] for turn in dial] + [[0] * max_seq_len] * (max_turns - len(dial))
+                          for dial in input_feats]
+    else:
+        attention_mask = None
+    del input_feats
+
+    # Create torch data tensors
+    features['dialogue_ids'] = IdTensor(dial_ids)
+    features['input_ids'] = torch.tensor(input_ids)
+    features['token_type_ids'] = torch.tensor(token_type_ids) if token_type_ids else None
+    features['attention_mask'] = torch.tensor(attention_mask) if attention_mask else None
+    del input_ids, token_type_ids, attention_mask
+
+    # Extract all informable and requestable slots from the ontology
+    informable_slots = [f"{domain}-{slot}" for domain in ontology for slot in ontology[domain]
+                        if ontology[domain][slot]['possible_values']
+                        and ontology[domain][slot]['possible_values'] != ['?']]
+    requestable_slots = [f"{domain}-{slot}" for domain in ontology for slot in ontology[domain]
+                         if '?' in ontology[domain][slot]['possible_values']]
+    for slot in requestable_slots:
+        domain, slot = slot.split('-', 1)
+        ontology[domain][slot]['possible_values'].remove('?')
+
+    # Extract a list of domains from the ontology slots
+    domains = list(set(informable_slots + requestable_slots))
+    domains = list(set([slot.split('-', 1)[0] for slot in domains]))
+
+    # Create slot labels
+    for domslot in tqdm(informable_slots):
+        labels = []
+        for dial in data:
+            labs = []
+            for turn in dial:
+                value = [v for d, substate in turn['state'].items() for s, v in substate.items()
+                         if f'{d}-{s}' == domslot]
+                domain, slot = domslot.split('-', 1)
+                if turn['dataset_name'] in ontology[domain][slot]['dataset_names']:
+                    value = value[0] if value else 'none'
+                else:
+                    value = -1
+                if value in ontology[domain][slot]['possible_values'] and value != -1:
+                    value = ontology[domain][slot]['possible_values'].index(value)
+                else:
+                    value = -1  # If value is not in ontology then we do not penalise the model
+                labs.append(value)
+                if len(labs) >= max_turns:
+                    break
+            labs = labs + [-1] * (max_turns - len(labs))
+            labels.append(labs)
+
+        labels = torch.tensor(labels)
+        features['state_labels-' + domslot] = labels
+
+    # Create requestable slot labels
+    for domslot in tqdm(requestable_slots):
+        labels = []
+        for dial in data:
+            labs = []
+            for turn in dial:
+                domain, slot = domslot.split('-', 1)
+                if turn['dataset_name'] in ontology[domain][slot]['dataset_names']:
+                    acts = [act['intent'] for act in turn['dialogue_acts']
+                            if act['domain'] == domain and act['slot'] == slot]
+                    if acts:
+                        act_ = acts[0]
+                        if act_ == 'request':
+                            labs.append(1)
+                        else:
+                            labs.append(0)
+                    else:
+                        labs.append(0)
+                else:
+                    labs.append(-1)
+                if len(labs) >= max_turns:
+                    break
+            labs = labs + [-1] * (max_turns - len(labs))
+            labels.append(labs)
+
+        labels = torch.tensor(labels)
+        features['request_labels-' + domslot] = labels
+
+    # General act labels (1-goodbye, 2-thank you)
+    labels = []
+    for dial in tqdm(data):
+        labs = []
+        for turn in dial:
+            acts = [act['intent'] for act in turn['dialogue_acts'] if act['intent'] in ['bye', 'thank']]
+            if acts:
+                if 'bye' in acts:
+                    labs.append(1)
+                else:
+                    labs.append(2)
+            else:
+                labs.append(0)
+            if len(labs) >= max_turns:
+                break
+        labs = labs + [-1] * (max_turns - len(labs))
+        labels.append(labs)
+
+    labels = torch.tensor(labels)
+    features['general_act_labels'] = labels
+
+    # Create active domain labels
+    for domain in tqdm(domains):
+        labels = []
+        for dial in data:
+            labs = []
+            for turn in dial:
+                possible_domains = list()
+                for dom in ontology:
+                    for slt in ontology[dom]:
+                        if turn['dataset_name'] in ontology[dom][slt]['dataset_names']:
+                            possible_domains.append(dom)
+
+                if domain in turn['active_domains']:
+                    labs.append(1)
+                elif domain in possible_domains:
+                    labs.append(0)
+                else:
+                    labs.append(-1)
+                if len(labs) >= max_turns:
+                    break
+            labs = labs + [-1] * (max_turns - len(labs))
+            labels.append(labs)
+
+        labels = torch.tensor(labels)
+        features['active_domain_labels-' + domain] = labels
+
+    del labels
+
+    return features
+
+
+class UnifiedFormatDataset(Dataset):
+    """
+    Class for preprocessing, and storing data easily from the Convlab3 unified format.
+
+    Attributes:
+        dataset_dict (dict): Dictionary containing all the data in dataset
+        ontology (dict): Set of all domain-slot-value triplets in the ontology of the model
+        features (dict): Set of numeric features containing all inputs and labels formatted for the SetSUMBT model
+    """
+    def __init__(self,
+                 dataset_name: str,
+                 set_type: str,
+                 tokenizer: PreTrainedTokenizer,
+                 max_turns: int = 12,
+                 max_seq_len: int = 64,
+                 train_ratio: float = 1.0,
+                 seed: int = 0,
+                 data: dict = None,
+                 ontology: dict = None):
+        """
+        Args:
+            dataset_name (str): Name of the dataset/s to load (multiple to be seperated by +)
+            set_type (str): Subset of the dataset to load (train, validation or test)
+            tokenizer (transformers tokenizer): Tokenizer for the encoder model used
+            max_turns (int): Maximum numbers of turns in a dialogue
+            max_seq_len (int): Maximum number of tokens in a dialogue turn
+            train_ratio (float): Fraction of training data to use during training
+            seed (int): Seed governing random order of ids for subsampling
+            data (dict): Dataset features for loading from dict
+            ontology (dict): Ontology dict for loading from dict
+        """
+        if data is not None:
+            self.ontology = ontology
+            self.features = data
+        else:
+            if '+' in dataset_name:
+                dataset_args = [{"dataset_name": name} for name in dataset_name.split('+')]
+            else:
+                dataset_args = [{"dataset_name": dataset_name}]
+            self.dataset_dicts = [load_dataset(**dataset_args_) for dataset_args_ in dataset_args]
+            self.ontology = get_ontology_slots(dataset_name)
+            values = [get_values_from_data(dataset, set_type) for dataset in self.dataset_dicts]
+            self.ontology = ontology_add_values(self.ontology, combine_value_sets(values), set_type)
+            self.ontology = ontology_add_requestable_slots(self.ontology, get_requestable_slots(self.dataset_dicts))
+
+            if train_ratio != 1.0:
+                for dataset_args_ in dataset_args:
+                    dataset_args_['dial_ids_order'] = seed
+                    dataset_args_['split2ratio'] = {'train': train_ratio, 'validation': train_ratio}
+            self.dataset_dicts = [load_dataset(**dataset_args_) for dataset_args_ in dataset_args]
+
+            data = [load_dst_data(dataset_dict, data_split=set_type, speaker='all',
+                                  dialogue_acts=True, split_to_turn=False)
+                    for dataset_dict in self.dataset_dicts]
+            data_list = [data_[set_type] for data_ in data]
+
+            data = []
+            for idx, data_ in enumerate(data_list):
+                data += extract_dialogues(data_, dataset_args[idx]["dataset_name"])
+            self.features = convert_examples_to_features(data, self.ontology, tokenizer, max_turns, max_seq_len)
+
+    def __getitem__(self, index: int) -> dict:
+        """
+        Obtain dialogues with specific ids from dataset
+
+        Args:
+            index (int/list/tensor): Index/indices of dialogues to get
+
+        Returns:
+            features (dict): All inputs and labels required to train the model
+        """
+        return {label: self.features[label][index] for label in self.features
+                if self.features[label] is not None}
+
+    def __len__(self):
+        """
+        Get number of dialogues in the dataset
+
+        Returns:
+            len (int): Number of dialogues in the dataset object
+        """
+        return self.features['input_ids'].size(0)
+
+    def resample(self, size: int = None) -> Dataset:
+        """
+        Resample subset of the dataset
+
+        Args:
+            size (int): Number of dialogues to sample
+
+        Returns:
+            self (Dataset): Dataset object
+        """
+        # If no subset size is specified we resample a set with the same size as the full dataset
+        n_dialogues = self.__len__()
+        if not size:
+            size = n_dialogues
+
+        dialogues = torch.randint(low=0, high=n_dialogues, size=(size,))
+        self.features = self.__getitem__(dialogues)
+        
+        return self
+
+    def to(self, device):
+        """
+        Map all data to a device
+
+        Args:
+            device (torch device): Device to map data to
+        """
+        self.device = device
+        self.features = {label: self.features[label].to(device) for label in self.features
+                         if self.features[label] is not None}
+
+    @classmethod
+    def from_datadict(cls, data: dict, ontology: dict):
+        return cls(None, None, None, data=data, ontology=ontology)
+
+
+def get_dataloader(dataset_name: str,
+                   set_type: str,
+                   batch_size: int,
+                   tokenizer: PreTrainedTokenizer,
+                   max_turns: int = 12,
+                   max_seq_len: int = 64,
+                   device='cpu',
+                   resampled_size: int = None,
+                   train_ratio: float = 1.0,
+                   seed: int = 0) -> DataLoader:
+    '''
+    Module to create torch dataloaders
+
+    Args:
+        dataset_name (str): Name of the dataset to load
+        set_type (str): Subset of the dataset to load (train, validation or test)
+        batch_size (int): Batch size for the dataloader
+        tokenizer (transformers tokenizer): Tokenizer for the encoder model used
+        max_turns (int): Maximum numbers of turns in a dialogue
+        max_seq_len (int): Maximum number of tokens in a dialogue turn
+        device (torch device): Device to map data to
+        resampled_size (int): Number of dialogues to sample
+        train_ratio (float): Ratio of training data to use for training
+        seed (int): Seed governing random order of ids for subsampling
+
+    Returns:
+        loader (torch dataloader): Dataloader to train and evaluate the setsumbt model
+    '''
+    data = UnifiedFormatDataset(dataset_name, set_type, tokenizer, max_turns, max_seq_len, train_ratio=train_ratio,
+                                seed=seed)
+    data.to(device)
+
+    if resampled_size:
+        data = data.resample(resampled_size)
+
+    if set_type in ['test', 'validation']:
+        sampler = SequentialSampler(data)
+    else:
+        sampler = RandomSampler(data)
+    loader = DataLoader(data, sampler=sampler, batch_size=batch_size)
+
+    return loader
+
+
+def change_batch_size(loader: DataLoader, batch_size: int) -> DataLoader:
+    """
+    Change the batch size of a preloaded loader
+
+    Args:
+        loader (DataLoader): Dataloader to train and evaluate the setsumbt model
+        batch_size (int): Batch size for the dataloader
+
+    Returns:
+        loader (DataLoader): Dataloader to train and evaluate the setsumbt model
+    """
+
+    if 'SequentialSampler' in str(loader.sampler):
+        sampler = SequentialSampler(loader.dataset)
+    else:
+        sampler = RandomSampler(loader.dataset)
+    loader = DataLoader(loader.dataset, sampler=sampler, batch_size=batch_size)
+
+    return loader
+
+def dataloader_sample_dialogues(loader: DataLoader, sample_size: int) -> DataLoader:
+    """
+    Sample a subset of the dialogues in a dataloader
+
+    Args:
+        loader (DataLoader): Dataloader to train and evaluate the setsumbt model
+        sample_size (int): Number of dialogues to sample
+
+    Returns:
+        loader (DataLoader): Dataloader to train and evaluate the setsumbt model
+    """
+
+    dataset = loader.dataset.resample(sample_size)
+
+    if 'SequentialSampler' in str(loader.sampler):
+        sampler = SequentialSampler(dataset)
+    else:
+        sampler = RandomSampler(dataset)
+    loader = DataLoader(loader.dataset, sampler=sampler, batch_size=loader.batch_size)
+
+    return loader
diff --git a/convlab/dst/setsumbt/dataset/utils.py b/convlab/dst/setsumbt/dataset/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..96773d6b9b181925b3004e4971e440d9c7720bfb
--- /dev/null
+++ b/convlab/dst/setsumbt/dataset/utils.py
@@ -0,0 +1,442 @@
+# -*- coding: utf-8 -*-
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
+# Authors: Carel van Niekerk (niekerk@hhu.de)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Convlab3 Unified dataset data processing utilities"""
+
+import numpy
+import pdb
+
+from convlab.util import load_ontology, load_dst_data, load_nlu_data
+from convlab.dst.setsumbt.dataset.value_maps import VALUE_MAP, DOMAINS_MAP, QUANTITIES, TIME
+
+
+def get_ontology_slots(dataset_name: str) -> dict:
+    """
+    Function to extract slots, slot descriptions and categorical slot values from the dataset ontology.
+
+    Args:
+        dataset_name (str): Dataset name
+
+    Returns:
+        ontology_slots (dict): Ontology dictionary containing slots, descriptions and categorical slot values
+    """
+    dataset_names = dataset_name.split('+') if '+' in dataset_name else [dataset_name]
+    ontology_slots = dict()
+    for dataset_name in dataset_names:
+        ontology = load_ontology(dataset_name)
+        domains = [domain for domain in ontology['domains'] if domain not in ['booking', 'general']]
+        for domain in domains:
+            domain_name = DOMAINS_MAP.get(domain, domain.lower())
+            if domain_name not in ontology_slots:
+                ontology_slots[domain_name] = dict()
+            for slot, slot_info in ontology['domains'][domain]['slots'].items():
+                if slot not in ontology_slots[domain_name]:
+                    ontology_slots[domain_name][slot] = {'description': slot_info['description'],
+                                                         'possible_values': list(),
+                                                         'dataset_names': list()}
+                if slot_info['is_categorical']:
+                    ontology_slots[domain_name][slot]['possible_values'] += slot_info['possible_values']
+
+                ontology_slots[domain_name][slot]['possible_values'] = list(set(ontology_slots[domain_name][slot]['possible_values']))
+                ontology_slots[domain_name][slot]['dataset_names'].append(dataset_name)
+
+    return ontology_slots
+
+
+def get_values_from_data(dataset: dict, data_split: str = "train") -> dict:
+    """
+    Function to extract slots, slot descriptions and categorical slot values from the dataset ontology.
+
+    Args:
+        dataset (dict): Dataset dictionary obtained using the load_dataset function
+        data_split (str): Dataset split: train/validation/test
+
+    Returns:
+        value_sets (dict): Dictionary containing possible values obtained from dataset
+    """
+    data = load_dst_data(dataset, data_split='all', speaker='user')
+
+    # Remove test data from the data when building training/validation ontology
+    if data_split == 'train':
+        data = {key: itm for key, itm in data.items() if key == 'train'}
+    elif data_split == 'validation':
+        data = {key: itm for key, itm in data.items() if key in ['train', 'validation']}
+
+    value_sets = {}
+    for set_type, dataset in data.items():
+        for turn in dataset:
+            for domain, substate in turn['state'].items():
+                domain_name = DOMAINS_MAP.get(domain, domain.lower())
+                if domain_name not in value_sets:
+                    value_sets[domain_name] = {}
+                for slot, value in substate.items():
+                    if slot not in value_sets[domain_name]:
+                        value_sets[domain_name][slot] = []
+                    if value and value not in value_sets[domain_name][slot]:
+                        value_sets[domain_name][slot].append(value)
+            # pdb.set_trace()
+
+    return clean_values(value_sets)
+
+
+def combine_value_sets(value_sets: list) -> dict:
+    """
+    Function to combine value sets extracted from different datasets
+
+    Args:
+        value_sets (list): List of value sets extracted using the get_values_from_data function
+
+    Returns:
+        value_set (dict): Dictionary containing possible values obtained from datasets
+    """
+    value_set = value_sets[0]
+    for _value_set in value_sets[1:]:
+        for domain, domain_info in _value_set.items():
+            for slot, possible_values in domain_info.items():
+                if domain not in value_set:
+                    value_set[domain] = dict()
+                if slot not in value_set[domain]:
+                    value_set[domain][slot] = list()
+                value_set[domain][slot] += _value_set[domain][slot]
+                value_set[domain][slot] = list(set(value_set[domain][slot]))
+
+    return value_set
+
+
+def clean_values(value_sets: dict, value_map: dict = VALUE_MAP) -> dict:
+    """
+    Function to clean up the possible value sets extracted from the states in the dataset
+
+    Args:
+        value_sets (dict): Dictionary containing possible values obtained from dataset
+        value_map (dict): Label map to avoid duplication and typos in values
+
+    Returns:
+        clean_vals (dict): Cleaned Dictionary containing possible values obtained from dataset
+    """
+    clean_vals = {}
+    for domain, subset in value_sets.items():
+        clean_vals[domain] = {}
+        for slot, values in subset.items():
+            # Remove pipe separated values
+            values = list(set([val.split('|', 1)[0] for val in values]))
+
+            # Map values using value_map
+            for old, new in value_map.items():
+                values = list(set([val.replace(old, new) for val in values]))
+
+            # Remove empty and dontcare from possible value sets
+            values = [val for val in values if val not in ['', 'dontcare']]
+
+            # MultiWOZ specific value sets for quantity, time and boolean slots
+            if 'people' in slot or 'duration' in slot or 'stay' in slot:
+                values = QUANTITIES
+            elif 'time' in slot or 'leave' in slot or 'arrive' in slot:
+                values = TIME
+            elif 'parking' in slot or 'internet' in slot:
+                values = ['yes', 'no']
+
+            clean_vals[domain][slot] = values
+
+    return clean_vals
+
+
+def ontology_add_values(ontology_slots: dict, value_sets: dict, data_split: str = "train") -> dict:
+    """
+    Add value sets obtained from the dataset to the ontology
+    Args:
+        ontology_slots (dict): Ontology dictionary containing slots, descriptions and categorical slot values
+        value_sets (dict): Cleaned Dictionary containing possible values obtained from dataset
+        data_split (str): Dataset split: train/validation/test
+
+    Returns:
+        ontology_slots (dict): Ontology dictionary containing slots, slot descriptions and possible value sets
+    """
+    ontology = {}
+    for domain in sorted(ontology_slots):
+        if data_split in ['train', 'validation']:
+            if domain not in value_sets:
+                continue
+            possible_values = [v for slot, vals in value_sets[domain].items() for v in vals]
+            if len(possible_values) == 0:
+                continue
+        ontology[domain] = {}
+        for slot in sorted(ontology_slots[domain]):
+            if not ontology_slots[domain][slot]['possible_values']:
+                if domain in value_sets:
+                    if slot in value_sets[domain]:
+                        ontology_slots[domain][slot]['possible_values'] = value_sets[domain][slot]
+            if ontology_slots[domain][slot]['possible_values']:
+                values = sorted(ontology_slots[domain][slot]['possible_values'])
+                ontology_slots[domain][slot]['possible_values'] = ['none', 'do not care'] + values
+
+            ontology[domain][slot] = ontology_slots[domain][slot]
+
+    return ontology
+
+
+def get_requestable_slots(datasets: list) -> dict:
+    """
+    Function to get set of requestable slots from the dataset action labels.
+    Args:
+        datasets (dict): Dataset dictionary obtained using the load_dataset function
+
+    Returns:
+        slots (dict): Dictionary containing requestable domain-slot pairs
+    """
+    datasets = [load_nlu_data(dataset, data_split='all', speaker='user') for dataset in datasets]
+
+    slots = {}
+    for data in datasets:
+        for set_type, subset in data.items():
+            for turn in subset:
+                requests = [act for act in turn['dialogue_acts']['categorical'] if act['intent'] == 'request']
+                requests += [act for act in turn['dialogue_acts']['non-categorical'] if act['intent'] == 'request']
+                requests += [act for act in turn['dialogue_acts']['binary'] if act['intent'] == 'request']
+                requests = [(act['domain'], act['slot']) for act in requests]
+                for domain, slot in requests:
+                    domain_name = DOMAINS_MAP.get(domain, domain.lower())
+                    if domain_name not in slots:
+                        slots[domain_name] = []
+                    slots[domain_name].append(slot)
+
+    slots = {domain: list(set(slot_list)) for domain, slot_list in slots.items()}
+
+    return slots
+
+
+def ontology_add_requestable_slots(ontology_slots: dict, requestable_slots: dict) -> dict:
+    """
+    Add requestable slots obtained from the dataset to the ontology
+    Args:
+        ontology_slots (dict): Ontology dictionary containing slots, descriptions and categorical slot values
+        requestable_slots (dict): Dictionary containing requestable domain-slot pairs
+
+    Returns:
+        ontology_slots (dict): Ontology dictionary containing slots, slot descriptions and
+        possible value sets including requests
+    """
+    for domain in ontology_slots:
+        for slot in ontology_slots[domain]:
+            if domain in requestable_slots:
+                if slot in requestable_slots[domain]:
+                    ontology_slots[domain][slot]['possible_values'].append('?')
+
+    return ontology_slots
+
+
+def extract_turns(dialogue: list, dataset_name: str, dialogue_id: str) -> list:
+    """
+    Extract the required information from the data provided by unified loader
+    Args:
+        dialogue (list): List of turns within a dialogue
+        dataset_name (str): Name of the dataset to which the dialogue belongs
+        dialogue_str (str): ID of the dialogue
+
+    Returns:
+        turns (list): List of turns within a dialogue
+    """
+    turns = []
+    turn_info = {}
+    for turn in dialogue:
+        if turn['speaker'] == 'system':
+            turn_info['system_utterance'] = turn['utterance']
+
+        # System utterance in the first turn is always empty as conversation is initiated by the user
+        if turn['utt_idx'] == 1:
+            turn_info['system_utterance'] = ''
+
+        if turn['speaker'] == 'user':
+            turn_info['user_utterance'] = turn['utterance']
+
+            # Inform acts not required by model
+            turn_info['dialogue_acts'] = [act for act in turn['dialogue_acts']['categorical']
+                                          if act['intent'] not in ['inform']]
+            turn_info['dialogue_acts'] += [act for act in turn['dialogue_acts']['non-categorical']
+                                           if act['intent'] not in ['inform']]
+            turn_info['dialogue_acts'] += [act for act in turn['dialogue_acts']['binary']
+                                           if act['intent'] not in ['inform']]
+
+            turn_info['state'] = turn['state']
+            turn_info['dataset_name'] = dataset_name
+            turn_info['dialogue_id'] = dialogue_id
+
+        if 'system_utterance' in turn_info and 'user_utterance' in turn_info:
+            turns.append(turn_info)
+            turn_info = {}
+
+    return turns
+
+
+def clean_states(turns: list) -> list:
+    """
+    Clean the state within each turn of a dialogue (cleaning values and mapping to options used in ontology)
+    Args:
+        turns (list): List of turns within a dialogue
+
+    Returns:
+        clean_turns (list): List of turns within a dialogue
+    """
+    clean_turns = []
+    for turn in turns:
+        clean_state = {}
+        clean_acts = []
+        for act in turn['dialogue_acts']:
+            domain = act['domain']
+            act['domain'] = DOMAINS_MAP.get(domain, domain.lower())
+            clean_acts.append(act)
+        for domain, subset in turn['state'].items():
+            domain_name = DOMAINS_MAP.get(domain, domain.lower())
+            clean_state[domain_name] = {}
+            for slot, value in subset.items():
+                # Remove pipe separated values
+                value = value.split('|', 1)[0]
+
+                # Map values using value_map
+                for old, new in VALUE_MAP.items():
+                    value = value.replace(old, new)
+
+                # Map dontcare to "do not care" and empty to 'none'
+                value = value.replace('dontcare', 'do not care')
+                value = value if value else 'none'
+
+                # Map quantity values to the integer quantity value
+                if 'people' in slot or 'duration' in slot or 'stay' in slot:
+                    try:
+                        if value not in ['do not care', 'none']:
+                            value = int(value)
+                            value = str(value) if value < 10 else QUANTITIES[-1]
+                    except:
+                        value = value
+                # Map time values to the most appropriate value in the standard time set
+                elif 'time' in slot or 'leave' in slot or 'arrive' in slot:
+                    try:
+                        if value not in ['do not care', 'none']:
+                            # Strip after/before from time value
+                            value = value.replace('after ', '').replace('before ', '')
+                            # Extract hours and minutes from different possible formats
+                            if ':' not in value and len(value) == 4:
+                                h, m = value[:2], value[2:]
+                            elif len(value) == 1:
+                                h = int(value)
+                                m = 0
+                            elif 'pm' in value:
+                                h = int(value.replace('pm', '')) + 12
+                                m = 0
+                            elif 'am' in value:
+                                h = int(value.replace('pm', ''))
+                                m = 0
+                            elif ':' in value:
+                                h, m = value.split(':')
+                            elif ';' in value:
+                                h, m = value.split(';')
+                            # Map to closest 5 minutes
+                            if int(m) % 5 != 0:
+                                m = round(int(m) / 5) * 5
+                                h = int(h)
+                                if m == 60:
+                                    m = 0
+                                    h += 1
+                                if h >= 24:
+                                    h -= 24
+                            # Set in standard 24 hour format
+                            h, m = int(h), int(m)
+                            value = '%02i:%02i' % (h, m)
+                    except:
+                        value = value
+                # Map boolean slots to yes/no value
+                elif 'parking' in slot or 'internet' in slot:
+                    if value not in ['do not care', 'none']:
+                        if value == 'free':
+                            value = 'yes'
+                        elif True in [v in value.lower() for v in ['yes', 'no']]:
+                            value = [v for v in ['yes', 'no'] if v in value][0]
+
+                clean_state[domain_name][slot] = value
+        turn['state'] = clean_state
+        turn['dialogue_acts'] = clean_acts
+        clean_turns.append(turn)
+
+    return clean_turns
+
+
+def get_active_domains(turns: list) -> list:
+    """
+    Get active domains at each turn in a dialogue
+    Args:
+        turns (list): List of turns within a dialogue
+
+    Returns:
+        turns (list): List of turns within a dialogue
+    """
+    for turn_id in range(len(turns)):
+        # At first turn all domains with not none values in the state are active
+        if turn_id == 0:
+            domains = [d for d, substate in turns[turn_id]['state'].items() for s, v in substate.items() if v != 'none']
+            domains += [act['domain'] for act in turns[turn_id]['dialogue_acts'] if act['domain'] in turns[turn_id]['state']]
+            domains = [DOMAINS_MAP.get(domain, domain.lower()) for domain in domains]
+            turns[turn_id]['active_domains'] = list(set(domains))
+        else:
+            # Use changes in domains to identify active domains
+            domains = []
+            for domain, substate in turns[turn_id]['state'].items():
+                domain_name = DOMAINS_MAP.get(domain, domain.lower())
+                for slot, value in substate.items():
+                    if value != turns[turn_id - 1]['state'][domain][slot]:
+                        val = value
+                    else:
+                        val = 'none'
+                    if value == 'none':
+                        val = 'none'
+                    if val != 'none':
+                        domains.append(domain_name)
+            # Add all domains activated by a user action
+            domains += [act['domain'] for act in turns[turn_id]['dialogue_acts']
+                        if act['domain'] in turns[turn_id]['state']]
+            turns[turn_id]['active_domains'] = list(set(domains))
+
+    return turns
+
+
+class IdTensor:
+    def __init__(self, values):
+        self.values = numpy.array(values)
+
+    def __getitem__(self, index: int):
+        return self.values[index].tolist()
+
+    def to(self, device):
+        return self
+
+
+def extract_dialogues(data: list, dataset_name: str) -> list:
+    """
+    Extract all dialogues from dataset
+    Args:
+        data (list): List of all dialogues in a subset of the data
+        dataset_name (str): Name of the dataset to which the dialogues belongs
+
+    Returns:
+        dialogues (list): List of all extracted dialogues
+    """
+    dialogues = []
+    for dial in data:
+        dial_id = dial['dialogue_id']
+        turns = extract_turns(dial['turns'], dataset_name, dial_id)
+        turns = clean_states(turns)
+        turns = get_active_domains(turns)
+        dialogues.append(turns)
+
+    return dialogues
diff --git a/convlab/dst/setsumbt/dataset/value_maps.py b/convlab/dst/setsumbt/dataset/value_maps.py
new file mode 100644
index 0000000000000000000000000000000000000000..619600a7b0a57096918058ff117aa2ca5aac864a
--- /dev/null
+++ b/convlab/dst/setsumbt/dataset/value_maps.py
@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
+# Authors: Carel van Niekerk (niekerk@hhu.de)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Convlab3 Unified dataset value maps"""
+
+
+# MultiWOZ specific label map to avoid duplication and typos in values
+VALUE_MAP = {'guesthouse': 'guest house', 'belfry': 'belfray', '-': ' ', '&': 'and', 'b and b': 'bed and breakfast',
+             'cityroomz': 'city roomz', '  ': ' ', 'acorn house': 'acorn guest house', 'marriot': 'marriott',
+             'worth house': 'the worth house', 'alesbray lodge guest house': 'aylesbray lodge',
+             'huntingdon hotel': 'huntingdon marriott hotel', 'huntingd': 'huntingdon marriott hotel',
+             'jamaicanchinese': 'chinese', 'barbequemodern european': 'modern european',
+             'north americanindian': 'north american', 'caribbeanindian': 'indian', 'sheeps': "sheep's"}
+
+
+# Domain map for SGD and TM Data
+DOMAINS_MAP = {'Alarm_1': 'alarm', 'Banks_1': 'banks', 'Banks_2': 'banks', 'Buses_1': 'bus', 'Buses_2': 'bus',
+               'Buses_3': 'bus', 'Calendar_1': 'calendar', 'Events_1': 'events', 'Events_2': 'events',
+               'Events_3': 'events', 'Flights_1': 'flights', 'Flights_2': 'flights', 'Flights_3': 'flights',
+               'Flights_4': 'flights', 'Homes_1': 'homes', 'Homes_2': 'homes', 'Hotels_1': 'hotel',
+               'Hotels_2': 'hotel', 'Hotels_3': 'hotel', 'Hotels_4': 'hotel', 'Media_1': 'media',
+               'Media_2': 'media', 'Media_3': 'media', 'Messaging_1': 'messaging', 'Movies_1': 'movies',
+               'Movies_2': 'movies', 'Movies_3': 'movies', 'Music_1': 'music', 'Music_2': 'music', 'Music_3': 'music',
+               'Payment_1': 'payment', 'RentalCars_1': 'rentalcars', 'RentalCars_2': 'rentalcars',
+               'RentalCars_3': 'rentalcars', 'Restaurants_1': 'restaurant', 'Restaurants_2': 'restaurant',
+               'RideSharing_1': 'ridesharing', 'RideSharing_2': 'ridesharing', 'Services_1': 'services',
+               'Services_2': 'services', 'Services_3': 'services', 'Services_4': 'services', 'Trains_1': 'train',
+               'Travel_1': 'travel', 'Weather_1': 'weather', 'movie_ticket': 'movies',
+               'restaurant_reservation': 'restaurant', 'coffee_ordering': 'coffee', 'pizza_ordering': 'takeout',
+               'auto_repair': 'car_repairs', 'flights': 'flights', 'food-ordering': 'takeout', 'hotels': 'hotel',
+               'movies': 'movies', 'music': 'music', 'restaurant-search': 'restaurant', 'sports': 'sports',
+               'movie': 'movies'}
+
+
+# Generic value sets for quantity and time slots
+QUANTITIES = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10 or more']
+TIME = [[(i, j) for i in range(24)] for j in range(0, 60, 5)]
+TIME = ['%02i:%02i' % t for l in TIME for t in l]
\ No newline at end of file
diff --git a/convlab/dst/setsumbt/distillation_setup.py b/convlab/dst/setsumbt/distillation_setup.py
index e0d87bb964041cae19d58351cd6b31f6d836f125..2279e22265ea417ebe9a13e63837a625f858e73d 100644
--- a/convlab/dst/setsumbt/distillation_setup.py
+++ b/convlab/dst/setsumbt/distillation_setup.py
@@ -1,53 +1,51 @@
+# -*- coding: utf-8 -*-
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
+# Authors: Carel van Niekerk (niekerk@hhu.de)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Get ensemble predictions and build distillation dataloaders"""
+
 from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
 import os
+import json
 
 import torch
-import transformers
-from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler
-from transformers import RobertaConfig, BertConfig
+from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
 from tqdm import tqdm
 
-import convlab
-from convlab.dst.setsumbt.multiwoz.dataset.multiwoz21 import EnsembleMultiWoz21
+from convlab.dst.setsumbt.dataset.unified_format import UnifiedFormatDataset, change_batch_size
 from convlab.dst.setsumbt.modeling import EnsembleSetSUMBT
+from convlab.dst.setsumbt.modeling import training
 
-DEVICE = 'cuda'
-
-
-def args_parser():
-    parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
-    parser.add_argument('--model_path', type=str)
-    parser.add_argument('--model_type', type=str)
-    parser.add_argument('--set_type', type=str)
-    parser.add_argument('--batch_size', type=int)
-    parser.add_argument('--ensemble_size', type=int)
-    parser.add_argument('--reduction', type=str, default='mean')
-    parser.add_argument('--get_ensemble_distributions', action='store_true')
-    parser.add_argument('--build_dataloaders', action='store_true')
-    
-    return parser.parse_args()
-
-
-def main():
-    args = args_parser()
+DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
 
-    if args.get_ensemble_distributions:
-        get_ensemble_distributions(args)
-    elif args.build_dataloaders:
-        path = os.path.join(args.model_path, 'dataloaders', f'{args.set_type}.data')
-        data = torch.load(path)
-        loader = get_loader(data, args.set_type, args.batch_size)
 
-        path = os.path.join(args.model_path, 'dataloaders', f'{args.set_type}.dataloader')
-        torch.save(loader, path)
-    else:
-        raise NameError("NotImplemented")
+def get_loader(data: dict, ontology: dict, set_type: str = 'train', batch_size: int = 3) -> DataLoader:
+    """
+    Build dataloader from ensemble prediction data
 
+    Args:
+        data: Dictionary of ensemble predictions
+        ontology: Data ontology
+        set_type: Data subset (train/validation/test)
+        batch_size: Number of dialogues per batch
 
-def get_loader(data, set_type='train', batch_size=3):
+    Returns:
+        loader: Data loader object
+    """
     data = flatten_data(data)
     data = do_label_padding(data)
-    data = EnsembleMultiWoz21(data)
+    data = UnifiedFormatDataset.from_datadict(data, ontology)
     if set_type == 'train':
         sampler = RandomSampler(data)
     else:
@@ -57,7 +55,16 @@ def get_loader(data, set_type='train', batch_size=3):
     return loader
 
 
-def do_label_padding(data):
+def do_label_padding(data: dict) -> dict:
+    """
+    Add padding to the ensemble predictions (used as labels in distillation)
+
+    Args:
+        data: Dictionary of ensemble predictions
+
+    Returns:
+        data: Padded ensemble predictions
+    """
     if 'attention_mask' in data:
         dialogs, turns = torch.where(data['attention_mask'].sum(-1) == 0.0)
     else:
@@ -70,13 +77,17 @@ def do_label_padding(data):
     return data
 
 
-map_dict = {'belief_state': 'belief', 'greeting_act_belief': 'goodbye_belief',
-            'state_labels': 'labels', 'request_labels': 'request',
-            'domain_labels': 'active', 'greeting_labels': 'goodbye'}
-def flatten_data(data):
+def flatten_data(data: dict) -> dict:
+    """
+    Map data to flattened feature format used in training
+    Args:
+        data: Ensemble prediction data
+
+    Returns:
+        data: Flattened ensemble prediction data
+    """
     data_new = dict()
     for label, feats in data.items():
-        label = map_dict.get(label, label)
         if type(feats) == dict:
             for label_, feats_ in feats.items():
                 data_new[label + '-' + label_] = feats_
@@ -87,13 +98,11 @@ def flatten_data(data):
 
 
 def get_ensemble_distributions(args):
-    if args.model_type == 'roberta':
-        config = RobertaConfig
-    elif args.model_type == 'bert':
-        config = BertConfig
-    config = config.from_pretrained(args.model_path)
-    config.ensemble_size = args.ensemble_size
-
+    """
+    Load data and get ensemble predictions
+    Args:
+        args: Runtime arguments
+    """
     device = DEVICE
 
     model = EnsembleSetSUMBT.from_pretrained(args.model_path)
@@ -107,16 +116,10 @@ def get_ensemble_distributions(args):
     dataloader = torch.load(dataloader)
     database = torch.load(database)
 
-    # Get slot and value embeddings
-    slots = {slot: val for slot, val in database.items()}
-    values = {slot: val[1] for slot, val in database.items()}
-    del database
+    if dataloader.batch_size != args.batch_size:
+        dataloader = change_batch_size(dataloader, args.batch_size)
 
-    # Load model ontology
-    model.add_slot_candidates(slots)
-    for slot in model.informable_slot_ids:
-        model.add_value_candidates(slot, values[slot], replace=True)
-    del slots, values
+    training.set_ontology_embeddings(model, database)
 
     print('Environment set up.')
 
@@ -125,18 +128,24 @@ def get_ensemble_distributions(args):
     attention_mask = []
     state_labels = {slot: [] for slot in model.informable_slot_ids}
     request_labels = {slot: [] for slot in model.requestable_slot_ids}
-    domain_labels = {domain: [] for domain in model.domain_ids}
-    greeting_labels = []
+    active_domain_labels = {domain: [] for domain in model.domain_ids}
+    general_act_labels = []
+
+    is_noisy = [] if 'is_noisy' in dataloader.dataset.features else None
+
     belief_state = {slot: [] for slot in model.informable_slot_ids}
-    request_belief = {slot: [] for slot in model.requestable_slot_ids}
-    domain_belief = {domain: [] for domain in model.domain_ids}
-    greeting_act_belief = []
+    request_probs = {slot: [] for slot in model.requestable_slot_ids}
+    active_domain_probs = {domain: [] for domain in model.domain_ids}
+    general_act_probs = []
     model.eval()
     for batch in tqdm(dataloader, desc='Batch:'):
         ids = batch['input_ids']
         tt_ids = batch['token_type_ids'] if 'token_type_ids' in batch else None
         mask = batch['attention_mask'] if 'attention_mask' in batch else None
 
+        if 'is_noisy' in batch:
+            is_noisy.append(batch['is_noisy'])
+
         input_ids.append(ids)
         token_type_ids.append(tt_ids)
         attention_mask.append(mask)
@@ -146,61 +155,123 @@ def get_ensemble_distributions(args):
         mask = mask.to(device) if mask is not None else None
 
         for slot in state_labels:
-            state_labels[slot].append(batch['labels-' + slot])
-        if model.config.predict_intents:
+            state_labels[slot].append(batch['state_labels-' + slot])
+        if model.config.predict_actions:
             for slot in request_labels:
-                request_labels[slot].append(batch['request-' + slot])
-            for domain in domain_labels:
-                domain_labels[domain].append(batch['active-' + domain])
-            greeting_labels.append(batch['goodbye'])
+                request_labels[slot].append(batch['request_labels-' + slot])
+            for domain in active_domain_labels:
+                active_domain_labels[domain].append(batch['active_domain_labels-' + domain])
+            general_act_labels.append(batch['general_act_labels'])
 
         with torch.no_grad():
-            p, p_req, p_dom, p_bye, _ = model(ids, mask, tt_ids,
-                                            reduction=args.reduction)
+            p, p_req, p_dom, p_gen, _ = model(ids, mask, tt_ids, reduction=args.reduction)
 
         for slot in belief_state:
             belief_state[slot].append(p[slot].cpu())
-        if model.config.predict_intents:
-            for slot in request_belief:
-                request_belief[slot].append(p_req[slot].cpu())
-            for domain in domain_belief:
-                domain_belief[domain].append(p_dom[domain].cpu())
-            greeting_act_belief.append(p_bye.cpu())
+        if model.config.predict_actions:
+            for slot in request_probs:
+                request_probs[slot].append(p_req[slot].cpu())
+            for domain in active_domain_probs:
+                active_domain_probs[domain].append(p_dom[domain].cpu())
+            general_act_probs.append(p_gen.cpu())
     
     input_ids = torch.cat(input_ids, 0) if input_ids[0] is not None else None
     token_type_ids = torch.cat(token_type_ids, 0) if token_type_ids[0] is not None else None
     attention_mask = torch.cat(attention_mask, 0) if attention_mask[0] is not None else None
+    is_noisy = torch.cat(is_noisy, 0) if is_noisy is not None else None
 
     state_labels = {slot: torch.cat(l, 0) for slot, l in state_labels.items()}
-    if model.config.predict_intents:
+    if model.config.predict_actions:
         request_labels = {slot: torch.cat(l, 0) for slot, l in request_labels.items()}
-        domain_labels = {domain: torch.cat(l, 0) for domain, l in domain_labels.items()}
-        greeting_labels = torch.cat(greeting_labels, 0)
+        active_domain_labels = {domain: torch.cat(l, 0) for domain, l in active_domain_labels.items()}
+        general_act_labels = torch.cat(general_act_labels, 0)
     
     belief_state = {slot: torch.cat(p, 0) for slot, p in belief_state.items()}
-    if model.config.predict_intents:
-        request_belief = {slot: torch.cat(p, 0) for slot, p in request_belief.items()}
-        domain_belief = {domain: torch.cat(p, 0) for domain, p in domain_belief.items()}
-        greeting_act_belief = torch.cat(greeting_act_belief, 0)
+    if model.config.predict_actions:
+        request_probs = {slot: torch.cat(p, 0) for slot, p in request_probs.items()}
+        active_domain_probs = {domain: torch.cat(p, 0) for domain, p in active_domain_probs.items()}
+        general_act_probs = torch.cat(general_act_probs, 0)
 
     data = {'input_ids': input_ids}
     if token_type_ids is not None:
         data['token_type_ids'] = token_type_ids
     if attention_mask is not None:
         data['attention_mask'] = attention_mask
+    if is_noisy is not None:
+        data['is_noisy'] = is_noisy
     data['state_labels'] = state_labels
     data['belief_state'] = belief_state
-    if model.config.predict_intents:
+    if model.config.predict_actions:
         data['request_labels'] = request_labels
-        data['domain_labels'] = domain_labels
-        data['greeting_labels'] = greeting_labels
-        data['request_belief'] = request_belief
-        data['domain_belief'] = domain_belief
-        data['greeting_act_belief'] = greeting_act_belief
+        data['active_domain_labels'] = active_domain_labels
+        data['general_act_labels'] = general_act_labels
+        data['request_probs'] = request_probs
+        data['active_domain_probs'] = active_domain_probs
+        data['general_act_probs'] = general_act_probs
 
     file = os.path.join(args.model_path, 'dataloaders', f'{args.set_type}.data')
     torch.save(data, file)
 
 
+def ensemble_distribution_data_to_predictions_format(model_path: str, set_type: str):
+    """
+    Convert ensemble predictions to predictions file format.
+
+    Args:
+        model_path: Path to ensemble location.
+        set_type: Evaluation dataset (train/dev/test).
+    """
+    data = torch.load(os.path.join(model_path, 'dataloaders', f"{set_type}.data"))
+
+    # Get oracle labels
+    if 'request_probs' in data:
+        data_new = {'state_labels': data['state_labels'],
+                    'request_labels': data['request_labels'],
+                    'active_domain_labels': data['active_domain_labels'],
+                    'general_act_labels': data['general_act_labels']}
+    else:
+        data_new = {'state_labels': data['state_labels']}
+
+    # Marginalising across ensemble distributions
+    data_new['belief_states'] = {slot: distribution.mean(-2) for slot, distribution in data['belief_state'].items()}
+    if 'request_probs' in data:
+        data_new['request_probs'] = {slot: distribution.mean(-1)
+                                     for slot, distribution in data['request_probs'].items()}
+        data_new['active_domain_probs'] = {domain: distribution.mean(-1)
+                                           for domain, distribution in data['active_domain_probs'].items()}
+        data_new['general_act_probs'] = data['general_act_probs'].mean(-2)
+
+    # Save predictions file
+    predictions_dir = os.path.join(model_path, 'predictions')
+    if not os.path.exists(predictions_dir):
+        os.mkdir(predictions_dir)
+    torch.save(data_new, os.path.join(predictions_dir, f"{set_type}.predictions"))
+
+
 if __name__ == "__main__":
-    main()
+    parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
+    parser.add_argument('--model_path', type=str)
+    parser.add_argument('--set_type', type=str)
+    parser.add_argument('--batch_size', type=int, default=3)
+    parser.add_argument('--reduction', type=str, default='none')
+    parser.add_argument('--get_ensemble_distributions', action='store_true')
+    parser.add_argument('--convert_distributions_to_predictions', action='store_true')
+    parser.add_argument('--build_dataloaders', action='store_true')
+    args = parser.parse_args()
+
+    if args.get_ensemble_distributions:
+        get_ensemble_distributions(args)
+    if args.convert_distributions_to_predictions:
+        ensemble_distribution_data_to_predictions_format(args.model_path, args.set_type)
+    if args.build_dataloaders:
+        path = os.path.join(args.model_path, 'dataloaders', f'{args.set_type}.data')
+        data = torch.load(path)
+
+        reader = open(os.path.join(args.model_path, 'database', f'{args.set_type}.json'), 'r')
+        ontology = json.load(reader)
+        reader.close()
+
+        loader = get_loader(data, ontology, args.set_type, args.batch_size)
+
+        path = os.path.join(args.model_path, 'dataloaders', f'{args.set_type}.dataloader')
+        torch.save(loader, path)
diff --git a/convlab/dst/setsumbt/do/calibration.py b/convlab/dst/setsumbt/do/calibration.py
deleted file mode 100644
index 27ee058eca882ce7e10937f9640d143b88e57f5e..0000000000000000000000000000000000000000
--- a/convlab/dst/setsumbt/do/calibration.py
+++ /dev/null
@@ -1,481 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2021 DSML Group, Heinrich Heine University, Düsseldorf
-# Authors: Carel van Niekerk (niekerk@hhu.de)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Run SetSUMBT Calibration"""
-
-import logging
-import random
-import os
-from shutil import copy2 as copy
-
-import torch
-from transformers import (BertModel, BertConfig, BertTokenizer,
-                          RobertaModel, RobertaConfig, RobertaTokenizer,
-                          AdamW, get_linear_schedule_with_warmup)
-from tqdm import tqdm, trange
-from tensorboardX import SummaryWriter
-from torch.distributions import Categorical
-
-from convlab.dst.setsumbt.modeling.bert_nbt import BertSetSUMBT
-from convlab.dst.setsumbt.modeling.roberta_nbt import RobertaSetSUMBT
-from convlab.dst.setsumbt.multiwoz import multiwoz21
-from convlab.dst.setsumbt.multiwoz import ontology as embeddings
-from convlab.dst.setsumbt.utils import get_args, upload_local_directory_to_gcs, update_args
-from convlab.dst.setsumbt.modeling import calibration_utils
-from convlab.dst.setsumbt.modeling import ensemble_utils
-from convlab.dst.setsumbt.loss.ece import ece, jg_ece, l2_acc
-
-
-# Datasets
-DATASETS = {
-    'multiwoz21': multiwoz21
-}
-
-MODELS = {
-    'bert': (BertSetSUMBT, BertModel, BertConfig, BertTokenizer),
-    'roberta': (RobertaSetSUMBT, RobertaModel, RobertaConfig, RobertaTokenizer)
-}
-
-
-def main(args=None, config=None):
-    # Get arguments
-    if args is None:
-        args, config = get_args(MODELS)
-
-    # Select Dataset object
-    if args.dataset in DATASETS:
-        Dataset = DATASETS[args.dataset]
-    else:
-        raise NameError('NotImplemented')
-
-    if args.model_type in MODELS:
-        SetSumbtModel, CandidateEncoderModel, ConfigClass, Tokenizer = MODELS[args.model_type]
-    else:
-        raise NameError('NotImplemented')
-
-    # Set up output directory
-    OUTPUT_DIR = args.output_dir
-    if not os.path.exists(OUTPUT_DIR):
-        os.mkdir(OUTPUT_DIR)
-    args.output_dir = OUTPUT_DIR
-    if not os.path.exists(os.path.join(OUTPUT_DIR, 'predictions')):
-        os.mkdir(os.path.join(OUTPUT_DIR, 'predictions'))
-
-    paths = os.listdir(args.output_dir) if os.path.exists(
-        args.output_dir) else []
-    if 'pytorch_model.bin' in paths and 'config.json' in paths:
-        args.model_name_or_path = args.output_dir
-        config = ConfigClass.from_pretrained(args.model_name_or_path)
-    else:
-        paths = os.listdir(args.output_dir) if os.path.exists(
-            args.output_dir) else []
-        paths = [os.path.join(args.output_dir, p)
-                 for p in paths if 'checkpoint-' in p]
-        if paths:
-            paths = paths[0]
-            args.model_name_or_path = paths
-            config = ConfigClass.from_pretrained(args.model_name_or_path)
-
-    if args.ensemble_size > 0:
-        paths = os.listdir(args.output_dir) if os.path.exists(
-            args.output_dir) else []
-        paths = [os.path.join(args.output_dir, p)
-                 for p in paths if 'ensemble_' in p]
-        if paths:
-            args.model_name_or_path = args.output_dir
-            config = ConfigClass.from_pretrained(args.model_name_or_path)
-
-    args = update_args(args, config)
-
-    # Set up data directory
-    DATA_DIR = args.data_dir
-    Dataset.set_datadir(DATA_DIR)
-    embeddings.set_datadir(DATA_DIR)
-
-    if args.shrink_active_domains and args.dataset == 'multiwoz21':
-        Dataset.set_active_domains(
-            ['attraction', 'hotel', 'restaurant', 'taxi', 'train'])
-
-    # Download and preprocess
-    Dataset.create_examples(
-        args.max_turn_len, args.predict_intents, args.force_processing)
-
-    # Create logger
-    global logger
-    logger = logging.getLogger(__name__)
-    logger.setLevel(logging.INFO)
-
-    formatter = logging.Formatter(
-        '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-
-    if 'stream' not in args.logging_path:
-        fh = logging.FileHandler(args.logging_path)
-        fh.setLevel(logging.INFO)
-        fh.setFormatter(formatter)
-        logger.addHandler(fh)
-    else:
-        ch = logging.StreamHandler()
-        ch.setLevel(level=logging.INFO)
-        ch.setFormatter(formatter)
-        logger.addHandler(ch)
-
-    # Get device
-    if torch.cuda.is_available() and args.n_gpu > 0:
-        device = torch.device('cuda')
-    else:
-        device = torch.device('cpu')
-        args.n_gpu = 0
-
-    if args.n_gpu == 0:
-        args.fp16 = False
-
-    # Set up model training/evaluation
-    calibration.set_logger(logger, None)
-    calibration.set_seed(args)
-
-    if args.ensemble_size > 0:
-        ensemble.set_logger(logger, tb_writer)
-        ensemble_utils.set_seed(args)
-
-    # Perform tasks
-
-    if os.path.exists(os.path.join(OUTPUT_DIR, 'predictions', 'test.predictions')):
-        pred = torch.load(os.path.join(
-            OUTPUT_DIR, 'predictions', 'test.predictions'))
-        labels = pred['labels']
-        belief_states = pred['belief_states']
-        if 'request_labels' in pred:
-            request_labels = pred['request_labels']
-            request_belief = pred['request_belief']
-            domain_labels = pred['domain_labels']
-            domain_belief = pred['domain_belief']
-            greeting_labels = pred['greeting_labels']
-            greeting_belief = pred['greeting_belief']
-        else:
-            request_belief = None
-        del pred
-    elif args.ensemble_size > 0:
-        # Get training batch loaders and ontology embeddings
-        if os.path.exists(os.path.join(OUTPUT_DIR, 'database', 'test.db')):
-            test_slots = torch.load(os.path.join(
-                OUTPUT_DIR, 'database', 'test.db'))
-        else:
-            # Create Tokenizer and embedding model for Data Loaders and ontology
-            encoder = CandidateEncoderModel.from_pretrained(
-                config.candidate_embedding_model_name)
-            tokenizer = Tokenizer(config.candidate_embedding_model_name)
-            embeddings.get_slot_candidate_embeddings(
-                'test', args, tokenizer, encoder)
-            test_slots = torch.load(os.path.join(
-                OUTPUT_DIR, 'database', 'test.db'))
-
-        exists = False
-        if os.path.exists(os.path.join(OUTPUT_DIR, 'dataloaders', 'test.dataloader')):
-            test_dataloader = torch.load(os.path.join(
-                OUTPUT_DIR, 'dataloaders', 'test.dataloader'))
-            if test_dataloader.batch_size == args.test_batch_size:
-                exists = True
-        if not exists:
-            tokenizer = Tokenizer(config.candidate_embedding_model_name)
-            test_dataloader = Dataset.get_dataloader('test', args.test_batch_size, tokenizer, args.max_dialogue_len,
-                                                     config.max_turn_len)
-            torch.save(test_dataloader, os.path.join(
-                OUTPUT_DIR, 'dataloaders', 'test.dataloader'))
-
-        config, models = ensemble.get_models(
-            args.model_name_or_path, device, ConfigClass, SetSumbtModel)
-
-        belief_states, labels = ensemble_utils.get_predictions(
-            args, models, device, test_dataloader, test_slots)
-        torch.save({'belief_states': belief_states, 'labels': labels},
-                   os.path.join(OUTPUT_DIR, 'predictions', 'test.predictions'))
-    else:
-        # Get training batch loaders and ontology embeddings
-        if os.path.exists(os.path.join(OUTPUT_DIR, 'database', 'test.db')):
-            test_slots = torch.load(os.path.join(
-                OUTPUT_DIR, 'database', 'test.db'))
-        else:
-            # Create Tokenizer and embedding model for Data Loaders and ontology
-            encoder = CandidateEncoderModel.from_pretrained(
-                config.candidate_embedding_model_name)
-            tokenizer = Tokenizer(config.candidate_embedding_model_name)
-            embeddings.get_slot_candidate_embeddings(
-                'test', args, tokenizer, encoder)
-            test_slots = torch.load(os.path.join(
-                OUTPUT_DIR, 'database', 'test.db'))
-
-        exists = False
-        if os.path.exists(os.path.join(OUTPUT_DIR, 'dataloaders', 'test.dataloader')):
-            test_dataloader = torch.load(os.path.join(
-                OUTPUT_DIR, 'dataloaders', 'test.dataloader'))
-            if test_dataloader.batch_size == args.test_batch_size:
-                exists = True
-        if not exists:
-            tokenizer = Tokenizer(config.candidate_embedding_model_name)
-            test_dataloader = Dataset.get_dataloader('test', args.test_batch_size, tokenizer, args.max_dialogue_len,
-                                                     config.max_turn_len)
-            torch.save(test_dataloader, os.path.join(
-                OUTPUT_DIR, 'dataloaders', 'test.dataloader'))
-
-        # Initialise Model
-        model = SetSumbtModel.from_pretrained(
-            args.model_name_or_path, config=config)
-        model = model.to(device)
-
-        # Get slot and value embeddings
-        slots = {slot: test_slots[slot] for slot in test_slots}
-        values = {slot: test_slots[slot][1] for slot in test_slots}
-
-        # Load model ontology
-        model.add_slot_candidates(slots)
-        for slot in model.informable_slot_ids:
-            model.add_value_candidates(slot, values[slot], replace=True)
-
-        belief_states = calibration.get_predictions(
-            args, model, device, test_dataloader)
-        belief_states, labels, request_belief, request_labels, domain_belief, domain_labels, greeting_belief, greeting_labels = belief_states
-        out = {'belief_states': belief_states, 'labels': labels,
-               'request_belief': request_belief, 'request_labels': request_labels,
-               'domain_belief': domain_belief, 'domain_labels': domain_labels,
-               'greeting_belief': greeting_belief, 'greeting_labels': greeting_labels}
-        torch.save(out, os.path.join(
-            OUTPUT_DIR, 'predictions', 'test.predictions'))
-
-    # err = [ece(belief_states[slot].reshape(-1, belief_states[slot].size(-1)), labels[slot].reshape(-1), 10)
-    #         for slot in belief_states]
-    # err = max(err)
-    # logger.info('ECE: %f' % err)
-
-    # Calculate calibration metrics
-
-    jg = jg_ece(belief_states, labels, 10)
-    logger.info('Joint Goal ECE: %f' % jg)
-
-    binary_states = {}
-    for slot, p in belief_states.items():
-        shp = p.shape
-        p = p.reshape(-1, p.size(-1))
-        p_ = torch.ones(p.shape).to(p.device) * 1e-8
-        p_[range(p.size(0)), p.argmax(-1)] = 1.0 - 1e-8
-        binary_states[slot] = p_.reshape(shp)
-    jg = jg_ece(binary_states, labels, 10)
-    logger.info('Joint Goal Binary ECE: %f' % jg)
-
-    bs = {slot: torch.cat((p[:, :, 0].unsqueeze(-1), p[:, :, 1:].max(-1)
-                          [0].unsqueeze(-1)), -1) for slot, p in belief_states.items()}
-    ls = {}
-    for slot, l in labels.items():
-        y = torch.zeros((l.size(0), l.size(1))).to(l.device)
-        dials, turns = torch.where(l > 0)
-        y[dials, turns] = 1.0
-        dials, turns = torch.where(l < 0)
-        y[dials, turns] = -1.0
-        ls[slot] = y
-
-    jg = jg_ece(bs, ls, 10)
-    logger.info('Slot presence ECE: %f' % jg)
-
-    binary_states = {}
-    for slot, p in bs.items():
-        shp = p.shape
-        p = p.reshape(-1, p.size(-1))
-        p_ = torch.ones(p.shape).to(p.device) * 1e-8
-        p_[range(p.size(0)), p.argmax(-1)] = 1.0 - 1e-8
-        binary_states[slot] = p_.reshape(shp)
-    jg = jg_ece(binary_states, ls, 10)
-    logger.info('Slot presence Binary ECE: %f' % jg)
-
-    jg_acc = 0.0
-    padding = torch.cat([item.unsqueeze(-1)
-                        for _, item in labels.items()], -1).sum(-1) * -1.0
-    padding = (padding == len(labels))
-    padding = padding.reshape(-1)
-    for slot in belief_states:
-        topn = args.accuracy_topn
-        p_ = belief_states[slot]
-        gold = labels[slot]
-
-        if p_.size(-1) <= topn:
-            topn = p_.size(-1) - 1
-        if topn <= 0:
-            topn = 1
-
-        if topn > 1:
-            labs = p_.reshape(-1, p_.size(-1)).argsort(dim=-1, descending=True)
-            labs = labs[:, :topn]
-        else:
-            labs = p_.reshape(-1, p_.size(-1)).argmax(dim=-1).unsqueeze(-1)
-        acc = [lab in s for lab, s, pad in zip(
-            gold.reshape(-1), labs, padding) if not pad]
-        acc = torch.tensor(acc).float()
-
-        jg_acc += acc
-
-    n_turns = jg_acc.size(0)
-    sl_acc = sum(jg_acc / len(belief_states)).float()
-    jg_acc = sum((jg_acc / len(belief_states)).int()).float()
-
-    sl_acc /= n_turns
-    jg_acc /= n_turns
-
-    logger.info('Joint Goal Accuracy: %f, Slot Accuracy %f' % (jg_acc, sl_acc))
-
-    l2 = l2_acc(belief_states, labels, remove_belief=False)
-    logger.info(f'Model L2 Norm Goal Accuracy: {l2}')
-    l2 = l2_acc(belief_states, labels, remove_belief=True)
-    logger.info(f'Binary Model L2 Norm Goal Accuracy: {l2}')
-
-    for slot in belief_states:
-        p = belief_states[slot]
-        p = p.reshape(-1, p.size(-1))
-        p = torch.cat(
-            (p[:, 0].unsqueeze(-1), p[:, 1:].max(-1)[0].unsqueeze(-1)), -1)
-        belief_states[slot] = p
-
-        l = labels[slot].reshape(-1)
-        l[l > 0] = 1
-        labels[slot] = l
-
-    f1 = 0.0
-    for slot in belief_states:
-        prd = belief_states[slot].argmax(-1)
-        tp = ((prd == 1) * (labels[slot] == 1)).sum()
-        fp = ((prd == 1) * (labels[slot] == 0)).sum()
-        fn = ((prd == 0) * (labels[slot] == 1)).sum()
-        if tp > 0:
-            f1 += tp / (tp + 0.5 * (fp + fn))
-    f1 /= len(belief_states)
-    logger.info(f'Trucated Goal F1 Score: {f1}')
-
-    l2 = l2_acc(belief_states, labels, remove_belief=False)
-    logger.info(f'Model L2 Norm Trucated Goal Accuracy: {l2}')
-    l2 = l2_acc(belief_states, labels, remove_belief=True)
-    logger.info(f'Binary Model L2 Norm Trucated Goal Accuracy: {l2}')
-
-    if request_belief is not None:
-        tp, fp, fn = 0.0, 0.0, 0.0
-        for slot in request_belief:
-            p = request_belief[slot]
-            l = request_labels[slot]
-
-            tp += (p.round().int() * (l == 1)).reshape(-1).float()
-            fp += (p.round().int() * (l == 0)).reshape(-1).float()
-            fn += ((1 - p.round().int()) * (l == 1)).reshape(-1).float()
-        tp /= len(request_belief)
-        fp /= len(request_belief)
-        fn /= len(request_belief)
-        f1 = tp.sum() / (tp.sum() + 0.5 * (fp.sum() + fn.sum()))
-        logger.info('Request F1 Score: %f' % f1.item())
-
-        for slot in request_belief:
-            p = request_belief[slot]
-            p = p.unsqueeze(-1)
-            p = torch.cat((1 - p, p), -1)
-            request_belief[slot] = p
-        jg = jg_ece(request_belief, request_labels, 10)
-        logger.info('Request Joint Goal ECE: %f' % jg)
-
-        binary_states = {}
-        for slot, p in request_belief.items():
-            shp = p.shape
-            p = p.reshape(-1, p.size(-1))
-            p_ = torch.ones(p.shape).to(p.device) * 1e-8
-            p_[range(p.size(0)), p.argmax(-1)] = 1.0 - 1e-8
-            binary_states[slot] = p_.reshape(shp)
-        jg = jg_ece(binary_states, request_labels, 10)
-        logger.info('Request Joint Goal Binary ECE: %f' % jg)
-
-        tp, fp, fn = 0.0, 0.0, 0.0
-        for dom in domain_belief:
-            p = domain_belief[dom]
-            l = domain_labels[dom]
-
-            tp += (p.round().int() * (l == 1)).reshape(-1).float()
-            fp += (p.round().int() * (l == 0)).reshape(-1).float()
-            fn += ((1 - p.round().int()) * (l == 1)).reshape(-1).float()
-        tp /= len(domain_belief)
-        fp /= len(domain_belief)
-        fn /= len(domain_belief)
-        f1 = tp.sum() / (tp.sum() + 0.5 * (fp.sum() + fn.sum()))
-        logger.info('Domain F1 Score: %f' % f1.item())
-
-        for dom in domain_belief:
-            p = domain_belief[dom]
-            p = p.unsqueeze(-1)
-            p = torch.cat((1 - p, p), -1)
-            domain_belief[dom] = p
-        jg = jg_ece(domain_belief, domain_labels, 10)
-        logger.info('Domain Joint Goal ECE: %f' % jg)
-
-        binary_states = {}
-        for slot, p in domain_belief.items():
-            shp = p.shape
-            p = p.reshape(-1, p.size(-1))
-            p_ = torch.ones(p.shape).to(p.device) * 1e-8
-            p_[range(p.size(0)), p.argmax(-1)] = 1.0 - 1e-8
-            binary_states[slot] = p_.reshape(shp)
-        jg = jg_ece(binary_states, domain_labels, 10)
-        logger.info('Domain Joint Goal Binary ECE: %f' % jg)
-
-        tp = ((greeting_belief.argmax(-1) > 0) *
-              (greeting_labels > 0)).reshape(-1).float().sum()
-        fp = ((greeting_belief.argmax(-1) > 0) *
-              (greeting_labels == 0)).reshape(-1).float().sum()
-        fn = ((greeting_belief.argmax(-1) == 0) *
-              (greeting_labels > 0)).reshape(-1).float().sum()
-        f1 = tp / (tp + 0.5 * (fp + fn))
-        logger.info('Greeting F1 Score: %f' % f1.item())
-
-        err = ece(greeting_belief.reshape(-1, greeting_belief.size(-1)),
-                  greeting_labels.reshape(-1), 10)
-        logger.info('Greetings ECE: %f' % err)
-
-        greeting_belief = greeting_belief.reshape(-1, greeting_belief.size(-1))
-        binary_states = torch.ones(greeting_belief.shape).to(
-            greeting_belief.device) * 1e-8
-        binary_states[range(greeting_belief.size(0)),
-                      greeting_belief.argmax(-1)] = 1.0 - 1e-8
-        err = ece(binary_states, greeting_labels.reshape(-1), 10)
-        logger.info('Greetings Binary ECE: %f' % err)
-
-        for slot in request_belief:
-            p = request_belief[slot].unsqueeze(-1)
-            request_belief[slot] = torch.cat((1 - p, p), -1)
-
-        l2 = l2_acc(request_belief, request_labels, remove_belief=False)
-        logger.info(f'Model L2 Norm Request Accuracy: {l2}')
-        l2 = l2_acc(request_belief, request_labels, remove_belief=True)
-        logger.info(f'Binary Model L2 Norm Request Accuracy: {l2}')
-
-        for slot in domain_belief:
-            p = domain_belief[slot].unsqueeze(-1)
-            domain_belief[slot] = torch.cat((1 - p, p), -1)
-
-        l2 = l2_acc(domain_belief, domain_labels, remove_belief=False)
-        logger.info(f'Model L2 Norm Domain Accuracy: {l2}')
-        l2 = l2_acc(domain_belief, domain_labels, remove_belief=True)
-        logger.info(f'Binary Model L2 Norm Domain Accuracy: {l2}')
-
-        greeting_labels = {'bye': greeting_labels}
-        greeting_belief = {'bye': greeting_belief}
-
-        l2 = l2_acc(greeting_belief, greeting_labels, remove_belief=False)
-        logger.info(f'Model L2 Norm Greeting Accuracy: {l2}')
-        l2 = l2_acc(greeting_belief, greeting_labels, remove_belief=False)
-        logger.info(f'Binary Model L2 Norm Greeting Accuracy: {l2}')
-
-
-if __name__ == "__main__":
-    main()
diff --git a/convlab/dst/setsumbt/do/evaluate.py b/convlab/dst/setsumbt/do/evaluate.py
new file mode 100644
index 0000000000000000000000000000000000000000..2fe351b3d5c2af187da58ffcc46e8184013bbcdb
--- /dev/null
+++ b/convlab/dst/setsumbt/do/evaluate.py
@@ -0,0 +1,296 @@
+# -*- coding: utf-8 -*-
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
+# Authors: Carel van Niekerk (niekerk@hhu.de)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Run SetSUMBT Calibration"""
+
+import logging
+import os
+
+import torch
+from transformers import (BertModel, BertConfig, BertTokenizer,
+                          RobertaModel, RobertaConfig, RobertaTokenizer)
+
+from convlab.dst.setsumbt.modeling import BertSetSUMBT, RobertaSetSUMBT
+from convlab.dst.setsumbt.dataset import unified_format
+from convlab.dst.setsumbt.dataset import ontology as embeddings
+from convlab.dst.setsumbt.utils import get_args, update_args
+from convlab.dst.setsumbt.modeling import evaluation_utils
+from convlab.dst.setsumbt.loss.uncertainty_measures import ece, jg_ece, l2_acc
+from convlab.dst.setsumbt.modeling import training
+
+
+# Available model
+MODELS = {
+    'bert': (BertSetSUMBT, BertModel, BertConfig, BertTokenizer),
+    'roberta': (RobertaSetSUMBT, RobertaModel, RobertaConfig, RobertaTokenizer)
+}
+
+
+def main(args=None, config=None):
+    # Get arguments
+    if args is None:
+        args, config = get_args(MODELS)
+
+    if args.model_type in MODELS:
+        SetSumbtModel, CandidateEncoderModel, ConfigClass, Tokenizer = MODELS[args.model_type]
+    else:
+        raise NameError('NotImplemented')
+
+    # Set up output directory
+    OUTPUT_DIR = args.output_dir
+    args.output_dir = OUTPUT_DIR
+    if not os.path.exists(os.path.join(OUTPUT_DIR, 'predictions')):
+        os.mkdir(os.path.join(OUTPUT_DIR, 'predictions'))
+
+    # Set pretrained model path to the trained checkpoint
+    paths = os.listdir(args.output_dir) if os.path.exists(args.output_dir) else []
+    if 'pytorch_model.bin' in paths and 'config.json' in paths:
+        args.model_name_or_path = args.output_dir
+        config = ConfigClass.from_pretrained(args.model_name_or_path)
+    else:
+        paths = [os.path.join(args.output_dir, p) for p in paths if 'checkpoint-' in p]
+        if paths:
+            paths = paths[0]
+            args.model_name_or_path = paths
+            config = ConfigClass.from_pretrained(args.model_name_or_path)
+
+    args = update_args(args, config)
+
+    # Create logger
+    global logger
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.INFO)
+
+    formatter = logging.Formatter('%(asctime)s - %(message)s', '%H:%M %m-%d-%y')
+
+    fh = logging.FileHandler(args.logging_path)
+    fh.setLevel(logging.INFO)
+    fh.setFormatter(formatter)
+    logger.addHandler(fh)
+
+    # Get device
+    if torch.cuda.is_available() and args.n_gpu > 0:
+        device = torch.device('cuda')
+    else:
+        device = torch.device('cpu')
+        args.n_gpu = 0
+
+    if args.n_gpu == 0:
+        args.fp16 = False
+
+    # Set up model training/evaluation
+    evaluation_utils.set_seed(args)
+
+    # Perform tasks
+    if os.path.exists(os.path.join(OUTPUT_DIR, 'predictions', 'test.predictions')):
+        pred = torch.load(os.path.join(OUTPUT_DIR, 'predictions', 'test.predictions'))
+        state_labels = pred['state_labels']
+        belief_states = pred['belief_states']
+        if 'request_labels' in pred:
+            request_labels = pred['request_labels']
+            request_probs = pred['request_probs']
+            active_domain_labels = pred['active_domain_labels']
+            active_domain_probs = pred['active_domain_probs']
+            general_act_labels = pred['general_act_labels']
+            general_act_probs = pred['general_act_probs']
+        else:
+            request_probs = None
+        del pred
+    else:
+        # Get training batch loaders and ontology embeddings
+        if os.path.exists(os.path.join(OUTPUT_DIR, 'dataloaders', 'test.dataloader')):
+            test_dataloader = torch.load(os.path.join(OUTPUT_DIR, 'dataloaders', 'test.dataloader'))
+            if test_dataloader.batch_size != args.test_batch_size:
+                test_dataloader = unified_format.change_batch_size(test_dataloader, args.test_batch_size)
+        else:
+            tokenizer = Tokenizer(config.candidate_embedding_model_name)
+            test_dataloader = unified_format.get_dataloader(args.dataset, 'test',
+                                                            args.test_batch_size, tokenizer, args.max_dialogue_len,
+                                                            config.max_turn_len)
+            torch.save(test_dataloader, os.path.join(OUTPUT_DIR, 'dataloaders', 'test.dataloader'))
+
+        if os.path.exists(os.path.join(OUTPUT_DIR, 'database', 'test.db')):
+            test_slots = torch.load(os.path.join(OUTPUT_DIR, 'database', 'test.db'))
+        else:
+            encoder = CandidateEncoderModel.from_pretrained(config.candidate_embedding_model_name)
+            test_slots = embeddings.get_slot_candidate_embeddings(test_dataloader.dataset.ontology,
+                                                                  'test', args, tokenizer, encoder)
+
+        # Initialise Model
+        model = SetSumbtModel.from_pretrained(args.model_name_or_path, config=config)
+        model = model.to(device)
+
+        training.set_ontology_embeddings(model, test_slots)
+
+        belief_states = evaluation_utils.get_predictions(args, model, device, test_dataloader)
+        state_labels = belief_states[1]
+        request_probs = belief_states[2]
+        request_labels = belief_states[3]
+        active_domain_probs = belief_states[4]
+        active_domain_labels = belief_states[5]
+        general_act_probs = belief_states[6]
+        general_act_labels = belief_states[7]
+        belief_states = belief_states[0]
+        out = {'belief_states': belief_states, 'state_labels': state_labels, 'request_probs': request_probs,
+               'request_labels': request_labels, 'active_domain_probs': active_domain_probs,
+               'active_domain_labels': active_domain_labels, 'general_act_probs': general_act_probs,
+               'general_act_labels': general_act_labels}
+        torch.save(out, os.path.join(OUTPUT_DIR, 'predictions', 'test.predictions'))
+
+    # Calculate calibration metrics
+    jg = jg_ece(belief_states, state_labels, 10)
+    logger.info('Joint Goal ECE: %f' % jg)
+
+    jg_acc = 0.0
+    padding = torch.cat([item.unsqueeze(-1) for _, item in state_labels.items()], -1).sum(-1) * -1.0
+    padding = (padding == len(state_labels))
+    padding = padding.reshape(-1)
+    for slot in belief_states:
+        p_ = belief_states[slot]
+        gold = state_labels[slot]
+
+        pred = p_.reshape(-1, p_.size(-1)).argmax(dim=-1).unsqueeze(-1)
+        acc = [lab in s for lab, s, pad in zip(gold.reshape(-1), pred, padding) if not pad]
+        acc = torch.tensor(acc).float()
+
+        jg_acc += acc
+
+    n_turns = jg_acc.size(0)
+    jg_acc = sum((jg_acc / len(belief_states)).int()).float()
+
+    jg_acc /= n_turns
+
+    logger.info(f'Joint Goal Accuracy: {jg_acc}')
+
+    l2 = l2_acc(belief_states, state_labels, remove_belief=False)
+    logger.info(f'Model L2 Norm Goal Accuracy: {l2}')
+    l2 = l2_acc(belief_states, state_labels, remove_belief=True)
+    logger.info(f'Binary Model L2 Norm Goal Accuracy: {l2}')
+
+    padding = torch.cat([item.unsqueeze(-1) for _, item in state_labels.items()], -1).sum(-1) * -1.0
+    padding = (padding == len(state_labels))
+    padding = padding.reshape(-1)
+
+    tp, fp, fn, tn, n = 0.0, 0.0, 0.0, 0.0, 0.0
+    for slot in belief_states:
+        p_ = belief_states[slot]
+        gold = state_labels[slot].reshape(-1)
+        p_ = p_.reshape(-1, p_.size(-1))
+
+        p_ = p_[~padding].argmax(-1)
+        gold = gold[~padding]
+
+        tp += (p_ == gold)[gold != 0].int().sum().item()
+        fp += (p_ != 0)[gold == 0].int().sum().item()
+        fp += (p_ != gold)[gold != 0].int().sum().item()
+        fp -= (p_ == 0)[gold != 0].int().sum().item()
+        fn += (p_ == 0)[gold != 0].int().sum().item()
+        tn += (p_ == 0)[gold == 0].int().sum().item()
+        n += p_.size(0)
+
+    acc = (tp + tn) / n
+    prec = tp / (tp + fp)
+    rec = tp / (tp + fn)
+    f1 = 2 * (prec * rec) / (prec + rec)
+
+    logger.info(f"Slot Accuracy: {acc}, Slot F1: {f1}, Slot Precision: {prec}, Slot Recall: {rec}")
+
+    if request_probs is not None:
+        tp, fp, fn = 0.0, 0.0, 0.0
+        for slot in request_probs:
+            p = request_probs[slot]
+            l = request_labels[slot]
+
+            tp += (p.round().int() * (l == 1)).reshape(-1).float()
+            fp += (p.round().int() * (l == 0)).reshape(-1).float()
+            fn += ((1 - p.round().int()) * (l == 1)).reshape(-1).float()
+        tp /= len(request_probs)
+        fp /= len(request_probs)
+        fn /= len(request_probs)
+        f1 = tp.sum() / (tp.sum() + 0.5 * (fp.sum() + fn.sum()))
+        logger.info('Request F1 Score: %f' % f1.item())
+
+        for slot in request_probs:
+            p = request_probs[slot]
+            p = p.unsqueeze(-1)
+            p = torch.cat((1 - p, p), -1)
+            request_probs[slot] = p
+        jg = jg_ece(request_probs, request_labels, 10)
+        logger.info('Request Joint Goal ECE: %f' % jg)
+
+        tp, fp, fn = 0.0, 0.0, 0.0
+        for dom in active_domain_probs:
+            p = active_domain_probs[dom]
+            l = active_domain_labels[dom]
+
+            tp += (p.round().int() * (l == 1)).reshape(-1).float()
+            fp += (p.round().int() * (l == 0)).reshape(-1).float()
+            fn += ((1 - p.round().int()) * (l == 1)).reshape(-1).float()
+        tp /= len(active_domain_probs)
+        fp /= len(active_domain_probs)
+        fn /= len(active_domain_probs)
+        f1 = tp.sum() / (tp.sum() + 0.5 * (fp.sum() + fn.sum()))
+        logger.info('Domain F1 Score: %f' % f1.item())
+
+        for dom in active_domain_probs:
+            p = active_domain_probs[dom]
+            p = p.unsqueeze(-1)
+            p = torch.cat((1 - p, p), -1)
+            active_domain_probs[dom] = p
+        jg = jg_ece(active_domain_probs, active_domain_labels, 10)
+        logger.info('Domain Joint Goal ECE: %f' % jg)
+
+        tp = ((general_act_probs.argmax(-1) > 0) *
+              (general_act_labels > 0)).reshape(-1).float().sum()
+        fp = ((general_act_probs.argmax(-1) > 0) *
+              (general_act_labels == 0)).reshape(-1).float().sum()
+        fn = ((general_act_probs.argmax(-1) == 0) *
+              (general_act_labels > 0)).reshape(-1).float().sum()
+        f1 = tp / (tp + 0.5 * (fp + fn))
+        logger.info('General Act F1 Score: %f' % f1.item())
+
+        err = ece(general_act_probs.reshape(-1, general_act_probs.size(-1)),
+                  general_act_labels.reshape(-1), 10)
+        logger.info('General Act ECE: %f' % err)
+
+        for slot in request_probs:
+            p = request_probs[slot].unsqueeze(-1)
+            request_probs[slot] = torch.cat((1 - p, p), -1)
+
+        l2 = l2_acc(request_probs, request_labels, remove_belief=False)
+        logger.info(f'Model L2 Norm Request Accuracy: {l2}')
+        l2 = l2_acc(request_probs, request_labels, remove_belief=True)
+        logger.info(f'Binary Model L2 Norm Request Accuracy: {l2}')
+
+        for slot in active_domain_probs:
+            p = active_domain_probs[slot].unsqueeze(-1)
+            active_domain_probs[slot] = torch.cat((1 - p, p), -1)
+
+        l2 = l2_acc(active_domain_probs, active_domain_labels, remove_belief=False)
+        logger.info(f'Model L2 Norm Domain Accuracy: {l2}')
+        l2 = l2_acc(active_domain_probs, active_domain_labels, remove_belief=True)
+        logger.info(f'Binary Model L2 Norm Domain Accuracy: {l2}')
+
+        general_act_labels = {'general': general_act_labels}
+        general_act_probs = {'general': general_act_probs}
+
+        l2 = l2_acc(general_act_probs, general_act_labels, remove_belief=False)
+        logger.info(f'Model L2 Norm General Act Accuracy: {l2}')
+        l2 = l2_acc(general_act_probs, general_act_labels, remove_belief=False)
+        logger.info(f'Binary Model L2 Norm General Act Accuracy: {l2}')
+
+
+if __name__ == "__main__":
+    main()
diff --git a/convlab/dst/setsumbt/do/nbt.py b/convlab/dst/setsumbt/do/nbt.py
index 821dca598c814240f39e359cedec7ef795a341b5..d250f29e0186732b27fb284273d9f9ff4f166d2f 100644
--- a/convlab/dst/setsumbt/do/nbt.py
+++ b/convlab/dst/setsumbt/do/nbt.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2021 DSML Group, Heinrich Heine University, Düsseldorf
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
 # Authors: Carel van Niekerk (niekerk@hhu.de)
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,33 +16,29 @@
 """Run SetSUMBT training/eval"""
 
 import logging
-import random
 import os
 from shutil import copy2 as copy
+import json
+from copy import deepcopy
+import pdb
 
 import torch
-from torch.nn import DataParallel
+import transformers
 from transformers import (BertModel, BertConfig, BertTokenizer,
-                          RobertaModel, RobertaConfig, RobertaTokenizer,
-                          AdamW, get_linear_schedule_with_warmup)
-from tqdm import tqdm, trange
-import numpy as np
+                          RobertaModel, RobertaConfig, RobertaTokenizer)
 from tensorboardX import SummaryWriter
+from tqdm import tqdm
 
-from convlab.dst.setsumbt.modeling.bert_nbt import BertSetSUMBT
-from convlab.dst.setsumbt.modeling.roberta_nbt import RobertaSetSUMBT
-from convlab.dst.setsumbt.multiwoz import multiwoz21
+from convlab.dst.setsumbt.modeling import BertSetSUMBT, RobertaSetSUMBT
+from convlab.dst.setsumbt.dataset import unified_format
 from convlab.dst.setsumbt.modeling import training
-from convlab.dst.setsumbt.multiwoz import ontology as embeddings
+from convlab.dst.setsumbt.dataset import ontology as embeddings
 from convlab.dst.setsumbt.utils import get_args, update_args
-from convlab.dst.setsumbt.modeling import ensemble_utils
+from convlab.dst.setsumbt.modeling.ensemble_nbt import setup_ensemble
+from convlab.util.custom_util import model_downloader
 
 
-# Datasets
-DATASETS = {
-    'multiwoz21': multiwoz21
-}
-
+# Available model
 MODELS = {
     'bert': (BertSetSUMBT, BertModel, BertConfig, BertTokenizer),
     'roberta': (RobertaSetSUMBT, RobertaModel, RobertaConfig, RobertaTokenizer)
@@ -54,12 +50,6 @@ def main(args=None, config=None):
     if args is None:
         args, config = get_args(MODELS)
 
-    # Select Dataset object
-    if args.dataset in DATASETS:
-        Dataset = DATASETS[args.dataset]
-    else:
-        raise NameError('NotImplemented')
-
     if args.model_type in MODELS:
         SetSumbtModel, CandidateEncoderModel, ConfigClass, Tokenizer = MODELS[args.model_type]
     else:
@@ -67,6 +57,23 @@ def main(args=None, config=None):
 
     # Set up output directory
     OUTPUT_DIR = args.output_dir
+
+    # Download model if needed
+    if not os.path.exists(OUTPUT_DIR):
+        # Get path /.../convlab/dst/setsumbt/multiwoz/models
+        download_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        download_path = os.path.join(download_path, 'models')
+        if not os.path.exists(download_path):
+            os.mkdir(download_path)
+        model_downloader(download_path, OUTPUT_DIR)
+        # Downloadable model path format http://.../model_name.zip
+        OUTPUT_DIR = OUTPUT_DIR.split('/')[-1].replace('.zip', '')
+        OUTPUT_DIR = os.path.join(download_path, OUTPUT_DIR)
+
+        args.tensorboard_path = os.path.join(OUTPUT_DIR, args.tensorboard_path.split('/')[-1])
+        args.logging_path = os.path.join(OUTPUT_DIR, args.logging_path.split('/')[-1])
+        os.mkdir(os.path.join(OUTPUT_DIR, 'dataloaders'))
+
     if not os.path.exists(OUTPUT_DIR):
         os.makedirs(OUTPUT_DIR)
         os.mkdir(os.path.join(OUTPUT_DIR, 'database'))
@@ -74,53 +81,21 @@ def main(args=None, config=None):
     args.output_dir = OUTPUT_DIR
 
     # Set pretrained model path to the trained checkpoint
-    if args.do_train:
-        paths = os.listdir(args.output_dir) if os.path.exists(
-            args.output_dir) else []
-        paths = [os.path.join(args.output_dir, p)
-                 for p in paths if 'checkpoint-' in p]
+    paths = os.listdir(args.output_dir) if os.path.exists(args.output_dir) else []
+    if 'pytorch_model.bin' in paths and 'config.json' in paths:
+        args.model_name_or_path = args.output_dir
+        config = ConfigClass.from_pretrained(args.model_name_or_path,
+                                             local_files_only=args.transformers_local_files_only)
+    else:
+        paths = [os.path.join(args.output_dir, p) for p in paths if 'checkpoint-' in p]
         if paths:
             paths = paths[0]
             args.model_name_or_path = paths
-            config = ConfigClass.from_pretrained(args.model_name_or_path)
-        else:
-            paths = os.listdir(args.output_dir) if os.path.exists(
-                args.output_dir) else []
-            if 'pytorch_model.bin' in paths and 'config.json' in paths:
-                args.model_name_or_path = args.output_dir
-                config = ConfigClass.from_pretrained(args.model_name_or_path)
-    else:
-        paths = os.listdir(args.output_dir) if os.path.exists(
-            args.output_dir) else []
-        if 'pytorch_model.bin' in paths and 'config.json' in paths:
-            args.model_name_or_path = args.output_dir
-            config = ConfigClass.from_pretrained(args.model_name_or_path)
-        else:
-            paths = os.listdir(args.output_dir) if os.path.exists(
-                args.output_dir) else []
-            paths = [os.path.join(args.output_dir, p)
-                     for p in paths if 'checkpoint-' in p]
-            if paths:
-                paths = paths[0]
-                args.model_name_or_path = paths
-                config = ConfigClass.from_pretrained(args.model_name_or_path)
+            config = ConfigClass.from_pretrained(args.model_name_or_path,
+                                                 local_files_only=args.transformers_local_files_only)
 
     args = update_args(args, config)
 
-    # Set up data directory
-    DATA_DIR = args.data_dir
-    Dataset.set_datadir(DATA_DIR)
-    embeddings.set_datadir(DATA_DIR)
-
-    # If use shrinked domains, remove bus and hospital domains from the training data and model ontology
-    if args.shrink_active_domains and args.dataset == 'multiwoz21':
-        Dataset.set_active_domains(
-            ['attraction', 'hotel', 'restaurant', 'taxi', 'train'])
-
-    # Download and preprocess
-    Dataset.create_examples(
-        args.max_turn_len, args.predict_actions, args.force_processing)
-
     # Create TensorboardX writer
     tb_writer = SummaryWriter(logdir=args.tensorboard_path)
 
@@ -129,19 +104,12 @@ def main(args=None, config=None):
     logger = logging.getLogger(__name__)
     logger.setLevel(logging.INFO)
 
-    formatter = logging.Formatter(
-        '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    formatter = logging.Formatter('%(asctime)s - %(message)s', '%H:%M %m-%d-%y')
 
-    if 'stream' not in args.logging_path:
-        fh = logging.FileHandler(args.logging_path)
-        fh.setLevel(logging.INFO)
-        fh.setFormatter(formatter)
-        logger.addHandler(fh)
-    else:
-        ch = logging.StreamHandler()
-        ch.setLevel(level=logging.INFO)
-        ch.setFormatter(formatter)
-        logger.addHandler(ch)
+    fh = logging.FileHandler(args.logging_path)
+    fh.setLevel(logging.INFO)
+    fh.setFormatter(formatter)
+    logger.addHandler(fh)
 
     # Get device
     if torch.cuda.is_available() and args.n_gpu > 0:
@@ -154,103 +122,123 @@ def main(args=None, config=None):
         args.fp16 = False
 
     # Initialise Model
-    model = SetSumbtModel.from_pretrained(
-        args.model_name_or_path, config=config)
+    transformers.utils.logging.set_verbosity_info()
+    model = SetSumbtModel.from_pretrained(args.model_name_or_path, config=config,
+                                          local_files_only=args.transformers_local_files_only)
     model = model.to(device)
 
     # Create Tokenizer and embedding model for Data Loaders and ontology
-    encoder = model.roberta if args.model_type == 'roberta' else None
-    encoder = model.bert if args.model_type == 'bert' else encoder
-
-    tokenizer = Tokenizer.from_pretrained(config.tokenizer_name, config=config)
+    encoder = CandidateEncoderModel.from_pretrained(config.candidate_embedding_model_name,
+                                                    local_files_only=args.transformers_local_files_only)
+    tokenizer = Tokenizer.from_pretrained(config.tokenizer_name, config=config,
+                                          local_files_only=args.transformers_local_files_only)
 
     # Set up model training/evaluation
     training.set_logger(logger, tb_writer)
     training.set_seed(args)
     embeddings.set_seed(args)
 
+    transformers.utils.logging.set_verbosity_error()
     if args.ensemble_size > 1:
-        ensemble_utils.set_logger(logger, tb_writer)
-        ensemble.set_seed(args)
-        logger.info('Building %i resampled dataloaders each of size %i' % (args.ensemble_size,
-                                                                           args.data_sampling_size))
-        dataloaders = ensemble_utils.build_train_loaders(args, tokenizer, Dataset)
+        # Build all dataloaders
+        train_dataloader = unified_format.get_dataloader(args.dataset,
+                                                         'train',
+                                                         args.train_batch_size,
+                                                         tokenizer,
+                                                         args.max_dialogue_len,
+                                                         args.max_turn_len,
+                                                         train_ratio=args.dataset_train_ratio,
+                                                         seed=args.seed)
+        torch.save(train_dataloader, os.path.join(OUTPUT_DIR, 'dataloaders', 'train.dataloader'))
+        dev_dataloader = unified_format.get_dataloader(args.dataset,
+                                                       'validation',
+                                                       args.dev_batch_size,
+                                                       tokenizer,
+                                                       args.max_dialogue_len,
+                                                       args.max_turn_len,
+                                                       train_ratio=args.dataset_train_ratio,
+                                                       seed=args.seed)
+        torch.save(dev_dataloader, os.path.join(OUTPUT_DIR, 'dataloaders', 'dev.dataloader'))
+        test_dataloader = unified_format.get_dataloader(args.dataset,
+                                                        'test',
+                                                        args.test_batch_size,
+                                                        tokenizer,
+                                                        args.max_dialogue_len,
+                                                        args.max_turn_len,
+                                                        train_ratio=args.dataset_train_ratio,
+                                                        seed=args.seed)
+        torch.save(test_dataloader, os.path.join(OUTPUT_DIR, 'dataloaders', 'test.dataloader'))
+
+        embeddings.get_slot_candidate_embeddings(train_dataloader.dataset.ontology, 'train', args, tokenizer, encoder)
+        embeddings.get_slot_candidate_embeddings(dev_dataloader.dataset.ontology, 'dev', args, tokenizer, encoder)
+        embeddings.get_slot_candidate_embeddings(test_dataloader.dataset.ontology, 'test', args, tokenizer, encoder)
+
+        setup_ensemble(OUTPUT_DIR, args.ensemble_size)
+
+        logger.info(f'Building {args.ensemble_size} resampled dataloaders each of size {args.data_sampling_size}.')
+        dataloaders = [unified_format.dataloader_sample_dialogues(deepcopy(train_dataloader), args.data_sampling_size)
+                       for _ in tqdm(range(args.ensemble_size))]
         logger.info('Dataloaders built.')
+
         for i, loader in enumerate(dataloaders):
-            path = os.path.join(OUTPUT_DIR, 'ensemble-%i' % i)
+            path = os.path.join(OUTPUT_DIR, 'ens-%i' % i)
             if not os.path.exists(path):
                 os.mkdir(path)
-            path = os.path.join(path, 'train.dataloader')
+            path = os.path.join(path, 'dataloaders', 'train.dataloader')
             torch.save(loader, path)
         logger.info('Dataloaders saved.')
 
-        train_slots = embeddings.get_slot_candidate_embeddings(
-            'train', args, tokenizer, encoder)
-        dev_slots = embeddings.get_slot_candidate_embeddings(
-            'dev', args, tokenizer, encoder)
-        test_slots = embeddings.get_slot_candidate_embeddings(
-            'test', args, tokenizer, encoder)
-
-        train_dataloader = Dataset.get_dataloader(
-            'train', args.train_batch_size, tokenizer, args.max_dialogue_len, config.max_turn_len)
-        torch.save(dev_dataloader, os.path.join(
-            OUTPUT_DIR, 'dataloaders', 'train.dataloader'))
-        dev_dataloader = Dataset.get_dataloader(
-            'dev', args.dev_batch_size, tokenizer, args.max_dialogue_len, config.max_turn_len)
-        torch.save(dev_dataloader, os.path.join(
-            OUTPUT_DIR, 'dataloaders', 'dev.dataloader'))
-        test_dataloader = Dataset.get_dataloader(
-            'test', args.test_batch_size, tokenizer, args.max_dialogue_len, config.max_turn_len)
-        torch.save(test_dataloader, os.path.join(
-            OUTPUT_DIR, 'dataloaders', 'test.dataloader'))
-
         # Do not perform standard training after ensemble setup is created
         return 0
 
     # Perform tasks
     # TRAINING
     if args.do_train:
-        # Get training batch loaders and ontology embeddings
-        if os.path.exists(os.path.join(OUTPUT_DIR, 'database', 'train.db')):
-            train_slots = torch.load(os.path.join(
-                OUTPUT_DIR, 'database', 'train.db'))
-        else:
-            train_slots = embeddings.get_slot_candidate_embeddings(
-                'train', args, tokenizer, encoder)
-        if os.path.exists(os.path.join(OUTPUT_DIR, 'database', 'dev.db')):
-            dev_slots = torch.load(os.path.join(
-                OUTPUT_DIR, 'database', 'dev.db'))
-        else:
-            dev_slots = embeddings.get_slot_candidate_embeddings(
-                'dev', args, tokenizer, encoder)
-
-        exists = False
         if os.path.exists(os.path.join(OUTPUT_DIR, 'dataloaders', 'train.dataloader')):
-            train_dataloader = torch.load(os.path.join(
-                OUTPUT_DIR, 'dataloaders', 'train.dataloader'))
-            if train_dataloader.batch_size == args.train_batch_size:
-                exists = True
-        if not exists:
+            train_dataloader = torch.load(os.path.join(OUTPUT_DIR, 'dataloaders', 'train.dataloader'))
+            if train_dataloader.batch_size != args.train_batch_size:
+                train_dataloader = unified_format.change_batch_size(train_dataloader, args.train_batch_size)
+        else:
             if args.data_sampling_size <= 0:
                 args.data_sampling_size = None
-            train_dataloader = Dataset.get_dataloader('train', args.train_batch_size, tokenizer, args.max_dialogue_len,
-                                                      config.max_turn_len, resampled_size=args.data_sampling_size)
-            torch.save(train_dataloader, os.path.join(
-                OUTPUT_DIR, 'dataloaders', 'train.dataloader'))
+            train_dataloader = unified_format.get_dataloader(args.dataset,
+                                                             'train',
+                                                             args.train_batch_size,
+                                                             tokenizer,
+                                                             args.max_dialogue_len,
+                                                             config.max_turn_len,
+                                                             resampled_size=args.data_sampling_size,
+                                                             train_ratio=args.dataset_train_ratio,
+                                                             seed=args.seed)
+            torch.save(train_dataloader, os.path.join(OUTPUT_DIR, 'dataloaders', 'train.dataloader'))
+
+        # Get training batch loaders and ontology embeddings
+        if os.path.exists(os.path.join(OUTPUT_DIR, 'database', 'train.db')):
+            train_slots = torch.load(os.path.join(OUTPUT_DIR, 'database', 'train.db'))
+        else:
+            train_slots = embeddings.get_slot_candidate_embeddings(train_dataloader.dataset.ontology,
+                                                                   'train', args, tokenizer, encoder)
 
         # Get development set batch loaders= and ontology embeddings
         if args.do_eval:
-            exists = False
             if os.path.exists(os.path.join(OUTPUT_DIR, 'dataloaders', 'dev.dataloader')):
-                dev_dataloader = torch.load(os.path.join(
-                    OUTPUT_DIR, 'dataloaders', 'dev.dataloader'))
-                if dev_dataloader.batch_size == args.dev_batch_size:
-                    exists = True
-            if not exists:
-                dev_dataloader = Dataset.get_dataloader('dev', args.dev_batch_size, tokenizer, args.max_dialogue_len,
-                                                        config.max_turn_len)
-                torch.save(dev_dataloader, os.path.join(
-                    OUTPUT_DIR, 'dataloaders', 'dev.dataloader'))
+                dev_dataloader = torch.load(os.path.join(OUTPUT_DIR, 'dataloaders', 'dev.dataloader'))
+                if dev_dataloader.batch_size != args.dev_batch_size:
+                    dev_dataloader = unified_format.change_batch_size(dev_dataloader, args.dev_batch_size)
+            else:
+                dev_dataloader = unified_format.get_dataloader(args.dataset,
+                                                               'validation',
+                                                               args.dev_batch_size,
+                                                               tokenizer,
+                                                               args.max_dialogue_len,
+                                                               config.max_turn_len)
+                torch.save(dev_dataloader, os.path.join(OUTPUT_DIR, 'dataloaders', 'dev.dataloader'))
+
+            if os.path.exists(os.path.join(OUTPUT_DIR, 'database', 'dev.db')):
+                dev_slots = torch.load(os.path.join(OUTPUT_DIR, 'database', 'dev.db'))
+            else:
+                dev_slots = embeddings.get_slot_candidate_embeddings(dev_dataloader.dataset.ontology,
+                                                                     'dev', args, tokenizer, encoder)
         else:
             dev_dataloader = None
             dev_slots = None
@@ -259,94 +247,80 @@ def main(args=None, config=None):
         training.set_ontology_embeddings(model, train_slots)
 
         # TRAINING !!!!!!!!!!!!!!!!!!
-        training.train(args, model, device, train_dataloader, dev_dataloader, train_slots, dev_slots,
-                       embeddings=embeddings, tokenizer=tokenizer)
+        training.train(args, model, device, train_dataloader, dev_dataloader, train_slots, dev_slots)
 
         # Copy final best model to the output dir
         checkpoints = os.listdir(OUTPUT_DIR)
         checkpoints = [p for p in checkpoints if 'checkpoint' in p]
         checkpoints = sorted([int(p.split('-')[-1]) for p in checkpoints])
-        best_checkpoint = checkpoints[-1]
-        best_checkpoint = os.path.join(
-            OUTPUT_DIR, f'checkpoint-{best_checkpoint}')
-        copy(os.path.join(best_checkpoint, 'pytorch_model.bin'),
-             os.path.join(OUTPUT_DIR, 'pytorch_model.bin'))
-        copy(os.path.join(best_checkpoint, 'config.json'),
-             os.path.join(OUTPUT_DIR, 'config.json'))
+        best_checkpoint = os.path.join(OUTPUT_DIR, f'checkpoint-{checkpoints[-1]}')
+        copy(os.path.join(best_checkpoint, 'pytorch_model.bin'), os.path.join(OUTPUT_DIR, 'pytorch_model.bin'))
+        copy(os.path.join(best_checkpoint, 'config.json'), os.path.join(OUTPUT_DIR, 'config.json'))
 
         # Load best model for evaluation
-        model = SumbtModel.from_pretrained(OUTPUT_DIR)
+        model = SetSumbtModel.from_pretrained(OUTPUT_DIR)
         model = model.to(device)
 
     # Evaluation on the development set
     if args.do_eval:
-        # Get development set batch loaders= and ontology embeddings
-        if os.path.exists(os.path.join(OUTPUT_DIR, 'database', 'dev.db')):
-            dev_slots = torch.load(os.path.join(
-                OUTPUT_DIR, 'database', 'dev.db'))
+        if os.path.exists(os.path.join(OUTPUT_DIR, 'dataloaders', 'dev.dataloader')):
+            dev_dataloader = torch.load(os.path.join(OUTPUT_DIR, 'dataloaders', 'dev.dataloader'))
+            if dev_dataloader.batch_size != args.dev_batch_size:
+                dev_dataloader = unified_format.change_batch_size(dev_dataloader, args.dev_batch_size)
         else:
-            dev_slots = embeddings.get_slot_candidate_embeddings(
-                'dev', args, tokenizer, encoder)
+            dev_dataloader = unified_format.get_dataloader(args.dataset,
+                                                           'validation',
+                                                           args.dev_batch_size,
+                                                           tokenizer,
+                                                           args.max_dialogue_len,
+                                                           config.max_turn_len)
+            torch.save(dev_dataloader, os.path.join(OUTPUT_DIR, 'dataloaders', 'dev.dataloader'))
 
-        exists = False
-        if os.path.exists(os.path.join(OUTPUT_DIR, 'dataloaders', 'dev.dataloader')):
-            dev_dataloader = torch.load(os.path.join(
-                OUTPUT_DIR, 'dataloaders', 'dev.dataloader'))
-            if dev_dataloader.batch_size == args.dev_batch_size:
-                exists = True
-        if not exists:
-            dev_dataloader = Dataset.get_dataloader('dev', args.dev_batch_size, tokenizer, args.max_dialogue_len,
-                                                    config.max_turn_len)
-            torch.save(dev_dataloader, os.path.join(
-                OUTPUT_DIR, 'dataloaders', 'dev.dataloader'))
+        if os.path.exists(os.path.join(OUTPUT_DIR, 'database', 'dev.db')):
+            dev_slots = torch.load(os.path.join(OUTPUT_DIR, 'database', 'dev.db'))
+        else:
+            dev_slots = embeddings.get_slot_candidate_embeddings(dev_dataloader.dataset.ontology,
+                                                                 'dev', args, tokenizer, encoder)
 
         # Load model ontology
         training.set_ontology_embeddings(model, dev_slots)
 
         # EVALUATION
-        jg_acc, sl_acc, req_f1, dom_f1, bye_f1, loss = training.evaluate(
-            args, model, device, dev_dataloader)
-        if req_f1:
-            logger.info('Development loss: %f, Joint Goal Accuracy: %f, Slot Accuracy: %f, Request F1 Score: %f, Domain F1 Score: %f, Goodbye F1 Score: %f'
-                        % (loss, jg_acc, sl_acc, req_f1, dom_f1, bye_f1))
-        else:
-            logger.info('Development loss: %f, Joint Goal Accuracy: %f, Slot Accuracy: %f'
-                        % (loss, jg_acc, sl_acc))
+        jg_acc, sl_acc, req_f1, dom_f1, gen_f1, loss = training.evaluate(args, model, device, dev_dataloader)
+        training.log_info('dev', loss, jg_acc, sl_acc, req_f1, dom_f1, gen_f1)
 
     # Evaluation on the test set
     if args.do_test:
-        # Get test set batch loaders= and ontology embeddings
-        if os.path.exists(os.path.join(OUTPUT_DIR, 'database', 'test.db')):
-            test_slots = torch.load(os.path.join(
-                OUTPUT_DIR, 'database', 'test.db'))
+        if os.path.exists(os.path.join(OUTPUT_DIR, 'dataloaders', 'test.dataloader')):
+            test_dataloader = torch.load(os.path.join(OUTPUT_DIR, 'dataloaders', 'test.dataloader'))
+            if test_dataloader.batch_size != args.test_batch_size:
+                test_dataloader = unified_format.change_batch_size(test_dataloader, args.test_batch_size)
         else:
-            test_slots = embeddings.get_slot_candidate_embeddings(
-                'test', args, tokenizer, encoder)
+            test_dataloader = unified_format.get_dataloader(args.dataset, 'test',
+                                                            args.test_batch_size, tokenizer, args.max_dialogue_len,
+                                                            config.max_turn_len)
+            torch.save(test_dataloader, os.path.join(OUTPUT_DIR, 'dataloaders', 'test.dataloader'))
 
-        exists = False
-        if os.path.exists(os.path.join(OUTPUT_DIR, 'dataloaders', 'test.dataloader')):
-            test_dataloader = torch.load(os.path.join(
-                OUTPUT_DIR, 'dataloaders', 'test.dataloader'))
-            if test_dataloader.batch_size == args.test_batch_size:
-                exists = True
-        if not exists:
-            test_dataloader = Dataset.get_dataloader('test', args.test_batch_size, tokenizer, args.max_dialogue_len,
-                                                     config.max_turn_len)
-            torch.save(test_dataloader, os.path.join(
-                OUTPUT_DIR, 'dataloaders', 'test.dataloader'))
+        if os.path.exists(os.path.join(OUTPUT_DIR, 'database', 'test.db')):
+            test_slots = torch.load(os.path.join(OUTPUT_DIR, 'database', 'test.db'))
+        else:
+            test_slots = embeddings.get_slot_candidate_embeddings(test_dataloader.dataset.ontology,
+                                                                  'test', args, tokenizer, encoder)
 
         # Load model ontology
         training.set_ontology_embeddings(model, test_slots)
 
         # TESTING
-        jg_acc, sl_acc, req_f1, dom_f1, bye_f1, loss = training.evaluate(
-            args, model, device, test_dataloader)
-        if req_f1:
-            logger.info('Test loss: %f, Joint Goal Accuracy: %f, Slot Accuracy: %f, Request F1 Score: %f, Domain F1 Score: %f, Goodbye F1 Score: %f'
-                        % (loss, jg_acc, sl_acc, req_f1, dom_f1, bye_f1))
-        else:
-            logger.info('Test loss: %f, Joint Goal Accuracy: %f, Slot Accuracy: %f'
-                        % (loss, jg_acc, sl_acc))
+        jg_acc, sl_acc, req_f1, dom_f1, gen_f1, loss, output = training.evaluate(args, model, device, test_dataloader,
+                                                                                 return_eval_output=True)
+
+        if not os.path.exists(os.path.join(OUTPUT_DIR, 'predictions')):
+            os.mkdir(os.path.join(OUTPUT_DIR, 'predictions'))
+        writer = open(os.path.join(OUTPUT_DIR, 'predictions', 'test.json'), 'w')
+        json.dump(output, writer)
+        writer.close()
+
+        training.log_info('test', loss, jg_acc, sl_acc, req_f1, dom_f1, gen_f1)
 
     tb_writer.close()
 
diff --git a/convlab/dst/setsumbt/get_golden_labels.py b/convlab/dst/setsumbt/get_golden_labels.py
new file mode 100644
index 0000000000000000000000000000000000000000..7fb2841d0d503181119c791a7046fd7e0025d236
--- /dev/null
+++ b/convlab/dst/setsumbt/get_golden_labels.py
@@ -0,0 +1,138 @@
+import json
+from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
+import os
+
+from tqdm import tqdm
+
+from convlab.util import load_dataset
+from convlab.util import load_dst_data
+from convlab.dst.setsumbt.dataset.value_maps import VALUE_MAP, DOMAINS_MAP, QUANTITIES, TIME
+
+
+def extract_data(dataset_names: str) -> list:
+    dataset_dicts = [load_dataset(dataset_name=name) for name in dataset_names.split('+')]
+    data = []
+    for dataset_dict in dataset_dicts:
+        dataset = load_dst_data(dataset_dict, data_split='test', speaker='all', dialogue_acts=True, split_to_turn=False)
+        for dial in dataset['test']:
+            data.append(dial)
+
+    return data
+
+def clean_state(state):
+    clean_state = dict()
+    for domain, subset in state.items():
+        clean_state[domain] = {}
+        for slot, value in subset.items():
+            # Remove pipe separated values
+            value = value.split('|')
+
+            # Map values using value_map
+            for old, new in VALUE_MAP.items():
+                value = [val.replace(old, new) for val in value]
+            value = '|'.join(value)
+
+            # Map dontcare to "do not care" and empty to 'none'
+            value = value.replace('dontcare', 'do not care')
+            value = value if value else 'none'
+
+            # Map quantity values to the integer quantity value
+            if 'people' in slot or 'duration' in slot or 'stay' in slot:
+                try:
+                    if value not in ['do not care', 'none']:
+                        value = int(value)
+                        value = str(value) if value < 10 else QUANTITIES[-1]
+                except:
+                    value = value
+            # Map time values to the most appropriate value in the standard time set
+            elif 'time' in slot or 'leave' in slot or 'arrive' in slot:
+                try:
+                    if value not in ['do not care', 'none']:
+                        # Strip after/before from time value
+                        value = value.replace('after ', '').replace('before ', '')
+                        # Extract hours and minutes from different possible formats
+                        if ':' not in value and len(value) == 4:
+                            h, m = value[:2], value[2:]
+                        elif len(value) == 1:
+                            h = int(value)
+                            m = 0
+                        elif 'pm' in value:
+                            h = int(value.replace('pm', '')) + 12
+                            m = 0
+                        elif 'am' in value:
+                            h = int(value.replace('pm', ''))
+                            m = 0
+                        elif ':' in value:
+                            h, m = value.split(':')
+                        elif ';' in value:
+                            h, m = value.split(';')
+                        # Map to closest 5 minutes
+                        if int(m) % 5 != 0:
+                            m = round(int(m) / 5) * 5
+                            h = int(h)
+                            if m == 60:
+                                m = 0
+                                h += 1
+                            if h >= 24:
+                                h -= 24
+                        # Set in standard 24 hour format
+                        h, m = int(h), int(m)
+                        value = '%02i:%02i' % (h, m)
+                except:
+                    value = value
+            # Map boolean slots to yes/no value
+            elif 'parking' in slot or 'internet' in slot:
+                if value not in ['do not care', 'none']:
+                    if value == 'free':
+                        value = 'yes'
+                    elif True in [v in value.lower() for v in ['yes', 'no']]:
+                        value = [v for v in ['yes', 'no'] if v in value][0]
+
+            value = value if value != 'none' else ''
+
+            clean_state[domain][slot] = value
+
+    return clean_state
+
+def extract_states(data):
+    states_data = {}
+    for dial in data:
+        states = []
+        for turn in dial['turns']:
+            if 'state' in turn:
+                state = clean_state(turn['state'])
+                states.append(state)
+        states_data[dial['dialogue_id']] = states
+
+    return states_data
+
+
+def get_golden_state(prediction, data):
+    state = data[prediction['dial_idx']][prediction['utt_idx']]
+    pred = prediction['predictions']['state']
+    pred = {domain: {slot: pred.get(DOMAINS_MAP.get(domain, domain.lower()), dict()).get(slot, '')
+                     for slot in state[domain]} for domain in state}
+    prediction['state'] = state
+    prediction['predictions']['state'] = pred
+
+    return prediction
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
+    parser.add_argument('--dataset_name', type=str, help='Name of dataset', default="multiwoz21")
+    parser.add_argument('--model_path', type=str, help='Path to model dir')
+    args = parser.parse_args()
+
+    data = extract_data(args.dataset_name)
+    data = extract_states(data)
+
+    reader = open(os.path.join(args.model_path, "predictions", "test.json"), 'r')
+    predictions = json.load(reader)
+    reader.close()
+
+    predictions = [get_golden_state(pred, data) for pred in tqdm(predictions)]
+
+    writer = open(os.path.join(args.model_path, "predictions", f"test_{args.dataset_name}.json"), 'w')
+    json.dump(predictions, writer)
+    writer.close()
diff --git a/convlab/dst/setsumbt/loss/__init__.py b/convlab/dst/setsumbt/loss/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..475f7646126ea03b630efcbbc688f86c5a8ec16e
--- /dev/null
+++ b/convlab/dst/setsumbt/loss/__init__.py
@@ -0,0 +1,4 @@
+from convlab.dst.setsumbt.loss.bayesian_matching import BayesianMatchingLoss, BinaryBayesianMatchingLoss
+from convlab.dst.setsumbt.loss.kl_distillation import KLDistillationLoss, BinaryKLDistillationLoss
+from convlab.dst.setsumbt.loss.labelsmoothing import LabelSmoothingLoss, BinaryLabelSmoothingLoss
+from convlab.dst.setsumbt.loss.endd_loss import RKLDirichletMediatorLoss, BinaryRKLDirichletMediatorLoss
diff --git a/convlab/dst/setsumbt/loss/bayesian.py b/convlab/dst/setsumbt/loss/bayesian.py
deleted file mode 100644
index e52d8d07733383c7f95b6825b4ab5e5e1c7a0977..0000000000000000000000000000000000000000
--- a/convlab/dst/setsumbt/loss/bayesian.py
+++ /dev/null
@@ -1,144 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2020 DSML Group, Heinrich Heine University, Düsseldorf
-# Authors: Carel van Niekerk (niekerk@hhu.de)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Bayesian Matching Activation and Loss Functions"""
-
-import torch
-from torch import digamma, lgamma
-from torch.nn import Module
-
-
-# Inverse Linear activation function
-def invlinear(x):
-    z = (1.0 / (1.0 - x)) * (x < 0)
-    z += (1.0 + x) * (x >= 0)
-    return z
-
-# Exponential activation function
-def exponential(x):
-    return torch.exp(x)
-
-
-# Dirichlet activation function for the model
-def dirichlet(a):
-    p = exponential(a)
-    repeat_dim = (1,)*(len(p.shape)-1) + (p.size(-1),)
-    p = p / p.sum(-1).unsqueeze(-1).repeat(repeat_dim)
-    return p
-
-
-# Pytorch BayesianMatchingLoss nn.Module
-class BayesianMatchingLoss(Module):
-
-    def __init__(self, lamb=0.01, ignore_index=-1):
-        super(BayesianMatchingLoss, self).__init__()
-
-        self.lamb = lamb
-        self.ignore_index = ignore_index
-    
-    def forward(self, alpha, labels, prior=None):
-        # Assert input sizes
-        assert alpha.dim() == 2                 # Observations, predictive distribution
-        assert labels.dim() == 1                # Label for each observation
-        assert labels.size(0) == alpha.size(0)  # Equal number of observation
-
-        # Confirm predictive distribution dimension
-        if labels.max() <= alpha.size(-1):
-            dimension = alpha.size(-1)
-        else:
-            raise NameError('Label dimension %i is larger than prediction dimension %i.' % (labels.max(), alpha.size(-1)))
-        
-        # Remove observations with no labels
-        if prior is not None:
-            prior = prior[labels != self.ignore_index]
-        alpha = exponential(alpha[labels != self.ignore_index])
-        labels = labels[labels != self.ignore_index]
-        
-        # Initialise and reshape prior parameters
-        if prior is None:
-            prior = torch.ones(dimension)
-        prior = prior.to(alpha.device)
-
-        # KL divergence term
-        lb = lgamma(alpha.sum(-1)) - lgamma(prior.sum(-1)) + (lgamma(prior) - lgamma(alpha)).sum(-1)
-        e = digamma(alpha) - digamma(alpha.sum(-1)).unsqueeze(-1).repeat((1, alpha.size(-1)))
-        e = ((alpha - prior) * e).sum(-1)
-        kl = lb + e
-        kl *= self.lamb
-        del lb, e, prior
-
-        # Expected log likelihood
-        expected_likelihood = digamma(alpha[range(labels.size(0)), labels]) - digamma(alpha.sum(1))
-        del alpha, labels
-
-        # Apply ELBO loss and mean reduction
-        loss = (kl - expected_likelihood).mean()
-        del kl, expected_likelihood
-
-        return loss
-
-
-# Pytorch BayesianMatchingLoss nn.Module
-class BinaryBayesianMatchingLoss(Module):
-
-    def __init__(self, lamb=0.01, ignore_index=-1):
-        super(BinaryBayesianMatchingLoss, self).__init__()
-
-        self.lamb = lamb
-        self.ignore_index = ignore_index
-    
-    def forward(self, alpha, labels, prior=None):
-        # Assert input sizes
-        assert alpha.dim() == 1                 # Observations, predictive distribution
-        assert labels.dim() == 1                # Label for each observation
-        assert labels.size(0) == alpha.size(0)  # Equal number of observation
-
-        # Confirm predictive distribution dimension
-        if labels.max() <= 2:
-            dimension = 2
-        else:
-            raise NameError('Label dimension %i is larger than prediction dimension %i.' % (labels.max(), alpha.size(-1)))
-        
-        # Remove observations with no labels
-        if prior is not None:
-            prior = prior[labels != self.ignore_index]
-        alpha = alpha[labels != self.ignore_index]
-        alpha_sum = 1 + (1 / self.lamb)
-        alpha = (torch.sigmoid(alpha) * alpha_sum).reshape(-1, 1)
-        alpha = torch.cat((alpha_sum - alpha, alpha), 1)
-        labels = labels[labels != self.ignore_index]
-        
-        # Initialise and reshape prior parameters
-        if prior is None:
-            prior = torch.ones(dimension)
-        prior = prior.to(alpha.device)
-
-        # KL divergence term
-        lb = lgamma(alpha.sum(-1)) - lgamma(prior.sum(-1)) + (lgamma(prior) - lgamma(alpha)).sum(-1)
-        e = digamma(alpha) - digamma(alpha.sum(-1)).unsqueeze(-1).repeat((1, alpha.size(-1)))
-        e = ((alpha - prior) * e).sum(-1)
-        kl = lb + e
-        kl *= self.lamb
-        del lb, e, prior
-
-        # Expected log likelihood
-        expected_likelihood = digamma(alpha[range(labels.size(0)), labels.long()]) - digamma(alpha.sum(1))
-        del alpha, labels
-
-        # Apply ELBO loss and mean reduction
-        loss = (kl - expected_likelihood).mean()
-        del kl, expected_likelihood
-
-        return loss
diff --git a/convlab/dst/setsumbt/loss/bayesian_matching.py b/convlab/dst/setsumbt/loss/bayesian_matching.py
new file mode 100644
index 0000000000000000000000000000000000000000..3e91444d60afeeb6e2ca54192dd2283810fc5135
--- /dev/null
+++ b/convlab/dst/setsumbt/loss/bayesian_matching.py
@@ -0,0 +1,115 @@
+# -*- coding: utf-8 -*-
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
+# Authors: Carel van Niekerk (niekerk@hhu.de)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Bayesian Matching Activation and Loss Functions (see https://arxiv.org/pdf/2002.07965.pdf for details)"""
+
+import torch
+from torch import digamma, lgamma
+from torch.nn import Module
+
+
+class BayesianMatchingLoss(Module):
+    """Bayesian matching loss (https://arxiv.org/pdf/2002.07965.pdf) implementation"""
+
+    def __init__(self, lamb: float = 0.001, ignore_index: int = -1) -> Module:
+        """
+        Args:
+            lamb (float): Weighting factor for the KL Divergence component
+            ignore_index (int): Specifies a target value that is ignored and does not contribute to the input gradient.
+        """
+        super(BayesianMatchingLoss, self).__init__()
+
+        self.lamb = lamb
+        self.ignore_index = ignore_index
+    
+    def forward(self, inputs: torch.Tensor, labels: torch.Tensor, prior: torch.Tensor = None) -> torch.Tensor:
+        """
+        Args:
+            inputs (Tensor): Predictive distribution
+            labels (Tensor): Label indices
+            prior (Tensor): Prior distribution over label classes
+
+        Returns:
+            loss (Tensor): Loss value
+        """
+        # Assert input sizes
+        assert inputs.dim() == 2                 # Observations, predictive distribution
+        assert labels.dim() == 1                # Label for each observation
+        assert labels.size(0) == inputs.size(0)  # Equal number of observation
+
+        # Confirm predictive distribution dimension
+        if labels.max() <= inputs.size(-1):
+            dimension = inputs.size(-1)
+        else:
+            raise NameError(f'Label dimension {labels.max()} is larger than prediction dimension {inputs.size(-1)}.')
+        
+        # Remove observations to be ignored in loss calculation
+        if prior is not None:
+            prior = prior[labels != self.ignore_index]
+        inputs = torch.exp(inputs[labels != self.ignore_index])
+        labels = labels[labels != self.ignore_index]
+        
+        # Initialise and reshape prior parameters
+        if prior is None:
+            prior = torch.ones(dimension).to(inputs.device)
+        prior = prior.to(inputs.device)
+
+        # KL divergence term (divergence of predictive distribution from prior over label classes - regularisation term)
+        log_gamma_term = lgamma(inputs.sum(-1)) - lgamma(prior.sum(-1)) + (lgamma(prior) - lgamma(inputs)).sum(-1)
+        div_term = digamma(inputs) - digamma(inputs.sum(-1)).unsqueeze(-1).repeat((1, inputs.size(-1)))
+        div_term = ((inputs - prior) * div_term).sum(-1)
+        kl_term = log_gamma_term + div_term
+        kl_term *= self.lamb
+        del log_gamma_term, div_term, prior
+
+        # Expected log likelihood
+        expected_likelihood = digamma(inputs[range(labels.size(0)), labels]) - digamma(inputs.sum(-1))
+        del inputs, labels
+
+        # Apply ELBO loss and mean reduction
+        loss = (kl_term - expected_likelihood).mean()
+        del kl_term, expected_likelihood
+
+        return loss
+
+
+class BinaryBayesianMatchingLoss(BayesianMatchingLoss):
+    """Bayesian matching loss (https://arxiv.org/pdf/2002.07965.pdf) implementation"""
+
+    def __init__(self, lamb: float = 0.001, ignore_index: int = -1) -> Module:
+        """
+        Args:
+            lamb (float): Weighting factor for the KL Divergence component
+            ignore_index (int): Specifies a target value that is ignored and does not contribute to the input gradient.
+        """
+        super(BinaryBayesianMatchingLoss, self).__init__(lamb, ignore_index)
+
+    def forward(self, inputs: torch.Tensor, labels: torch.Tensor, prior: torch.Tensor = None) -> torch.Tensor:
+        """
+        Args:
+            inputs (Tensor): Predictive distribution
+            labels (Tensor): Label indices
+            prior (Tensor): Prior distribution over label classes
+
+        Returns:
+            loss (Tensor): Loss value
+        """
+        
+        # Create 2D input dirichlet distribution
+        input_sum = 1 + (1 / self.lamb)
+        inputs = (torch.sigmoid(inputs) * input_sum).reshape(-1, 1)
+        inputs = torch.cat((input_sum - inputs, inputs), 1)
+
+        return super().forward(inputs, labels, prior=prior)
diff --git a/convlab/dst/setsumbt/loss/distillation.py b/convlab/dst/setsumbt/loss/distillation.py
deleted file mode 100644
index 3cf13f10635376467f3b137adaa83367f26603ef..0000000000000000000000000000000000000000
--- a/convlab/dst/setsumbt/loss/distillation.py
+++ /dev/null
@@ -1,201 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2020 DSML Group, Heinrich Heine University, Düsseldorf
-# Authors: Carel van Niekerk (niekerk@hhu.de)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Bayesian Matching Activation and Loss Functions"""
-
-import torch
-from torch import lgamma, log
-from torch.nn import Module
-from torch.nn.functional import kl_div
-
-from convlab.dst.setsumbt.loss.bayesian import BayesianMatchingLoss
-
-
-# Pytorch BayesianMatchingLoss nn.Module
-class DistillationKL(Module):
-
-    def __init__(self, lamb=1e-4, ignore_index=-1):
-        super(DistillationKL, self).__init__()
-
-        self.lamb = lamb
-        self.ignore_index = ignore_index
-    
-    def forward(self, alpha, labels, temp=1.0):
-        # Assert input sizes
-        assert alpha.dim() == 2                 # Observations, predictive distribution
-        assert labels.dim() == 2                # Label for each observation
-        assert labels.size(0) == alpha.size(0)  # Equal number of observation
-
-        # Confirm predictive distribution dimension
-        if labels.size(-1) == alpha.size(-1):
-            dimension = alpha.size(-1)
-        else:
-            raise NameError('Label dimension %i is larger than prediction dimension %i.' % (labels.size(-1), alpha.size(-1)))
-        
-        alpha = torch.log(torch.softmax(alpha / temp, -1))
-        ids = torch.where(labels[:, 0] != self.ignore_index)[0]
-        alpha = alpha[ids]
-        labels = labels[ids]
-
-        labels = ((1 - self.lamb) * labels) + (self.lamb * (1 / labels.size(-1)))
-
-        kl = kl_div(alpha, labels, reduction='none').sum(-1).mean()
-        return kl    
-
-
-# Pytorch BayesianMatchingLoss nn.Module
-class BinaryDistillationKL(Module):
-
-    def __init__(self, lamb=1e-4, ignore_index=-1):
-        super(BinaryDistillationKL, self).__init__()
-
-        self.lamb = lamb
-        self.ignore_index = ignore_index
-    
-    def forward(self, alpha, labels, temp=0.0):
-        # Assert input sizes
-        assert alpha.dim() == 1                 # Observations, predictive distribution
-        assert labels.dim() == 1                # Label for each observation
-        assert labels.size(0) == alpha.size(0)  # Equal number of observation
-
-        # Confirm predictive distribution dimension
-        # if labels.size(-1) == alpha.size(-1):
-        #     dimension = alpha.size(-1)
-        # else:
-        #     raise NameError('Label dimension %i is larger than prediction dimension %i.' % (labels.size(-1), alpha.size(-1)))
-        
-        alpha = torch.sigmoid(alpha / temp).unsqueeze(-1)
-        ids = torch.where(labels != self.ignore_index)[0]
-        alpha = alpha[ids]
-        labels = labels[ids]
-
-        alpha = torch.log(torch.cat((1 - alpha, alpha), 1))
-        
-        labels = labels.unsqueeze(-1)
-        labels = torch.cat((1 - labels, labels), -1)
-        labels = ((1 - self.lamb) * labels) + (self.lamb * (1 / labels.size(-1)))
-
-        kl = kl_div(alpha, labels, reduction='none').sum(-1).mean()
-        return kl  
-
-
-# def smart_sort(x, permutation):
-#     assert x.dim() == permutation.dim()
-#     if x.dim() == 3:
-#         d1, d2, d3 = x.size()
-#         ret = x[torch.arange(d1).unsqueeze(-1).unsqueeze(-1).repeat((1, d2, d3)).flatten(),
-#                 torch.arange(d2).unsqueeze(0).unsqueeze(-1).repeat((d1, 1, d3)).flatten(),
-#                 permutation.flatten()].view(d1, d2, d3)
-#         return ret
-#     elif x.dim() == 2:
-#         d1, d2 = x.size()
-#         ret = x[torch.arange(d1).unsqueeze(-1).repeat((1, d2)).flatten(),
-#                 permutation.flatten()].view(d1, d2)
-#         return ret
-
-
-# # Pytorch BayesianMatchingLoss nn.Module
-# class DistillationNLL(Module):
-
-#     def __init__(self, lamb=1e-4, ignore_index=-1):
-#         super(DistillationNLL, self).__init__()
-
-#         self.lamb = lamb
-#         self.ignore_index = ignore_index
-#         self.loss_add = BayesianMatchingLoss(lamb=0.001)
-    
-#     def forward(self, alpha, labels, temp=1.0):
-#         # Assert input sizes
-#         assert alpha.dim() == 2                 # Observations, predictive distribution
-#         assert labels.dim() == 3                # Label for each observation
-#         assert labels.size(0) == alpha.size(0)  # Equal number of observation
-
-#         # Confirm predictive distribution dimension
-#         if labels.size(-1) == alpha.size(-1):
-#             dimension = alpha.size(-1)
-#         else:
-#             raise NameError('Label dimension %i is larger than prediction dimension %i.' % (labels.size(-1), alpha.size(-1)))
-        
-#         alpha = torch.exp(alpha / temp)
-#         ids = torch.where(labels[:, 0, 0] != self.ignore_index)[0]
-#         alpha = alpha[ids]
-#         labels = labels[ids]
-
-#         best_labels = labels.mean(-2).argmax(-1)
-#         loss2 = self.loss_add(alpha, best_labels)
-
-#         topn = labels.mean(-2).argsort(-1, descending=True)
-#         n = 10
-#         alpha = smart_sort(alpha, topn)[:, :n]
-#         labels = smart_sort(labels, topn.unsqueeze(-2).repeat((1, labels.size(-2), 1)))
-#         labels = labels[:, :, :n]
-#         labels = labels / labels.sum(-1).unsqueeze(-1).repeat((1, 1, labels.size(-1)))
-
-#         labels = log(((1 - self.lamb) * labels) + (self.lamb * (1 / labels.size(-1))))
-
-#         loss = (alpha - 1) * labels.mean(-2)
-#         # loss = (alpha - 1) * labels
-#         loss = lgamma(alpha.sum(-1)) - lgamma(alpha).sum(-1) + loss.sum(-1) 
-#         loss = -1.0 * loss.mean()
-#         # loss = -1.0 * loss.mean() / alpha.size(-1)
-
-#         return loss      
-
-
-# # Pytorch BayesianMatchingLoss nn.Module
-# class BinaryDistillationNLL(Module):
-
-#     def __init__(self, lamb=1e-4, ignore_index=-1):
-#         super(BinaryDistillationNLL, self).__init__()
-
-#         self.lamb = lamb
-#         self.ignore_index = ignore_index
-    
-#     def forward(self, alpha, labels, temp=0.0):
-#         # Assert input sizes
-#         assert alpha.dim() == 1                 # Observations, predictive distribution
-#         assert labels.dim() == 2                # Label for each observation
-#         assert labels.size(0) == alpha.size(0)  # Equal number of observation
-
-#         # Confirm predictive distribution dimension
-#         # if labels.size(-1) == alpha.size(-1):
-#         #     dimension = alpha.size(-1)
-#         # else:
-#         #     raise NameError('Label dimension %i is larger than prediction dimension %i.' % (labels.size(-1), alpha.size(-1)))
-        
-#         # Remove observations with no labels
-#         ids = torch.where(labels[:, 0] != self.ignore_index)[0]
-#         # alpha_sum = 1 + (1 / self.lamb)
-#         alpha_sum = 10.0
-#         alpha = (torch.sigmoid(alpha) * alpha_sum).reshape(-1, 1)
-#         alpha = alpha[ids]
-#         labels = labels[ids]
-
-#         if temp != 1.0:
-#             alpha = torch.log(alpha + 1e-4)
-#             alpha = torch.exp(alpha / temp)
-
-#         alpha = torch.cat((alpha_sum - alpha, alpha), 1)
-        
-#         labels = labels.unsqueeze(-1)
-#         labels = torch.cat((1 - labels, labels), -1)
-#         # labels[labels[:, 0, 0] == self.ignore_index] = 1
-#         labels = log(((1 - self.lamb) * labels) + (self.lamb * (1 / labels.size(-1))))
-
-#         loss = (alpha - 1) * labels.mean(-2)
-#         loss = lgamma(alpha.sum(-1)) - lgamma(alpha).sum(-1) + loss.sum(-1)
-#         loss = -1.0 * loss.mean()
-
-#         return loss    
diff --git a/convlab/dst/setsumbt/loss/endd_loss.py b/convlab/dst/setsumbt/loss/endd_loss.py
index d84c3f720d4970c520d97aa9e65a12647468d3ae..9bd794bf4569f54f5896e1e88ed1edeadc0fe1e2 100644
--- a/convlab/dst/setsumbt/loss/endd_loss.py
+++ b/convlab/dst/setsumbt/loss/endd_loss.py
@@ -1,30 +1,46 @@
 import torch
+from torch.nn import Module
+from torch.nn.functional import kl_div
 
 EPS = torch.finfo(torch.float32).eps
 
+
 @torch.no_grad()
-def compute_mkl(ensemble_probs, ensemble_mean_probs, ensemble_logprobs):
-    mkl = torch.nn.functional.kl_div(ensemble_logprobs, ensemble_mean_probs.unsqueeze(1).expand_as(ensemble_probs),
-                                     reduction='none').sum(-1).mean(1)
-    return mkl
+def compute_mkl(ensemble_mean_probs: torch.Tensor, ensemble_logprobs: torch.Tensor) -> torch.Tensor:
+    """
+    Computing MKL in ensemble.
+
+    Args:
+        ensemble_mean_probs (Tensor): Marginal predictive distribution of the ensemble
+        ensemble_logprobs (Tensor): Log predictive distributions of individual ensemble members
+
+    Returns:
+        mkl (Tensor): MKL
+    """
+    mkl = kl_div(ensemble_logprobs, ensemble_mean_probs.unsqueeze(1).expand_as(ensemble_logprobs),reduction='none')
+    return mkl.sum(-1).mean(1)
+
 
 @torch.no_grad()
-def compute_ensemble_stats(ensemble_logits):
-    # ensemble_probs = torch.softmax(ensemble_logits, dim=-1)
-    # ensemble_mean_probs = ensemble_probs.mean(dim=1)
-    # ensemble_logprobs = torch.log_softmax(ensemble_logits, dim=-1)
-    ensemble_probs = ensemble_logits
+def compute_ensemble_stats(ensemble_probs: torch.Tensor) -> dict:
+    """
+    Compute a range of ensemble uncertainty measures
+
+    Args:
+        ensemble_probs (Tensor): Predictive distributions of the ensemble members
+
+    Returns:
+        stats (dict): Dictionary of ensemble uncertainty measures
+    """
     ensemble_mean_probs = ensemble_probs.mean(dim=1)
-    num_classes = ensemble_logits.size(-1)
-    ensemble_logprobs = torch.log(ensemble_logits + (1e-4 / num_classes))
+    num_classes = ensemble_probs.size(-1)
+    ensemble_logprobs = torch.log(ensemble_probs + (1e-4 / num_classes))
 
     entropy_of_expected = torch.distributions.Categorical(probs=ensemble_mean_probs).entropy()
     expected_entropy = torch.distributions.Categorical(probs=ensemble_probs).entropy().mean(dim=1)
     mutual_info = entropy_of_expected - expected_entropy
 
-    mkl = compute_mkl(ensemble_probs, ensemble_mean_probs, ensemble_logprobs)
-
-    # num_classes = ensemble_logits.size(-1)
+    mkl = compute_mkl(ensemble_mean_probs, ensemble_logprobs)
 
     ensemble_precision = (num_classes - 1) / (2 * mkl.unsqueeze(1) + EPS)
 
@@ -39,108 +55,226 @@ def compute_ensemble_stats(ensemble_logits):
     }
     return stats
 
-def entropy(probs, dim: int = -1):
+
+def entropy(probs: torch.Tensor, dim: int = -1) -> torch.Tensor:
+    """
+    Compute entropy in a predictive distribution
+
+    Args:
+        probs (Tensor): Predictive distributions
+        dim (int): Dimension representing the predictive probabilities for a single prediction
+
+    Returns:
+        entropy (Tensor): Entropy
+    """
     return -(probs * (probs + EPS).log()).sum(dim=dim)
 
 
-def compute_dirichlet_uncertainties(dirichlet_params, precisions, expected_dirichlet):
+def compute_dirichlet_uncertainties(dirichlet_params: torch.Tensor,
+                                    precisions: torch.Tensor,
+                                    expected_dirichlet: torch.Tensor) -> tuple:
     """
     Function which computes measures of uncertainty for Dirichlet model.
-    :param dirichlet_params:  Tensor of size [batch_size, n_classes] of Dirichlet concentration parameters.
-    :param precisions: Tensor of size [batch_size, 1] of Dirichlet Precisions
-    :param expected_dirichlet: Tensor of size [batch_size, n_classes] of probablities of expected categorical under Dirichlet.
-    :return: Tensors of token level uncertainties of size [batch_size]
+
+    Args:
+        dirichlet_params (Tensor): Dirichlet concentration parameters.
+        precisions (Tensor): Dirichlet Precisions
+        expected_dirichlet (Tensor): Probabities of expected categorical under Dirichlet.
+
+    Returns:
+        stats (tuple): Token level uncertainties
     """
     batch_size, n_classes = dirichlet_params.size()
 
     entropy_of_expected = entropy(expected_dirichlet)
 
-    expected_entropy = (
-            -expected_dirichlet * (torch.digamma(dirichlet_params + 1) - torch.digamma(precisions + 1))).sum(dim=-1)
+    expected_entropy = -expected_dirichlet * (torch.digamma(dirichlet_params + 1) - torch.digamma(precisions + 1))
+    expected_entropy = expected_entropy.sum(dim=-1)
 
-    mutual_information = -((expected_dirichlet + EPS) * (
-            torch.log(expected_dirichlet + EPS) - torch.digamma(dirichlet_params + 1 + EPS) + torch.digamma(
-        precisions + 1 + EPS))).sum(dim=-1)
-    # assert torch.allclose(mutual_information, entropy_of_expected - expected_entropy, atol=1e-4, rtol=0)
+    mutual_information = torch.log(expected_dirichlet + EPS) - torch.digamma(dirichlet_params + 1 + EPS)
+    mutual_information += torch.digamma(precisions + 1 + EPS)
+    mutual_information *= -(expected_dirichlet + EPS)
+    mutual_information = mutual_information.sum(dim=-1)
 
     epkl = (n_classes - 1) / precisions.squeeze(-1)
 
-    mkl = (expected_dirichlet * (
-            torch.log(expected_dirichlet + EPS) - torch.digamma(dirichlet_params + EPS) + torch.digamma(
-        precisions + EPS))).sum(dim=-1)
+    mkl = torch.log(expected_dirichlet + EPS) - torch.digamma(dirichlet_params + EPS)
+    mkl += torch.digamma(precisions + EPS)
+    mkl *= expected_dirichlet
+    mkl = mkl.sum(dim=-1)
+
+    stats = (entropy_of_expected.clamp(min=0), expected_entropy.clamp(min=0), mutual_information.clamp(min=0))
+    stats += (epkl.clamp(min=0), mkl.clamp(min=0))
+
+    return stats
+
+
+def get_dirichlet_parameters(logits: torch.Tensor,
+                             parametrization,
+                             add_to_alphas: float = 0,
+                             dtype=torch.double) -> tuple:
+    """
+    Get dirichlet parameters from model logits
 
-    return entropy_of_expected.clamp(min=0), \
-           expected_entropy.clamp(min=0), \
-           mutual_information.clamp(min=0), \
-           epkl.clamp(min=0), \
-           mkl.clamp(min=0)
+    Args:
+        logits (Tensor): Model logits
+        parametrization (function): Mapping from logits to concentration parameters
+        add_to_alphas (float): Addition constant for stability
+        dtype (data type): Data type of the parameters
 
-def get_dirichlet_parameters(logits, parametrization, add_to_alphas=0, dtype=torch.double):
+    Return:
+        params (tuple): Concentration and precision parameters of the model Dirichlet
+    """
     max_val = torch.finfo(dtype).max / logits.size(-1) - 1
     alphas = torch.clip(parametrization(logits.to(dtype=dtype)) + add_to_alphas, max=max_val)
     precision = torch.sum(alphas, dim=-1, dtype=dtype)
     return alphas, precision
 
 
-def logits_to_mutual_info(logits):
-    alphas, precision = get_dirichlet_parameters(logits, torch.exp, 1.0)
+def logits_to_mutual_info(logits: torch.Tensor) -> torch.Tensor:
+    """
+    Map modfel logits to mutual information of model Dirichlet
 
-    unsqueezed_precision = precision.unsqueeze(1)
-    normalized_probs = alphas / unsqueezed_precision
+    Args:
+        logits (Tensor): Model logits
 
-    entropy_of_expected, expected_entropy, mutual_information, epkl, mkl = compute_dirichlet_uncertainties(alphas,
-                                                                                                           unsqueezed_precision,
-                                                                                                           normalized_probs)
-    
-    # Max entropy is log(K) for K classes. Hence relative MI is calculated as MI/log(K)
-    # mutual_information /= torch.log(torch.tensor(logits.size(-1)))
-    
-    return mutual_information
+    Returns:
+        mutual_information (Tensor): Mutual information of the model Dirichlet
+    """
+    alphas, precision = get_dirichlet_parameters(logits, torch.exp, 1.0)
 
+    normalized_probs = alphas / precision.unsqueeze(1)
 
-def rkl_dirichlet_mediator_loss(logits, ensemble_stats, model_offset, target_offset, parametrization=torch.exp):
-    turns = torch.where(ensemble_stats[:, 0, 0] != -1)[0]
-    logits = logits[turns]
-    ensemble_stats = ensemble_stats[turns]
+    _, _, mutual_information, _, _ = compute_dirichlet_uncertainties(alphas, precision.unsqueeze(1), normalized_probs)
     
-    ensemble_stats = compute_ensemble_stats(ensemble_stats)
-
-    alphas, precision = get_dirichlet_parameters(logits, parametrization, model_offset)
-
-    unsqueezed_precision = precision.unsqueeze(1)
-    normalized_probs = alphas / unsqueezed_precision
-
-    entropy_of_expected, expected_entropy, mutual_information, epkl, mkl = compute_dirichlet_uncertainties(alphas,
-                                                                                                           unsqueezed_precision,
-                                                                                                           normalized_probs)
-
-    stats = {
-        'alpha_min': alphas.min(),
-        'alpha_mean': alphas.mean(),
-        'precision': precision,
-        'entropy_of_expected': entropy_of_expected,
-        'mutual_info': mutual_information,
-        'mkl': mkl,
-    }
-
-    num_classes = alphas.size(-1)
-
-    ensemble_precision = ensemble_stats['precision']
-
-    ensemble_precision += target_offset * num_classes
-    ensemble_probs = ensemble_stats['mean_probs']
-
-    expected_KL_term = -1.0 * torch.sum(ensemble_probs * (torch.digamma(alphas + EPS)
-                                                          - torch.digamma(precision.unsqueeze(-1) + EPS)), dim=-1)
-    assert torch.isfinite(expected_KL_term).all(), (torch.max(alphas), torch.max(precision), alphas.dtype)
-
-    differential_negentropy_term = torch.sum(torch.lgamma(alphas + EPS), dim=-1) - torch.lgamma(precision + EPS) \
-                                   - torch.sum(
-        (alphas - 1) * (torch.digamma(alphas + EPS) - torch.digamma(precision.unsqueeze(-1) + EPS)), dim=-1)
-    assert torch.isfinite(differential_negentropy_term).all()
-
-    cost = expected_KL_term - differential_negentropy_term / ensemble_precision.squeeze(-1)
+    return mutual_information
 
-    assert torch.isfinite(cost).all()
-    return torch.mean(cost), stats, ensemble_stats
 
+class RKLDirichletMediatorLoss(Module):
+    """Reverse KL Dirichlet Mediator Loss (https://arxiv.org/abs/2105.06987)"""
+
+    def __init__(self,
+                 model_offset: float = 1.0,
+                 target_offset: float = 1,
+                 ignore_index: int = -1,
+                 parameterization=torch.exp):
+        """
+        Args:
+            model_offset (float): Offset of model Dirichlet for stability
+            target_offset (float): Offset of target Dirichlet for stability
+            ignore_index (int): Specifies a target value that is ignored and does not contribute to the input gradient.
+            parameterization (function): Mapping from logits to concentration parameters
+        """
+        super(RKLDirichletMediatorLoss, self).__init__()
+
+        self.model_offset = model_offset
+        self.target_offset = target_offset
+        self.ignore_index = ignore_index
+        self.parameterization = parameterization
+
+    def logits_to_mutual_info(self, logits: torch.Tensor) -> torch.Tensor:
+        """
+        Map modfel logits to mutual information of model Dirichlet
+
+        Args:
+            logits (Tensor): Model logits
+
+        Returns:
+            mutual_information (Tensor): Mutual information of the model Dirichlet
+        """
+        return logits_to_mutual_info(logits)
+
+    def forward(self, logits: torch.Tensor, targets: torch.Tensor) -> torch.Tensor:
+        """
+        Args:
+            logits (Tensor): Model logits
+            targets (Tensor): Ensemble predictive distributions
+
+        Returns:
+            loss (Tensor): RKL dirichlet mediator loss value
+        """
+
+        # Remove padding
+        turns = torch.where(targets[:, 0, 0] != self.ignore_index)[0]
+        logits = logits[turns]
+        targets = targets[turns]
+
+        ensemble_stats = compute_ensemble_stats(targets)
+
+        alphas, precision = get_dirichlet_parameters(logits, self.parameterization, self.model_offset)
+
+        normalized_probs = alphas / precision.unsqueeze(1)
+
+        stats = compute_dirichlet_uncertainties(alphas, precision.unsqueeze(1), normalized_probs)
+        entropy_of_expected, expected_entropy, mutual_information, epkl, mkl = stats
+
+        stats = {
+            'alpha_min': alphas.min(),
+            'alpha_mean': alphas.mean(),
+            'precision': precision,
+            'entropy_of_expected': entropy_of_expected,
+            'mutual_info': mutual_information,
+            'mkl': mkl,
+        }
+
+        num_classes = alphas.size(-1)
+
+        ensemble_precision = ensemble_stats['precision']
+
+        ensemble_precision += self.target_offset * num_classes
+        ensemble_probs = ensemble_stats['mean_probs']
+
+        expected_kl_term = torch.digamma(alphas + EPS) - torch.digamma(precision.unsqueeze(-1) + EPS)
+        expected_kl_term = -1.0 * torch.sum(ensemble_probs * expected_kl_term, dim=-1)
+        assert torch.isfinite(expected_kl_term).all(), (torch.max(alphas), torch.max(precision), alphas.dtype)
+
+        differential_negentropy_term_ = torch.digamma(alphas + EPS) - torch.digamma(precision.unsqueeze(-1) + EPS)
+        differential_negentropy_term_ *= alphas - 1.0
+        differential_negentropy_term = torch.sum(torch.lgamma(alphas + EPS), dim=-1) - torch.lgamma(precision + EPS)
+        differential_negentropy_term -= torch.sum(differential_negentropy_term_, dim=-1)
+        assert torch.isfinite(differential_negentropy_term).all()
+
+        loss = expected_kl_term - differential_negentropy_term / ensemble_precision.squeeze(-1)
+        assert torch.isfinite(loss).all()
+
+        return torch.mean(loss), stats, ensemble_stats
+
+
+class BinaryRKLDirichletMediatorLoss(RKLDirichletMediatorLoss):
+    """Reverse KL Dirichlet Mediator Loss (https://arxiv.org/abs/2105.06987)"""
+
+    def __init__(self,
+                 model_offset: float = 1.0,
+                 target_offset: float = 1,
+                 ignore_index: int = -1,
+                 parameterization=torch.exp):
+        """
+        Args:
+            model_offset (float): Offset of model Dirichlet for stability
+            target_offset (float): Offset of target Dirichlet for stability
+            ignore_index (int): Specifies a target value that is ignored and does not contribute to the input gradient.
+            parameterization (function): Mapping from logits to concentration parameters
+        """
+        super(BinaryRKLDirichletMediatorLoss, self).__init__(model_offset, target_offset,
+                                                             ignore_index, parameterization)
+
+    def forward(self, logits: torch.Tensor, targets: torch.Tensor) -> torch.Tensor:
+        """
+        Args:
+            logits (Tensor): Model logits
+            targets (Tensor): Ensemble predictive distributions
+
+        Returns:
+            loss (Tensor): RKL dirichlet mediator loss value
+        """
+        # Convert single target probability p to distribution [1-p, p]
+        targets = targets.reshape(-1, targets.size(-1), 1)
+        targets = torch.cat([1 - targets, targets], -1)
+        targets[targets[:, 0, 1] == self.ignore_index] = self.ignore_index
+
+        # Convert input logits into predictive distribution [1-z, z]
+        logits = torch.sigmoid(logits).unsqueeze(1)
+        logits = torch.cat((1 - logits, logits), 1)
+        logits = -1.0 * torch.log((1 / (logits + 1e-8)) - 1)  # Inverse sigmoid
+
+        return super().forward(logits, targets)
diff --git a/convlab/dst/setsumbt/loss/kl_distillation.py b/convlab/dst/setsumbt/loss/kl_distillation.py
new file mode 100644
index 0000000000000000000000000000000000000000..9aee234ab68054f2b4a83d6feb5e453384d89e94
--- /dev/null
+++ b/convlab/dst/setsumbt/loss/kl_distillation.py
@@ -0,0 +1,104 @@
+# -*- coding: utf-8 -*-
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
+# Authors: Carel van Niekerk (niekerk@hhu.de)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""KL Divergence Ensemble Distillation loss"""
+
+import torch
+from torch.nn import Module
+from torch.nn.functional import kl_div
+
+
+class KLDistillationLoss(Module):
+    """Ensemble Distillation loss using KL Divergence (https://arxiv.org/pdf/1503.02531.pdf) implementation"""
+
+    def __init__(self, lamb: float = 1e-4, ignore_index: int = -1) -> Module:
+        """
+        Args:
+            lamb (float): Target smoothing parameter
+            ignore_index (int): Specifies a target value that is ignored and does not contribute to the input gradient.
+        """
+        super(KLDistillationLoss, self).__init__()
+
+        self.lamb = lamb
+        self.ignore_index = ignore_index
+    
+    def forward(self, inputs: torch.Tensor, targets: torch.Tensor, temp: float = 1.0) -> torch.Tensor:
+        """
+        Args:
+            inputs (Tensor): Predictive distribution
+            targets (Tensor): Target distribution (ensemble marginal)
+            temp (float): Temperature scaling coefficient for predictive distribution
+
+        Returns:
+            loss (Tensor): Loss value
+        """
+        # Assert input sizes
+        assert inputs.dim() == 2                  # Observations, predictive distribution
+        assert targets.dim() == 2                # Label for each observation
+        assert targets.size(0) == inputs.size(0)  # Equal number of observation
+
+        # Confirm predictive distribution dimension
+        if targets.size(-1) != inputs.size(-1):
+            name_error = f'Target dimension {targets.size(-1)} is not the same as the prediction dimension '
+            name_error += f'{inputs.size(-1)}.'
+            raise NameError(name_error)
+
+        # Remove observations to be ignored in loss calculation
+        inputs = torch.log(torch.softmax(inputs / temp, -1))
+        ids = torch.where(targets[:, 0] != self.ignore_index)[0]
+        inputs = inputs[ids]
+        targets = targets[ids]
+
+        # Target smoothing
+        targets = ((1 - self.lamb) * targets) + (self.lamb / targets.size(-1))
+
+        return kl_div(inputs, targets, reduction='none').sum(-1).mean()
+
+
+# Pytorch BayesianMatchingLoss nn.Module
+class BinaryKLDistillationLoss(KLDistillationLoss):
+    """Binary Ensemble Distillation loss using KL Divergence (https://arxiv.org/pdf/1503.02531.pdf) implementation"""
+
+    def __init__(self, lamb: float = 1e-4, ignore_index: int = -1) -> Module:
+        """
+        Args:
+            lamb (float): Target smoothing parameter
+            ignore_index (int): Specifies a target value that is ignored and does not contribute to the input gradient.
+        """
+        super(BinaryKLDistillationLoss, self).__init__(lamb, ignore_index)
+
+    def forward(self, inputs: torch.Tensor, targets: torch.Tensor, temp: float = 1.0) -> torch.Tensor:
+        """
+        Args:
+            inputs (Tensor): Predictive distribution
+            targets (Tensor): Target distribution (ensemble marginal)
+            temp (float): Temperature scaling coefficient for predictive distribution
+
+        Returns:
+            loss (Tensor): Loss value
+        """
+        # Assert input sizes
+        assert inputs.dim() == 1                 # Observations, predictive distribution
+        assert targets.dim() == 1                # Label for each observation
+        assert targets.size(0) == inputs.size(0)  # Equal number of observation
+        
+        # Convert input and target to 2D binary distribution for KL divergence computation
+        inputs = torch.sigmoid(inputs / temp).unsqueeze(-1)
+        inputs = torch.log(torch.cat((1 - inputs, inputs), 1))
+
+        targets = targets.unsqueeze(-1)
+        targets = torch.cat((1 - targets, targets), -1)
+
+        return super().forward(input, targets, temp)
diff --git a/convlab/dst/setsumbt/loss/labelsmoothing.py b/convlab/dst/setsumbt/loss/labelsmoothing.py
index 8fcc60afd50603cb5c2c84fd698fd11ce7fb7415..61d4b353303451eac7eb09592bdb2c5200328250 100644
--- a/convlab/dst/setsumbt/loss/labelsmoothing.py
+++ b/convlab/dst/setsumbt/loss/labelsmoothing.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2020 DSML Group, Heinrich Heine University, Düsseldorf
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
 # Authors: Carel van Niekerk (niekerk@hhu.de)
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,7 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Inhibited Softmax Activation and Loss Functions"""
+"""Label smoothing loss function"""
 
 
 import torch
@@ -23,66 +23,97 @@ from torch.nn.functional import kl_div
 
 class LabelSmoothingLoss(Module):
     """
-    With label smoothing,
-    KL-divergence between q_{smoothed ground truth prob.}(w)
-    and p_{prob. computed by model}(w) is minimized.
+    Label smoothing loss minimises the KL-divergence between q_{smoothed ground truth prob}(w)
+    and p_{prob. computed by model}(w).
     """
-    def __init__(self, label_smoothing=0.05, ignore_index=-1):
+
+    def __init__(self, label_smoothing: float = 0.05, ignore_index: int = -1) -> Module:
+        """
+        Args:
+            label_smoothing (float): Label smoothing constant
+            ignore_index (int): Specifies a target value that is ignored and does not contribute to the input gradient.
+        """
         super(LabelSmoothingLoss, self).__init__()
 
         assert 0.0 < label_smoothing <= 1.0
         self.ignore_index = ignore_index
         self.label_smoothing = float(label_smoothing)
 
-    def forward(self, logits, targets):
+    def forward(self, inputs: torch.Tensor, labels: torch.Tensor) -> torch.Tensor:
         """
-        output (FloatTensor): batch_size x n_classes
-        target (LongTensor): batch_size
+        Args:
+            input (Tensor): Predictive distribution
+            labels (Tensor): Label indices
+
+        Returns:
+            loss (Tensor): Loss value
         """
-        assert logits.dim() == 2
-        assert targets.dim() == 1
-        assert self.label_smoothing <= ((logits.size(-1) - 1) / logits.size(-1))
+        # Assert input sizes
+        assert inputs.dim() == 2
+        assert labels.dim() == 1
+        assert self.label_smoothing <= ((inputs.size(-1) - 1) / inputs.size(-1))
 
-        logits = logits[targets != self.ignore_index]
-        targets = targets[targets != self.ignore_index]
+        # Confirm predictive distribution dimension
+        if labels.max() <= inputs.size(-1):
+            dimension = inputs.size(-1)
+        else:
+            raise NameError(f'Label dimension {labels.max()} is larger than prediction dimension {inputs.size(-1)}.')
 
-        logits = torch.log(torch.softmax(logits, -1))
-        labels = torch.ones(logits.size()).float().to(logits.device)
-        labels *= self.label_smoothing / (logits.size(-1) - 1)
-        labels[range(labels.size(0)), targets] = 1.0 - self.label_smoothing
+        # Remove observations to be ignored in loss calculation
+        inputs = inputs[labels != self.ignore_index]
+        labels = labels[labels != self.ignore_index]
 
-        kl = kl_div(logits, labels, reduction='none').sum(-1).mean()
-        del logits, targets, labels
-        return kl
+        if labels.size(0) == 0.0:
+            return torch.zeros(1).float().to(labels.device).mean()
 
+        # Create target distribution
+        inputs = torch.log(torch.softmax(inputs, -1))
+        targets = torch.ones(inputs.size()).float().to(inputs.device)
+        targets *= self.label_smoothing / (dimension - 1)
+        targets[range(labels.size(0)), labels] = 1.0 - self.label_smoothing
 
-class BinaryLabelSmoothingLoss(Module):
+        return kl_div(inputs, targets, reduction='none').sum(-1).mean()
+
+
+class BinaryLabelSmoothingLoss(LabelSmoothingLoss):
     """
-    With label smoothing,
-    KL-divergence between q_{smoothed ground truth prob.}(w)
-    and p_{prob. computed by model}(w) is minimized.
+    Label smoothing loss minimises the KL-divergence between q_{smoothed ground truth prob}(w)
+    and p_{prob. computed by model}(w).
     """
-    def __init__(self, label_smoothing=0.05):
-        super(BinaryLabelSmoothingLoss, self).__init__()
 
-        assert 0.0 < label_smoothing <= 1.0
-        self.label_smoothing = float(label_smoothing)
+    def __init__(self, label_smoothing: float = 0.05, ignore_index: int = -1) -> Module:
+        """
+        Args:
+            label_smoothing (float): Label smoothing constant
+            ignore_index (int): Specifies a target value that is ignored and does not contribute to the input gradient.
+        """
+        super(BinaryLabelSmoothingLoss, self).__init__(label_smoothing, ignore_index)
 
-    def forward(self, logits, targets):
+    def forward(self, inputs: torch.Tensor, labels: torch.Tensor) -> torch.Tensor:
         """
-        output (FloatTensor): batch_size x n_classes
-        target (LongTensor): batch_size
+        Args:
+            input (Tensor): Predictive distribution
+            labels (Tensor): Label indices
+
+        Returns:
+            loss (Tensor): Loss value
         """
-        assert logits.dim() == 1
-        assert targets.dim() == 1
+        # Assert input sizes
+        assert inputs.dim() == 1
+        assert labels.dim() == 1
         assert self.label_smoothing <= 0.5
 
-        logits = torch.sigmoid(logits).reshape(-1, 1)
-        logits = torch.log(torch.cat((1 - logits, logits), 1))
-        labels = torch.ones(logits.size()).float().to(logits.device)
-        labels *= self.label_smoothing
-        labels[range(labels.size(0)), targets.long()] = 1.0 - self.label_smoothing
+        # Remove observations to be ignored in loss calculation
+        inputs = inputs[labels != self.ignore_index]
+        labels = labels[labels != self.ignore_index]
+
+        if labels.size(0) == 0.0:
+            return torch.zeros(1).float().to(labels.device).mean()
+
+        inputs = torch.sigmoid(inputs).reshape(-1, 1)
+        inputs = torch.log(torch.cat((1 - inputs, inputs), 1))
+        targets = torch.ones(inputs.size()).float().to(inputs.device)
+        targets *= self.label_smoothing
+        targets[range(labels.size(0)), labels.long()] = 1.0 - self.label_smoothing
 
-        kl = kl_div(logits, labels, reduction='none').sum(-1).mean()
-        del logits, targets
-        return kl
+        return kl_div(inputs, targets, reduction='none').sum(-1).mean()
diff --git a/convlab/dst/setsumbt/loss/ece.py b/convlab/dst/setsumbt/loss/uncertainty_measures.py
similarity index 50%
rename from convlab/dst/setsumbt/loss/ece.py
rename to convlab/dst/setsumbt/loss/uncertainty_measures.py
index 034b9aa0bf5882aea49b08a64d7f93164208b5d9..87c89dd31c724cc7d599230c6d4a15faee9b680e 100644
--- a/convlab/dst/setsumbt/loss/ece.py
+++ b/convlab/dst/setsumbt/loss/uncertainty_measures.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2020 DSML Group, Heinrich Heine University, Düsseldorf
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
 # Authors: Carel van Niekerk (niekerk@hhu.de)
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,14 +13,24 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Expected calibration error"""
+"""Uncertainty evaluation metrics for dialogue belief tracking"""
 
 import torch
 
 
-def fill_bins(n_bins, logits):
-    assert logits.dim() == 2
-    logits = logits.max(-1)[0]
+def fill_bins(n_bins: int, probs: torch.Tensor) -> list:
+    """
+    Function to split observations into bins based on predictive probabilities
+
+    Args:
+        n_bins (int): Number of bins
+        probs (Tensor): Predictive probabilities for the observations
+
+    Returns:
+        bins (list): List of observation ids for each bin
+    """
+    assert probs.dim() == 2
+    probs = probs.max(-1)[0]
 
     step = 1.0 / n_bins
     bin_ranges = torch.arange(0.0, 1.0 + 1e-10, step)
@@ -28,29 +38,49 @@ def fill_bins(n_bins, logits):
     for b in range(n_bins):
         lower, upper = bin_ranges[b], bin_ranges[b + 1]
         if b == 0:
-            ids = torch.where((logits >= lower) * (logits <= upper))[0]
+            ids = torch.where((probs >= lower) * (probs <= upper))[0]
         else:
-            ids = torch.where((logits > lower) * (logits <= upper))[0]
+            ids = torch.where((probs > lower) * (probs <= upper))[0]
         bins.append(ids)
     return bins
 
 
-def bin_confidence(bins, logits):
-    logits = logits.max(-1)[0]
+def bin_confidence(bins: list, probs: torch.Tensor) -> torch.Tensor:
+    """
+    Compute the confidence score within each bin
+
+    Args:
+        bins (list): List of observation ids for each bin
+        probs (Tensor): Predictive probabilities for the observations
+
+    Returns:
+        scores (Tensor): Average confidence score within each bin
+    """
+    probs = probs.max(-1)[0]
 
     scores = []
     for b in bins:
         if b is not None:
-            l = logits[b]
-            scores.append(l.mean())
+            scores.append(probs[b].mean())
         else:
             scores.append(-1)
     scores = torch.tensor(scores)
     return scores
 
 
-def bin_accuracy(bins, logits, y_true):
-    y_pred = logits.argmax(-1)
+def bin_accuracy(bins: list, probs: torch.Tensor, y_true: torch.Tensor) -> torch.Tensor:
+    """
+    Compute the accuracy score for observations in each bin
+
+    Args:
+        bins (list): List of observation ids for each bin
+        probs (Tensor): Predictive probabilities for the observations
+        y_true (Tensor): Labels for the observations
+
+    Returns:
+        acc (Tensor): Accuracies for the observations in each bin
+    """
+    y_pred = probs.argmax(-1)
 
     acc = []
     for b in bins:
@@ -68,13 +98,24 @@ def bin_accuracy(bins, logits, y_true):
     return acc
 
 
-def ece(logits, y_true, n_bins):
-    bins = fill_bins(n_bins, logits)
+def ece(probs: torch.Tensor, y_true: torch.Tensor, n_bins: int) -> float:
+    """
+    Expected calibration error calculation
 
-    scores = bin_confidence(bins, logits)
-    acc = bin_accuracy(bins, logits, y_true)
+    Args:
+        probs (Tensor): Predictive probabilities for the observations
+        y_true (Tensor): Labels for the observations
+        n_bins (int): Number of bins
 
-    n = logits.size(0)
+    Returns:
+        ece (float): Expected calibration error
+    """
+    bins = fill_bins(n_bins, probs)
+
+    scores = bin_confidence(bins, probs)
+    acc = bin_accuracy(bins, probs, y_true)
+
+    n = probs.size(0)
     bk = torch.tensor([b.size(0) for b in bins])
 
     ece = torch.abs(scores - acc) * bk / n
@@ -84,34 +125,30 @@ def ece(logits, y_true, n_bins):
     return ece
 
 
-def jg_ece(logits, y_true, n_bins):
-    y_pred = {slot: logits[slot].reshape(-1, logits[slot].size(-1)).argmax(-1) for slot in logits}
+def jg_ece(belief_state: dict, y_true: dict, n_bins: int) -> float:
+    """
+        Joint goal expected calibration error calculation
+
+        Args:
+            belief_state (dict): Belief state probabilities for the dialogue turns
+            y_true (dict): Labels for the state in dialogue turns
+            n_bins (int): Number of bins
+
+        Returns:
+            ece (float): Joint goal expected calibration error
+        """
+    y_pred = {slot: bs.reshape(-1, bs.size(-1)).argmax(-1) for slot, bs in belief_state.items()}
     goal_acc = {slot: (y_pred[slot] == y_true[slot].reshape(-1)).int() for slot in y_pred}
     goal_acc = sum([goal_acc[slot] for slot in goal_acc])
     goal_acc = (goal_acc == len(y_true)).int()
 
-    scores = [logits[slot].reshape(-1, logits[slot].size(-1)).max(-1)[0].unsqueeze(0) for slot in logits]
+    # Confidence score is minimum across slots as a single bad predictions leads to incorrect prediction in state
+    scores = [bs.reshape(-1, bs.size(-1)).max(-1)[0].unsqueeze(0) for slot, bs in belief_state.items()]
     scores = torch.cat(scores, 0).min(0)[0]
 
-    step = 1.0 / n_bins
-    bin_ranges = torch.arange(0.0, 1.0 + 1e-10, step)
-    bins = []
-    for b in range(n_bins):
-        lower, upper = bin_ranges[b], bin_ranges[b + 1]
-        if b == 0:
-            ids = torch.where((scores >= lower) * (scores <= upper))[0]
-        else:
-            ids = torch.where((scores > lower) * (scores <= upper))[0]
-        bins.append(ids)
+    bins = fill_bins(n_bins, scores.unsqueeze(-1))
 
-    conf = []
-    for b in bins:
-        if b is not None:
-            l = scores[b]
-            conf.append(l.mean())
-        else:
-            conf.append(-1)
-    conf = torch.tensor(conf)
+    conf = bin_confidence(bins, scores.unsqueeze(-1))
 
     slot = [s for s in y_true][0]
     acc = []
@@ -127,7 +164,7 @@ def jg_ece(logits, y_true, n_bins):
             acc.append(-1)
     acc = torch.tensor(acc)
 
-    n = logits[slot].reshape(-1, logits[slot].size(-1)).size(0)
+    n = belief_state[slot].reshape(-1, belief_state[slot].size(-1)).size(0)
     bk = torch.tensor([b.size(0) for b in bins])
 
     ece = torch.abs(conf - acc) * bk / n
@@ -137,12 +174,22 @@ def jg_ece(logits, y_true, n_bins):
     return ece
 
 
-def l2_acc(belief_state, labels, remove_belief=False):
+def l2_acc(belief_state: dict, labels: dict, remove_belief: bool = False) -> float:
+    """
+    Compute L2 Error of belief state prediction
+
+    Args:
+        belief_state (dict): Belief state probabilities for the dialogue turns
+        labels (dict): Labels for the state in dialogue turns
+        remove_belief (bool): Convert belief state to dialogue state
+
+    Returns:
+        err (float): L2 Error of belief state prediction
+    """
     # Get ids used for removing padding turns.
     padding = labels[list(labels.keys())[0]].reshape(-1)
     padding = torch.where(padding != -1)[0]
 
-    # l2 = []
     state = []
     labs = []
     for slot, bs in belief_state.items():
@@ -163,13 +210,8 @@ def l2_acc(belief_state, labels, remove_belief=False):
         y = torch.zeros(bs.shape).cuda()
         y[range(y.size(0)), lab] = 1.0
 
-        # err = torch.sqrt(((y - bs) ** 2).sum(-1))
-        # l2.append(err.unsqueeze(-1))
-
         state.append(bs)
         labs.append(y)
-    
-    # err = torch.cat(l2, -1).max(-1)[0]
 
     # Concatenate all slots into a single belief state
     state = torch.cat(state, -1)
diff --git a/convlab/dst/setsumbt/modeling/__init__.py b/convlab/dst/setsumbt/modeling/__init__.py
index 011a1a774e2d1a22e46e242d1812549895f2246b..59f1439948421ac365e4602b7800c94d3b8b32dd 100644
--- a/convlab/dst/setsumbt/modeling/__init__.py
+++ b/convlab/dst/setsumbt/modeling/__init__.py
@@ -1,3 +1,5 @@
 from convlab.dst.setsumbt.modeling.bert_nbt import BertSetSUMBT
 from convlab.dst.setsumbt.modeling.roberta_nbt import RobertaSetSUMBT
-from convlab.dst.setsumbt.modeling.ensemble_nbt import EnsembleSetSUMBT, DropoutEnsembleSetSUMBT
+from convlab.dst.setsumbt.modeling.ensemble_nbt import EnsembleSetSUMBT
+
+from convlab.dst.setsumbt.modeling.temperature_scheduler import LinearTemperatureScheduler
diff --git a/convlab/dst/setsumbt/modeling/bert_nbt.py b/convlab/dst/setsumbt/modeling/bert_nbt.py
index 8b402b6be09684b27bb73acf17e578bc0e3b4bbd..6762fb3891b4720c3889d8c0809b8791f3bf7633 100644
--- a/convlab/dst/setsumbt/modeling/bert_nbt.py
+++ b/convlab/dst/setsumbt/modeling/bert_nbt.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2021 DSML Group, Heinrich Heine University, Düsseldorf
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
 # Authors: Carel van Niekerk (niekerk@hhu.de)
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,11 +16,10 @@
 """BERT SetSUMBT"""
 
 import torch
-import transformers
 from torch.autograd import Variable
 from transformers import BertModel, BertPreTrainedModel
 
-from convlab.dst.setsumbt.modeling.functional import _initialise, _nbt_forward
+from convlab.dst.setsumbt.modeling.setsumbt import SetSUMBTHead
 
 
 class BertSetSUMBT(BertPreTrainedModel):
@@ -35,59 +34,37 @@ class BertSetSUMBT(BertPreTrainedModel):
             for p in self.bert.parameters():
                 p.requires_grad = False
 
-        _initialise(self, config)
-
-    # Add new slot candidates to the model
-    def add_slot_candidates(self, slot_candidates):
-        """slot_candidates is a list of tuples for each slot.
-        - The tuples contains the slot embedding, informable value embeddings and a request indicator.
-        - If the informable value embeddings is None the slot is not informable
-        - If the request indicator is false the slot is not requestable"""
-        if self.slot_embeddings.size(0) != 0:
-            embeddings = self.slot_embeddings.detach()
-        else:
-            embeddings = torch.zeros(0)
-
-        for slot in slot_candidates:
-            if slot in self.slot_ids:
-                index = self.slot_ids[slot]
-                embeddings[index, :] = slot_candidates[slot][0]
-            else:
-                index = embeddings.size(0)
-                emb = slot_candidates[slot][0].unsqueeze(0).to(embeddings.device)
-                embeddings = torch.cat((embeddings, emb), 0)
-                self.slot_ids[slot] = index
-                setattr(self, slot + '_value_embeddings', Variable(torch.zeros(0), requires_grad=False))
-            # Add slot to relevant requestable and informable slot lists
-            if slot_candidates[slot][2]:
-                self.requestable_slot_ids[slot] = index
-            if slot_candidates[slot][1] is not None:
-                self.informable_slot_ids[slot] = index
-            
-            domain = slot.split('-', 1)[0]
-            if domain not in self.domain_ids:
-                self.domain_ids[domain] = []
-            self.domain_ids[domain].append(index)
-            self.domain_ids[domain] = list(set(self.domain_ids[domain]))
-        
-        self.slot_embeddings = Variable(embeddings, requires_grad=False)
-
-
-    # Add new value candidates to the model
-    def add_value_candidates(self, slot, value_candidates, replace=False):
-        embeddings = getattr(self, slot + '_value_embeddings')
-
-        if embeddings.size(0) == 0 or replace:
-            embeddings = value_candidates
-        else:
-            embeddings = torch.cat((embeddings, value_candidates), 0)
-        
-        setattr(self, slot + '_value_embeddings', embeddings)
-
-    
-    def forward(self, input_ids, token_type_ids, attention_mask, hidden_state=None, inform_labels=None,
-                request_labels=None, domain_labels=None, goodbye_labels=None,
-                get_turn_pooled_representation=False, calculate_inform_mutual_info=False):
+        self.setsumbt = SetSUMBTHead(config)
+        self.add_slot_candidates = self.setsumbt.add_slot_candidates
+        self.add_value_candidates = self.setsumbt.add_value_candidates
+
+    def forward(self,
+                input_ids: torch.Tensor,
+                attention_mask: torch.Tensor,
+                token_type_ids: torch.Tensor = None,
+                hidden_state: torch.Tensor = None,
+                state_labels: torch.Tensor = None,
+                request_labels: torch.Tensor = None,
+                active_domain_labels: torch.Tensor = None,
+                general_act_labels: torch.Tensor = None,
+                get_turn_pooled_representation: bool = False,
+                calculate_state_mutual_info: bool = False):
+        """
+        Args:
+            input_ids: Input token ids
+            attention_mask: Input padding mask
+            token_type_ids: Token type indicator
+            hidden_state: Latent internal dialogue belief state
+            state_labels: Dialogue state labels
+            request_labels: User request action labels
+            active_domain_labels: Current active domain labels
+            general_act_labels: General user action labels
+            get_turn_pooled_representation: Return pooled representation of the current dialogue turn
+            calculate_state_mutual_info: Return mutual information in the dialogue state
+
+        Returns:
+            out: Tuple containing loss, predictive distributions, model statistics and state mutual information
+        """
 
         # Encode Dialogues
         batch_size, dialogue_size, turn_size = input_ids.size()
@@ -103,9 +80,10 @@ class BertSetSUMBT(BertPreTrainedModel):
         turn_embeddings = turn_embeddings.reshape(batch_size * dialogue_size, turn_size, -1)
 
         if get_turn_pooled_representation:
-            return _nbt_forward(self, turn_embeddings, bert_output.pooler_output, attention_mask, batch_size,
-                                dialogue_size, turn_size, hidden_state, inform_labels, request_labels, domain_labels,
-                                goodbye_labels, calculate_inform_mutual_info) + (bert_output.pooler_output,)
-        return _nbt_forward(self, turn_embeddings, bert_output.pooler_output, attention_mask, batch_size, dialogue_size,
-                            turn_size, hidden_state, inform_labels, request_labels, domain_labels, goodbye_labels,
-                            calculate_inform_mutual_info)
+            return self.setsumbt(turn_embeddings, bert_output.pooler_output, attention_mask,
+                                 batch_size, dialogue_size, hidden_state, state_labels,
+                                 request_labels, active_domain_labels, general_act_labels,
+                                 calculate_state_mutual_info) + (bert_output.pooler_output,)
+        return self.setsumbt(turn_embeddings, bert_output.pooler_output, attention_mask, batch_size,
+                             dialogue_size, hidden_state, state_labels, request_labels, active_domain_labels,
+                             general_act_labels, calculate_state_mutual_info)
diff --git a/convlab/dst/setsumbt/modeling/calibration_utils.py b/convlab/dst/setsumbt/modeling/calibration_utils.py
deleted file mode 100644
index 8514ac8d259162c5bcc55607bb8356de6d4b47c7..0000000000000000000000000000000000000000
--- a/convlab/dst/setsumbt/modeling/calibration_utils.py
+++ /dev/null
@@ -1,134 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2020 DSML Group, Heinrich Heine University, Düsseldorf
-# Authors: Carel van Niekerk (niekerk@hhu.de)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Discriminative models calibration"""
-
-import random
-
-import torch
-import numpy as np
-from tqdm import tqdm
-
-
-# Load logger and tensorboard summary writer
-def set_logger(logger_, tb_writer_):
-    global logger, tb_writer
-    logger = logger_
-    tb_writer = tb_writer_
-
-
-# Set seeds
-def set_seed(args):
-    random.seed(args.seed)
-    np.random.seed(args.seed)
-    torch.manual_seed(args.seed)
-    if args.n_gpu > 0:
-        torch.cuda.manual_seed_all(args.seed)
-    logger.info('Seed set to %d.' % args.seed)
-
-
-def get_predictions(args, model, device, dataloader):
-    logger.info("  Num Batches = %d", len(dataloader))
-
-    model.eval()
-    if args.dropout_iterations > 1:
-        model.train()
-    
-    belief_states = {slot: [] for slot in model.informable_slot_ids}
-    request_belief = {slot: [] for slot in model.requestable_slot_ids}
-    domain_belief = {dom: [] for dom in model.domain_ids}
-    greeting_belief = []
-    labels = {slot: [] for slot in model.informable_slot_ids}
-    request_labels = {slot: [] for slot in model.requestable_slot_ids}
-    domain_labels = {dom: [] for dom in model.domain_ids}
-    greeting_labels = []
-    epoch_iterator = tqdm(dataloader, desc="Iteration")
-    for step, batch in enumerate(epoch_iterator):
-        with torch.no_grad():    
-            input_ids = batch['input_ids'].to(device)
-            token_type_ids = batch['token_type_ids'].to(device) if 'token_type_ids' in batch else None
-            attention_mask = batch['attention_mask'].to(device) if 'attention_mask' in batch else None
-
-            if args.dropout_iterations > 1:
-                p = {slot: [] for slot in model.informable_slot_ids}
-                for _ in range(args.dropout_iterations):
-                    p_, p_req_, p_dom_, p_bye_, _ = model(input_ids=input_ids,
-                                                        token_type_ids=token_type_ids,
-                                                        attention_mask=attention_mask)
-                    for slot in model.informable_slot_ids:
-                        p[slot].append(p_[slot].unsqueeze(0))
-                
-                mu = {slot: torch.cat(p[slot], 0).mean(0) for slot in model.informable_slot_ids}
-                sig = {slot: torch.cat(p[slot], 0).var(0) for slot in model.informable_slot_ids}
-                p = {slot: mu[slot] / torch.sqrt(1 + sig[slot]) for slot in model.informable_slot_ids}
-                p = {slot: normalise(p[slot]) for slot in model.informable_slot_ids}
-            else:
-                p, p_req, p_dom, p_bye, _ = model(input_ids=input_ids,
-                                                token_type_ids=token_type_ids,
-                                                attention_mask=attention_mask)
-            
-            for slot in model.informable_slot_ids:
-                p_ = p[slot]
-                labs = batch['labels-' + slot].to(device)
-                
-                belief_states[slot].append(p_)
-                labels[slot].append(labs)
-            
-            if p_req is not None:
-                for slot in model.requestable_slot_ids:
-                    p_ = p_req[slot]
-                    labs = batch['request-' + slot].to(device)
-
-                    request_belief[slot].append(p_)
-                    request_labels[slot].append(labs)
-                
-                for domain in model.domain_ids:
-                    p_ = p_dom[domain]
-                    labs = batch['active-' + domain].to(device)
-
-                    domain_belief[domain].append(p_)
-                    domain_labels[domain].append(labs)
-                
-                greeting_belief.append(p_bye)
-                greeting_labels.append(batch['goodbye'].to(device))
-    
-    for slot in belief_states:
-        belief_states[slot] = torch.cat(belief_states[slot], 0)
-        labels[slot] = torch.cat(labels[slot], 0)
-    if p_req is not None:
-        for slot in request_belief:
-            request_belief[slot] = torch.cat(request_belief[slot], 0)
-            request_labels[slot] = torch.cat(request_labels[slot], 0)
-        for domain in domain_belief:
-            domain_belief[domain] = torch.cat(domain_belief[domain], 0)
-            domain_labels[domain] = torch.cat(domain_labels[domain], 0)
-        greeting_belief = torch.cat(greeting_belief, 0)
-        greeting_labels = torch.cat(greeting_labels, 0)
-    else:
-        request_belief, request_labels, domain_belief, domain_labels, greeting_belief, greeting_labels = [None]*6
-
-    return belief_states, labels, request_belief, request_labels, domain_belief, domain_labels, greeting_belief, greeting_labels
-
-
-def normalise(p):
-    p_shape = p.size()
-
-    p = p.reshape(-1, p_shape[-1]) + 1e-10
-    p_sum = p.sum(-1).unsqueeze(1).repeat((1, p_shape[-1]))
-    p /= p_sum
-
-    p = p.reshape(p_shape)
-
-    return p
diff --git a/convlab/dst/setsumbt/modeling/ensemble_nbt.py b/convlab/dst/setsumbt/modeling/ensemble_nbt.py
index 9f101d128c6b8c9093c4959834cf5aac35b322a2..6d3d8035a4d6f47f2ea8551050ca8da682ea0376 100644
--- a/convlab/dst/setsumbt/modeling/ensemble_nbt.py
+++ b/convlab/dst/setsumbt/modeling/ensemble_nbt.py
@@ -16,9 +16,9 @@
 """Ensemble SetSUMBT"""
 
 import os
+from shutil import copy2 as copy
 
 import torch
-import transformers
 from torch.nn import Module
 from transformers import RobertaConfig, BertConfig
 
@@ -29,8 +29,13 @@ MODELS = {'bert': BertSetSUMBT, 'roberta': RobertaSetSUMBT}
 
 
 class EnsembleSetSUMBT(Module):
+    """Ensemble SetSUMBT Model for joint ensemble prediction"""
 
     def __init__(self, config):
+        """
+        Args:
+            config (configuration): Model configuration class
+        """
         super(EnsembleSetSUMBT, self).__init__()
         self.config = config
 
@@ -38,175 +43,138 @@ class EnsembleSetSUMBT(Module):
         model_cls = MODELS[self.config.model_type]
         for attr in [f'model_{i}' for i in range(self.config.ensemble_size)]:
             setattr(self, attr, model_cls(config))
-    
 
-    # Load all ensemble memeber parameters
-    def load(self, path, config=None):
-        if config is None:
-            config = self.config
-        
+    def _load(self, path: str):
+        """
+        Load parameters
+        Args:
+            path: Location of model parameters
+        """
         for attr in [f'model_{i}' for i in range(self.config.ensemble_size)]:
             idx = attr.split('_', 1)[-1]
-            state_dict = torch.load(os.path.join(path, f'pytorch_model_{idx}.bin'))
+            state_dict = torch.load(os.path.join(path, f'ens-{idx}/pytorch_model.bin'))
             getattr(self, attr).load_state_dict(state_dict)
-    
 
-    # Add new slot candidates to the ensemble members
-    def add_slot_candidates(self, slot_candidates):
+    def add_slot_candidates(self, slot_candidates: tuple):
+        """
+        Add slots to the model ontology, the tuples should contain the slot embedding, informable value embeddings
+        and a request indicator, if the informable value embeddings is None the slot is not informable and if
+        the request indicator is false the slot is not requestable.
+
+        Args:
+            slot_candidates: Tuple containing slot embedding, informable value embeddings and a request indicator
+        """
         for attr in [f'model_{i}' for i in range(self.config.ensemble_size)]:
             getattr(self, attr).add_slot_candidates(slot_candidates)
-        self.requestable_slot_ids = self.model_0.requestable_slot_ids
-        self.informable_slot_ids = self.model_0.informable_slot_ids
-        self.domain_ids = self.model_0.domain_ids
-
-
-    # Add new value candidates to the ensemble members
-    def add_value_candidates(self, slot, value_candidates, replace=False):
+        self.requestable_slot_ids = self.model_0.setsumbt.requestable_slot_ids
+        self.informable_slot_ids = self.model_0.setsumbt.informable_slot_ids
+        self.domain_ids = self.model_0.setsumbt.domain_ids
+
+    def add_value_candidates(self, slot: str, value_candidates: torch.Tensor, replace: bool = False):
+        """
+        Add value candidates for a slot
+
+        Args:
+            slot: Slot name
+            value_candidates: Value candidate embeddings
+            replace: If true existing value candidates are replaced
+        """
         for attr in [f'model_{i}' for i in range(self.config.ensemble_size)]:
             getattr(self, attr).add_value_candidates(slot, value_candidates, replace)
-        
 
-    # Forward pass of full ensemble
-    def forward(self, input_ids, attention_mask, token_type_ids=None, reduction='mean'):
-        logits, request_logits, domain_logits, goodbye_scores = [], [], [], []
-        logits = {slot: [] for slot in self.model_0.informable_slot_ids}
-        request_logits = {slot: [] for slot in self.model_0.requestable_slot_ids}
-        domain_logits = {dom: [] for dom in self.model_0.domain_ids}
-        goodbye_scores = []
+    def forward(self,
+                input_ids: torch.Tensor,
+                attention_mask: torch.Tensor,
+                token_type_ids: torch.Tensor = None,
+                reduction: str = 'mean') -> tuple:
+        """
+        Args:
+            input_ids: Input token ids
+            attention_mask: Input padding mask
+            token_type_ids: Token type indicator
+            reduction: Reduction of ensemble member predictive distributions (mean, none)
+
+        Returns:
+
+        """
+        belief_state_probs = {slot: [] for slot in self.informable_slot_ids}
+        request_probs = {slot: [] for slot in self.requestable_slot_ids}
+        active_domain_probs = {dom: [] for dom in self.domain_ids}
+        general_act_probs = []
         for attr in [f'model_{i}' for i in range(self.config.ensemble_size)]:
             # Prediction from each ensemble member
-            l, r, d, g, _ = getattr(self, attr)(input_ids=input_ids,
+            b, r, d, g, _ = getattr(self, attr)(input_ids=input_ids,
                                                 token_type_ids=token_type_ids,
                                                 attention_mask=attention_mask)
-            for slot in logits:
-                logits[slot].append(l[slot].unsqueeze(-2))
-            if self.config.predict_intents:
-                for slot in request_logits:
-                    request_logits[slot].append(r[slot].unsqueeze(-1))
-                for dom in domain_logits:
-                    domain_logits[dom].append(d[dom].unsqueeze(-1))
-                goodbye_scores.append(g.unsqueeze(-2))
+            for slot in belief_state_probs:
+                belief_state_probs[slot].append(b[slot].unsqueeze(-2))
+            if self.config.predict_actions:
+                for slot in request_probs:
+                    request_probs[slot].append(r[slot].unsqueeze(-1))
+                for dom in active_domain_probs:
+                    active_domain_probs[dom].append(d[dom].unsqueeze(-1))
+                general_act_probs.append(g.unsqueeze(-2))
         
-        logits = {slot: torch.cat(l, -2) for slot, l in logits.items()}
-        if self.config.predict_intents:
-            request_logits = {slot: torch.cat(l, -1) for slot, l in request_logits.items()}
-            domain_logits = {dom: torch.cat(l, -1) for dom, l in domain_logits.items()}
-            goodbye_scores = torch.cat(goodbye_scores, -2)
+        belief_state_probs = {slot: torch.cat(l, -2) for slot, l in belief_state_probs.items()}
+        if self.config.predict_actions:
+            request_probs = {slot: torch.cat(l, -1) for slot, l in request_probs.items()}
+            active_domain_probs = {dom: torch.cat(l, -1) for dom, l in active_domain_probs.items()}
+            general_act_probs = torch.cat(general_act_probs, -2)
         else:
-            request_logits = {}
-            domain_logits = {}
-            goodbye_scores = torch.tensor(0.0)
+            request_probs = {}
+            active_domain_probs = {}
+            general_act_probs = torch.tensor(0.0)
 
         # Apply reduction of ensemble to single posterior
         if reduction == 'mean':
-            logits = {slot: l.mean(-2) for slot, l in logits.items()}
-            request_logits = {slot: l.mean(-1) for slot, l in request_logits.items()}
-            domain_logits = {dom: l.mean(-1) for dom, l in domain_logits.items()}
-            goodbye_scores = goodbye_scores.mean(-2)
+            belief_state_probs = {slot: l.mean(-2) for slot, l in belief_state_probs.items()}
+            request_probs = {slot: l.mean(-1) for slot, l in request_probs.items()}
+            active_domain_probs = {dom: l.mean(-1) for dom, l in active_domain_probs.items()}
+            general_act_probs = general_act_probs.mean(-2)
         elif reduction != 'none':
             raise(NameError('Not Implemented!'))
 
-        return logits, request_logits, domain_logits, goodbye_scores, _
+        return belief_state_probs, request_probs, active_domain_probs, general_act_probs, _
     
 
     @classmethod
     def from_pretrained(cls, path):
-        if not os.path.exists(os.path.join(path, 'config.json')):
+        config_path = os.path.join(path, 'ens-0', 'config.json')
+        if not os.path.exists(config_path):
             raise(NameError('Could not find config.json in model path.'))
-        if not os.path.exists(os.path.join(path, 'pytorch_model_0.bin')):
-            raise(NameError('Could not find a model binary in the model path.'))
         
         try:
-            config = RobertaConfig.from_pretrained(path)
+            config = RobertaConfig.from_pretrained(config_path)
         except:
-            config = BertConfig.from_pretrained(path)
+            config = BertConfig.from_pretrained(config_path)
+
+        config.ensemble_size = len([dir for dir in os.listdir(path) if 'ens-' in dir])
         
         model = cls(config)
-        model.load(path)
+        model._load(path)
 
         return model
 
 
-class DropoutEnsembleSetSUMBT(Module):
-
-    def __init__(self, config):
-        super(DropoutEnsembleBeliefTracker, self).__init__()
-        self.config = config
-
-        model_cls = MODELS[self.config.model_type]
-        self.model = model_cls(config)
-        self.model.train()
-    
-
-    def load(self, path, config=None):
-        if config is None:
-            config = self.config
-        state_dict = torch.load(os.path.join(path, f'pytorch_model.bin'))
-        self.model.load_state_dict(state_dict)
-    
-
-    # Add new slot candidates to the model
-    def add_slot_candidates(self, slot_candidates):
-        self.model.add_slot_candidates(slot_candidates)
-        self.requestable_slot_ids = self.model.requestable_slot_ids
-        self.informable_slot_ids = self.model.informable_slot_ids
-        self.domain_ids = self.model.domain_ids
-
-
-    # Add new value candidates to the model
-    def add_value_candidates(self, slot, value_candidates, replace=False):
-        self.model.add_value_candidates(slot, value_candidates, replace)
-        
-    
-    def forward(self, input_ids, attention_mask, token_type_ids=None, reduction='mean'):
-
-        input_ids = input_ids.unsqueeze(0).repeat((self.config.ensemble_size, 1, 1, 1))
-        input_ids = input_ids.reshape(-1, input_ids.size(-2), input_ids.size(-1))
-        if attention_mask is not None:
-            attention_mask = attention_mask.unsqueeze(0).repeat((10, 1, 1, 1))
-            attention_mask = attention_mask.reshape(-1, attention_mask.size(-2), attention_mask.size(-1))
-        if token_type_ids is not None:
-            token_type_ids = token_type_ids.unsqueeze(0).repeat((10, 1, 1, 1))
-            token_type_ids = token_type_ids.reshape(-1, token_type_ids.size(-2), token_type_ids.size(-1))
-        
-        self.model.train()
-        logits, request_logits, domain_logits, goodbye_scores, _ = self.model(input_ids=input_ids,
-                                                                            attention_mask=attention_mask,
-                                                                            token_type_ids=token_type_ids)
-        
-        logits = {s: l.reshape(self.config.ensemble_size, -1, l.size(-2), l.size(-1)).transpose(0, 1).transpose(1, 2)
-                for s, l in logits.items()}
-        request_logits = {s: l.reshape(self.config.ensemble_size, -1, l.size(-1)).transpose(0, 1).transpose(1, 2)
-                        for s, l in request_logits.items()}
-        domain_logits = {s: l.reshape(self.config.ensemble_size, -1, l.size(-1)).transpose(0, 1).transpose(1, 2)
-                        for s, l in domain_logits.items()}
-        goodbye_scores = goodbye_scores.reshape(self.config.ensemble_size, -1, goodbye_scores.size(-2), goodbye_scores.size(-1))
-        goodbye_scores = goodbye_scores.transpose(0, 1).transpose(1, 2)
-
-        if reduction == 'mean':
-            logits = {slot: l.mean(-2) for slot, l in logits.items()}
-            request_logits = {slot: l.mean(-1) for slot, l in request_logits.items()}
-            domain_logits = {dom: l.mean(-1) for dom, l in domain_logits.items()}
-            goodbye_scores = goodbye_scores.mean(-2)
-        elif reduction != 'none':
-            raise(NameError('Not Implemented!'))
-
-        return logits, request_logits, domain_logits, goodbye_scores, _
-    
-
-    @classmethod
-    def from_pretrained(cls, path):
-        if not os.path.exists(os.path.join(path, 'config.json')):
-            raise(NameError('Could not find config.json in model path.'))
-        if not os.path.exists(os.path.join(path, 'pytorch_model.bin')):
-            raise(NameError('Could not find a model binary in the model path.'))
-        
-        try:
-            config = RobertaConfig.from_pretrained(path)
-        except:
-            config = BertConfig.from_pretrained(path)
-        
-        model = cls(config)
-        model.load(path)
-
-        return model
+def setup_ensemble(model_path: str, ensemble_size: int):
+    """
+    Setup ensemble model directory structure.
+
+    Args:
+        model_path: Path to ensemble model directory
+        ensemble_size: Number of ensemble members
+    """
+    for i in range(ensemble_size):
+        path = os.path.join(model_path, f'ens-{i}')
+        if not os.path.exists(path):
+            os.mkdir(path)
+            os.mkdir(os.path.join(path, 'dataloaders'))
+            os.mkdir(os.path.join(path, 'database'))
+            # Add development set dataloader to each ensemble member directory
+            for set_type in ['dev']:
+                copy(os.path.join(model_path, 'dataloaders', f'{set_type}.dataloader'),
+                     os.path.join(path, 'dataloaders', f'{set_type}.dataloader'))
+            # Add training and development set ontologies to each ensemble member directory
+            for set_type in ['train', 'dev']:
+                copy(os.path.join(model_path, 'database', f'{set_type}.db'),
+                     os.path.join(path, 'database', f'{set_type}.db'))
diff --git a/convlab/dst/setsumbt/modeling/evaluation_utils.py b/convlab/dst/setsumbt/modeling/evaluation_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..c73d4b6d32a485a2cf2b5948dbd6a9a4d7f346cb
--- /dev/null
+++ b/convlab/dst/setsumbt/modeling/evaluation_utils.py
@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
+# Authors: Carel van Niekerk (niekerk@hhu.de)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Evaluation Utilities"""
+
+import random
+
+import torch
+import numpy as np
+from tqdm import tqdm
+
+
+def set_seed(args):
+    """
+    Set random seeds
+
+    Args:
+        args (Arguments class): Arguments class containing seed and number of gpus to use
+    """
+    random.seed(args.seed)
+    np.random.seed(args.seed)
+    torch.manual_seed(args.seed)
+    if args.n_gpu > 0:
+        torch.cuda.manual_seed_all(args.seed)
+
+
+def get_predictions(args, model, device: torch.device, dataloader: torch.utils.data.DataLoader) -> tuple:
+    """
+    Get model predictions
+
+    Args:
+        args: Runtime arguments
+        model: SetSUMBT Model
+        device: Torch device
+        dataloader: Dataloader containing eval data
+    """
+    model.eval()
+    
+    belief_states = {slot: [] for slot in model.setsumbt.informable_slot_ids}
+    request_probs = {slot: [] for slot in model.setsumbt.requestable_slot_ids}
+    active_domain_probs = {dom: [] for dom in model.setsumbt.domain_ids}
+    general_act_probs = []
+    state_labels = {slot: [] for slot in model.setsumbt.informable_slot_ids}
+    request_labels = {slot: [] for slot in model.setsumbt.requestable_slot_ids}
+    active_domain_labels = {dom: [] for dom in model.setsumbt.domain_ids}
+    general_act_labels = []
+    epoch_iterator = tqdm(dataloader, desc="Iteration")
+    for step, batch in enumerate(epoch_iterator):
+        with torch.no_grad():    
+            input_ids = batch['input_ids'].to(device)
+            token_type_ids = batch['token_type_ids'].to(device) if 'token_type_ids' in batch else None
+            attention_mask = batch['attention_mask'].to(device) if 'attention_mask' in batch else None
+
+            p, p_req, p_dom, p_gen, _ = model(input_ids=input_ids, token_type_ids=token_type_ids,
+                                              attention_mask=attention_mask)
+
+            for slot in belief_states:
+                p_ = p[slot]
+                labs = batch['state_labels-' + slot].to(device)
+                
+                belief_states[slot].append(p_)
+                state_labels[slot].append(labs)
+            
+            if p_req is not None:
+                for slot in request_probs:
+                    p_ = p_req[slot]
+                    labs = batch['request_labels-' + slot].to(device)
+
+                    request_probs[slot].append(p_)
+                    request_labels[slot].append(labs)
+                
+                for domain in active_domain_probs:
+                    p_ = p_dom[domain]
+                    labs = batch['active_domain_labels-' + domain].to(device)
+
+                    active_domain_probs[domain].append(p_)
+                    active_domain_labels[domain].append(labs)
+                
+                general_act_probs.append(p_gen)
+                general_act_labels.append(batch['general_act_labels'].to(device))
+    
+    for slot in belief_states:
+        belief_states[slot] = torch.cat(belief_states[slot], 0)
+        state_labels[slot] = torch.cat(state_labels[slot], 0)
+    if p_req is not None:
+        for slot in request_probs:
+            request_probs[slot] = torch.cat(request_probs[slot], 0)
+            request_labels[slot] = torch.cat(request_labels[slot], 0)
+        for domain in active_domain_probs:
+            active_domain_probs[domain] = torch.cat(active_domain_probs[domain], 0)
+            active_domain_labels[domain] = torch.cat(active_domain_labels[domain], 0)
+        general_act_probs = torch.cat(general_act_probs, 0)
+        general_act_labels = torch.cat(general_act_labels, 0)
+    else:
+        request_probs, request_labels, active_domain_probs, active_domain_labels = [None] * 4
+        general_act_probs, general_act_labels = [None] * 2
+
+    out = (belief_states, state_labels, request_probs, request_labels)
+    out += (active_domain_probs, active_domain_labels, general_act_probs, general_act_labels)
+    return out
diff --git a/convlab/dst/setsumbt/modeling/functional.py b/convlab/dst/setsumbt/modeling/functional.py
deleted file mode 100644
index 0dd083d0da080ca089e81d9ae01e5f0954243f61..0000000000000000000000000000000000000000
--- a/convlab/dst/setsumbt/modeling/functional.py
+++ /dev/null
@@ -1,456 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2021 DSML Group, Heinrich Heine University, Düsseldorf
-# Authors: Carel van Niekerk (niekerk@hhu.de)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""SetSUMBT functionals"""
-
-import torch
-import transformers
-from torch.autograd import Variable
-from torch.nn import (MultiheadAttention, GRU, LSTM, Linear, LayerNorm, Dropout,
-                      CosineSimilarity, CrossEntropyLoss, PairwiseDistance,
-                      Sequential, ReLU, Conv1d, GELU, BCEWithLogitsLoss)
-from torch.nn.init import (xavier_normal_, constant_)
-
-from convlab.dst.setsumbt.loss.bayesian import BayesianMatchingLoss, BinaryBayesianMatchingLoss, dirichlet
-from convlab.dst.setsumbt.loss.labelsmoothing import LabelSmoothingLoss, BinaryLabelSmoothingLoss
-from convlab.dst.setsumbt.loss.distillation import DistillationKL, BinaryDistillationKL
-from convlab.dst.setsumbt.loss.endd_loss import rkl_dirichlet_mediator_loss, logits_to_mutual_info
-
-
-# Default belief tracker model intialisation function
-def _initialise(self, config):
-    # Slot Utterance matching attention
-    self.slot_attention = MultiheadAttention(
-        config.hidden_size, config.slot_attention_heads)
-
-    # Latent context tracker
-    # Initial state prediction
-    if not config.rnn_zero_init and config.nbt_type in ['gru', 'lstm']:
-        self.belief_init = Sequential(Linear(config.hidden_size, config.nbt_hidden_size),
-                                      ReLU(), Dropout(config.dropout_rate))
-
-    # Recurrent context tracker setup
-    if config.nbt_type == 'gru':
-        self.nbt = GRU(input_size=config.hidden_size,
-                       hidden_size=config.nbt_hidden_size,
-                       num_layers=config.nbt_layers,
-                       dropout=0.0 if config.nbt_layers == 1 else config.dropout_rate,
-                       batch_first=True)
-        # Initialise Parameters
-        xavier_normal_(self.nbt.weight_ih_l0)
-        xavier_normal_(self.nbt.weight_hh_l0)
-        constant_(self.nbt.bias_ih_l0, 0.0)
-        constant_(self.nbt.bias_hh_l0, 0.0)
-    elif config.nbt_type == 'lstm':
-        self.nbt = LSTM(input_size=config.hidden_size,
-                        hidden_size=config.nbt_hidden_size,
-                        num_layers=config.nbt_layers,
-                        dropout=0.0 if config.nbt_layers == 1 else config.dropout_rate,
-                        batch_first=True)
-        # Initialise Parameters
-        xavier_normal_(self.nbt.weight_ih_l0)
-        xavier_normal_(self.nbt.weight_hh_l0)
-        constant_(self.nbt.bias_ih_l0, 0.0)
-        constant_(self.nbt.bias_hh_l0, 0.0)
-    else:
-        raise NameError('Not Implemented')
-
-    # Feature decoder and layer norm
-    self.intermediate = Linear(config.nbt_hidden_size, config.hidden_size)
-    self.layer_norm = LayerNorm(config.hidden_size)
-
-    # Dropout
-    self.dropout = Dropout(config.dropout_rate)
-
-    # Set pooler for set similarity model
-    if self.config.set_similarity:
-        # 1D convolutional set pooler
-        if self.config.set_pooling == 'cnn':
-            self.conv_pooler = Conv1d(
-                self.config.hidden_size, self.config.hidden_size, 3)
-        # Deep averaging network set pooler
-        elif self.config.set_pooling == 'dan':
-            self.avg_net = Sequential(Linear(self.config.hidden_size, 2 * self.config.hidden_size), GELU(),
-                                      Linear(2 * self.config.hidden_size, self.config.hidden_size))
-
-    # Model ontology placeholders
-    self.slot_embeddings = Variable(torch.zeros(0), requires_grad=False)
-    self.slot_ids = dict()
-    self.requestable_slot_ids = dict()
-    self.informable_slot_ids = dict()
-    self.domain_ids = dict()
-
-    # Matching network similarity measure
-    if config.distance_measure == 'cosine':
-        self.distance = CosineSimilarity(dim=-1, eps=1e-8)
-    elif config.distance_measure == 'euclidean':
-        self.distance = PairwiseDistance(p=2.0, eps=1e-06, keepdim=False)
-    else:
-        raise NameError('NotImplemented')
-
-    # Belief state loss function
-    if config.loss_function == 'crossentropy':
-        self.loss = CrossEntropyLoss(ignore_index=-1)
-    elif config.loss_function == 'bayesianmatching':
-        self.loss = BayesianMatchingLoss(ignore_index=-1, lamb=config.kl_scaling_factor)
-    elif config.loss_function == 'labelsmoothing':
-        self.loss = LabelSmoothingLoss(ignore_index=-1, label_smoothing=config.label_smoothing)
-    elif config.loss_function == 'distillation':
-        self.loss = DistillationKL(ignore_index=-1, lamb=config.ensemble_smoothing)
-        self.temp = 1.0
-    elif config.loss_function == 'distribution_distillation':
-        self.loss = rkl_dirichlet_mediator_loss
-        self.temp = 1.0
-    else:
-        raise NameError('NotImplemented')
-
-    # Intent and domain prediction heads
-    if config.predict_actions:
-        self.request_gate = Linear(config.hidden_size, 1)
-        self.goodbye_gate = Linear(config.hidden_size, 3)
-        self.domain_gate = Linear(config.hidden_size, 1)
-
-        # Intent and domain loss function
-        self.request_weight = float(self.config.user_request_loss_weight)
-        self.goodbye_weight = float(self.config.user_general_act_loss_weight)
-        self.domain_weight = float(self.config.active_domain_loss_weight)
-        if config.loss_function == 'crossentropy':
-            self.request_loss = BCEWithLogitsLoss()
-            self.goodbye_loss = CrossEntropyLoss(ignore_index=-1)
-            self.domain_loss = BCEWithLogitsLoss()
-        elif config.loss_function == 'labelsmoothing':
-            self.request_loss = BinaryLabelSmoothingLoss(label_smoothing=config.label_smoothing)
-            self.goodbye_loss = LabelSmoothingLoss(ignore_index=-1, label_smoothing=config.label_smoothing)
-            self.domain_loss = BinaryLabelSmoothingLoss(label_smoothing=config.label_smoothing)
-        elif config.loss_function == 'bayesianmatching':
-            self.request_loss = BinaryBayesianMatchingLoss(ignore_index=-1, lamb=config.kl_scaling_factor)
-            self.goodbye_loss = BayesianMatchingLoss(ignore_index=-1, lamb=config.kl_scaling_factor)
-            self.domain_loss = BinaryBayesianMatchingLoss(ignore_index=-1, lamb=config.kl_scaling_factor)
-        elif config.loss_function == 'distillation':
-            self.request_loss = BinaryDistillationKL(ignore_index=-1, lamb=config.ensemble_smoothing)
-            self.goodbye_loss = DistillationKL(ignore_index=-1, lamb=config.ensemble_smoothing)
-            self.domain_loss = BinaryDistillationKL(ignore_index=-1, lamb=config.ensemble_smoothing)
-
-
-# Default belief tracker forward pass.
-def _nbt_forward(self, turn_embeddings,
-                 turn_pooled_representation,
-                 attention_mask,
-                 batch_size,
-                 dialogue_size,
-                 turn_size,
-                 hidden_state,
-                 inform_labels,
-                 request_labels,
-                 domain_labels,
-                 goodbye_labels,
-                 calculate_inform_mutual_info):
-    hidden_size = turn_embeddings.size(-1)
-    # Initialise loss
-    loss = 0.0
-
-    # Goodbye predictions
-    goodbye_probs = None
-    if self.config.predict_actions:
-        # General action prediction
-        goodbye_scores = self.goodbye_gate(
-            turn_pooled_representation.reshape(batch_size * dialogue_size, hidden_size))
-
-        # Compute loss for general action predictions (weighted loss)
-        if goodbye_labels is not None:
-            if self.config.loss_function == 'distillation':
-                goodbye_labels = goodbye_labels.reshape(-1, goodbye_labels.size(-1))
-                loss += self.goodbye_loss(goodbye_scores, goodbye_labels, self.temp) * self.goodbye_weight
-            elif self.config.loss_function == 'distribution_distillation':
-                goodbye_labels = goodbye_labels.reshape(-1, goodbye_labels.size(-2), goodbye_labels.size(-1))
-                loss += self.loss(goodbye_scores, goodbye_labels, 1.0, 1.0)[0] * self.goodbye_weight
-            else:
-                goodbye_labels = goodbye_labels.reshape(-1)
-                loss += self.goodbye_loss(goodbye_scores, goodbye_labels) * self.request_weight
-
-        # Compute general action probabilities
-        if self.config.loss_function in ['crossentropy', 'labelsmoothing', 'distillation', 'distribution_distillation']:
-            goodbye_probs = torch.softmax(goodbye_scores, -1).reshape(batch_size, dialogue_size, -1)
-        elif self.config.loss_function in ['bayesianmatching']:
-            goodbye_probs = dirichlet(goodbye_scores.reshape(batch_size, dialogue_size, -1))
-
-    # Slot utterance matching
-    num_slots = self.slot_embeddings.size(0)
-    slot_embeddings = self.slot_embeddings.reshape(-1, hidden_size)
-    slot_embeddings = slot_embeddings.unsqueeze(1).repeat((1, batch_size * dialogue_size, 1)).to(turn_embeddings.device)
-
-    if self.config.set_similarity:
-        # Slot mask shape [num_slots * slot_len, batch_size * dialogue_size, 768]
-        slot_mask = (slot_embeddings != 0.0).float()
-
-    # Turn embeddings shape [turn_size, batch_size * dialogue_size, 768]
-    turn_embeddings = turn_embeddings.transpose(0, 1)
-    # Compute key padding mask
-    key_padding_mask = (attention_mask[:, :, 0] == 0.0)
-    key_padding_mask[key_padding_mask[:, 0] == True, :] = False
-    # Multi head attention of slot over tokens
-    hidden, _ = self.slot_attention(query=slot_embeddings,
-                                    key=turn_embeddings,
-                                    value=turn_embeddings,
-                                    key_padding_mask=key_padding_mask)  # [num_slots, batch_size * dialogue_size, 768]
-
-    # Set embeddings for all masked tokens to 0
-    attention_mask = attention_mask[:, 0, :].unsqueeze(0).repeat((slot_embeddings.size(0), 1, 1))
-    hidden = hidden * attention_mask
-    if self.config.set_similarity:
-        hidden = hidden * slot_mask
-    # Hidden layer shape [num_dials, num_slots, num_turns, 768]
-    hidden = hidden.transpose(0, 1).reshape(batch_size, dialogue_size, slot_embeddings.size(0), -1).transpose(1, 2)
-
-    # Latent context tracking
-    # [batch_size * num_slots, dialogue_size, 768]
-    hidden = hidden.reshape(batch_size * slot_embeddings.size(0), dialogue_size, -1)
-
-    if self.config.nbt_type == 'gru':
-        self.nbt.flatten_parameters()
-        if hidden_state is None:
-            if self.config.rnn_zero_init:
-                context = torch.zeros(self.config.nbt_layers, batch_size * slot_embeddings.size(0),
-                                      self.config.nbt_hidden_size)
-                context = context.to(turn_embeddings.device)
-            else:
-                context = self.belief_init(hidden[:, 0, :]).unsqueeze(0).repeat((self.config.nbt_layers, 1, 1))
-        else:
-            context = hidden_state.to(hidden.device)
-
-        # [batch_size, dialogue_size, nbt_hidden_size]
-        belief_embedding, context = self.nbt(hidden, context)
-    elif self.config.nbt_type == 'lstm':
-        self.nbt.flatten_parameters()
-        if self.config.rnn_zero_init:
-            context = (torch.zeros(self.config.nbt_layers, batch_size * num_slots, self.config.nbt_hidden_size),
-                       torch.zeros(self.config.nbt_layers, batch_size * num_slots, self.config.nbt_hidden_size))
-            context = (context[0].to(turn_embeddings.device),
-                       context[1].to(turn_embeddings.device))
-        else:
-            context = (self.belief_init(hidden[:, 0, :]).unsqueeze(0).repeat((self.config.nbt_layers, 1, 1)),
-                       torch.zeros(self.config.nbt_layers, batch_size * num_slots, self.config.nbt_hidden_size))
-            context = (context[0], context[1].to(turn_embeddings.device))
-
-        # [batch_size, dialogue_size, nbt_hidden_size]
-        belief_embedding, context = self.nbt(hidden, context)
-
-    # Decode features
-    belief_embedding = belief_embedding.reshape(batch_size, slot_embeddings.size(0), dialogue_size, -1).transpose(1, 2)
-    if self.config.set_similarity:
-        belief_embedding = belief_embedding.reshape(batch_size, dialogue_size, num_slots, -1,
-                                                    self.config.nbt_hidden_size)
-    # [batch_size, dialogue_size, num_slots, *slot_desc_len, 768]
-    # Normalisation and regularisation
-    belief_embedding = self.layer_norm(self.intermediate(belief_embedding))
-    belief_embedding = self.dropout(belief_embedding)
-
-    # Pooling of the set of latent context representation
-    if self.config.set_similarity:
-        slot_mask = slot_mask.transpose(0, 1).reshape(batch_size, dialogue_size, num_slots, -1, hidden_size)
-        belief_embedding = belief_embedding * slot_mask
-
-        # Apply pooler to latent context sequence
-        if self.config.set_pooling == 'mean':
-            belief_embedding = belief_embedding.sum(-2) / slot_mask.sum(-2)
-            belief_embedding = belief_embedding.reshape(batch_size, dialogue_size, num_slots, -1)
-        elif self.config.set_pooling == 'cnn':
-            belief_embedding = belief_embedding.reshape(-1, slot_mask.size(-2), hidden_size).transpose(1, 2)
-            belief_embedding = self.conv_pooler(belief_embedding)
-            # Mean pooling after CNN
-            belief_embedding = belief_embedding.mean(-1).reshape(batch_size, dialogue_size, num_slots, -1)
-        elif self.config.set_pooling == 'dan':
-            # sqrt N reduction
-            belief_embedding = belief_embedding.sum(-2) / torch.sqrt(torch.tensor(slot_mask.sum(-2)))
-            # Deep averaging feature extractor
-            belief_embedding = self.avg_net(belief_embedding)
-            belief_embedding = belief_embedding.reshape(batch_size, dialogue_size, num_slots, -1)
-
-    # Perform classification
-    if self.config.predict_actions:
-        # User request prediction
-        request_probs = dict()
-        for slot, slot_id in self.requestable_slot_ids.items():
-            request_scores = self.request_gate(belief_embedding[:, :, slot_id, :])
-
-            # Store output probabilities
-            request_scores = request_scores.reshape(batch_size, dialogue_size)
-            mask = attention_mask[0, :, 0].reshape(batch_size, dialogue_size)
-            batches, dialogues = torch.where(mask == 0.0)
-            # Set request scores to 0.0 for padded turns
-            request_scores[batches, dialogues] = 0.0
-            if self.config.loss_function in ['crossentropy', 'labelsmoothing', 'bayesianmatching',
-                                             'distillation', 'distribution_distillation']:
-                request_probs[slot] = torch.sigmoid(request_scores)
-
-            if request_labels is not None:
-                # Compute request gate loss
-                request_scores = request_scores.reshape(-1)
-                if self.config.loss_function == 'distillation':
-                    loss += self.request_loss(request_scores, request_labels[slot].reshape(-1),
-                                              self.temp) * self.request_weight
-                elif self.config.loss_function == 'distribution_distillation':
-                    scores, labs = convert_probs_to_logits(request_scores, request_labels[slot])
-                    loss += self.loss(scores, labs, 1.0, 1.0)[0] * self.request_weight
-                else:
-                    labs = request_labels[slot].reshape(-1)
-                    request_scores = request_scores[labs != -1]
-                    labs = labs[labs != -1].float()
-                    loss += self.request_loss(request_scores, labs) * self.request_weight
-
-        # Active domain prediction
-        domain_probs = dict()
-        for domain, slot_ids in self.domain_ids.items():
-            belief = belief_embedding[:, :, slot_ids, :]
-            if len(slot_ids) > 1:
-                # SqrtN reduction across all slots within a domain
-                belief = belief.sum(2) / ((belief != 0.0).float().sum(2) ** 0.5)
-            domain_scores = self.domain_gate(belief)
-
-            # Store output probabilities
-            domain_scores = domain_scores.reshape(batch_size, dialogue_size)
-            mask = attention_mask[0, :, 0].reshape(batch_size, dialogue_size)
-            batches, dialogues = torch.where(mask == 0.0)
-            domain_scores[batches, dialogues] = 0.0
-            if self.config.loss_function in ['crossentropy', 'labelsmoothing', 'bayesianmatching', 'distillation',
-                                             'distribution_distillation']:
-                domain_probs[domain] = torch.sigmoid(domain_scores)
-
-            if domain_labels is not None:
-                # Compute domain prediction loss
-                domain_scores = domain_scores.reshape(-1)
-                if self.config.loss_function == 'distillation':
-                    loss += self.domain_loss(domain_scores, domain_labels[domain].reshape(-1),
-                                             self.temp) * self.domain_weight
-                elif self.config.loss_function == 'distribution_distillation':
-                    scores, labs = convert_probs_to_logits(domain_scores, domain_labels[domain])
-                    loss += self.loss(scores, labs, 1.0, 1.0)[0] * self.request_weight
-                else:
-                    labs = domain_labels[domain].reshape(-1)
-                    domain_scores = domain_scores[labs != -1]
-                    labs = labs[labs != -1].float()
-                    loss += self.domain_loss(domain_scores, labs) * self.domain_weight
-    else:
-        request_probs, domain_probs = None, None
-
-    # Informable slot predictions
-    inform_probs = dict()
-    out_dict = dict()
-    mutual_info = dict()
-    stats = dict()
-    for slot, slot_id in self.informable_slot_ids.items():
-        # Get slot belief embedding and value candidates
-        candidate_embeddings = getattr(self, slot + '_value_embeddings').to(turn_embeddings.device)
-        belief = belief_embedding[:, :, slot_id, :]
-        slot_size = candidate_embeddings.size(0)
-
-        # Use similaroty matching to produce belief state
-        if self.config.distance_measure in ['cosine', 'euclidean']:
-            belief = belief.unsqueeze(2).repeat((1, 1, slot_size, 1))
-            belief = belief.reshape(-1, self.config.hidden_size)
-
-            # Pooling of set of value candidate description representation
-            if self.config.set_similarity and self.config.set_pooling == 'mean':
-                candidate_mask = (candidate_embeddings != 0.0).float()
-                candidate_embeddings = candidate_embeddings.sum(1) / candidate_mask.sum(1)
-            elif self.config.set_similarity and self.config.set_pooling == 'cnn':
-                candidate_embeddings = candidate_embeddings.transpose(1, 2)
-                candidate_embeddings = self.conv_pooler(candidate_embeddings).mean(-1)
-            elif self.config.set_similarity and self.config.set_pooling == 'dan':
-                candidate_mask = (candidate_embeddings != 0.0).float()
-                candidate_embeddings = candidate_embeddings.sum(1) / torch.sqrt(torch.tensor(candidate_mask.sum(1)))
-                candidate_embeddings = self.avg_net(candidate_embeddings)
-
-            candidate_embeddings = candidate_embeddings.unsqueeze(0).unsqueeze(0).repeat((batch_size,
-                                                                                          dialogue_size, 1, 1))
-            candidate_embeddings = candidate_embeddings.reshape(-1, self.config.hidden_size)
-
-        # Score value candidates
-        if self.config.distance_measure == 'cosine':
-            scores = self.distance(belief, candidate_embeddings)
-            # *27 here rescales the cosine similarity for better learning
-            scores = scores.reshape(batch_size * dialogue_size, -1) * 27.0
-        elif self.config.distance_measure == 'euclidean':
-            scores = -1.0 * self.distance(belief, candidate_embeddings)
-            scores = scores.reshape(batch_size * dialogue_size, -1)
-
-        # Calculate belief state
-        if self.config.loss_function in ['crossentropy', 'inhibitedce',
-                                         'labelsmoothing', 'distillation', 'distribution_distillation']:
-            probs_ = torch.softmax(scores.reshape(batch_size, dialogue_size, -1), -1)
-        elif self.config.loss_function in ['bayesianmatching']:
-            probs_ = dirichlet(scores.reshape(batch_size, dialogue_size, -1))
-
-        # Compute knowledge uncertainty in the beleif states
-        if calculate_inform_mutual_info and self.config.loss_function == 'distribution_distillation':
-            mutual_info[slot] = logits_to_mutual_info(scores).reshape(batch_size, dialogue_size)
-
-        # Set padded turn probabilities to zero
-        mask = attention_mask[self.slot_ids[slot],:, 0].reshape(batch_size, dialogue_size)
-        batches, dialogues = torch.where(mask == 0.0)
-        probs_[batches, dialogues, :] = 0.0
-        inform_probs[slot] = probs_
-
-        # Calculate belief state loss
-        if inform_labels is not None and slot in inform_labels:
-            if self.config.loss_function == 'bayesianmatching':
-                prior = torch.ones(scores.size(-1)).float().to(scores.device)
-                prior = prior * self.config.prior_constant
-                prior = prior.unsqueeze(0).repeat((scores.size(0), 1))
-
-                loss += self.loss(scores, inform_labels[slot].reshape(-1), prior=prior)
-            elif self.config.loss_function == 'distillation':
-                labels = inform_labels[slot]
-                labels = labels.reshape(-1, labels.size(-1))
-                loss += self.loss(scores, labels, self.temp)
-            elif self.config.loss_function == 'distribution_distillation':
-                labels = inform_labels[slot]
-                labels = labels.reshape(-1, labels.size(-2), labels.size(-1))
-                loss_, model_stats, ensemble_stats = self.loss(scores, labels, 1.0, 1.0)
-                loss += loss_
-
-                # Calculate stats regarding model precisions
-                precision = model_stats['precision']
-                ensemble_precision = ensemble_stats['precision']
-                stats[slot] = {'model_precision_min': precision.min(),
-                               'model_precision_max': precision.max(),
-                               'model_precision_mean': precision.mean(),
-                               'ensemble_precision_min': ensemble_precision.min(),
-                               'ensemble_precision_max': ensemble_precision.max(),
-                               'ensemble_precision_mean': ensemble_precision.mean()}
-            else:
-                loss += self.loss(scores, inform_labels[slot].reshape(-1))
-
-    # Return model outputs
-    out = inform_probs, request_probs, domain_probs, goodbye_probs, context
-    if inform_labels is not None or request_labels is not None or domain_labels is not None or goodbye_labels is not None:
-        out = (loss,) + out + (stats,)
-    if calculate_inform_mutual_info:
-        out = out + (mutual_info,)
-    return out
-
-
-# Convert binary scores and labels to 2 class classification problem for distribution distillation
-def convert_probs_to_logits(scores, labels):
-    # Convert single target probability p to distribution [1-p, p]
-    labels = labels.reshape(-1, labels.size(-1), 1)
-    labels = torch.cat([1 - labels, labels], -1)
-
-    # Convert input scores into predictive distribution [1-z, z]
-    scores = torch.sigmoid(scores).unsqueeze(1)
-    scores = torch.cat((1 - scores, scores), 1)
-    scores = -1.0 * torch.log((1 / (scores + 1e-8)) - 1)  # Inverse sigmoid
-
-    return scores, labels
diff --git a/convlab/dst/setsumbt/modeling/roberta_nbt.py b/convlab/dst/setsumbt/modeling/roberta_nbt.py
index 36920c5ca550a3295f31aaca53c2ebed8c22be37..f72d17fafa50553434b6d4dcd20b8e53d143892f 100644
--- a/convlab/dst/setsumbt/modeling/roberta_nbt.py
+++ b/convlab/dst/setsumbt/modeling/roberta_nbt.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2021 DSML Group, Heinrich Heine University, Düsseldorf
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
 # Authors: Carel van Niekerk (niekerk@hhu.de)
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,16 +16,19 @@
 """RoBERTa SetSUMBT"""
 
 import torch
-import transformers
-from torch.autograd import Variable
 from transformers import RobertaModel, RobertaPreTrainedModel
 
-from convlab.dst.setsumbt.modeling.functional import _initialise, _nbt_forward
+from convlab.dst.setsumbt.modeling.setsumbt import SetSUMBTHead
 
 
 class RobertaSetSUMBT(RobertaPreTrainedModel):
+    """Roberta based SetSUMBT model"""
 
     def __init__(self, config):
+        """
+        Args:
+            config (configuration): Model configuration class
+        """
         super(RobertaSetSUMBT, self).__init__(config)
         self.config = config
 
@@ -35,60 +38,37 @@ class RobertaSetSUMBT(RobertaPreTrainedModel):
             for p in self.roberta.parameters():
                 p.requires_grad = False
 
-        _initialise(self, config)
+        self.setsumbt = SetSUMBTHead(config)
+        self.add_slot_candidates = self.setsumbt.add_slot_candidates
+        self.add_value_candidates = self.setsumbt.add_value_candidates
     
+    def forward(self,
+                input_ids: torch.Tensor,
+                attention_mask: torch.Tensor,
+                token_type_ids: torch.Tensor = None,
+                hidden_state: torch.Tensor = None,
+                state_labels: torch.Tensor = None,
+                request_labels: torch.Tensor = None,
+                active_domain_labels: torch.Tensor = None,
+                general_act_labels: torch.Tensor = None,
+                get_turn_pooled_representation: bool = False,
+                calculate_state_mutual_info: bool = False):
+        """
+        Args:
+            input_ids: Input token ids
+            attention_mask: Input padding mask
+            token_type_ids: Token type indicator
+            hidden_state: Latent internal dialogue belief state
+            state_labels: Dialogue state labels
+            request_labels: User request action labels
+            active_domain_labels: Current active domain labels
+            general_act_labels: General user action labels
+            get_turn_pooled_representation: Return pooled representation of the current dialogue turn
+            calculate_state_mutual_info: Return mutual information in the dialogue state
 
-    # Add new slot candidates to the model
-    def add_slot_candidates(self, slot_candidates):
-        """slot_candidates is a list of tuples for each slot.
-        - The tuples contains the slot embedding, informable value embeddings and a request indicator.
-        - If the informable value embeddings is None the slot is not informable
-        - If the request indicator is false the slot is not requestable"""
-        if self.slot_embeddings.size(0) != 0:
-            embeddings = self.slot_embeddings.detach()
-        else:
-            embeddings = torch.zeros(0)
-
-        for slot in slot_candidates:
-            if slot in self.slot_ids:
-                index = self.slot_ids[slot]
-                embeddings[index, :] = slot_candidates[slot][0]
-            else:
-                index = embeddings.size(0)
-                emb = slot_candidates[slot][0].unsqueeze(0).to(embeddings.device)
-                embeddings = torch.cat((embeddings, emb), 0)
-                self.slot_ids[slot] = index
-                setattr(self, slot + '_value_embeddings', Variable(torch.zeros(0), requires_grad=False))
-            # Add slot to relevant requestable and informable slot lists
-            if slot_candidates[slot][2]:
-                self.requestable_slot_ids[slot] = index
-            if slot_candidates[slot][1] is not None:
-                self.informable_slot_ids[slot] = index
-            
-            domain = slot.split('-', 1)[0]
-            if domain not in self.domain_ids:
-                self.domain_ids[domain] = []
-            self.domain_ids[domain].append(index)
-            self.domain_ids[domain] = list(set(self.domain_ids[domain]))
-        
-        self.slot_embeddings = Variable(embeddings, requires_grad=False)
-
-
-    # Add new value candidates to the model
-    def add_value_candidates(self, slot, value_candidates, replace=False):
-        embeddings = getattr(self, slot + '_value_embeddings')
-
-        if embeddings.size(0) == 0 or replace:
-            embeddings = value_candidates
-        else:
-            embeddings = torch.cat((embeddings, value_candidates.to(embeddings.device)), 0)
-        
-        setattr(self, slot + '_value_embeddings', embeddings)
-        
-    
-    def forward(self, input_ids, attention_mask, token_type_ids=None, hidden_state=None, inform_labels=None,
-                request_labels=None, domain_labels=None, goodbye_labels=None,
-                get_turn_pooled_representation=False, calculate_inform_mutual_info=False):
+        Returns:
+            out: Tuple containing loss, predictive distributions, model statistics and state mutual information
+        """
         if token_type_ids is not None:
             token_type_ids = None
 
@@ -106,9 +86,10 @@ class RobertaSetSUMBT(RobertaPreTrainedModel):
         turn_embeddings = turn_embeddings.reshape(batch_size * dialogue_size, turn_size, -1)
         
         if get_turn_pooled_representation:
-            return _nbt_forward(self, turn_embeddings, roberta_output.pooler_output, attention_mask, batch_size, dialogue_size,
-                                turn_size, hidden_state, inform_labels, request_labels, domain_labels, goodbye_labels,
-                                calculate_inform_mutual_info) + (roberta_output.pooler_output,)
-        return _nbt_forward(self, turn_embeddings, roberta_output.pooler_output, attention_mask, batch_size, dialogue_size,
-                            turn_size, hidden_state, inform_labels, request_labels, domain_labels, goodbye_labels,
-                            calculate_inform_mutual_info)
+            return self.setsumbt(turn_embeddings, roberta_output.pooler_output, attention_mask,
+                                 batch_size, dialogue_size, hidden_state, state_labels,
+                                 request_labels, active_domain_labels, general_act_labels,
+                                 calculate_state_mutual_info) + (roberta_output.pooler_output,)
+        return self.setsumbt(turn_embeddings, roberta_output.pooler_output, attention_mask, batch_size,
+                             dialogue_size, hidden_state, state_labels, request_labels, active_domain_labels,
+                             general_act_labels, calculate_state_mutual_info)
diff --git a/convlab/dst/setsumbt/modeling/setsumbt.py b/convlab/dst/setsumbt/modeling/setsumbt.py
new file mode 100644
index 0000000000000000000000000000000000000000..0249649f0840d66b0cec8a65c91aded906f62f85
--- /dev/null
+++ b/convlab/dst/setsumbt/modeling/setsumbt.py
@@ -0,0 +1,564 @@
+# -*- coding: utf-8 -*-
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
+# Authors: Carel van Niekerk (niekerk@hhu.de)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""SetSUMBT Prediction Head"""
+
+import torch
+from torch.autograd import Variable
+from torch.nn import (Module, MultiheadAttention, GRU, LSTM, Linear, LayerNorm, Dropout,
+                      CosineSimilarity, CrossEntropyLoss, PairwiseDistance,
+                      Sequential, ReLU, Conv1d, GELU, BCEWithLogitsLoss)
+from torch.nn.init import (xavier_normal_, constant_)
+
+from convlab.dst.setsumbt.loss import (BayesianMatchingLoss, BinaryBayesianMatchingLoss,
+                                       KLDistillationLoss, BinaryKLDistillationLoss,
+                                       LabelSmoothingLoss, BinaryLabelSmoothingLoss,
+                                       RKLDirichletMediatorLoss, BinaryRKLDirichletMediatorLoss)
+
+
+class SlotUtteranceMatching(Module):
+    """Slot Utterance matching attention based information extractor"""
+
+    def __init__(self, hidden_size: int = 768, attention_heads: int = 12):
+        """
+        Args:
+            hidden_size (int): Dimension of token embeddings
+            attention_heads (int): Number of attention heads to use in attention module
+        """
+        super(SlotUtteranceMatching, self).__init__()
+
+        self.attention = MultiheadAttention(hidden_size, attention_heads)
+
+    def forward(self,
+                turn_embeddings: torch.Tensor,
+                attention_mask: torch.Tensor,
+                slot_embeddings: torch.Tensor) -> torch.Tensor:
+        """
+        Args:
+            turn_embeddings: Embeddings for each token in each turn [n_turns, turn_length, hidden_size]
+            attention_mask: Padding mask for each turn [n_turns, turn_length, hidden_size]
+            slot_embeddings: Embeddings for each token in the slot descriptions
+
+        Returns:
+            hidden: Information extracted from turn related to slot descriptions
+        """
+        turn_embeddings = turn_embeddings.transpose(0, 1)
+
+        key_padding_mask = (attention_mask[:, :, 0] == 0.0)
+        key_padding_mask[key_padding_mask[:, 0], :] = False
+
+        hidden, _ = self.attention(query=slot_embeddings, key=turn_embeddings, value=turn_embeddings,
+                                   key_padding_mask=key_padding_mask)
+
+        attention_mask = attention_mask[:, 0, :].unsqueeze(0).repeat((slot_embeddings.size(0), 1, 1))
+        hidden = hidden * attention_mask
+
+        return hidden
+
+
+class RecurrentNeuralBeliefTracker(Module):
+    """Recurrent latent neural belief tracking module"""
+
+    def __init__(self,
+                 nbt_type: str = 'gru',
+                 rnn_zero_init: bool = False,
+                 input_size: int = 768,
+                 hidden_size: int = 300,
+                 hidden_layers: int = 1,
+                 dropout_rate: float = 0.3):
+        """
+        Args:
+            nbt_type: Type of recurrent neural network (gru/lstm)
+            rnn_zero_init: Use zero initialised state for the RNN
+            input_size: Embedding size of the inputs
+            hidden_size: Hidden size of the RNN
+            hidden_layers: Number of RNN Layers
+            dropout_rate: Dropout rate
+        """
+        super(RecurrentNeuralBeliefTracker, self).__init__()
+
+        if rnn_zero_init:
+            self.belief_init = Sequential(Linear(input_size, hidden_size), ReLU(), Dropout(dropout_rate))
+        else:
+            self.belief_init = None
+
+        self.nbt_type = nbt_type
+        self.hidden_layers = hidden_layers
+        self.hidden_size = hidden_size
+        if nbt_type == 'gru':
+            self.nbt = GRU(input_size=input_size,
+                           hidden_size=hidden_size,
+                           num_layers=hidden_layers,
+                           dropout=0.0 if hidden_layers == 1 else dropout_rate,
+                           batch_first=True)
+        elif nbt_type == 'lstm':
+            self.nbt = LSTM(input_size=input_size,
+                            hidden_size=hidden_size,
+                            num_layers=hidden_layers,
+                            dropout=0.0 if hidden_layers == 1 else dropout_rate,
+                            batch_first=True)
+        else:
+            raise NameError('Not Implemented')
+
+        # Initialise Parameters
+        xavier_normal_(self.nbt.weight_ih_l0)
+        xavier_normal_(self.nbt.weight_hh_l0)
+        constant_(self.nbt.bias_ih_l0, 0.0)
+        constant_(self.nbt.bias_hh_l0, 0.0)
+
+        # Intermediate feature mapping and layer normalisation
+        self.intermediate = Linear(hidden_size, input_size)
+        self.layer_norm = LayerNorm(input_size)
+        self.dropout = Dropout(dropout_rate)
+
+    def forward(self, inputs: torch.Tensor, hidden_state: torch.Tensor = None) -> torch.Tensor:
+        """
+        Args:
+            inputs: Latent turn level information
+            hidden_state: Latent internal belief state
+
+        Returns:
+            belief_embedding: Belief state embeddings
+            context: Latent internal belief state
+        """
+        self.nbt.flatten_parameters()
+        if hidden_state is None:
+            if self.belief_init is None:
+                context = torch.zeros(self.hidden_layers, inputs.size(0), self.hidden_size).to(inputs.device)
+            else:
+                context = self.belief_init(inputs[:, 0, :]).unsqueeze(0).repeat((self.hidden_layers, 1, 1))
+            if self.nbt_type == "lstm":
+                context = (context, torch.zeros(self.hidden_layers, inputs.size(0), self.hidden_size).to(inputs.device))
+        else:
+            context = hidden_state.to(inputs.device)
+
+        # [batch_size, dialogue_size, nbt_hidden_size]
+        belief_embedding, context = self.nbt(inputs, context)
+
+        # Normalisation and regularisation
+        belief_embedding = self.layer_norm(self.intermediate(belief_embedding))
+        belief_embedding = self.dropout(belief_embedding)
+
+        return belief_embedding, context
+
+
+class SetPooler(Module):
+    """Token set pooler"""
+
+    def __init__(self, pooling_strategy: str = 'cnn', hidden_size: int = 768):
+        """
+        Args:
+            pooling_strategy: Type of set pooler (cnn/dan/mean)
+            hidden_size: Token embedding size
+        """
+        super(SetPooler, self).__init__()
+
+        self.pooling_strategy = pooling_strategy
+        if pooling_strategy == 'cnn':
+            self.cnn_filter_size = 3
+            self.pooler = Conv1d(hidden_size, hidden_size, self.cnn_filter_size)
+        elif pooling_strategy == 'dan':
+            self.pooler = Sequential(Linear(hidden_size, hidden_size), GELU(), Linear(2 * hidden_size, hidden_size))
+
+    def forward(self, inputs, attention_mask):
+        """
+        Args:
+            inputs: Token set embeddings
+            attention_mask: Padding mask for the set of tokens
+
+        Returns:
+
+        """
+        if self.pooling_strategy == "mean":
+            hidden = inputs.sum(1) / attention_mask.sum(1)
+        elif self.pooling_strategy == "cnn":
+            hidden = self.pooler(inputs.transpose(1, 2)).mean(-1)
+        elif self.pooling_strategy == 'dan':
+            hidden = inputs.sum(1) / torch.sqrt(torch.tensor(attention_mask.sum(1)))
+            hidden = self.pooler(hidden)
+
+        return hidden
+
+
+class SetSUMBTHead(Module):
+    """SetSUMBT Prediction Head for Language Models"""
+
+    def __init__(self, config):
+        """
+        Args:
+            config (configuration): Model configuration class
+        """
+        super(SetSUMBTHead, self).__init__()
+        self.config = config
+        # Slot Utterance matching attention
+        self.slot_utterance_matching = SlotUtteranceMatching(config.hidden_size, config.slot_attention_heads)
+
+        # Latent context tracker
+        self.nbt = RecurrentNeuralBeliefTracker(config.nbt_type, config.rnn_zero_init, config.hidden_size,
+                                                config.nbt_hidden_size, config.nbt_layers, config.dropout_rate)
+
+        # Set pooler for set similarity model
+        if self.config.set_similarity:
+            self.set_pooler = SetPooler(config.set_pooling, config.hidden_size)
+
+        # Model ontology placeholders
+        self.slot_embeddings = Variable(torch.zeros(0), requires_grad=False)
+        self.slot_ids = dict()
+        self.requestable_slot_ids = dict()
+        self.informable_slot_ids = dict()
+        self.domain_ids = dict()
+
+        # Matching network similarity measure
+        if config.distance_measure == 'cosine':
+            self.distance = CosineSimilarity(dim=-1, eps=1e-8)
+        elif config.distance_measure == 'euclidean':
+            self.distance = PairwiseDistance(p=2.0, eps=1e-6, keepdim=False)
+        else:
+            raise NameError('NotImplemented')
+
+        # User goal prediction loss function
+        if config.loss_function == 'crossentropy':
+            self.loss = CrossEntropyLoss(ignore_index=-1)
+        elif config.loss_function == 'bayesianmatching':
+            self.loss = BayesianMatchingLoss(ignore_index=-1, lamb=config.kl_scaling_factor)
+        elif config.loss_function == 'labelsmoothing':
+            self.loss = LabelSmoothingLoss(ignore_index=-1, label_smoothing=config.label_smoothing)
+        elif config.loss_function == 'distillation':
+            self.loss = KLDistillationLoss(ignore_index=-1, lamb=config.ensemble_smoothing)
+            self.temp = 1.0
+        elif config.loss_function == 'distribution_distillation':
+            self.loss = RKLDirichletMediatorLoss(ignore_index=-1)
+        else:
+            raise NameError('NotImplemented')
+
+        # Intent and domain prediction heads
+        if config.predict_actions:
+            self.request_gate = Linear(config.hidden_size, 1)
+            self.general_act_gate = Linear(config.hidden_size, 3)
+            self.active_domain_gate = Linear(config.hidden_size, 1)
+
+            # Intent and domain loss function
+            self.request_weight = float(self.config.user_request_loss_weight)
+            self.general_act_weight = float(self.config.user_general_act_loss_weight)
+            self.active_domain_weight = float(self.config.active_domain_loss_weight)
+            if config.loss_function == 'crossentropy':
+                self.request_loss = BCEWithLogitsLoss()
+                self.general_act_loss = CrossEntropyLoss(ignore_index=-1)
+                self.active_domain_loss = BCEWithLogitsLoss()
+            elif config.loss_function == 'labelsmoothing':
+                self.request_loss = BinaryLabelSmoothingLoss(label_smoothing=config.label_smoothing)
+                self.general_act_loss = LabelSmoothingLoss(ignore_index=-1, label_smoothing=config.label_smoothing)
+                self.active_domain_loss = BinaryLabelSmoothingLoss(label_smoothing=config.label_smoothing)
+            elif config.loss_function == 'bayesianmatching':
+                self.request_loss = BinaryBayesianMatchingLoss(ignore_index=-1, lamb=config.kl_scaling_factor)
+                self.general_act_loss = BayesianMatchingLoss(ignore_index=-1, lamb=config.kl_scaling_factor)
+                self.active_domain_loss = BinaryBayesianMatchingLoss(ignore_index=-1, lamb=config.kl_scaling_factor)
+            elif config.loss_function == 'distillation':
+                self.request_loss = BinaryKLDistillationLoss(ignore_index=-1, lamb=config.ensemble_smoothing)
+                self.general_act_loss = KLDistillationLoss(ignore_index=-1, lamb=config.ensemble_smoothing)
+                self.active_domain_loss = BinaryKLDistillationLoss(ignore_index=-1, lamb=config.ensemble_smoothing)
+            elif config.loss_function == 'distribution_distillation':
+                self.request_loss = BinaryRKLDirichletMediatorLoss(ignore_index=-1)
+                self.general_act_loss = RKLDirichletMediatorLoss(ignore_index=-1)
+                self.active_domain_loss = BinaryRKLDirichletMediatorLoss(ignore_index=-1)
+
+    def add_slot_candidates(self, slot_candidates: tuple):
+        """
+        Add slots to the model ontology, the tuples should contain the slot embedding, informable value embeddings
+        and a request indicator, if the informable value embeddings is None the slot is not informable and if
+        the request indicator is false the slot is not requestable.
+
+        Args:
+            slot_candidates: Tuple containing slot embedding, informable value embeddings and a request indicator
+        """
+        if self.slot_embeddings.size(0) != 0:
+            embeddings = self.slot_embeddings.detach()
+        else:
+            embeddings = torch.zeros(0)
+
+        for slot in slot_candidates:
+            if slot in self.slot_ids:
+                index = self.slot_ids[slot]
+                embeddings[index, :] = slot_candidates[slot][0]
+            else:
+                index = embeddings.size(0)
+                emb = slot_candidates[slot][0].unsqueeze(0).to(embeddings.device)
+                embeddings = torch.cat((embeddings, emb), 0)
+                self.slot_ids[slot] = index
+                setattr(self, slot + '_value_embeddings', Variable(torch.zeros(0), requires_grad=False))
+            # Add slot to relevant requestable and informable slot lists
+            if slot_candidates[slot][2]:
+                self.requestable_slot_ids[slot] = index
+            if slot_candidates[slot][1] is not None:
+                self.informable_slot_ids[slot] = index
+
+            domain = slot.split('-', 1)[0]
+            if domain not in self.domain_ids:
+                self.domain_ids[domain] = []
+            self.domain_ids[domain].append(index)
+            self.domain_ids[domain] = list(set(self.domain_ids[domain]))
+
+        self.slot_embeddings = Variable(embeddings, requires_grad=False)
+
+    def add_value_candidates(self, slot: str, value_candidates: torch.Tensor, replace: bool = False):
+        """
+        Add value candidates for a slot
+
+        Args:
+            slot: Slot name
+            value_candidates: Value candidate embeddings
+            replace: If true existing value candidates are replaced
+        """
+        embeddings = getattr(self, slot + '_value_embeddings')
+
+        if embeddings.size(0) == 0 or replace:
+            embeddings = value_candidates
+        else:
+            embeddings = torch.cat((embeddings, value_candidates.to(embeddings.device)), 0)
+
+        setattr(self, slot + '_value_embeddings', embeddings)
+
+    def forward(self,
+                turn_embeddings: torch.Tensor,
+                turn_pooled_representation: torch.Tensor,
+                attention_mask: torch.Tensor,
+                batch_size: int,
+                dialogue_size: int,
+                hidden_state: torch.Tensor = None,
+                state_labels: torch.Tensor = None,
+                request_labels: torch.Tensor = None,
+                active_domain_labels: torch.Tensor = None,
+                general_act_labels: torch.Tensor = None,
+                calculate_state_mutual_info: bool = False):
+        """
+        Args:
+            turn_embeddings: Token embeddings in the current turn
+            turn_pooled_representation: Pooled representation of the current dialogue turn
+            attention_mask: Padding mask for the current dialogue turn
+            batch_size: Number of dialogues in the batch
+            dialogue_size: Number of turns in each dialogue
+            hidden_state: Latent internal dialogue belief state
+            state_labels: Dialogue state labels
+            request_labels: User request action labels
+            active_domain_labels: Current active domain labels
+            general_act_labels: General user action labels
+            calculate_state_mutual_info: Return mutual information in the dialogue state
+
+        Returns:
+            out: Tuple containing loss, predictive distributions, model statistics and state mutual information
+        """
+        hidden_size = turn_embeddings.size(-1)
+        # Initialise loss
+        loss = 0.0
+
+        # General Action predictions
+        general_act_probs = None
+        if self.config.predict_actions:
+            # General action prediction
+            general_act_logits = self.general_act_gate(turn_pooled_representation.reshape(batch_size * dialogue_size,
+                                                                                          hidden_size))
+
+            # Compute loss for general action predictions (weighted loss)
+            if general_act_labels is not None:
+                if self.config.loss_function == 'distillation':
+                    general_act_labels = general_act_labels.reshape(-1, general_act_labels.size(-1))
+                    loss += self.general_act_loss(general_act_logits, general_act_labels,
+                                                  self.temp) * self.general_act_weight
+                elif self.config.loss_function == 'distribution_distillation':
+                    general_act_labels = general_act_labels.reshape(-1, general_act_labels.size(-2),
+                                                                    general_act_labels.size(-1))
+                    loss += self.general_act_loss(general_act_logits, general_act_labels)[0] * self.general_act_weight
+                else:
+                    general_act_labels = general_act_labels.reshape(-1)
+                    loss += self.general_act_loss(general_act_logits, general_act_labels) * self.general_act_weight
+
+            # Compute general action probabilities
+            general_act_probs = torch.softmax(general_act_logits, -1).reshape(batch_size, dialogue_size, -1)
+
+        # Slot utterance matching
+        num_slots = self.slot_embeddings.size(0)
+        slot_embeddings = self.slot_embeddings.reshape(-1, hidden_size)
+        slot_embeddings = slot_embeddings.unsqueeze(1).repeat((1, batch_size * dialogue_size, 1))
+        slot_embeddings = slot_embeddings.to(turn_embeddings.device)
+
+        if self.config.set_similarity:
+            # Slot mask shape [num_slots * slot_len, batch_size * dialogue_size, 768]
+            slot_mask = (slot_embeddings != 0.0).float()
+
+        hidden = self.slot_utterance_matching(turn_embeddings, attention_mask, slot_embeddings)
+
+        if self.config.set_similarity:
+            hidden = hidden * slot_mask
+        # Hidden layer shape [num_dials, num_slots, num_turns, 768]
+        hidden = hidden.transpose(0, 1).reshape(batch_size, dialogue_size, slot_embeddings.size(0), -1).transpose(1, 2)
+
+        # Latent context tracking
+        # [batch_size * num_slots, dialogue_size, 768]
+        hidden = hidden.reshape(batch_size * slot_embeddings.size(0), dialogue_size, -1)
+        belief_embedding, hidden_state = self.nbt(hidden, hidden_state)
+
+        belief_embedding = belief_embedding.reshape(batch_size, slot_embeddings.size(0),
+                                                    dialogue_size, -1).transpose(1, 2)
+        if self.config.set_similarity:
+            belief_embedding = belief_embedding.reshape(batch_size, dialogue_size, num_slots, -1,
+                                                        self.config.hidden_size)
+        # [batch_size, dialogue_size, num_slots, *slot_desc_len, 768]
+
+        # Pooling of the set of latent context representation
+        if self.config.set_similarity:
+            slot_mask = slot_mask.transpose(0, 1).reshape(batch_size, dialogue_size, num_slots, -1, hidden_size)
+            belief_embedding = belief_embedding * slot_mask
+
+            belief_embedding = self.set_pooler(belief_embedding.reshape(-1, slot_mask.size(-2), hidden_size),
+                                               slot_mask.reshape(-1, slot_mask.size(-2), hidden_size))
+            belief_embedding = belief_embedding.reshape(batch_size, dialogue_size, num_slots, -1)
+
+        # Perform classification
+        # Get padded batch, dialogue idx pairs
+        batches, dialogues = torch.where(attention_mask[:, 0, 0].reshape(batch_size, dialogue_size) == 0.0)
+        
+        if self.config.predict_actions:
+            # User request prediction
+            request_probs = dict()
+            for slot, slot_id in self.requestable_slot_ids.items():
+                request_logits = self.request_gate(belief_embedding[:, :, slot_id, :])
+
+                # Store output probabilities
+                request_logits = request_logits.reshape(batch_size, dialogue_size)
+                # Set request scores to 0.0 for padded turns
+                request_logits[batches, dialogues] = 0.0
+                request_probs[slot] = torch.sigmoid(request_logits)
+
+                if request_labels is not None:
+                    # Compute request gate loss
+                    request_logits = request_logits.reshape(-1)
+                    if self.config.loss_function == 'distillation':
+                        loss += self.request_loss(request_logits, request_labels[slot].reshape(-1),
+                                                  self.temp) * self.request_weight
+                    elif self.config.loss_function == 'distribution_distillation':
+                        loss += self.request_loss(request_logits, request_labels[slot])[0] * self.request_weight
+                    else:
+                        labs = request_labels[slot].reshape(-1)
+                        request_logits = request_logits[labs != -1]
+                        labs = labs[labs != -1].float()
+                        loss += self.request_loss(request_logits, labs) * self.request_weight
+
+            # Active domain prediction
+            active_domain_probs = dict()
+            for domain, slot_ids in self.domain_ids.items():
+                belief = belief_embedding[:, :, slot_ids, :]
+                if len(slot_ids) > 1:
+                    # SqrtN reduction across all slots within a domain
+                    belief = belief.sum(2) / ((belief != 0.0).float().sum(2) ** 0.5)
+                active_domain_logits = self.active_domain_gate(belief)
+
+                # Store output probabilities
+                active_domain_logits = active_domain_logits.reshape(batch_size, dialogue_size)
+                active_domain_logits[batches, dialogues] = 0.0
+                active_domain_probs[domain] = torch.sigmoid(active_domain_logits)
+
+                if active_domain_labels is not None and domain in active_domain_labels:
+                    # Compute domain prediction loss
+                    active_domain_logits = active_domain_logits.reshape(-1)
+                    if self.config.loss_function == 'distillation':
+                        loss += self.active_domain_loss(active_domain_logits, active_domain_labels[domain].reshape(-1),
+                                                        self.temp) * self.active_domain_weight
+                    elif self.config.loss_function == 'distribution_distillation':
+                        loss += self.active_domain_loss(active_domain_logits,
+                                                        active_domain_labels[domain])[0] * self.active_domain_weight
+                    else:
+                        labs = active_domain_labels[domain].reshape(-1)
+                        active_domain_logits = active_domain_logits[labs != -1]
+                        labs = labs[labs != -1].float()
+                        loss += self.active_domain_loss(active_domain_logits, labs) * self.active_domain_weight
+        else:
+            request_probs, active_domain_probs = None, None
+
+        # Dialogue state predictions
+        belief_state_probs = dict()
+        belief_state_mutual_info = dict()
+        belief_state_stats = dict()
+        for slot, slot_id in self.informable_slot_ids.items():
+            # Get slot belief embedding and value candidates
+            candidate_embeddings = getattr(self, slot + '_value_embeddings').to(turn_embeddings.device)
+            belief = belief_embedding[:, :, slot_id, :]
+            slot_size = candidate_embeddings.size(0)
+
+            belief = belief.unsqueeze(2).repeat((1, 1, slot_size, 1))
+            belief = belief.reshape(-1, self.config.hidden_size)
+
+            if self.config.set_similarity:
+                candidate_embeddings = self.set_pooler(candidate_embeddings, (candidate_embeddings != 0.0).float())
+            candidate_embeddings = candidate_embeddings.unsqueeze(0).unsqueeze(0).repeat((batch_size,
+                                                                                          dialogue_size, 1, 1))
+            candidate_embeddings = candidate_embeddings.reshape(-1, self.config.hidden_size)
+
+            # Score value candidates
+            if self.config.distance_measure == 'cosine':
+                logits = self.distance(belief, candidate_embeddings)
+                # *27 here rescales the cosine similarity for better learning
+                logits = logits.reshape(batch_size * dialogue_size, -1) * 27.0
+            elif self.config.distance_measure == 'euclidean':
+                logits = -1.0 * self.distance(belief, candidate_embeddings)
+                logits = logits.reshape(batch_size * dialogue_size, -1)
+
+            # Calculate belief state
+            probs_ = torch.softmax(logits.reshape(batch_size, dialogue_size, -1), -1)
+
+            # Compute knowledge uncertainty in the beleif states
+            if calculate_state_mutual_info and self.config.loss_function == 'distribution_distillation':
+                belief_state_mutual_info[slot] = self.loss.logits_to_mutual_info(logits).reshape(batch_size, dialogue_size)
+
+            # Set padded turn probabilities to zero
+            probs_[batches, dialogues, :] = 0.0
+            belief_state_probs[slot] = probs_
+
+            # Calculate belief state loss
+            if state_labels is not None and slot in state_labels:
+                if self.config.loss_function == 'bayesianmatching':
+                    prior = torch.ones(logits.size(-1)).float().to(logits.device)
+                    prior = prior * self.config.prior_constant
+                    prior = prior.unsqueeze(0).repeat((logits.size(0), 1))
+
+                    loss += self.loss(logits, state_labels[slot].reshape(-1), prior=prior)
+                elif self.config.loss_function == 'distillation':
+                    labels = state_labels[slot]
+                    labels = labels.reshape(-1, labels.size(-1))
+                    loss += self.loss(logits, labels, self.temp)
+                elif self.config.loss_function == 'distribution_distillation':
+                    labels = state_labels[slot]
+                    labels = labels.reshape(-1, labels.size(-2), labels.size(-1))
+                    loss_, model_stats, ensemble_stats = self.loss(logits, labels)
+                    loss += loss_
+
+                    # Calculate stats regarding model precisions
+                    precision = model_stats['precision']
+                    ensemble_precision = ensemble_stats['precision']
+                    belief_state_stats[slot] = {'model_precision_min': precision.min(),
+                                                'model_precision_max': precision.max(),
+                                                'model_precision_mean': precision.mean(),
+                                                'ensemble_precision_min': ensemble_precision.min(),
+                                                'ensemble_precision_max': ensemble_precision.max(),
+                                                'ensemble_precision_mean': ensemble_precision.mean()}
+                else:
+                    loss += self.loss(logits, state_labels[slot].reshape(-1))
+
+        # Return model outputs
+        out = belief_state_probs, request_probs, active_domain_probs, general_act_probs, hidden_state
+        if state_labels is not None or request_labels is not None:
+            out = (loss,) + out + (belief_state_stats,)
+        if calculate_state_mutual_info:
+            out = out + (belief_state_mutual_info,)
+        return out
diff --git a/convlab/dst/setsumbt/modeling/temperature_scheduler.py b/convlab/dst/setsumbt/modeling/temperature_scheduler.py
index fab205befe3350c9beb9d81566c813cf00b55cf2..654e83c5d1ad9dc908213cca8967a84893395b04 100644
--- a/convlab/dst/setsumbt/modeling/temperature_scheduler.py
+++ b/convlab/dst/setsumbt/modeling/temperature_scheduler.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2021 DSML Group, Heinrich Heine University, Düsseldorf
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
 # Authors: Carel van Niekerk (niekerk@hhu.de)
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,50 +13,70 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Temperature Scheduler Class"""
-import torch
+"""Linear Temperature Scheduler Class"""
+
 
 # Temp scheduler class for ensemble distillation
-class TemperatureScheduler:
+class LinearTemperatureScheduler:
+    """
+    Temperature scheduler object used for distribution temperature scheduling in distillation
 
-    def __init__(self, total_steps, base_temp=2.5, cycle_len=0.1):
-        self.state = {}
+    Attributes:
+        state (dict): Internal state of scheduler
+    """
+    def __init__(self,
+                 total_steps: int,
+                 base_temp: float = 2.5,
+                 cycle_len: float = 0.1):
+        """
+        Args:
+            total_steps (int): Total number of training steps
+            base_temp (float): Starting temperature
+            cycle_len (float): Fraction of total steps used for scheduling cycle
+        """
+        self.state = dict()
         self.state['total_steps'] = total_steps
         self.state['current_step'] = 0
         self.state['base_temp'] = base_temp
         self.state['current_temp'] = base_temp
         self.state['cycles'] = [int(total_steps * cycle_len / 2), int(total_steps * cycle_len)]
+        self.state['rate'] = (self.state['base_temp'] - 1.0) / (self.state['cycles'][1] - self.state['cycles'][0])
     
     def step(self):
+        """
+        Update temperature based on the schedule
+        """
         self.state['current_step'] += 1
         assert self.state['current_step'] <= self.state['total_steps']
         if self.state['current_step'] > self.state['cycles'][0]:
             if self.state['current_step'] < self.state['cycles'][1]:
-                rate = (self.state['base_temp'] - 1.0) / (self.state['cycles'][1] - self.state['cycles'][0])
-                self.state['current_temp'] -= rate
+                self.state['current_temp'] -= self.state['rate']
             else:
                 self.state['current_temp'] = 1.0
     
     def temp(self):
+        """
+        Get current temperature
+
+        Returns:
+            temp (float): Current temperature for distribution scaling
+        """
         return float(self.state['current_temp'])
     
     def state_dict(self):
-        return self.state
-    
-    def load_state_dict(self, sd):
-        self.state = sd
+        """
+        Return scheduler state
 
-
-# if __name__ == "__main__":
-#     temp_scheduler = TemperatureScheduler(100)
-#     print(temp_scheduler.state_dict())
-
-#     temp = []
-#     for i in range(100):
-#         temp.append(temp_scheduler.temp())
-#         temp_scheduler.step()
+        Returns:
+            state (dict): Dictionary format state of the scheduler
+        """
+        return self.state
     
-#     temp_scheduler.load_state_dict(temp_scheduler.state_dict())
-#     print(temp_scheduler.state_dict())
+    def load_state_dict(self, state_dict: dict):
+        """
+        Load scheduler state from dictionary
 
-#     print(temp)
+        Args:
+            state_dict (dict): Dictionary format state of the scheduler
+        """
+        self.state = state_dict
diff --git a/convlab/dst/setsumbt/modeling/training.py b/convlab/dst/setsumbt/modeling/training.py
index 259c6e1da061ad6800f92e9237324181678459cb..590b2ac7372b26262625d08691a8528ffddd82d2 100644
--- a/convlab/dst/setsumbt/modeling/training.py
+++ b/convlab/dst/setsumbt/modeling/training.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2021 DSML Group, Heinrich Heine University, Düsseldorf
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
 # Authors: Carel van Niekerk (niekerk@hhu.de)
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,17 +13,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Training utils"""
+"""Training and evaluation utils"""
 
 import random
 import os
 import logging
+from copy import deepcopy
 
 import torch
 from torch.nn import DataParallel
 from torch.distributions import Categorical
 import numpy as np
-from transformers import AdamW, get_linear_schedule_with_warmup
+from transformers import get_linear_schedule_with_warmup
+from torch.optim import AdamW
 from tqdm import tqdm, trange
 try:
     from apex import amp
@@ -31,7 +33,7 @@ except:
     print('Apex not used')
 
 from convlab.dst.setsumbt.utils import clear_checkpoints
-from convlab.dst.setsumbt.modeling.temperature_scheduler import TemperatureScheduler
+from convlab.dst.setsumbt.modeling import LinearTemperatureScheduler
 
 
 # Load logger and tensorboard summary writer
@@ -59,18 +61,131 @@ def set_ontology_embeddings(model, slots, load_slots=True):
     if load_slots:
         slots = {slot: embs for slot, embs in slots.items()}
         model.add_slot_candidates(slots)
-    for slot in model.informable_slot_ids:
+    try:
+        informable_slot_ids = model.setsumbt.informable_slot_ids
+    except:
+        informable_slot_ids = model.informable_slot_ids
+    for slot in informable_slot_ids:
         model.add_value_candidates(slot, values[slot], replace=True)
 
 
-def train(args, model, device, train_dataloader, dev_dataloader, slots, slots_dev, embeddings=None, tokenizer=None):
-    """Train model!"""
+def log_info(global_step, loss, jg_acc=None, sl_acc=None, req_f1=None, dom_f1=None, gen_f1=None, stats=None):
+    """
+    Log training statistics.
+
+    Args:
+        global_step: Number of global training steps completed
+        loss: Training loss
+        jg_acc: Joint goal accuracy
+        sl_acc: Slot accuracy
+        req_f1: Request prediction F1 score
+        dom_f1: Active domain prediction F1 score
+        gen_f1: General action prediction F1 score
+        stats: Uncertainty measure statistics of model
+    """
+    if type(global_step) == int:
+        info = f"{global_step} steps complete, "
+        info += f"Loss since last update: {loss}. Validation set stats: "
+    elif global_step == 'training_complete':
+        info = f"Training Complete, "
+        info += f"Validation set stats: "
+    elif global_step == 'dev':
+        info = f"Validation set stats: Loss: {loss}, "
+    elif global_step == 'test':
+        info = f"Test set stats: Loss: {loss}, "
+    info += f"Joint Goal Acc: {jg_acc}, Slot Acc: {sl_acc}, "
+    if req_f1 is not None:
+        info += f"Request F1 Score: {req_f1}, Active Domain F1 Score: {dom_f1}, "
+        info += f"General Action F1 Score: {gen_f1}"
+    logger.info(info)
+
+    if type(global_step) == int:
+        tb_writer.add_scalar('JointGoalAccuracy/Dev', jg_acc, global_step)
+        tb_writer.add_scalar('SlotAccuracy/Dev', sl_acc, global_step)
+        if req_f1 is not None:
+            tb_writer.add_scalar('RequestF1Score/Dev', req_f1, global_step)
+            tb_writer.add_scalar('ActiveDomainF1Score/Dev', dom_f1, global_step)
+            tb_writer.add_scalar('GeneralActionF1Score/Dev', gen_f1, global_step)
+        tb_writer.add_scalar('Loss/Dev', loss, global_step)
+
+        if stats:
+            for slot, stats_slot in stats.items():
+                for key, item in stats_slot.items():
+                    tb_writer.add_scalar(f'{key}_{slot}/Dev', item, global_step)
+
+
+def get_input_dict(batch: dict,
+                   predict_actions: bool,
+                   model_informable_slot_ids: list,
+                   model_requestable_slot_ids: list = None,
+                   model_domain_ids: list = None,
+                   device = 'cpu') -> dict:
+    """
+    Produce model input arguments
+
+    Args:
+        batch: Batch of data from the dataloader
+        predict_actions: Model should predict user actions if set true
+        model_informable_slot_ids: List of model dialogue state slots
+        model_requestable_slot_ids: List of model requestable slots
+        model_domain_ids: List of model domains
+        device: Current torch device in use
+
+    Returns:
+        input_dict: Dictrionary containing model inputs for the batch
+    """
+    input_dict = dict()
+
+    input_dict['input_ids'] = batch['input_ids'].to(device)
+    input_dict['token_type_ids'] = batch['token_type_ids'].to(device) if 'token_type_ids' in batch else None
+    input_dict['attention_mask'] = batch['attention_mask'].to(device) if 'attention_mask' in batch else None
+
+    if any('belief_state' in key for key in batch):
+        input_dict['state_labels'] = {slot: batch['belief_state-' + slot].to(device)
+                                      for slot in model_informable_slot_ids
+                                      if ('belief_state-' + slot) in batch}
+        if predict_actions:
+            input_dict['request_labels'] = {slot: batch['request_probs-' + slot].to(device)
+                                            for slot in model_requestable_slot_ids
+                                            if ('request_probs-' + slot) in batch}
+            input_dict['active_domain_labels'] = {domain: batch['active_domain_probs-' + domain].to(device)
+                                                  for domain in model_domain_ids
+                                                  if ('active_domain_probs-' + domain) in batch}
+            input_dict['general_act_labels'] = batch['general_act_probs'].to(device)
+    else:
+        input_dict['state_labels'] = {slot: batch['state_labels-' + slot].to(device)
+                                      for slot in model_informable_slot_ids if ('state_labels-' + slot) in batch}
+        if predict_actions:
+            input_dict['request_labels'] = {slot: batch['request_labels-' + slot].to(device)
+                                            for slot in model_requestable_slot_ids
+                                            if ('request_labels-' + slot) in batch}
+            input_dict['active_domain_labels'] = {domain: batch['active_domain_labels-' + domain].to(device)
+                                                  for domain in model_domain_ids
+                                                  if ('active_domain_labels-' + domain) in batch}
+            input_dict['general_act_labels'] = batch['general_act_labels'].to(device)
+
+    return input_dict
+
+
+def train(args, model, device, train_dataloader, dev_dataloader, slots: dict, slots_dev: dict):
+    """
+    Train the SetSUMBT model.
+
+    Args:
+        args: Runtime arguments
+        model: SetSUMBT Model instance to train
+        device: Torch device to use during training
+        train_dataloader: Dataloader containing the training data
+        dev_dataloader: Dataloader containing the validation set data
+        slots: Model ontology used for training
+        slots_dev: Model ontology used for evaluating on the validation set
+    """
 
     # Calculate the total number of training steps to be performed
     if args.max_training_steps > 0:
         t_total = args.max_training_steps
-        args.num_train_epochs = args.max_training_steps // (
-            (len(train_dataloader) // args.gradient_accumulation_steps) + 1)
+        args.num_train_epochs = (len(train_dataloader) // args.gradient_accumulation_steps) + 1
+        args.num_train_epochs = args.max_training_steps // args.num_train_epochs
     else:
         t_total = (len(train_dataloader) // args.gradient_accumulation_steps) * args.num_train_epochs
 
@@ -88,12 +203,12 @@ def train(args, model, device, train_dataloader, dev_dataloader, slots, slots_de
         {
             "params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],
             "weight_decay": 0.0,
-            "lr":args.learning_rate
+            "lr": args.learning_rate
         },
     ]
 
     # Initialise the optimizer
-    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, correct_bias=False)
+    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate)
 
     # Initialise linear lr scheduler
     num_warmup_steps = int(t_total * args.warmup_proportion)
@@ -109,8 +224,7 @@ def train(args, model, device, train_dataloader, dev_dataloader, slots, slots_de
 
     # Set up fp16 and multi gpu usage
     if args.fp16:
-        model, optimizer = amp.initialize(
-            model, optimizer, opt_level=args.fp16_opt_level)
+        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)
     if args.n_gpu > 1:
         model = DataParallel(model)
 
@@ -118,7 +232,7 @@ def train(args, model, device, train_dataloader, dev_dataloader, slots, slots_de
     best_model = {'joint goal accuracy': 0.0,
                   'request f1 score': 0.0,
                   'active domain f1 score': 0.0,
-                  'goodbye act f1 score': 0.0,
+                  'general act f1 score': 0.0,
                   'train loss': np.inf}
     if os.path.isfile(os.path.join(args.model_name_or_path, 'optimizer.pt')):
         logger.info("Optimizer loaded from previous run.")
@@ -136,27 +250,27 @@ def train(args, model, device, train_dataloader, dev_dataloader, slots, slots_de
             model.eval()
             set_ontology_embeddings(model.module if args.n_gpu > 1 else model, slots_dev, load_slots=False)
 
-            jg_acc, sl_acc, req_f1, dom_f1, bye_f1, loss, stats = train_eval(args, model, device, dev_dataloader)
+            jg_acc, sl_acc, req_f1, dom_f1, gen_f1, _, _ = evaluate(args, model, device, dev_dataloader, is_train=True)
 
             # Set model back to training mode
             model.train()
             model.zero_grad()
             set_ontology_embeddings(model.module if args.n_gpu > 1 else model, slots, load_slots=False)
         else:
-            jg_acc, req_f1, dom_f1, bye_f1 = 0.0, 0.0, 0.0, 0.0
+            jg_acc, req_f1, dom_f1, gen_f1 = 0.0, 0.0, 0.0, 0.0
 
         best_model['joint goal accuracy'] = jg_acc
         best_model['request f1 score'] = req_f1
         best_model['active domain f1 score'] = dom_f1
-        best_model['goodbye act f1 score'] = bye_f1
+        best_model['general act f1 score'] = gen_f1
 
     # Log training set up
-    logger.info("Device: %s, Number of GPUs: %s, FP16 training: %s" % (device, args.n_gpu, args.fp16))
+    logger.info(f"Device: {device}, Number of GPUs: {args.n_gpu}, FP16 training: {args.fp16}")
     logger.info("***** Running training *****")
-    logger.info("  Num Batches = %d" % len(train_dataloader))
-    logger.info("  Num Epochs = %d" % args.num_train_epochs)
-    logger.info("  Gradient Accumulation steps = %d" % args.gradient_accumulation_steps)
-    logger.info("  Total optimization steps = %d" % t_total)
+    logger.info(f"  Num Batches = {len(train_dataloader)}")
+    logger.info(f"  Num Epochs = {args.num_train_epochs}")
+    logger.info(f"  Gradient Accumulation steps = {args.gradient_accumulation_steps}")
+    logger.info(f"  Total optimization steps = {t_total}")
 
     # Initialise training parameters
     global_step = 0
@@ -173,11 +287,11 @@ def train(args, model, device, train_dataloader, dev_dataloader, slots, slots_de
             steps_trained_in_current_epoch = global_step % (len(train_dataloader) // args.gradient_accumulation_steps)
 
             logger.info("  Continuing training from checkpoint, will skip to saved global_step")
-            logger.info("  Continuing training from epoch %d" % epochs_trained)
-            logger.info("  Continuing training from global step %d" % global_step)
-            logger.info("  Will skip the first %d steps in the first epoch" % steps_trained_in_current_epoch)
+            logger.info(f"  Continuing training from epoch {epochs_trained}")
+            logger.info(f"  Continuing training from global step {global_step}")
+            logger.info(f"  Will skip the first {steps_trained_in_current_epoch} steps in the first epoch")
         except ValueError:
-            logger.info("  Starting fine-tuning.")
+            logger.info(f"  Starting fine-tuning.")
 
     # Prepare model for training
     tr_loss, logging_loss = 0.0, 0.0
@@ -196,43 +310,15 @@ def train(args, model, device, train_dataloader, dev_dataloader, slots, slots_de
                 continue
 
             # Extract all label dictionaries from the batch
-            if 'goodbye_belief' in batch:
-                labels = {slot: batch['belief-' + slot].to(device) for slot in model.informable_slot_ids
-                          if ('belief-' + slot) in batch}
-                request_labels = {slot: batch['request_belief-' + slot].to(device)
-                                  for slot in model.requestable_slot_ids
-                                  if ('request_belief-' + slot) in batch} if args.predict_actions else None
-                domain_labels = {domain: batch['domain_belief-' + domain].to(device) for domain in model.domain_ids
-                                 if ('domain_belief-' + domain) in batch} if args.predict_actions else None
-                goodbye_labels = batch['goodbye_belief'].to(
-                    device) if args.predict_actions else None
-            else:
-                labels = {slot: batch['labels-' + slot].to(device) for slot in model.informable_slot_ids
-                          if ('labels-' + slot) in batch}
-                request_labels = {slot: batch['request-' + slot].to(device) for slot in model.requestable_slot_ids
-                                  if ('request-' + slot) in batch} if args.predict_actions else None
-                domain_labels = {domain: batch['active-' + domain].to(device) for domain in model.domain_ids
-                                 if ('active-' + domain) in batch} if args.predict_actions else None
-                goodbye_labels = batch['goodbye'].to(
-                    device) if args.predict_actions else None
-
-            # Extract all model inputs from batch
-            input_ids = batch['input_ids'].to(device)
-            token_type_ids = batch['token_type_ids'].to(device) if 'token_type_ids' in batch else None
-            attention_mask = batch['attention_mask'].to(device) if 'attention_mask' in batch else None
+            input_dict = get_input_dict(batch, args.predict_actions, model.setsumbt.informable_slot_ids,
+                                        model.setsumbt.requestable_slot_ids, model.setsumbt.domain_ids, device)
 
             # Set up temperature scaling for the model
             if temp_scheduler is not None:
-                model.temp = temp_scheduler.temp()
+                model.setsumbt.temp = temp_scheduler.temp()
 
             # Forward pass to obtain loss
-            loss, _, _, _, _, _, stats = model(input_ids=input_ids,
-                                               token_type_ids=token_type_ids,
-                                               attention_mask=attention_mask,
-                                               inform_labels=labels,
-                                               request_labels=request_labels,
-                                               domain_labels=domain_labels,
-                                               goodbye_labels=goodbye_labels)
+            loss, _, _, _, _, _, stats = model(**input_dict)
 
             if args.n_gpu > 1:
                 loss = loss.mean()
@@ -258,7 +344,6 @@ def train(args, model, device, train_dataloader, dev_dataloader, slots, slots_de
                 tb_writer.add_scalar('LearningRate', lr, global_step)
 
                 if stats:
-                    # print(stats.keys())
                     for slot, stats_slot in stats.items():
                         for key, item in stats_slot.items():
                             tb_writer.add_scalar(f'{key}_{slot}/Train', item, global_step)
@@ -273,7 +358,6 @@ def train(args, model, device, train_dataloader, dev_dataloader, slots, slots_de
 
                 tr_loss += loss.float().item()
                 epoch_iterator.set_postfix(loss=loss.float().item())
-                loss = 0.0
                 global_step += 1
 
             # Save model checkpoint
@@ -286,52 +370,34 @@ def train(args, model, device, train_dataloader, dev_dataloader, slots, slots_de
                     model.eval()
                     set_ontology_embeddings(model.module if args.n_gpu > 1 else model, slots_dev, load_slots=False)
 
-                    jg_acc, sl_acc, req_f1, dom_f1, bye_f1, loss, stats = train_eval(args, model, device, dev_dataloader)
+                    jg_acc, sl_acc, req_f1, dom_f1, gen_f1, loss, stats = evaluate(args, model, device, dev_dataloader,
+                                                                                   is_train=True)
                     # Log model eval information
-                    if req_f1 is not None:
-                        logger.info('%i steps complete, Loss since last update = %f, Dev Joint goal acc = %f, Dev Slot acc = %f, Dev Request F1 Score = %f, Dev Domain F1 Score = %f, Dev Goodbye F1 Score = %f'
-                                    % (global_step, logging_loss / args.save_steps, jg_acc, sl_acc, req_f1, dom_f1, bye_f1))
-                        tb_writer.add_scalar('JointGoalAccuracy/Dev', jg_acc, global_step)
-                        tb_writer.add_scalar('SlotAccuracy/Dev', sl_acc, global_step)
-                        tb_writer.add_scalar('RequestF1Score/Dev', req_f1, global_step)
-                        tb_writer.add_scalar('DomainF1Score/Dev', dom_f1, global_step)
-                        tb_writer.add_scalar('GoodbyeF1Score/Dev', bye_f1, global_step)
-                    else:
-                        logger.info('%i steps complete, Loss since last update = %f, Dev Joint goal acc = %f, Dev Slot acc = %f'
-                                    % (global_step, logging_loss / args.save_steps, jg_acc, sl_acc))
-                        tb_writer.add_scalar('JointGoalAccuracy/Dev', jg_acc, global_step)
-                        tb_writer.add_scalar('SlotAccuracy/Dev', sl_acc, global_step)
-                    tb_writer.add_scalar('Loss/Dev', loss, global_step)
-                    if stats:
-                        for slot, stats_slot in stats.items():
-                            for key, item in stats_slot.items():
-                                tb_writer.add_scalar(f'{key}_{slot}/Dev', item, global_step)
+                    log_info(global_step, logging_loss / args.save_steps, jg_acc, sl_acc, req_f1, dom_f1, gen_f1, stats)
 
                     # Set model back to training mode
                     model.train()
                     model.zero_grad()
                     set_ontology_embeddings(model.module if args.n_gpu > 1 else model, slots, load_slots=False)
                 else:
-                    jg_acc, req_f1 = 0.0, None
-                    logger.info('%i steps complete, Loss since last update = %f' % (global_step, logging_loss / args.save_steps))
+                    log_info(global_step, logging_loss / args.save_steps)
 
                 logging_loss = tr_loss
 
                 # Compute the score of the best model
                 try:
-                    best_score = (best_model['request f1 score'] * model.config.user_request_loss_weight) + \
-                        (best_model['active domain f1 score'] * model.config.active_domain_loss_weight) + \
-                        (best_model['goodbye act f1 score'] *
-                         model.config.user_general_act_loss_weight)
+                    best_score = best_model['request f1 score'] * model.config.user_request_loss_weight
+                    best_score += best_model['active domain f1 score'] * model.config.active_domain_loss_weight
+                    best_score += best_model['general act f1 score'] * model.config.user_general_act_loss_weight
                 except AttributeError:
                     best_score = 0.0
                 best_score += best_model['joint goal accuracy']
 
                 # Compute the score of the current model
                 try:
-                    current_score = (req_f1 * model.config.user_request_loss_weight) + \
-                        (dom_f1 * model.config.active_domain_loss_weight) + \
-                        (bye_f1 * model.config.user_general_act_loss_weight) if req_f1 is not None else 0.0
+                    current_score = req_f1 * model.config.user_request_loss_weight if req_f1 is not None else 0.0
+                    current_score += dom_f1 * model.config.active_domain_loss_weight if dom_f1 is not None else 0.0
+                    current_score += gen_f1 * model.config.user_general_act_loss_weight if gen_f1 is not None else 0.0
                 except AttributeError:
                     current_score = 0.0
                 current_score += jg_acc
@@ -353,10 +419,10 @@ def train(args, model, device, train_dataloader, dev_dataloader, slots, slots_de
                     if req_f1:
                         best_model['request f1 score'] = req_f1
                         best_model['active domain f1 score'] = dom_f1
-                        best_model['goodbye act f1 score'] = bye_f1
+                        best_model['general act f1 score'] = gen_f1
                     best_model['train loss'] = tr_loss / global_step
 
-                    output_dir = os.path.join(args.output_dir, "checkpoint-{}".format(global_step))
+                    output_dir = os.path.join(args.output_dir, f"checkpoint-{global_step}")
                     if not os.path.exists(output_dir):
                         os.makedirs(output_dir, exist_ok=True)
 
@@ -386,14 +452,15 @@ def train(args, model, device, train_dataloader, dev_dataloader, slots, slots_de
                 epoch_iterator.close()
                 break
 
-        logger.info('Epoch %i complete, average training loss = %f' % (e + 1, tr_loss / global_step))
+        steps_trained_in_current_epoch = 0
+        logger.info(f'Epoch {e + 1} complete, average training loss = {tr_loss / global_step}')
 
         if args.max_training_steps > 0 and global_step > args.max_training_steps:
             train_iterator.close()
             break
         if args.patience > 0 and steps_since_last_update >= args.patience:
             train_iterator.close()
-            logger.info('Model has not improved for at least %i steps. Training stopped!' % args.patience)
+            logger.info(f'Model has not improved for at least {args.patience} steps. Training stopped!')
             break
 
     # Evaluate final model
@@ -401,30 +468,25 @@ def train(args, model, device, train_dataloader, dev_dataloader, slots, slots_de
         model.eval()
         set_ontology_embeddings(model.module if args.n_gpu > 1 else model, slots_dev, load_slots=False)
 
-        jg_acc, sl_acc, req_f1, dom_f1, bye_f1, loss, stats = train_eval(args, model, device, dev_dataloader)
-        if req_f1 is not None:
-            logger.info('Training complete, Training Loss = %f, Dev Joint goal acc = %f, Dev Slot acc = %f, Dev Request F1 Score = %f, Dev Domain F1 Score = %f, Dev Goodbye F1 Score = %f'
-                        % (tr_loss / global_step, jg_acc, sl_acc, req_f1, dom_f1, bye_f1))
-        else:
-            logger.info('Training complete, Training Loss = %f, Dev Joint goal acc = %f, Dev Slot acc = %f'
-                        % (tr_loss / global_step, jg_acc, sl_acc))
+        jg_acc, sl_acc, req_f1, dom_f1, gen_f1, loss, stats = evaluate(args, model, device, dev_dataloader,
+                                                                       is_train=True)
+
+        log_info('training_complete', tr_loss / global_step, jg_acc, sl_acc, req_f1, dom_f1, gen_f1)
     else:
-        jg_acc = 0.0
         logger.info('Training complete!')
 
     # Store final model
     try:
-        best_score = (best_model['request f1 score'] * model.config.user_request_loss_weight) + \
-            (best_model['active domain f1 score'] * model.config.active_domain_loss_weight) + \
-            (best_model['goodbye act f1 score'] *
-             model.config.user_general_act_loss_weight)
+        best_score = best_model['request f1 score'] * model.config.user_request_loss_weight
+        best_score += best_model['active domain f1 score'] * model.config.active_domain_loss_weight
+        best_score += best_model['general act f1 score'] * model.config.user_general_act_loss_weight
     except AttributeError:
         best_score = 0.0
     best_score += best_model['joint goal accuracy']
     try:
-        current_score = (req_f1 * model.config.user_request_loss_weight) + \
-                        (dom_f1 * model.config.active_domain_loss_weight) + \
-                        (bye_f1 * model.config.user_general_act_loss_weight) if req_f1 is not None else 0.0
+        current_score = req_f1 * model.config.user_request_loss_weight if req_f1 is not None else 0.0
+        current_score += dom_f1 * model.config.active_domain_loss_weight if dom_f1 is not None else 0.0
+        current_score += gen_f1 * model.config.user_general_act_loss_weight if gen_f1 is not None else 0.0
     except AttributeError:
         current_score = 0.0
     current_score += jg_acc
@@ -456,225 +518,88 @@ def train(args, model, device, train_dataloader, dev_dataloader, slots, slots_de
             torch.save(amp.state_dict(), os.path.join(output_dir, "amp.pt"))
         clear_checkpoints(args.output_dir)
     else:
-        logger.info(
-            'Final model not saved, since it is not the best performing model.')
-
-
-# Function for validation
-def train_eval(args, model, device, dev_dataloader):
-    """Evaluate Model during training!"""
-    accuracy_jg = []
-    accuracy_sl = []
-    accuracy_req = []
-    truepos_req, falsepos_req, falseneg_req = [], [], []
-    truepos_dom, falsepos_dom, falseneg_dom = [], [], []
-    truepos_bye, falsepos_bye, falseneg_bye = [], [], []
-    accuracy_dom = []
-    accuracy_bye = []
-    turns = []
-    for batch in dev_dataloader:
-        # Perform with no gradients stored
-        with torch.no_grad():
-            if 'goodbye_belief' in batch:
-                labels = {slot: batch['belief-' + slot].to(device) for slot in model.informable_slot_ids
-                          if ('belief-' + slot) in batch}
-                request_labels = {slot: batch['request_belief-' + slot].to(device) for slot in model.requestable_slot_ids
-                                  if ('request_belief-' + slot) in batch} if args.predict_actions else None
-                domain_labels = {domain: batch['domain_belief-' + domain].to(device) for domain in model.domain_ids
-                                 if ('domain_belief-' + domain) in batch} if args.predict_actions else None
-                goodbye_labels = batch['goodbye_belief'].to(
-                    device) if args.predict_actions else None
-            else:
-                labels = {slot: batch['labels-' + slot].to(device) for slot in model.informable_slot_ids
-                          if ('labels-' + slot) in batch}
-                request_labels = {slot: batch['request-' + slot].to(device) for slot in model.requestable_slot_ids
-                                  if ('request-' + slot) in batch} if args.predict_actions else None
-                domain_labels = {domain: batch['active-' + domain].to(device) for domain in model.domain_ids
-                                 if ('active-' + domain) in batch} if args.predict_actions else None
-                goodbye_labels = batch['goodbye'].to(
-                    device) if args.predict_actions else None
-
-            input_ids = batch['input_ids'].to(device)
-            token_type_ids = batch['token_type_ids'].to(
-                device) if 'token_type_ids' in batch else None
-            attention_mask = batch['attention_mask'].to(
-                device) if 'attention_mask' in batch else None
-
-            loss, p, p_req, p_dom, p_bye, _, stats = model(input_ids=input_ids,
-                                                           token_type_ids=token_type_ids,
-                                                           attention_mask=attention_mask,
-                                                           inform_labels=labels,
-                                                           request_labels=request_labels,
-                                                           domain_labels=domain_labels,
-                                                           goodbye_labels=goodbye_labels)
-
-        jg_acc = 0.0
-        req_acc = 0.0
-        req_tp, req_fp, req_fn = 0.0, 0.0, 0.0
-        dom_tp, dom_fp, dom_fn = 0.0, 0.0, 0.0
-        dom_acc = 0.0
-        for slot in model.informable_slot_ids:
-            labels = batch['labels-' + slot].to(device)
-            p_ = p[slot]
-
-            acc = (p_.argmax(-1) == labels).reshape(-1).float()
-            jg_acc += acc
-
-        if model.config.predict_actions:
-            for slot in model.requestable_slot_ids:
-                p_req_ = p_req[slot]
-                request_labels = batch['request-' + slot].to(device)
-
-                acc = (p_req_.round().int() == request_labels).reshape(-1).float()
-                tp = (p_req_.round().int() * (request_labels == 1)).reshape(-1).float()
-                fp = (p_req_.round().int() * (request_labels == 0)).reshape(-1).float()
-                fn = ((1 - p_req_.round().int()) * (request_labels == 1)).reshape(-1).float()
-                req_acc += acc
-                req_tp += tp
-                req_fp += fp
-                req_fn += fn
-
-            for domain in model.domain_ids:
-                p_dom_ = p_dom[domain]
-                domain_labels = batch['active-' + domain].to(device)
+        logger.info('Final model not saved, as it is not the best performing model.')
 
-                acc = (p_dom_.round().int() == domain_labels).reshape(-1).float()
-                tp = (p_dom_.round().int() * (domain_labels == 1)).reshape(-1).float()
-                fp = (p_dom_.round().int() * (domain_labels == 0)).reshape(-1).float()
-                fn = ((1 - p_dom_.round().int()) * (domain_labels == 1)).reshape(-1).float()
-                dom_acc += acc
-                dom_tp += tp
-                dom_fp += fp
-                dom_fn += fn
-
-            goodbye_labels = batch['goodbye'].to(device)
-            bye_acc = (p_bye.argmax(-1) == goodbye_labels).reshape(-1).float().sum()
-            bye_tp = ((p_bye.argmax(-1) > 0) * (goodbye_labels > 0)).reshape(-1).float().sum()
-            bye_fp = ((p_bye.argmax(-1) > 0) * (goodbye_labels == 0)).reshape(-1).float().sum()
-            bye_fn = ((p_bye.argmax(-1) == 0) * (goodbye_labels > 0)).reshape(-1).float().sum()
-        else:
-            req_acc, dom_acc, bye_acc = None, None, torch.tensor(0.0)
-            req_tp, req_fp, req_fn = None, None, None
-            dom_tp, dom_fp, dom_fn = None, None, None
-            bye_tp, bye_fp, bye_fn = torch.tensor(
-                0.0), torch.tensor(0.0), torch.tensor(0.0)
-
-        sl_acc = sum(jg_acc / len(model.informable_slot_ids)).float()
-        jg_acc = sum((jg_acc / len(model.informable_slot_ids)).int()).float()
-        req_acc = sum(req_acc / len(model.requestable_slot_ids)).float() if req_acc is not None else torch.tensor(0.0)
-        req_tp = sum(req_tp / len(model.requestable_slot_ids)).float() if req_tp is not None else torch.tensor(0.0)
-        req_fp = sum(req_fp / len(model.requestable_slot_ids)).float() if req_fp is not None else torch.tensor(0.0)
-        req_fn = sum(req_fn / len(model.requestable_slot_ids)).float() if req_fn is not None else torch.tensor(0.0)
-        dom_tp = sum(dom_tp / len(model.domain_ids)).float() if dom_tp is not None else torch.tensor(0.0)
-        dom_fp = sum(dom_fp / len(model.domain_ids)).float() if dom_fp is not None else torch.tensor(0.0)
-        dom_fn = sum(dom_fn / len(model.domain_ids)).float() if dom_fn is not None else torch.tensor(0.0)
-        dom_acc = sum(dom_acc / len(model.domain_ids)).float() if dom_acc is not None else torch.tensor(0.0)
-        n_turns = (labels >= 0).reshape(-1).sum().float().item()
-
-        accuracy_jg.append(jg_acc.item())
-        accuracy_sl.append(sl_acc.item())
-        accuracy_req.append(req_acc.item())
-        truepos_req.append(req_tp.item())
-        falsepos_req.append(req_fp.item())
-        falseneg_req.append(req_fn.item())
-        accuracy_dom.append(dom_acc.item())
-        truepos_dom.append(dom_tp.item())
-        falsepos_dom.append(dom_fp.item())
-        falseneg_dom.append(dom_fn.item())
-        accuracy_bye.append(bye_acc.item())
-        truepos_bye.append(bye_tp.item())
-        falsepos_bye.append(bye_fp.item())
-        falseneg_bye.append(bye_fn.item())
-        turns.append(n_turns)
-
-    # Global accuracy reduction across batches
-    turns = sum(turns)
-    jg_acc = sum(accuracy_jg) / turns
-    sl_acc = sum(accuracy_sl) / turns
-    if model.config.predict_actions:
-        req_acc = sum(accuracy_req) / turns
-        req_tp = sum(truepos_req)
-        req_fp = sum(falsepos_req)
-        req_fn = sum(falseneg_req)
-        req_f1 = req_tp / (req_tp + 0.5 * (req_fp + req_fn))
-        dom_acc = sum(accuracy_dom) / turns
-        dom_tp = sum(truepos_dom)
-        dom_fp = sum(falsepos_dom)
-        dom_fn = sum(falseneg_dom)
-        dom_f1 = dom_tp / (dom_tp + 0.5 * (dom_fp + dom_fn))
-        bye_tp = sum(truepos_bye)
-        bye_fp = sum(falsepos_bye)
-        bye_fn = sum(falseneg_bye)
-        bye_f1 = bye_tp / (bye_tp + 0.5 * (bye_fp + bye_fn))
-        bye_acc = sum(accuracy_bye) / turns
-    else:
-        req_acc, dom_acc, bye_acc = None, None, None
-        req_f1, dom_f1, bye_f1 = None, None, None
 
-    return jg_acc, sl_acc, req_f1, dom_f1, bye_f1, loss, stats
+def evaluate(args, model, device, dataloader, return_eval_output=False, is_train=False):
+    """
+    Evaluate model
 
+    Args:
+        args: Runtime arguments
+        model: SetSUMBT model instance
+        device: Torch device in use
+        dataloader: Dataloader of data to evaluate on
+        return_eval_output: If true return predicted and true states for all dialogues evaluated in semantic format
+        is_train: If true model is training and no logging is performed
 
-def evaluate(args, model, device, dataloader):
-    """Evaluate Model!"""
-    # Evaluate!
-    logger.info("***** Running evaluation *****")
-    logger.info("  Num Batches = %d", len(dataloader))
+    Returns:
+        out: Evaluated model statistics
+    """
+    return_eval_output = False if is_train else return_eval_output
+    if not is_train:
+        logger.info("***** Running evaluation *****")
+        logger.info("  Num Batches = %d", len(dataloader))
 
     tr_loss = 0.0
     model.eval()
+    if return_eval_output:
+        ontology = dataloader.dataset.ontology
 
-    # logits = {slot: [] for slot in model.informable_slot_ids}
     accuracy_jg = []
     accuracy_sl = []
-    accuracy_req = []
     truepos_req, falsepos_req, falseneg_req = [], [], []
     truepos_dom, falsepos_dom, falseneg_dom = [], [], []
-    truepos_bye, falsepos_bye, falseneg_bye = [], [], []
-    accuracy_dom = []
-    accuracy_bye = []
+    truepos_gen, falsepos_gen, falseneg_gen = [], [], []
     turns = []
-    epoch_iterator = tqdm(dataloader, desc="Iteration")
+    if return_eval_output:
+        evaluation_output = []
+    epoch_iterator = tqdm(dataloader, desc="Iteration") if not is_train else dataloader
     for batch in epoch_iterator:
         with torch.no_grad():
-            if 'goodbye_belief' in batch:
-                labels = {slot: batch['belief-' + slot].to(device) for slot in model.informable_slot_ids
-                          if ('belief-' + slot) in batch}
-                request_labels = {slot: batch['request_belief-' + slot].to(device) for slot in model.requestable_slot_ids
-                                  if ('request_belief-' + slot) in batch} if args.predict_actions else None
-                domain_labels = {domain: batch['domain_belief-' + domain].to(device) for domain in model.domain_ids
-                                 if ('domain_belief-' + domain) in batch} if args.predict_actions else None
-                goodbye_labels = batch['goodbye_belief'].to(
-                    device) if args.predict_actions else None
-            else:
-                labels = {slot: batch['labels-' + slot].to(device) for slot in model.informable_slot_ids
-                          if ('labels-' + slot) in batch}
-                request_labels = {slot: batch['request-' + slot].to(device) for slot in model.requestable_slot_ids
-                                  if ('request-' + slot) in batch} if args.predict_actions else None
-                domain_labels = {domain: batch['active-' + domain].to(device) for domain in model.domain_ids
-                                 if ('active-' + domain) in batch} if args.predict_actions else None
-                goodbye_labels = batch['goodbye'].to(
-                    device) if args.predict_actions else None
-
-            input_ids = batch['input_ids'].to(device)
-            token_type_ids = batch['token_type_ids'].to(device) if 'token_type_ids' in batch else None
-            attention_mask = batch['attention_mask'].to(device) if 'attention_mask' in batch else None
-
-            loss, p, p_req, p_dom, p_bye, _, _ = model(input_ids=input_ids,
-                                                       token_type_ids=token_type_ids,
-                                                       attention_mask=attention_mask,
-                                                       inform_labels=labels,
-                                                       request_labels=request_labels,
-                                                       domain_labels=domain_labels,
-                                                       goodbye_labels=goodbye_labels)
+            input_dict = get_input_dict(batch, args.predict_actions, model.setsumbt.informable_slot_ids,
+                                        model.setsumbt.requestable_slot_ids, model.setsumbt.domain_ids, device)
+
+            loss, p, p_req, p_dom, p_gen, _, stats = model(**input_dict)
 
         jg_acc = 0.0
+        num_inform_slots = 0.0
         req_acc = 0.0
         req_tp, req_fp, req_fn = 0.0, 0.0, 0.0
         dom_tp, dom_fp, dom_fn = 0.0, 0.0, 0.0
         dom_acc = 0.0
-        for slot in model.informable_slot_ids:
+
+        if return_eval_output:
+            eval_output_batch = []
+            for dial_id, dial in enumerate(input_dict['input_ids']):
+                for turn_id, turn in enumerate(dial):
+                    if turn.sum() != 0:
+                        eval_output_batch.append({'dial_idx': dial_id,
+                                                  'utt_idx': turn_id,
+                                                  'state': dict(),
+                                                  'predictions': {'state': dict()}
+                                                  })
+
+        for slot in model.setsumbt.informable_slot_ids:
             p_ = p[slot]
-            labels = batch['labels-' + slot].to(device)
+            state_labels = batch['state_labels-' + slot].to(device)
+
+            if return_eval_output:
+                prediction = p_.argmax(-1)
+
+                for sample in eval_output_batch:
+                    dom, slt = slot.split('-', 1)
+                    lab = state_labels[sample['dial_idx']][sample['utt_idx']].item()
+                    lab = ontology[dom][slt]['possible_values'][lab] if lab != -1 else 'NOT_IN_ONTOLOGY'
+                    pred = prediction[sample['dial_idx']][sample['utt_idx']].item()
+                    pred = ontology[dom][slt]['possible_values'][pred]
+
+                    if dom not in sample['state']:
+                        sample['state'][dom] = dict()
+                        sample['predictions']['state'][dom] = dict()
+
+                    sample['state'][dom][slt] = lab if lab != 'none' else ''
+                    sample['predictions']['state'][dom][slt] = pred if pred != 'none' else ''
 
             if args.temp_scaling > 0.0:
                 p_ = torch.log(p_ + 1e-10) / args.temp_scaling
@@ -683,28 +608,21 @@ def evaluate(args, model, device, dataloader):
                 p_ = torch.log(p_ + 1e-10) / 1.0
                 p_ = torch.softmax(p_, -1)
 
-            # logits[slot].append(p_)
-
-            if args.accuracy_samples > 0:
-                dist = Categorical(probs=p_.reshape(-1, p_.size(-1)))
-                lab_sample = dist.sample((args.accuracy_samples,))
-                lab_sample = lab_sample.transpose(0, 1)
-                acc = [lab in s for lab, s in zip(labels.reshape(-1), lab_sample)]
-                acc = torch.tensor(acc).float()
-            elif args.accuracy_topn > 0:
-                labs = p_.reshape(-1, p_.size(-1)).argsort(dim=-1, descending=True)
-                labs = labs[:, :args.accuracy_topn]
-                acc = [lab in s for lab, s in zip(labels.reshape(-1), labs)]
-                acc = torch.tensor(acc).float()
-            else:
-                acc = (p_.argmax(-1) == labels).reshape(-1).float()
+            acc = (p_.argmax(-1) == state_labels).reshape(-1).float()
 
             jg_acc += acc
+            num_inform_slots += (state_labels != -1).float().reshape(-1)
+
+        if return_eval_output:
+            for sample in eval_output_batch:
+                sample['dial_idx'] = batch['dialogue_ids'][sample['utt_idx']][sample['dial_idx']]
+                evaluation_output.append(deepcopy(sample))
+            eval_output_batch = []
 
         if model.config.predict_actions:
-            for slot in model.requestable_slot_ids:
+            for slot in model.setsumbt.requestable_slot_ids:
                 p_req_ = p_req[slot]
-                request_labels = batch['request-' + slot].to(device)
+                request_labels = batch['request_labels-' + slot].to(device)
 
                 acc = (p_req_.round().int() == request_labels).reshape(-1).float()
                 tp = (p_req_.round().int() * (request_labels == 1)).reshape(-1).float()
@@ -715,85 +633,83 @@ def evaluate(args, model, device, dataloader):
                 req_fp += fp
                 req_fn += fn
 
-            for domain in model.domain_ids:
+            domains = [domain for domain in model.setsumbt.domain_ids if f'active_domain_labels-{domain}' in batch]
+            for domain in domains:
                 p_dom_ = p_dom[domain]
-                domain_labels = batch['active-' + domain].to(device)
+                active_domain_labels = batch['active_domain_labels-' + domain].to(device)
 
-                acc = (p_dom_.round().int() == domain_labels).reshape(-1).float()
-                tp = (p_dom_.round().int() * (domain_labels == 1)).reshape(-1).float()
-                fp = (p_dom_.round().int() * (domain_labels == 0)).reshape(-1).float()
-                fn = ((1 - p_dom_.round().int()) * (domain_labels == 1)).reshape(-1).float()
+                acc = (p_dom_.round().int() == active_domain_labels).reshape(-1).float()
+                tp = (p_dom_.round().int() * (active_domain_labels == 1)).reshape(-1).float()
+                fp = (p_dom_.round().int() * (active_domain_labels == 0)).reshape(-1).float()
+                fn = ((1 - p_dom_.round().int()) * (active_domain_labels == 1)).reshape(-1).float()
                 dom_acc += acc
                 dom_tp += tp
                 dom_fp += fp
                 dom_fn += fn
 
-            goodbye_labels = batch['goodbye'].to(device)
-            bye_acc = (p_bye.argmax(-1) == goodbye_labels).reshape(-1).float().sum()
-            bye_tp = ((p_bye.argmax(-1) > 0) * (goodbye_labels > 0)).reshape(-1).float().sum()
-            bye_fp = ((p_bye.argmax(-1) > 0) * (goodbye_labels == 0)).reshape(-1).float().sum()
-            bye_fn = ((p_bye.argmax(-1) == 0) * (goodbye_labels > 0)).reshape(-1).float().sum()
+            general_act_labels = batch['general_act_labels'].to(device)
+            gen_tp = ((p_gen.argmax(-1) > 0) * (general_act_labels > 0)).reshape(-1).float().sum()
+            gen_fp = ((p_gen.argmax(-1) > 0) * (general_act_labels == 0)).reshape(-1).float().sum()
+            gen_fn = ((p_gen.argmax(-1) == 0) * (general_act_labels > 0)).reshape(-1).float().sum()
         else:
-            req_acc, dom_acc, bye_acc = None, None, torch.tensor(0.0)
             req_tp, req_fp, req_fn = None, None, None
             dom_tp, dom_fp, dom_fn = None, None, None
-            bye_tp, bye_fp, bye_fn = torch.tensor(
-                0.0), torch.tensor(0.0), torch.tensor(0.0)
-
-        sl_acc = sum(jg_acc / len(model.informable_slot_ids)).float()
-        jg_acc = sum((jg_acc / len(model.informable_slot_ids)).int()).float()
-        req_acc = sum(req_acc / len(model.requestable_slot_ids)).float() if req_acc is not None else torch.tensor(0.0)
-        req_tp = sum(req_tp / len(model.requestable_slot_ids)).float() if req_tp is not None else torch.tensor(0.0)
-        req_fp = sum(req_fp / len(model.requestable_slot_ids)).float() if req_fp is not None else torch.tensor(0.0)
-        req_fn = sum(req_fn / len(model.requestable_slot_ids)).float() if req_fn is not None else torch.tensor(0.0)
-        dom_tp = sum(dom_tp / len(model.domain_ids)).float() if dom_tp is not None else torch.tensor(0.0)
-        dom_fp = sum(dom_fp / len(model.domain_ids)).float() if dom_fp is not None else torch.tensor(0.0)
-        dom_fn = sum(dom_fn / len(model.domain_ids)).float() if dom_fn is not None else torch.tensor(0.0)
-        dom_acc = sum(dom_acc / len(model.domain_ids)).float() if dom_acc is not None else torch.tensor(0.0)
-        n_turns = (labels >= 0).reshape(-1).sum().float().item()
+            gen_tp, gen_fp, gen_fn = torch.tensor(0.0), torch.tensor(0.0), torch.tensor(0.0)
+
+        jg_acc = jg_acc[num_inform_slots > 0]
+        num_inform_slots = num_inform_slots[num_inform_slots > 0]
+        sl_acc = sum(jg_acc / num_inform_slots).float()
+        jg_acc = sum((jg_acc == num_inform_slots).int()).float()
+        if req_tp is not None and model.setsumbt.requestable_slot_ids:
+            req_tp = sum(req_tp / len(model.setsumbt.requestable_slot_ids)).float()
+            req_fp = sum(req_fp / len(model.setsumbt.requestable_slot_ids)).float()
+            req_fn = sum(req_fn / len(model.setsumbt.requestable_slot_ids)).float()
+        else:
+            req_tp, req_fp, req_fn = torch.tensor(0.0), torch.tensor(0.0), torch.tensor(0.0)
+        dom_tp = sum(dom_tp / len(model.setsumbt.domain_ids)).float() if dom_tp is not None else torch.tensor(0.0)
+        dom_fp = sum(dom_fp / len(model.setsumbt.domain_ids)).float() if dom_fp is not None else torch.tensor(0.0)
+        dom_fn = sum(dom_fn / len(model.setsumbt.domain_ids)).float() if dom_fn is not None else torch.tensor(0.0)
+        n_turns = num_inform_slots.size(0)
 
         accuracy_jg.append(jg_acc.item())
         accuracy_sl.append(sl_acc.item())
-        accuracy_req.append(req_acc.item())
         truepos_req.append(req_tp.item())
         falsepos_req.append(req_fp.item())
         falseneg_req.append(req_fn.item())
-        accuracy_dom.append(dom_acc.item())
         truepos_dom.append(dom_tp.item())
         falsepos_dom.append(dom_fp.item())
         falseneg_dom.append(dom_fn.item())
-        accuracy_bye.append(bye_acc.item())
-        truepos_bye.append(bye_tp.item())
-        falsepos_bye.append(bye_fp.item())
-        falseneg_bye.append(bye_fn.item())
+        truepos_gen.append(gen_tp.item())
+        falsepos_gen.append(gen_fp.item())
+        falseneg_gen.append(gen_fn.item())
         turns.append(n_turns)
         tr_loss += loss.item()
 
-    # for slot in logits:
-    #     logits[slot] = torch.cat(logits[slot], 0)
-
     # Global accuracy reduction across batches
     turns = sum(turns)
     jg_acc = sum(accuracy_jg) / turns
     sl_acc = sum(accuracy_sl) / turns
     if model.config.predict_actions:
-        req_acc = sum(accuracy_req) / turns
         req_tp = sum(truepos_req)
         req_fp = sum(falsepos_req)
         req_fn = sum(falseneg_req)
-        req_f1 = req_tp / (req_tp + 0.5 * (req_fp + req_fn))
-        dom_acc = sum(accuracy_dom) / turns
+        req_f1 = req_tp + 0.5 * (req_fp + req_fn)
+        req_f1 = req_tp / req_f1 if req_f1 != 0.0 else 0.0
         dom_tp = sum(truepos_dom)
         dom_fp = sum(falsepos_dom)
         dom_fn = sum(falseneg_dom)
-        dom_f1 = dom_tp / (dom_tp + 0.5 * (dom_fp + dom_fn))
-        bye_tp = sum(truepos_bye)
-        bye_fp = sum(falsepos_bye)
-        bye_fn = sum(falseneg_bye)
-        bye_f1 = bye_tp / (bye_tp + 0.5 * (bye_fp + bye_fn))
-        bye_acc = sum(accuracy_bye) / turns
+        dom_f1 = dom_tp + 0.5 * (dom_fp + dom_fn)
+        dom_f1 = dom_tp / dom_f1 if dom_f1 != 0.0 else 0.0
+        gen_tp = sum(truepos_gen)
+        gen_fp = sum(falsepos_gen)
+        gen_fn = sum(falseneg_gen)
+        gen_f1 = gen_tp + 0.5 * (gen_fp + gen_fn)
+        gen_f1 = gen_tp / gen_f1 if gen_f1 != 0.0 else 0.0
     else:
-        req_acc, dom_acc, bye_acc = None, None, None
-        req_f1, dom_f1, bye_f1 = None, None, None
+        req_f1, dom_f1, gen_f1 = None, None, None
 
-    return jg_acc, sl_acc, req_f1, dom_f1, bye_f1, tr_loss / len(dataloader)
+    if return_eval_output:
+        return jg_acc, sl_acc, req_f1, dom_f1, gen_f1, tr_loss / len(dataloader), evaluation_output
+    if is_train:
+        return jg_acc, sl_acc, req_f1, dom_f1, gen_f1, tr_loss / len(dataloader), stats
+    return jg_acc, sl_acc, req_f1, dom_f1, gen_f1, tr_loss / len(dataloader)
diff --git a/convlab/dst/setsumbt/multiwoz/Tracker.py b/convlab/dst/setsumbt/multiwoz/Tracker.py
deleted file mode 100644
index fed1a1a62334e9a721c1f5b7de442b17bfd14b76..0000000000000000000000000000000000000000
--- a/convlab/dst/setsumbt/multiwoz/Tracker.py
+++ /dev/null
@@ -1,455 +0,0 @@
-import os
-import json
-import copy
-import logging
-
-import torch
-import transformers
-from transformers import (BertModel, BertConfig, BertTokenizer,
-                          RobertaModel, RobertaConfig, RobertaTokenizer)
-from convlab.dst.setsumbt.modeling import (RobertaSetSUMBT,
-                                            BertSetSUMBT)
-
-from convlab.dst.dst import DST
-from convlab.util.multiwoz.state import default_state
-from convlab.util.multiwoz.multiwoz_slot_trans import REF_SYS_DA, REF_USR_DA
-from convlab.dst.rule.multiwoz import normalize_value
-from convlab.util.custom_util import model_downloader
-
-USE_CUDA = torch.cuda.is_available()
-
-# Map from SetSUMBT slot names to Convlab slot names
-SLOT_MAP = {'arrive by': 'arriveBy',
-            'leave at': 'leaveAt',
-            'price range': 'pricerange',
-            'trainid': 'trainID',
-            'reference': 'Ref',
-            'taxi types': 'car type'}
-
-
-class SetSUMBTTracker(DST):
-
-    def __init__(self, model_path="", model_type="roberta",
-                 get_turn_pooled_representation=False,
-                 get_confidence_scores=False,
-                 threshold='auto',
-                 return_entropy=False,
-                 return_mutual_info=False,
-                 store_full_belief_state=False):
-        super(SetSUMBTTracker, self).__init__()
-
-        self.model_type = model_type
-        self.model_path = model_path
-        self.get_turn_pooled_representation = get_turn_pooled_representation
-        self.get_confidence_scores = get_confidence_scores
-        self.threshold = threshold
-        self.return_entropy = return_entropy
-        self.return_mutual_info = return_mutual_info
-        self.store_full_belief_state = store_full_belief_state
-        if self.store_full_belief_state:
-            self.full_belief_state = {}
-        self.info_dict = {}
-
-        # Download model if needed
-        if not os.path.exists(self.model_path):
-            # Get path /.../convlab/dst/setsumbt/multiwoz/models
-            download_path = os.path.dirname(os.path.abspath(__file__))
-            download_path = os.path.join(download_path, 'models')
-            if not os.path.exists(download_path):
-                os.mkdir(download_path)
-            model_downloader(download_path, self.model_path)
-            # Downloadable model path format http://.../setsumbt_model_name.zip
-            self.model_path = self.model_path.split('/')[-1].split('_', 1)[-1].replace('.zip', '')
-            self.model_path = os.path.join(download_path, self.model_path)
-
-        # Select model type based on the encoder
-        if model_type == "roberta":
-            self.config = RobertaConfig.from_pretrained(self.model_path)
-            self.tokenizer = RobertaTokenizer
-            self.model = RobertaSetSUMBT
-        elif model_type == "bert":
-            self.config = BertConfig.from_pretrained(self.model_path)
-            self.tokenizer = BertTokenizer
-            self.model = BertSetSUMBT
-        else:
-            logging.debug("Name Error: Not Implemented")
-
-        self.device = torch.device('cuda') if USE_CUDA else torch.device('cpu')
-
-        # Value dict for value normalisation
-        path = os.path.dirname(
-            os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
-        path = os.path.join(path, 'data/multiwoz/value_dict.json')
-        self.value_dict = json.load(open(path))
-
-        self.load_weights()
-
-    def load_weights(self):
-        # Load tokenizer and model checkpoints
-        logging.info('Loading SetSUMBT pretrained model.')
-        self.tokenizer = self.tokenizer.from_pretrained(
-            self.config.tokenizer_name)
-        logging.info(
-            f'Model tokenizer loaded from {self.config.tokenizer_name}.')
-        self.model = self.model.from_pretrained(
-            self.model_path, config=self.config)
-        logging.info(f'Model loaded from {self.model_path}.')
-
-        # Transfer model to compute device and setup eval environment
-        self.model = self.model.to(self.device)
-        self.model.eval()
-        logging.info(f'Model transferred to device: {self.device}')
-
-        logging.info('Loading model ontology')
-        f = open(os.path.join(self.model_path, 'ontology.json'), 'r')
-        self.ontology = json.load(f)
-        f.close()
-
-        db = torch.load(os.path.join(self.model_path, 'ontology.db'))
-        # Get slot and value embeddings
-        slots = {slot: db[slot] for slot in db}
-        values = {slot: db[slot][1] for slot in db}
-        del db
-
-        # Load model ontology
-        self.model.add_slot_candidates(slots)
-        for slot in values:
-            self.model.add_value_candidates(slot, values[slot], replace=True)
-
-        if self.get_confidence_scores:
-            logging.info('Model will output action and state confidence scores.')
-        if self.get_confidence_scores:
-            self.get_thresholds(self.threshold)
-            logging.info('Uncertain Querying set up and thresholds set up at:')
-            logging.info(self.thresholds)
-        if self.return_entropy:
-            logging.info('Model will output state distribution entropy.')
-        if self.return_mutual_info:
-            logging.info('Model will output state distribution mutual information.')
-        logging.info('Ontology loaded successfully.')
-
-        self.det_dic = {}
-        for domain, dic in REF_USR_DA.items():
-            for key, value in dic.items():
-                assert '-' not in key
-                self.det_dic[key.lower()] = key + '-' + domain
-                self.det_dic[value.lower()] = key + '-' + domain
-
-    def get_thresholds(self, threshold='auto'):
-        self.thresholds = {}
-        for slot, value_candidates in self.ontology.items():
-            domain, slot = slot.split('-', 1)
-            slot = REF_SYS_DA[domain.capitalize()].get(slot, slot)
-            slot = slot.strip().split()[1] if 'book ' in slot else slot
-            slot = SLOT_MAP.get(slot, slot)
-
-            # Auto thresholds are set based on the number of value candidates per slot
-            if domain not in self.thresholds:
-                self.thresholds[domain] = {}
-            if threshold == 'auto':
-                thres = 1.0 / (float(len(value_candidates)) - 2.1)
-                self.thresholds[domain][slot] = max(0.05, thres)
-            else:
-                self.thresholds[domain][slot] = max(0.05, threshold)
-
-        return self.thresholds
-
-    def init_session(self):
-        self.state = default_state()
-        self.active_domains = {}
-        self.hidden_states = None
-        self.info_dict = {}
-
-    def update(self, user_act=''):
-        prev_state = self.state
-
-        # Convert dialogs into transformer input features (token_ids, masks, etc)
-        features = self.get_features(user_act)
-        # Model forward pass
-        pred_states, active_domains, user_acts, turn_pooled_representation, belief_state, entropy_, mutual_info_ = self.predict(
-            features)
-
-        if entropy_ is not None:
-            entropy = {}
-            for slot, e in entropy_.items():
-                domain, slot = slot.split('-', 1)
-                if domain not in entropy:
-                    entropy[domain] = {}
-                if 'book' in slot:
-                    assert slot.startswith('book ')
-                    slot = slot.strip().split()[1]
-                slot = SLOT_MAP.get(slot, slot)
-                entropy[domain][slot] = e
-            del entropy_
-        else:
-            entropy = None
-
-        if mutual_info_ is not None:
-            mutual_info = {}
-            for slot, mi in mutual_info_.items():
-                domain, slot = slot.split('-', 1)
-                if domain not in mutual_info:
-                    mutual_info[domain] = {}
-                if 'book' in slot:
-                    assert slot.startswith('book ')
-                    slot = slot.strip().split()[1]
-                slot = SLOT_MAP.get(slot, slot)
-                mutual_info[domain][slot] = mi[0, 0]
-        else:
-            mutual_info = None
-
-        if belief_state is not None:
-            bs_probs = {}
-            belief_state, request_dist, domain_dist, greeting_dist = belief_state
-            for slot, p in belief_state.items():
-                domain, slot = slot.split('-', 1)
-                if domain not in bs_probs:
-                    bs_probs[domain] = {}
-                if 'book' in slot:
-                    assert slot.startswith('book ')
-                    slot = slot.strip().split()[1]
-                slot = SLOT_MAP.get(slot, slot)
-                if slot not in bs_probs[domain]:
-                    bs_probs[domain][slot] = {}
-                bs_probs[domain][slot]['inform'] = p
-
-            for slot, p in request_dist.items():
-                domain, slot = slot.split('-', 1)
-                if domain not in bs_probs:
-                    bs_probs[domain] = {}
-                slot = SLOT_MAP.get(slot, slot)
-                if slot not in bs_probs[domain]:
-                    bs_probs[domain][slot] = {}
-                bs_probs[domain][slot]['request'] = p
-
-            for domain, p in domain_dist.items():
-                if domain not in bs_probs:
-                    bs_probs[domain] = {}
-                bs_probs[domain]['none'] = {'inform': p}
-
-            if 'general' not in bs_probs:
-                bs_probs['general'] = {}
-            bs_probs['general']['none'] = greeting_dist
-
-        new_domains = [d for d, active in active_domains.items() if active]
-        new_domains = [
-            d for d in new_domains if not self.active_domains.get(d, False)]
-        self.active_domains = active_domains
-
-        for domain in new_domains:
-            user_acts.append(['Inform', domain.capitalize(), 'none', 'none'])
-
-        new_belief_state = copy.deepcopy(prev_state['belief_state'])
-        # user_acts = []
-        for state, value in pred_states.items():
-            domain, slot = state.split('-', 1)
-            value = '' if value == 'none' else value
-            value = 'dontcare' if value == 'do not care' else value
-            value = 'guesthouse' if value == 'guest house' else value
-            if slot not in ['name', 'book']:
-                if domain not in new_belief_state:
-                    if domain == 'bus':
-                        continue
-                    else:
-                        logging.debug(
-                            'Error: domain <{}> not in belief state'.format(domain))
-            slot = REF_SYS_DA[domain.capitalize()].get(slot, slot)
-            assert 'semi' in new_belief_state[domain]
-            assert 'book' in new_belief_state[domain]
-            if 'book' in slot:
-                assert slot.startswith('book ')
-                slot = slot.strip().split()[1]
-            slot = SLOT_MAP.get(slot, slot)
-
-            # Uncertainty clipping of state
-            if belief_state is not None:
-                if bs_probs[domain][slot].get('inform', 1.0) < self.thresholds[domain][slot]:
-                    value = ''
-
-            domain_dic = new_belief_state[domain]
-            value = normalize_value(self.value_dict, domain, slot, value)
-            if slot in domain_dic['semi']:
-                new_belief_state[domain]['semi'][slot] = value
-                if prev_state['belief_state'][domain]['semi'][slot] != value:
-                    user_acts.append(['Inform', domain.capitalize(
-                    ), REF_USR_DA[domain.capitalize()].get(slot, slot), value])
-            elif slot in domain_dic['book']:
-                new_belief_state[domain]['book'][slot] = value
-                if prev_state['belief_state'][domain]['book'][slot] != value:
-                    user_acts.append(['Inform', domain.capitalize(
-                    ), REF_USR_DA[domain.capitalize()].get(slot, slot), value])
-            elif slot.lower() in domain_dic['book']:
-                new_belief_state[domain]['book'][slot.lower()] = value
-                if prev_state['belief_state'][domain]['book'][slot.lower()] != value:
-                    user_acts.append(['Inform', domain.capitalize(
-                    ), REF_USR_DA[domain.capitalize()].get(slot.lower(), slot.lower()), value])
-            else:
-                logging.debug(
-                    'unknown slot name <{}> with value <{}> of domain <{}>\nitem: {}\n\n'.format(
-                        slot, value, domain, state)
-                )
-
-        new_state = copy.deepcopy(dict(prev_state))
-        new_state['belief_state'] = new_belief_state
-        new_state['active_domains'] = self.active_domains
-        if belief_state is not None:
-            new_state['belief_state_probs'] = bs_probs
-        if entropy is not None:
-            new_state['entropy'] = entropy
-        if mutual_info is not None:
-            new_state['mutual_information'] = mutual_info
-
-        new_state['user_action'] = user_acts
-
-        user_requests = [[a, d, s, v]
-                         for a, d, s, v in user_acts if a == 'Request']
-        for act, domain, slot, value in user_requests:
-            k = REF_SYS_DA[domain].get(slot, slot)
-            domain = domain.lower()
-            if domain not in new_state['request_state']:
-                new_state['request_state'][domain] = {}
-            if k not in new_state['request_state'][domain]:
-                new_state['request_state'][domain][k] = 0
-
-        if turn_pooled_representation is not None:
-            new_state['turn_pooled_representation'] = turn_pooled_representation
-
-        self.state = new_state
-        self.info_dict = copy.deepcopy(dict(new_state))
-
-        return self.state
-
-    # Model prediction function
-
-    def predict(self, features):
-        # Forward Pass
-        mutual_info = None
-        with torch.no_grad():
-            turn_pooled_representation = None
-            if self.get_turn_pooled_representation:
-                belief_state, request, domain, goodbye, self.hidden_states, turn_pooled_representation = self.model(input_ids=features['input_ids'],
-                                                                                                                    token_type_ids=features[
-                                                                                                                        'token_type_ids'],
-                                                                                                                    attention_mask=features[
-                                                                                                                        'attention_mask'],
-                                                                                                                    hidden_state=self.hidden_states,
-                                                                                                                    get_turn_pooled_representation=True)
-            elif self.return_mutual_info:
-                belief_state, request, domain, goodbye, self.hidden_states, mutual_info = self.model(input_ids=features['input_ids'],
-                                                                                                     token_type_ids=features[
-                                                                                                         'token_type_ids'],
-                                                                                                     attention_mask=features[
-                                                                                                         'attention_mask'],
-                                                                                                     hidden_state=self.hidden_states,
-                                                                                                     get_turn_pooled_representation=False,
-                                                                                                     calculate_inform_mutual_info=True)
-            else:
-                belief_state, request, domain, goodbye, self.hidden_states = self.model(input_ids=features['input_ids'],
-                                                                                        token_type_ids=features['token_type_ids'],
-                                                                                        attention_mask=features['attention_mask'],
-                                                                                        hidden_state=self.hidden_states,
-                                                                                        get_turn_pooled_representation=False)
-
-        # Convert belief state into dialog state
-        predictions = {slot: state[0, 0, :].argmax().item()
-                       for slot, state in belief_state.items()}
-        predictions = {slot: self.ontology[slot][idx]
-                       for slot, idx in predictions.items()}
-        predictions = {s: v for s, v in predictions.items() if v != 'none'}
-
-        if self.store_full_belief_state:
-            self.full_belief_state = belief_state
-
-        # Obtain model output probabilities
-        if self.get_confidence_scores:
-            entropy = None
-            if self.return_entropy:
-                entropy = {slot: state[0, 0, :]
-                           for slot, state in belief_state.items()}
-                entropy = {slot: self.relative_entropy(
-                    p).item() for slot, p in entropy.items()}
-
-            # Confidence score is the max probability across all not "none" values candidates.
-            belief_state = {slot: state[0, 0, 1:].max().item()
-                            for slot, state in belief_state.items()}
-            request_dist = {SLOT_MAP.get(
-                slot, slot): p[0, 0].item() for slot, p in request.items()}
-            domain_dist = {domain: p[0, 0].item()
-                           for domain, p in domain.items()}
-            greeting_dist = {'bye': goodbye[0, 0, 1].item(
-            ), 'thank': goodbye[0, 0, 2].item()}
-            belief_state = (belief_state, request_dist,
-                            domain_dist, greeting_dist)
-        else:
-            belief_state = None
-            entropy = None
-
-        # Construct request action prediction
-        request = [slot for slot, p in request.items() if p[0, 0].item() > 0.5]
-        request = [slot.split('-', 1) for slot in request]
-        request = [[domain, SLOT_MAP.get(slot, slot)]
-                   for domain, slot in request]
-        request = [['Request', domain.capitalize(), REF_USR_DA[domain.capitalize()].get(
-            slot, slot), '?'] for domain, slot in request]
-
-        # Construct active domain set
-        domain = {domain: p[0, 0].item() > 0.5 for domain, p in domain.items()}
-
-        # Construct general domain action
-        goodbye = goodbye[0, 0, :].argmax(-1).item()
-        goodbye = [[], ['bye'], ['thank']][goodbye]
-        goodbye = [[act, 'general', 'none', 'none'] for act in goodbye]
-
-        user_acts = request + goodbye
-
-        return predictions, domain, user_acts, turn_pooled_representation, belief_state, entropy, mutual_info
-
-    def relative_entropy(self, probs):
-        entropy = probs * torch.log(probs + 1e-8)
-        entropy = -entropy.sum()
-        # Maximum entropy of a K dimentional distribution is ln(K)
-        entropy /= torch.log(torch.tensor(probs.size(-1)).float())
-
-        return entropy
-
-    # Convert dialog turns into model features
-    def get_features(self, user_act):
-        # Extract system utterance from dialog history
-        context = self.state['history']
-        if context:
-            if context[-1][0] != 'sys':
-                system_act = ''
-            else:
-                system_act = context[-1][-1]
-        else:
-            system_act = ''
-
-        # Tokenize dialog
-        features = self.tokenizer.encode_plus(user_act, system_act, add_special_tokens=True, max_length=self.config.max_turn_len,
-                                              padding='max_length', truncation='longest_first')
-
-        input_ids = torch.tensor(features['input_ids']).reshape(
-            1, 1, -1).to(self.device) if 'input_ids' in features else None
-        token_type_ids = torch.tensor(features['token_type_ids']).reshape(
-            1, 1, -1).to(self.device) if 'token_type_ids' in features else None
-        attention_mask = torch.tensor(features['attention_mask']).reshape(
-            1, 1, -1).to(self.device) if 'attention_mask' in features else None
-        features = {'input_ids': input_ids,
-                    'token_type_ids': token_type_ids, 'attention_mask': attention_mask}
-
-        return features
-
-
-# if __name__ == "__main__":
-#     tracker = SetSUMBTTracker(model_type='roberta', model_path='/gpfs/project/niekerk/results/nbt/convlab_setsumbt_acts')
-#                         # nlu_path='/gpfs/project/niekerk/data/bert_multiwoz_all_context.zip')
-#     tracker.init_session()
-#     state = tracker.update('hey. I need a cheap restaurant.')
-#     # tracker.state['history'].append(['usr', 'hey. I need a cheap restaurant.'])
-#     # tracker.state['history'].append(['sys', 'There are many cheap places, which food do you like?'])
-#     # state = tracker.update('If you have something Asian that would be great.')
-#     # tracker.state['history'].append(['usr', 'If you have something Asian that would be great.'])
-#     # tracker.state['history'].append(['sys', 'The Golden Wok is a nice cheap chinese restaurant.'])
-#     # state = tracker.update('Great. Where are they located?')
-#     # tracker.state['history'].append(['usr', 'Great. Where are they located?'])
-#     print(tracker.state)
diff --git a/convlab/dst/setsumbt/multiwoz/__init__.py b/convlab/dst/setsumbt/multiwoz/__init__.py
deleted file mode 100644
index a1f1fb894a0430545c62b78f9a6f4786c4e328a8..0000000000000000000000000000000000000000
--- a/convlab/dst/setsumbt/multiwoz/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from convlab.dst.setsumbt.multiwoz.dataset import multiwoz21, ontology
-from convlab.dst.setsumbt.multiwoz.Tracker import SetSUMBTTracker
\ No newline at end of file
diff --git a/convlab/dst/setsumbt/multiwoz/dataset/multiwoz21.py b/convlab/dst/setsumbt/multiwoz/dataset/multiwoz21.py
deleted file mode 100644
index 2c8e98f35429ce5194d82d07a1cf0de8fee54515..0000000000000000000000000000000000000000
--- a/convlab/dst/setsumbt/multiwoz/dataset/multiwoz21.py
+++ /dev/null
@@ -1,502 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2020 DSML Group, Heinrich Heine University, Düsseldorf
-# Authors: Carel van Niekerk (niekerk@hhu.de)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""MultiWOZ 2.1/2.3 Dialogue Dataset"""
-
-import os
-import json
-import requests
-import zipfile
-import io
-from shutil import copy2 as copy
-
-import torch
-from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler
-from tqdm import tqdm
-
-from convlab.dst.setsumbt.multiwoz.dataset.utils import (clean_text, ACTIVE_DOMAINS, get_domains, set_util_domains,
-                                                        fix_delexicalisation, extract_dialogue, PRICERANGE,
-                                                        BOOLEAN, DAYS, QUANTITIES, TIME, VALUE_MAP, map_values)
-
-
-# Set up global data_directory
-def set_datadir(dir):
-    global DATA_DIR
-    DATA_DIR = dir
-
-
-def set_active_domains(domains):
-    global ACTIVE_DOMAINS
-    ACTIVE_DOMAINS = [d for d in domains if d in ACTIVE_DOMAINS]
-    set_util_domains(ACTIVE_DOMAINS)
-
-
-# MultiWOZ2.1 download link
-URL = 'https://github.com/budzianowski/multiwoz/raw/master/data/MultiWOZ_2.1.zip'
-def set_url(url):
-    global URL
-    URL = url
-
-
-# Create Dialogue examples from the dataset
-def create_examples(max_utt_len, get_requestable_slots=False, force_processing=False):
-
-    # Load or download Raw Data
-    if not os.path.exists(DATA_DIR):
-        os.mkdir(DATA_DIR)
-    if not os.path.exists(os.path.join(DATA_DIR, 'data_raw.json')):
-        # Download data archive and extract
-        archive = _download()
-        data = _extract(archive)
-
-        writer = open(os.path.join(DATA_DIR, 'data_raw.json'), 'w')
-        json.dump(data, writer, indent = 2)
-        del archive, writer
-    else:
-        reader = open(os.path.join(DATA_DIR, 'data_raw.json'), 'r')
-        data = json.load(reader)
-
-    if force_processing or not os.path.exists(os.path.join(DATA_DIR, 'data_train.json')):
-        # Preprocess all dialogues
-        data_processed = _process(data['data'], data['system_acts'])
-        # Format data and split train, test and devlopment sets
-        train, dev, test = _split_data(data_processed, data['testListFile'],
-                                                            data['valListFile'], max_utt_len)
-
-        # Write data
-        writer = open(os.path.join(DATA_DIR, 'data_train.json'), 'w')
-        json.dump(train, writer, indent = 2)
-        writer = open(os.path.join(DATA_DIR, 'data_test.json'), 'w')
-        json.dump(test, writer, indent = 2)
-        writer = open(os.path.join(DATA_DIR, 'data_dev.json'), 'w')
-        json.dump(dev, writer, indent = 2)
-        writer.flush()
-        writer.close()
-        del writer
-
-        # Extract slots and slot value candidates from the dataset
-        for set_type in ['train', 'dev', 'test']:
-            _get_ontology(set_type, get_requestable_slots)
-        
-        script_path = os.path.abspath(__file__).replace('/multiwoz21.py', '')
-        file_name = 'mwoz21_ont_request.json' if get_requestable_slots else 'mwoz21_ont.json'
-        copy(os.path.join(script_path, file_name), os.path.join(DATA_DIR, 'ontology_test.json'))
-        copy(os.path.join(script_path, 'mwoz21_slot_descriptions.json'), os.path.join(DATA_DIR, 'slot_descriptions.json'))
-
-
-# Extract slots and slot value candidates from the dataset
-def _get_ontology(set_type, get_requestable_slots=False):
-
-    datasets = ['train']
-    if set_type in ['test', 'dev']:
-        datasets.append('dev')
-        datasets.append('test')
-
-    # Load examples
-    data = []
-    for dataset in datasets:
-        reader = open(os.path.join(DATA_DIR, 'data_%s.json' % dataset), 'r')
-        data += json.load(reader)
-
-    ontology = dict()
-    for dial in data:
-        for turn in dial['dialogue']:
-            for state in turn['dialogue_state']:
-                slot, value = state
-                value = map_values(value)
-                if slot not in ontology:
-                    ontology[slot] = [value]
-                else:
-                    ontology[slot].append(value)
-
-    requestable_slots = []
-    if get_requestable_slots:
-        for dial in data:
-            for turn in dial['dialogue']:
-                for act, dom, slot, val in turn['user_acts']:
-                    if act == 'request':
-                        requestable_slots.append(f'{dom}-{slot}')
-        requestable_slots = list(set(requestable_slots))
-
-    for slot in ontology:
-        if 'price' in slot:
-            ontology[slot] = PRICERANGE
-        if 'parking' in slot or 'internet' in slot:
-            ontology[slot] = BOOLEAN
-        if 'day' in slot:
-            ontology[slot] = DAYS
-        if 'people' in slot or 'duration' in slot or 'stay' in slot:
-            ontology[slot] = QUANTITIES
-        if 'time' in slot or 'leave' in slot or 'arrive' in slot:
-            ontology[slot] = TIME
-        if 'stars' in slot:
-            ontology[slot] += [str(i) for i in range(5)]
-
-    # Sort slot values and add none and dontcare values
-    for slot in ontology:
-        ontology[slot] = list(set(ontology[slot]))
-        ontology[slot] = ['none', 'do not care'] + sorted([s for s in ontology[slot] if s not in ['none', 'do not care']])
-    for slot in requestable_slots:
-        if slot in ontology:
-            ontology[slot].append('request')
-        else:
-            ontology[slot] = ['request']
-
-    writer = open(os.path.join(DATA_DIR, 'ontology_%s.json' % set_type), 'w')
-    json.dump(ontology, writer, indent=2)
-    writer.close()
-
-
-# Convert dialogue examples to model input features and labels
-def convert_examples_to_features(set_type, tokenizer, max_turns=12, max_seq_len=64):
-
-    features = dict()
-
-    # Load examples
-    reader = open(os.path.join(DATA_DIR, 'data_%s.json' % set_type), 'r')
-    data = json.load(reader)
-
-    # Get encoder input for system, user utterance pairs
-    input_feats = []
-    for dial in data:
-        dial_feats = []
-        for turn in dial['dialogue']:
-            if len(turn['system_transcript']) == 0:
-                usr = turn['transcript']
-                dial_feats.append(tokenizer.encode_plus(usr, add_special_tokens = True,
-                                                        max_length = max_seq_len, padding='max_length',
-                                                        truncation = 'longest_first'))
-            else:
-                usr = turn['transcript']
-                sys = turn['system_transcript']
-                dial_feats.append(tokenizer.encode_plus(usr, sys, add_special_tokens = True,
-                                                        max_length = max_seq_len, padding='max_length',
-                                                        truncation = 'longest_first'))
-            if len(dial_feats) >= max_turns:
-                break
-        input_feats.append(dial_feats)
-    del dial_feats
-
-    # Perform turn level padding
-    input_ids = [[turn['input_ids'] for turn in dial] + [[0] * max_seq_len] * (max_turns - len(dial)) for dial in input_feats]
-    if 'token_type_ids' in input_feats[0][0]:
-        token_type_ids = [[turn['token_type_ids'] for turn in dial] + [[0] * max_seq_len] * (max_turns - len(dial)) for dial in input_feats]
-    else:
-        token_type_ids = None
-    if 'attention_mask' in input_feats[0][0]:
-        attention_mask = [[turn['attention_mask'] for turn in dial] + [[0] * max_seq_len] * (max_turns - len(dial)) for dial in input_feats]
-    else:
-        attention_mask = None
-    del input_feats
-
-    # Create torch data tensors
-    features['input_ids'] = torch.tensor(input_ids)
-    features['token_type_ids'] = torch.tensor(token_type_ids) if token_type_ids else None
-    features['attention_mask'] = torch.tensor(attention_mask) if attention_mask else None
-    del input_ids, token_type_ids, attention_mask
-
-    # Load ontology
-    reader = open(os.path.join(DATA_DIR, 'ontology_%s.json' % set_type), 'r')
-    ontology = json.load(reader)
-    reader.close()
-
-    informable_slots = [slot for slot, values in ontology.items() if values != ['request']]
-    requestable_slots = [slot for slot, values in ontology.items() if 'request' in values]
-    for slot in requestable_slots:
-        ontology[slot].remove('request')
-    
-    domains = list(set(informable_slots + requestable_slots))
-    domains = list(set([slot.split('-', 1)[0] for slot in domains]))
-
-    # Create slot labels
-    for slot in informable_slots:
-        labels = []
-        for dial in data:
-            labs = []
-            for turn in dial['dialogue']:
-                slots_active = [s for s, v in turn['dialogue_state']]
-                if slot in slots_active:
-                    value = [v for s, v in turn['dialogue_state'] if s == slot][0]
-                else:
-                    value = 'none'
-                if value in ontology[slot]:
-                    value = ontology[slot].index(value)
-                else:
-                    value = map_values(value)
-                    if value in ontology[slot]:
-                        value = ontology[slot].index(value)
-                    else:
-                        value = -1 # If value is not in ontology then we do not penalise the model
-                labs.append(value)
-                if len(labs) >= max_turns:
-                    break
-            labs = labs + [-1] * (max_turns - len(labs))
-            labels.append(labs)
-
-        labels = torch.tensor(labels)
-        features['labels-' + slot] = labels
-
-    for slot in requestable_slots:
-        labels = []
-        for dial in data:
-            labs = []
-            for turn in dial['dialogue']:
-                slots_active = [[d, s] for i, d, s, v in turn['user_acts']]
-                if slot.split('-', 1) in slots_active:
-                    act_ = [i for i, d, s, v in turn['user_acts'] if f"{d}-{s}" == slot][0]
-                    if act_ == 'request':
-                        labs.append(1)
-                    else:
-                        labs.append(0)
-                else:
-                    labs.append(0)
-                if len(labs) >= max_turns:
-                    break
-            labs = labs + [-1] * (max_turns - len(labs))
-            labels.append(labs)
-        
-        labels = torch.tensor(labels)
-        features['request-' + slot] = labels
-    
-    # Greeting act labels (0-no greeting, 1-goodbye, 2-thank you)
-    labels = []
-    for dial in data:
-        labs = []
-        for turn in dial['dialogue']:
-            greeting_active = [i for i, d, s, v in turn['user_acts'] if i in ['bye', 'thank']]
-            if greeting_active:
-                if 'bye' in greeting_active:
-                    labs.append(1)
-                else :
-                    labs.append(2)
-            else:
-                labs.append(0)
-            if len(labs) >= max_turns:
-                break
-        labs = labs + [-1] * (max_turns - len(labs))
-        labels.append(labs)
-    
-    labels = torch.tensor(labels)
-    features['goodbye'] = labels
-
-    for domain in domains:
-        labels = []
-        for dial in data:
-            labs = []
-            for turn in dial['dialogue']:
-                if domain == turn['domain']:
-                    labs.append(1)
-                else:
-                    labs.append(0)
-                if len(labs) >= max_turns:
-                        break
-            labs = labs + [-1] * (max_turns - len(labs))
-            labels.append(labs)
-        
-        labels = torch.tensor(labels)
-        features['active-' + domain] = labels
-
-    del labels
-
-    return features
-
-
-# MultiWOZ2.1 Dataset object
-class MultiWoz21(Dataset):
-
-    def __init__(self, set_type, tokenizer, max_turns=12, max_seq_len=64):
-        self.features = convert_examples_to_features(set_type, tokenizer, max_turns, max_seq_len)
-
-    def __getitem__(self, index):
-        return {label: self.features[label][index] for label in self.features
-                if self.features[label] is not None}
-
-    def __len__(self):
-        return self.features['input_ids'].size(0)
-
-    def resample(self, size=None):
-        n_dialogues = self.__len__()
-        if not size:
-            size = n_dialogues
-
-        dialogues = torch.randint(low=0, high=n_dialogues, size=(size,))
-        self.features = {label: self.features[label][dialogues] for label in self.features
-                        if self.features[label] is not None}
-        
-        return self
-
-    def to(self, device):
-        self.device = device
-        self.features = {label: self.features[label].to(device) for label in self.features
-                         if self.features[label] is not None}
-
-
-# MultiWOZ2.1 Dataset object
-class EnsembleMultiWoz21(Dataset):
-    def __init__(self, data):
-        self.features = data
-
-    def __getitem__(self, index):
-        return {label: self.features[label][index] for label in self.features
-                if self.features[label] is not None}
-
-    def __len__(self):
-        return self.features['input_ids'].size(0)
-
-    def resample(self, size=None):
-        n_dialogues = self.__len__()
-        if not size:
-            size = n_dialogues
-
-        dialogues = torch.randint(low=0, high=n_dialogues, size=(size,))
-        self.features = {label: self.features[label][dialogues] for label in self.features
-                        if self.features[label] is not None}
-
-    def to(self, device):
-        self.device = device
-        self.features = {label: self.features[label].to(device) for label in self.features
-                         if self.features[label] is not None}
-
-
-# Module to create torch dataloaders
-def get_dataloader(set_type, batch_size, tokenizer, max_turns=12, max_seq_len=64, device=None, resampled_size=None):
-    data = MultiWoz21(set_type, tokenizer, max_turns, max_seq_len)
-    data.to('cpu')
-
-    if resampled_size:
-        data.resample(resampled_size)
-
-    if set_type in ['test', 'dev']:
-        sampler = SequentialSampler(data)
-    else:
-        sampler = RandomSampler(data)
-    loader = DataLoader(data, sampler=sampler, batch_size=batch_size)
-
-    return loader
-
-
-def _download(chunk_size=1048576):
-    """Download data archive.
-
-    Parameters:
-        chunk_size (int): Download chunk size. (default=1048576)
-    Returns:
-        archive: ZipFile archive object.
-    """
-    # Download the archive byte string
-    req = requests.get(URL, stream=True)
-    archive = b''
-    for n_chunks, chunk in tqdm(enumerate(req.iter_content(chunk_size=chunk_size)), desc='Download Chunk'):
-        if chunk:
-            archive += chunk
-
-    # Convert the bytestring into a zipfile object
-    archive = io.BytesIO(archive)
-    archive = zipfile.ZipFile(archive)
-
-    return archive
-
-
-def _extract(archive):
-    """Extract the json dictionaries from the archive.
-
-    Parameters:
-        archive: ZipFile archive object.
-    Returns:
-        data: Data dictionary.
-    """
-    files = [file for file in archive.filelist if ('.json' in file.filename or '.txt' in file.filename)
-            and 'MACOSX' not in file.filename]
-    objects = []
-    for file in tqdm(files, desc='File'):
-        data = archive.open(file).read()
-        # Get data objects from the files
-        try:
-            data = json.loads(data)
-        except json.decoder.JSONDecodeError:
-            data = data.decode().split('\n')
-        objects.append(data)
-
-    files = [file.filename.split('/')[-1].split('.')[0] for file in files]
-
-    data = {file: data for file, data in zip(files, objects)}
-    return data
-
-
-# Process files
-def _process(dialogue_data, acts_data):
-    print('Processing Dialogues')
-    out = {}
-    for dial_name in tqdm(dialogue_data):
-        dialogue = dialogue_data[dial_name]
-
-        prev_dom = ''
-        for turn_id, turn in enumerate(dialogue['log']):
-            dialogue['log'][turn_id]['text'] = clean_text(turn['text'])
-            if len(turn['metadata']) != 0:
-                crnt_dom = get_domains(dialogue['log'], turn_id, prev_dom)
-                prev_dom = crnt_dom
-                dialogue['log'][turn_id - 1]['domain'] = crnt_dom
-
-            dialogue['log'][turn_id] = fix_delexicalisation(turn)
-
-        out[dial_name] = dialogue
-
-    return out
-
-
-# Split data (train, dev, test)
-def _split_data(dial_data, test, dev, max_utt_len):
-    train_dials, test_dials, dev_dials = [], [], []
-    print('Formatting and Splitting Data')
-    for name in tqdm(dial_data):
-        dialogue = dial_data[name]
-        domains = []
-
-        dial = extract_dialogue(dialogue, max_utt_len)
-        if dial:
-            dialogue = dict()
-            dialogue['dialogue_idx'] = name
-            dialogue['domains'] = []
-            dialogue['dialogue'] = []
-
-            for turn_id, turn in enumerate(dial):
-                turn_dialog = dict()
-                turn_dialog['system_transcript'] = dial[turn_id - 1]['sys'] if turn_id > 0 else ''
-                turn_dialog['turn_idx'] = turn_id
-                turn_dialog['dialogue_state'] = turn['ds']
-                turn_dialog['transcript'] = turn['usr']
-                # turn_dialog['system_acts'] = dial[turn_id - 1]['sys_a'] if turn_id > 0 else []
-                turn_dialog['user_acts'] = turn['usr_a']
-                turn_dialog['domain'] = turn['domain']
-                dialogue['domains'].append(turn['domain'])
-                dialogue['dialogue'].append(turn_dialog)
-
-            dialogue['domains'] = [d for d in list(set(dialogue['domains'])) if d != '']
-            if True in [dom not in ACTIVE_DOMAINS for dom in dialogue['domains']]:
-                dialogue['domains'] = []
-            dialogue['domains'] = [dom for dom in dialogue['domains'] if dom in ACTIVE_DOMAINS]
-
-            if dialogue['domains']:
-                if name in test:
-                    test_dials.append(dialogue)
-                elif name in dev:
-                    dev_dials.append(dialogue)
-                else:
-                    train_dials.append(dialogue)
-
-    print('Number of Dialogues:\nTrain: %i\nDev: %i\nTest: %i' % (len(train_dials), len(dev_dials), len(test_dials)))
-
-    return train_dials, dev_dials, test_dials
diff --git a/convlab/dst/setsumbt/multiwoz/dataset/mwoz21_ont.json b/convlab/dst/setsumbt/multiwoz/dataset/mwoz21_ont.json
deleted file mode 100644
index b703793dc747535132b748d8b69838b4c151d8d5..0000000000000000000000000000000000000000
--- a/convlab/dst/setsumbt/multiwoz/dataset/mwoz21_ont.json
+++ /dev/null
@@ -1,2990 +0,0 @@
-{
-  "hotel-price range": [
-    "none",
-    "do not care",
-    "cheap",
-    "expensive",
-    "moderate"
-  ],
-  "hotel-type": [
-    "none",
-    "do not care",
-    "bed and breakfast",
-    "guest house",
-    "hotel"
-  ],
-  "hotel-parking": [
-    "none",
-    "do not care",
-    "no",
-    "yes"
-  ],
-  "hotel-book day": [
-    "none",
-    "do not care",
-    "friday",
-    "monday",
-    "saterday",
-    "sunday",
-    "thursday",
-    "tuesday",
-    "wednesday"
-  ],
-  "hotel-book people": [
-    "none",
-    "do not care",
-    "1",
-    "10 or more",
-    "2",
-    "3",
-    "4",
-    "5",
-    "6",
-    "7",
-    "8",
-    "9"
-  ],
-  "hotel-book stay": [
-    "none",
-    "do not care",
-    "1",
-    "10 or more",
-    "2",
-    "3",
-    "4",
-    "5",
-    "6",
-    "7",
-    "8",
-    "9"
-  ],
-  "train-destination": [
-    "none",
-    "do not care",
-    "bishops stortford",
-    "kings lynn",
-    "london liverpool street",
-    "centre",
-    "bishop stortford",
-    "liverpool",
-    "leicester",
-    "broxbourne",
-    "gourmet burger kitchen",
-    "copper kettle",
-    "bournemouth",
-    "stevenage",
-    "liverpool street",
-    "norwich",
-    "huntingdon marriott hotel",
-    "city centre north",
-    "taj tandoori",
-    "the copper kettle",
-    "peterborough",
-    "ely",
-    "lecester",
-    "london",
-    "willi",
-    "stansted airport",
-    "huntington marriott",
-    "cambridge",
-    "gonv",
-    "glastonbury",
-    "hol",
-    "north",
-    "birmingham new street",
-    "norway",
-    "petersborough",
-    "london kings cross",
-    "curry prince",
-    "bishops storford"
-  ],
-  "train-arrive by": [
-    "none",
-    "do not care",
-    "00:00",
-    "00:05",
-    "00:10",
-    "00:15",
-    "00:20",
-    "00:25",
-    "00:30",
-    "00:35",
-    "00:40",
-    "00:45",
-    "00:50",
-    "00:55",
-    "01:00",
-    "01:05",
-    "01:10",
-    "01:15",
-    "01:20",
-    "01:25",
-    "01:30",
-    "01:35",
-    "01:40",
-    "01:45",
-    "01:50",
-    "01:55",
-    "02:00",
-    "02:05",
-    "02:10",
-    "02:15",
-    "02:20",
-    "02:25",
-    "02:30",
-    "02:35",
-    "02:40",
-    "02:45",
-    "02:50",
-    "02:55",
-    "03:00",
-    "03:05",
-    "03:10",
-    "03:15",
-    "03:20",
-    "03:25",
-    "03:30",
-    "03:35",
-    "03:40",
-    "03:45",
-    "03:50",
-    "03:55",
-    "04:00",
-    "04:05",
-    "04:10",
-    "04:15",
-    "04:20",
-    "04:25",
-    "04:30",
-    "04:35",
-    "04:40",
-    "04:45",
-    "04:50",
-    "04:55",
-    "05:00",
-    "05:05",
-    "05:10",
-    "05:15",
-    "05:20",
-    "05:25",
-    "05:30",
-    "05:35",
-    "05:40",
-    "05:45",
-    "05:50",
-    "05:55",
-    "06:00",
-    "06:05",
-    "06:10",
-    "06:15",
-    "06:20",
-    "06:25",
-    "06:30",
-    "06:35",
-    "06:40",
-    "06:45",
-    "06:50",
-    "06:55",
-    "07:00",
-    "07:05",
-    "07:10",
-    "07:15",
-    "07:20",
-    "07:25",
-    "07:30",
-    "07:35",
-    "07:40",
-    "07:45",
-    "07:50",
-    "07:55",
-    "08:00",
-    "08:05",
-    "08:10",
-    "08:15",
-    "08:20",
-    "08:25",
-    "08:30",
-    "08:35",
-    "08:40",
-    "08:45",
-    "08:50",
-    "08:55",
-    "09:00",
-    "09:05",
-    "09:10",
-    "09:15",
-    "09:20",
-    "09:25",
-    "09:30",
-    "09:35",
-    "09:40",
-    "09:45",
-    "09:50",
-    "09:55",
-    "10:00",
-    "10:05",
-    "10:10",
-    "10:15",
-    "10:20",
-    "10:25",
-    "10:30",
-    "10:35",
-    "10:40",
-    "10:45",
-    "10:50",
-    "10:55",
-    "11:00",
-    "11:05",
-    "11:10",
-    "11:15",
-    "11:20",
-    "11:25",
-    "11:30",
-    "11:35",
-    "11:40",
-    "11:45",
-    "11:50",
-    "11:55",
-    "12:00",
-    "12:05",
-    "12:10",
-    "12:15",
-    "12:20",
-    "12:25",
-    "12:30",
-    "12:35",
-    "12:40",
-    "12:45",
-    "12:50",
-    "12:55",
-    "13:00",
-    "13:05",
-    "13:10",
-    "13:15",
-    "13:20",
-    "13:25",
-    "13:30",
-    "13:35",
-    "13:40",
-    "13:45",
-    "13:50",
-    "13:55",
-    "14:00",
-    "14:05",
-    "14:10",
-    "14:15",
-    "14:20",
-    "14:25",
-    "14:30",
-    "14:35",
-    "14:40",
-    "14:45",
-    "14:50",
-    "14:55",
-    "15:00",
-    "15:05",
-    "15:10",
-    "15:15",
-    "15:20",
-    "15:25",
-    "15:30",
-    "15:35",
-    "15:40",
-    "15:45",
-    "15:50",
-    "15:55",
-    "16:00",
-    "16:05",
-    "16:10",
-    "16:15",
-    "16:20",
-    "16:25",
-    "16:30",
-    "16:35",
-    "16:40",
-    "16:45",
-    "16:50",
-    "16:55",
-    "17:00",
-    "17:05",
-    "17:10",
-    "17:15",
-    "17:20",
-    "17:25",
-    "17:30",
-    "17:35",
-    "17:40",
-    "17:45",
-    "17:50",
-    "17:55",
-    "18:00",
-    "18:05",
-    "18:10",
-    "18:15",
-    "18:20",
-    "18:25",
-    "18:30",
-    "18:35",
-    "18:40",
-    "18:45",
-    "18:50",
-    "18:55",
-    "19:00",
-    "19:05",
-    "19:10",
-    "19:15",
-    "19:20",
-    "19:25",
-    "19:30",
-    "19:35",
-    "19:40",
-    "19:45",
-    "19:50",
-    "19:55",
-    "20:00",
-    "20:05",
-    "20:10",
-    "20:15",
-    "20:20",
-    "20:25",
-    "20:30",
-    "20:35",
-    "20:40",
-    "20:45",
-    "20:50",
-    "20:55",
-    "21:00",
-    "21:05",
-    "21:10",
-    "21:15",
-    "21:20",
-    "21:25",
-    "21:30",
-    "21:35",
-    "21:40",
-    "21:45",
-    "21:50",
-    "21:55",
-    "22:00",
-    "22:05",
-    "22:10",
-    "22:15",
-    "22:20",
-    "22:25",
-    "22:30",
-    "22:35",
-    "22:40",
-    "22:45",
-    "22:50",
-    "22:55",
-    "23:00",
-    "23:05",
-    "23:10",
-    "23:15",
-    "23:20",
-    "23:25",
-    "23:30",
-    "23:35",
-    "23:40",
-    "23:45",
-    "23:50",
-    "23:55"
-  ],
-  "train-departure": [
-    "none",
-    "do not care",
-    "bishops stortford",
-    "kings lynn",
-    "brookshite",
-    "london liverpool street",
-    "cam",
-    "liverpool",
-    "bro",
-    "leicester",
-    "broxbourne",
-    "norwhich",
-    "saint johns",
-    "stevenage",
-    "stansted",
-    "london liverpool",
-    "cambrid",
-    "city hall",
-    "rosas bed and breakfast",
-    "alpha-milton",
-    "wandlebury country park",
-    "norwich",
-    "liecester",
-    "stratford",
-    "peterborough",
-    "duxford",
-    "ely",
-    "london",
-    "stansted airport",
-    "lon",
-    "cambridge",
-    "panahar",
-    "cineworld",
-    "leicaster",
-    "birmingham",
-    "cafe uno",
-    "camboats",
-    "huntingdon",
-    "birmingham new street",
-    "arbu",
-    "alpha milton",
-    "east london",
-    "london kings cross",
-    "hamilton lodge",
-    "aylesbray lodge guest",
-    "el shaddai"
-  ],
-  "train-day": [
-    "none",
-    "do not care",
-    "friday",
-    "monday",
-    "saterday",
-    "sunday",
-    "thursday",
-    "tuesday",
-    "wednesday"
-  ],
-  "train-book people": [
-    "none",
-    "do not care",
-    "1",
-    "10 or more",
-    "2",
-    "3",
-    "4",
-    "5",
-    "6",
-    "7",
-    "8",
-    "9"
-  ],
-  "hotel-stars": [
-    "none",
-    "do not care",
-    "0",
-    "1",
-    "2",
-    "3",
-    "4",
-    "5"
-  ],
-  "hotel-internet": [
-    "none",
-    "do not care",
-    "no",
-    "yes"
-  ],
-  "hotel-name": [
-    "a and b guest house",
-    "city roomz",
-    "carolina bed and breakfast",
-    "limehouse",
-    "anatolia",
-    "hamilton lodge",
-    "the lensfield hotel",
-    "rosa's bed and breakfast",
-    "gall",
-    "aylesbray lodge",
-    "kirkwood",
-    "cambridge belfry",
-    "warkworth house",
-    "gonville",
-    "belfy hotel",
-    "nus",
-    "alexander",
-    "super 5",
-    "aylesbray lodge guest house",
-    "the gonvile hotel",
-    "allenbell",
-    "nothamilton lodge",
-    "ashley hotel",
-    "autumn house",
-    "hobsons house",
-    "hotel",
-    "ashely hotel",
-    "caridge belfrey",
-    "el shaddia guest house",
-    "avalon",
-    "cote",
-    "city centre north bed and breakfast",
-    "the cambridge belfry",
-    "home from home",
-    "wandlebury coutn",
-    "wankworth house",
-    "city stop rest",
-    "the worth house",
-    "cityroomz",
-    "huntingdon marriottt hotel",
-    "none",
-    "lensfield",
-    "rosas bed and breakfast",
-    "leverton house",
-    "gonville hotel",
-    "holiday inn cambridge",
-    "do not care",
-    "archway house",
-    "lan hon",
-    "levert",
-    "acorn guest house",
-    "cambridge",
-    "the ashley hotel",
-    "el shaddai",
-    "sleeperz",
-    "alpha milton guest house",
-    "doubletree by hilton cambridge",
-    "tandoori palace",
-    "express by",
-    "express by holiday inn cambridge",
-    "north bed and breakfast",
-    "holiday inn",
-    "arbury lodge guest house",
-    "alexander bed and breakfast",
-    "huntingdon marriott hotel",
-    "royal spice",
-    "sou",
-    "finches bed and breakfast",
-    "the alpha milton",
-    "bridge guest house",
-    "the acorn guest house",
-    "kirkwood house",
-    "eraina",
-    "la margherit",
-    "lensfield hotel",
-    "marriott hotel",
-    "nusha",
-    "city centre bed and breakfast",
-    "the allenbell",
-    "university arms hotel",
-    "clare",
-    "cherr",
-    "wartworth",
-    "acorn place",
-    "lovell lodge",
-    "whale"
-  ],
-  "train-leave at": [
-    "none",
-    "do not care",
-    "00:00",
-    "00:05",
-    "00:10",
-    "00:15",
-    "00:20",
-    "00:25",
-    "00:30",
-    "00:35",
-    "00:40",
-    "00:45",
-    "00:50",
-    "00:55",
-    "01:00",
-    "01:05",
-    "01:10",
-    "01:15",
-    "01:20",
-    "01:25",
-    "01:30",
-    "01:35",
-    "01:40",
-    "01:45",
-    "01:50",
-    "01:55",
-    "02:00",
-    "02:05",
-    "02:10",
-    "02:15",
-    "02:20",
-    "02:25",
-    "02:30",
-    "02:35",
-    "02:40",
-    "02:45",
-    "02:50",
-    "02:55",
-    "03:00",
-    "03:05",
-    "03:10",
-    "03:15",
-    "03:20",
-    "03:25",
-    "03:30",
-    "03:35",
-    "03:40",
-    "03:45",
-    "03:50",
-    "03:55",
-    "04:00",
-    "04:05",
-    "04:10",
-    "04:15",
-    "04:20",
-    "04:25",
-    "04:30",
-    "04:35",
-    "04:40",
-    "04:45",
-    "04:50",
-    "04:55",
-    "05:00",
-    "05:05",
-    "05:10",
-    "05:15",
-    "05:20",
-    "05:25",
-    "05:30",
-    "05:35",
-    "05:40",
-    "05:45",
-    "05:50",
-    "05:55",
-    "06:00",
-    "06:05",
-    "06:10",
-    "06:15",
-    "06:20",
-    "06:25",
-    "06:30",
-    "06:35",
-    "06:40",
-    "06:45",
-    "06:50",
-    "06:55",
-    "07:00",
-    "07:05",
-    "07:10",
-    "07:15",
-    "07:20",
-    "07:25",
-    "07:30",
-    "07:35",
-    "07:40",
-    "07:45",
-    "07:50",
-    "07:55",
-    "08:00",
-    "08:05",
-    "08:10",
-    "08:15",
-    "08:20",
-    "08:25",
-    "08:30",
-    "08:35",
-    "08:40",
-    "08:45",
-    "08:50",
-    "08:55",
-    "09:00",
-    "09:05",
-    "09:10",
-    "09:15",
-    "09:20",
-    "09:25",
-    "09:30",
-    "09:35",
-    "09:40",
-    "09:45",
-    "09:50",
-    "09:55",
-    "10:00",
-    "10:05",
-    "10:10",
-    "10:15",
-    "10:20",
-    "10:25",
-    "10:30",
-    "10:35",
-    "10:40",
-    "10:45",
-    "10:50",
-    "10:55",
-    "11:00",
-    "11:05",
-    "11:10",
-    "11:15",
-    "11:20",
-    "11:25",
-    "11:30",
-    "11:35",
-    "11:40",
-    "11:45",
-    "11:50",
-    "11:55",
-    "12:00",
-    "12:05",
-    "12:10",
-    "12:15",
-    "12:20",
-    "12:25",
-    "12:30",
-    "12:35",
-    "12:40",
-    "12:45",
-    "12:50",
-    "12:55",
-    "13:00",
-    "13:05",
-    "13:10",
-    "13:15",
-    "13:20",
-    "13:25",
-    "13:30",
-    "13:35",
-    "13:40",
-    "13:45",
-    "13:50",
-    "13:55",
-    "14:00",
-    "14:05",
-    "14:10",
-    "14:15",
-    "14:20",
-    "14:25",
-    "14:30",
-    "14:35",
-    "14:40",
-    "14:45",
-    "14:50",
-    "14:55",
-    "15:00",
-    "15:05",
-    "15:10",
-    "15:15",
-    "15:20",
-    "15:25",
-    "15:30",
-    "15:35",
-    "15:40",
-    "15:45",
-    "15:50",
-    "15:55",
-    "16:00",
-    "16:05",
-    "16:10",
-    "16:15",
-    "16:20",
-    "16:25",
-    "16:30",
-    "16:35",
-    "16:40",
-    "16:45",
-    "16:50",
-    "16:55",
-    "17:00",
-    "17:05",
-    "17:10",
-    "17:15",
-    "17:20",
-    "17:25",
-    "17:30",
-    "17:35",
-    "17:40",
-    "17:45",
-    "17:50",
-    "17:55",
-    "18:00",
-    "18:05",
-    "18:10",
-    "18:15",
-    "18:20",
-    "18:25",
-    "18:30",
-    "18:35",
-    "18:40",
-    "18:45",
-    "18:50",
-    "18:55",
-    "19:00",
-    "19:05",
-    "19:10",
-    "19:15",
-    "19:20",
-    "19:25",
-    "19:30",
-    "19:35",
-    "19:40",
-    "19:45",
-    "19:50",
-    "19:55",
-    "20:00",
-    "20:05",
-    "20:10",
-    "20:15",
-    "20:20",
-    "20:25",
-    "20:30",
-    "20:35",
-    "20:40",
-    "20:45",
-    "20:50",
-    "20:55",
-    "21:00",
-    "21:05",
-    "21:10",
-    "21:15",
-    "21:20",
-    "21:25",
-    "21:30",
-    "21:35",
-    "21:40",
-    "21:45",
-    "21:50",
-    "21:55",
-    "22:00",
-    "22:05",
-    "22:10",
-    "22:15",
-    "22:20",
-    "22:25",
-    "22:30",
-    "22:35",
-    "22:40",
-    "22:45",
-    "22:50",
-    "22:55",
-    "23:00",
-    "23:05",
-    "23:10",
-    "23:15",
-    "23:20",
-    "23:25",
-    "23:30",
-    "23:35",
-    "23:40",
-    "23:45",
-    "23:50",
-    "23:55"
-  ],
-  "restaurant-price range": [
-    "none",
-    "do not care",
-    "cheap",
-    "expensive",
-    "moderate"
-  ],
-  "restaurant-food": [
-    "british food",
-    "steakhouse",
-    "turkish",
-    "sushi",
-    "north american",
-    "scottish",
-    "french",
-    "austrian",
-    "korean",
-    "eastern european",
-    "swedish",
-    "gastro pub",
-    "modern eclectic",
-    "afternoon tea",
-    "welsh",
-    "christmas",
-    "tuscan",
-    "gastropub",
-    "sri lankan",
-    "molecular gastronomy",
-    "traditional american",
-    "italian",
-    "pizza",
-    "thai",
-    "south african",
-    "creative",
-    "english",
-    "asian",
-    "lebanese",
-    "hungarian",
-    "halal",
-    "portugese",
-    "modern english",
-    "african",
-    "light bites",
-    "malaysian",
-    "venetian",
-    "traditional",
-    "chinese",
-    "vegetarian",
-    "persian",
-    "thai and chinese",
-    "scandinavian",
-    "catalan",
-    "polynesian",
-    "crossover",
-    "canapes",
-    "cantonese",
-    "north african",
-    "seafood",
-    "brazilian",
-    "south indian",
-    "australasian",
-    "belgian",
-    "barbeque",
-    "the americas",
-    "indonesian",
-    "singaporean",
-    "irish",
-    "middle eastern",
-    "dojo noodle bar",
-    "caribbean",
-    "vietnamese",
-    "modern european",
-    "russian",
-    "none",
-    "german",
-    "world",
-    "japanese",
-    "moroccan",
-    "modern global",
-    "do not care",
-    "indian",
-    "british",
-    "american",
-    "danish",
-    "panasian",
-    "swiss",
-    "basque",
-    "north indian",
-    "modern american",
-    "australian",
-    "european",
-    "corsica",
-    "greek",
-    "northern european",
-    "mediterranean",
-    "portuguese",
-    "romanian",
-    "jamaican",
-    "polish",
-    "international",
-    "unusual",
-    "latin american",
-    "asian oriental",
-    "mexican",
-    "bistro",
-    "cuban",
-    "fusion",
-    "new zealand",
-    "spanish",
-    "eritrean",
-    "afghan",
-    "kosher"
-  ],
-  "attraction-name": [
-    "downing college",
-    "fitzwilliam",
-    "clare college",
-    "ruskin gallery",
-    "sidney sussex college",
-    "great saint mary's church",
-    "cherry hinton water play park",
-    "wandlebury country park",
-    "cafe uno",
-    "place",
-    "broughton",
-    "cineworld cinema",
-    "jesus college",
-    "vue cinema",
-    "history of science museum",
-    "mumford theatre",
-    "whale of time",
-    "fitzbillies",
-    "christs church",
-    "churchill college",
-    "museum of classical archaeology",
-    "gonville and caius college",
-    "pizza",
-    "kirkwood",
-    "saint catharines college",
-    "kings college",
-    "parkside",
-    "by",
-    "st catharines college",
-    "saint john's college",
-    "cherry hinton water park",
-    "st christs college",
-    "christ's college",
-    "bangkok city",
-    "scudamores punti co",
-    "free",
-    "great saint marys church",
-    "milton country park",
-    "the fez club",
-    "soultree",
-    "autu",
-    "whipple museum of the history of science",
-    "aylesbray lodge guest house",
-    "broughton house gallery",
-    "peoples portraits exhibition",
-    "primavera",
-    "kettles yard",
-    "all saint's church",
-    "cinema cinema",
-    "regency gallery",
-    "corpus christi",
-    "corn cambridge exchange",
-    "da vinci pizzeria",
-    "school",
-    "hobsons house",
-    "cambride and country folk museum",
-    "north",
-    "da v",
-    "cambridge corn exchange",
-    "soul tree nightclub",
-    "cambridge arts theater",
-    "saint catharine's college",
-    "byard art",
-    "cambridge punter",
-    "cambridge university botanic gardens",
-    "castle galleries",
-    "museum of archaelogy and anthropogy",
-    "no specific location",
-    "cherry hinton hall",
-    "gallery at 12 a high street",
-    "parkside pools",
-    "queen's college",
-    "little saint mary's church",
-    "gallery",
-    "home from home",
-    "tenpin",
-    "the wandlebury",
-    "county folk museum",
-    "swimming pool",
-    "christs college",
-    "cafe jello museum",
-    "scott polar",
-    "christ college",
-    "cambridge museum of technology",
-    "abbey pool and astroturf pitch",
-    "king hedges learner pool",
-    "the cambridge arts theatre",
-    "the castle galleries",
-    "cambridge and country folk museum",
-    "kohinoor",
-    "scudamores punting co",
-    "sidney sussex",
-    "the man on the moon",
-    "little saint marys church",
-    "queens",
-    "the place",
-    "old school",
-    "churchill",
-    "churchills college",
-    "hughes hall",
-    "churchhill college",
-    "riverboat georgina",
-    "none",
-    "belf",
-    "cambridge temporary art",
-    "abc theatre",
-    "cambridge contemporary art museum",
-    "man on the moon",
-    "the junction",
-    "cherry hinton water play",
-    "adc theatre",
-    "gonville hotel",
-    "magdalene college",
-    "peoples portraits exhibition at girton college",
-    "boat",
-    "centre",
-    "sheep's green and lammas land park fen causeway",
-    "do not care",
-    "the mumford theatre",
-    "archway house",
-    "queens' college",
-    "williams art and antiques",
-    "funky fun house",
-    "cherry hinton village centre",
-    "camboats",
-    "cambridge",
-    "old schools",
-    "kettle's yard",
-    "whale of a time",
-    "the churchill college",
-    "cafe jello gallery",
-    "aut",
-    "salsa",
-    "city",
-    "clare hall",
-    "boating",
-    "pembroke college",
-    "kings hedges learner pool",
-    "caffe uno",
-    "lammas land park",
-    "museum",
-    "the fitzwilliam museum",
-    "the cherry hinton village centre",
-    "the cambridge corn exchange",
-    "fitzwilliam museum",
-    "museum of archaelogy and anthropology",
-    "fez club",
-    "the cambridge punter",
-    "saint johns college",
-    "emmanuel college",
-    "cambridge belf",
-    "scudamore",
-    "lynne strover gallery",
-    "king's college",
-    "whippple museum",
-    "trinity college",
-    "college in the north",
-    "sheep's green",
-    "kambar",
-    "museum of archaelogy",
-    "adc",
-    "garde",
-    "club salsa",
-    "people's portraits exhibition at girton college",
-    "botanic gardens",
-    "carol",
-    "college",
-    "gallery at twelve a high street",
-    "abbey pool and astroturf",
-    "cambridge book and print gallery",
-    "jesus green outdoor pool",
-    "scott polar museum",
-    "saint barnabas press gallery",
-    "cambridge artworks",
-    "older churches",
-    "cambridge contemporary art",
-    "cherry hinton hall and grounds",
-    "univ",
-    "jesus green",
-    "ballare",
-    "abbey pool",
-    "cambridge botanic gardens",
-    "nusha",
-    "worth house",
-    "thanh",
-    "university arms hotel",
-    "cambridge arts theatre",
-    "cafe jello",
-    "cambridge and county folk museum",
-    "the cambridge artworks",
-    "all saints church",
-    "holy trinity church",
-    "contemporary art museum",
-    "architectural churches",
-    "queens college",
-    "trinity street college"
-  ],
-  "restaurant-name": [
-    "none",
-    "do not care",
-    "hotel du vin and bistro",
-    "ask",
-    "gourmet formal kitchen",
-    "the meze bar",
-    "lan hong house",
-    "cow pizza",
-    "one seven",
-    "prezzo",
-    "maharajah tandoori restaurant",
-    "alex",
-    "shanghai",
-    "golden wok",
-    "restaurant",
-    "fitzbillies",
-    "nil",
-    "copper kettle",
-    "meghna",
-    "hk fusion",
-    "bangkok city",
-    "hobsons house",
-    "tang chinese",
-    "anatolia",
-    "ugly duckling",
-    "anatolia and efes restaurant",
-    "sitar tandoori",
-    "city stop",
-    "ashley",
-    "pizza express fen ditton",
-    "molecular gastronomy",
-    "autumn house",
-    "el shaddia guesthouse",
-    "the grafton hotel",
-    "limehouse",
-    "gardenia",
-    "not metioned",
-    "hakka",
-    "michaelhouse cafe",
-    "pipasha",
-    "meze bar",
-    "archway",
-    "molecular gastonomy",
-    "yipee noodle bar",
-    "the peking",
-    "curry prince",
-    "midsummer house restaurant",
-    "pizza hut cherry hinton",
-    "the lucky star",
-    "stazione restaurant and coffee bar",
-    "shanghi family restaurant",
-    "good luck",
-    "j restaurant",
-    "bedouin",
-    "cott",
-    "little seoul",
-    "south",
-    "thanh binh",
-    "el",
-    "efes restaurant",
-    "kohinoor",
-    "clowns",
-    "india",
-    "the slug and lettuce",
-    "shiraz",
-    "barbakan",
-    "zizzi cambridge",
-    "restaurant one seven",
-    "slug and lettuce",
-    "travellers rest",
-    "binh",
-    "worth house",
-    "broughton house gallery",
-    "chiquito",
-    "the river bar steakhouse and grill",
-    "ros",
-    "golden house",
-    "india west",
-    "cam",
-    "panahar",
-    "restaurant 22",
-    "adden",
-    "indian",
-    "hu",
-    "jinling noodle bar",
-    "darrys cookhouse and wine shop",
-    "hobson house",
-    "cambridge be",
-    "el shaddai",
-    "ac",
-    "nandos",
-    "cambridge lodge",
-    "the cow pizza kitchen and bar",
-    "charlie",
-    "rajmahal",
-    "kymmoy",
-    "cambri",
-    "backstreet bistro",
-    "galleria",
-    "restaurant 2 two",
-    "chiquito restaurant bar",
-    "royal standard",
-    "lucky star",
-    "curry king",
-    "grafton hotel restaurant",
-    "mahal of cambridge",
-    "the bedouin",
-    "nus",
-    "the kohinoor",
-    "pizza hut fenditton",
-    "camboats",
-    "the gardenia",
-    "de luca cucina and bar",
-    "nusha",
-    "european",
-    "taj tandoori",
-    "tandoori palace",
-    "golden curry",
-    "efes",
-    "loch fyne",
-    "the maharajah tandoor",
-    "lovel",
-    "restaurant 17",
-    "clowns cafe",
-    "cambridge punter",
-    "bloomsbury restaurant",
-    "la mimosa",
-    "the cambridge chop house",
-    "funky",
-    "cotto",
-    "oak bistro",
-    "restaurant two two",
-    "pipasha restaurant",
-    "river bar steakhouse and grill",
-    "royal spice",
-    "the copper kettle",
-    "graffiti",
-    "nandos city centre",
-    "saffron brasserie",
-    "cambridge chop house",
-    "sitar",
-    "kitchen and bar",
-    "the good luck chinese food takeaway",
-    "clu",
-    "la tasca",
-    "cafe uno",
-    "cote",
-    "the varsity restaurant",
-    "bri",
-    "eraina",
-    "bridge",
-    "fin",
-    "cambridge lodge restaurant",
-    "grafton",
-    "hotpot",
-    "sala thong",
-    "margherita",
-    "wise buddha",
-    "the missing sock",
-    "seasame restaurant and bar",
-    "the dojo noodle bar",
-    "restaurant alimentum",
-    "gastropub",
-    "saigon city",
-    "la margherita",
-    "pizza hut",
-    "curry garden",
-    "ashley hotel",
-    "eraina and michaelhouse cafe",
-    "the golden curry",
-    "curry queen",
-    "cow pizza kitchen and bar",
-    "the peking restaurant:",
-    "hamilton lodge",
-    "alimentum",
-    "yippee noodle bar",
-    "2 two and cote",
-    "shanghai family restaurant",
-    "grafton hotel",
-    "yes",
-    "ali baba",
-    "dif",
-    "fitzbillies restaurant",
-    "peking restaurant",
-    "lev",
-    "nirala",
-    "the alex",
-    "tandoori",
-    "city stop restaurant",
-    "rice house",
-    "cityr",
-    "yu garden",
-    "meze bar restaurant",
-    "the",
-    "don pasquale pizzeria",
-    "rice boat",
-    "the hotpot",
-    "old school",
-    "the oak bistro",
-    "sesame restaurant and bar",
-    "pizza express",
-    "the gandhi",
-    "pizza hut fen ditton",
-    "charlie chan",
-    "da vinci pizzeria",
-    "dojo noodle bar",
-    "gourmet burger kitchen",
-    "the golden house",
-    "india house",
-    "hobso",
-    "missing sock",
-    "pizza hut city centre",
-    "parkside pools",
-    "riverside brasserie",
-    "caffe uno",
-    "primavera",
-    "the nirala",
-    "wagamama",
-    "au",
-    "ian hong house",
-    "frankie and bennys",
-    "4 kings parade city centre",
-    "shiraz restaurant",
-    "scudamores punt",
-    "mahal",
-    "saint johns chop house",
-    "de luca cucina and bar riverside brasserie",
-    "cocum",
-    "la raza"
-  ],
-  "attraction-type": [
-    "none",
-    "do not care",
-    "architecture",
-    "boat",
-    "boating",
-    "camboats",
-    "church",
-    "churchills college",
-    "cinema",
-    "college",
-    "concert",
-    "concerthall",
-    "entertainment",
-    "gallery",
-    "gastropub",
-    "hiking",
-    "hotel",
-    "multiple sports",
-    "museum",
-    "museum kettles yard",
-    "night club",
-    "outdoor",
-    "park",
-    "pool",
-    "special",
-    "sports",
-    "swimming pool",
-    "theater",
-    "theatre",
-    "concert hall",
-    "local site",
-    "nightclub",
-    "hotspot"
-  ],
-  "taxi-leave at": [
-    "none",
-    "do not care",
-    "00:00",
-    "00:05",
-    "00:10",
-    "00:15",
-    "00:20",
-    "00:25",
-    "00:30",
-    "00:35",
-    "00:40",
-    "00:45",
-    "00:50",
-    "00:55",
-    "01:00",
-    "01:05",
-    "01:10",
-    "01:15",
-    "01:20",
-    "01:25",
-    "01:30",
-    "01:35",
-    "01:40",
-    "01:45",
-    "01:50",
-    "01:55",
-    "02:00",
-    "02:05",
-    "02:10",
-    "02:15",
-    "02:20",
-    "02:25",
-    "02:30",
-    "02:35",
-    "02:40",
-    "02:45",
-    "02:50",
-    "02:55",
-    "03:00",
-    "03:05",
-    "03:10",
-    "03:15",
-    "03:20",
-    "03:25",
-    "03:30",
-    "03:35",
-    "03:40",
-    "03:45",
-    "03:50",
-    "03:55",
-    "04:00",
-    "04:05",
-    "04:10",
-    "04:15",
-    "04:20",
-    "04:25",
-    "04:30",
-    "04:35",
-    "04:40",
-    "04:45",
-    "04:50",
-    "04:55",
-    "05:00",
-    "05:05",
-    "05:10",
-    "05:15",
-    "05:20",
-    "05:25",
-    "05:30",
-    "05:35",
-    "05:40",
-    "05:45",
-    "05:50",
-    "05:55",
-    "06:00",
-    "06:05",
-    "06:10",
-    "06:15",
-    "06:20",
-    "06:25",
-    "06:30",
-    "06:35",
-    "06:40",
-    "06:45",
-    "06:50",
-    "06:55",
-    "07:00",
-    "07:05",
-    "07:10",
-    "07:15",
-    "07:20",
-    "07:25",
-    "07:30",
-    "07:35",
-    "07:40",
-    "07:45",
-    "07:50",
-    "07:55",
-    "08:00",
-    "08:05",
-    "08:10",
-    "08:15",
-    "08:20",
-    "08:25",
-    "08:30",
-    "08:35",
-    "08:40",
-    "08:45",
-    "08:50",
-    "08:55",
-    "09:00",
-    "09:05",
-    "09:10",
-    "09:15",
-    "09:20",
-    "09:25",
-    "09:30",
-    "09:35",
-    "09:40",
-    "09:45",
-    "09:50",
-    "09:55",
-    "10:00",
-    "10:05",
-    "10:10",
-    "10:15",
-    "10:20",
-    "10:25",
-    "10:30",
-    "10:35",
-    "10:40",
-    "10:45",
-    "10:50",
-    "10:55",
-    "11:00",
-    "11:05",
-    "11:10",
-    "11:15",
-    "11:20",
-    "11:25",
-    "11:30",
-    "11:35",
-    "11:40",
-    "11:45",
-    "11:50",
-    "11:55",
-    "12:00",
-    "12:05",
-    "12:10",
-    "12:15",
-    "12:20",
-    "12:25",
-    "12:30",
-    "12:35",
-    "12:40",
-    "12:45",
-    "12:50",
-    "12:55",
-    "13:00",
-    "13:05",
-    "13:10",
-    "13:15",
-    "13:20",
-    "13:25",
-    "13:30",
-    "13:35",
-    "13:40",
-    "13:45",
-    "13:50",
-    "13:55",
-    "14:00",
-    "14:05",
-    "14:10",
-    "14:15",
-    "14:20",
-    "14:25",
-    "14:30",
-    "14:35",
-    "14:40",
-    "14:45",
-    "14:50",
-    "14:55",
-    "15:00",
-    "15:05",
-    "15:10",
-    "15:15",
-    "15:20",
-    "15:25",
-    "15:30",
-    "15:35",
-    "15:40",
-    "15:45",
-    "15:50",
-    "15:55",
-    "16:00",
-    "16:05",
-    "16:10",
-    "16:15",
-    "16:20",
-    "16:25",
-    "16:30",
-    "16:35",
-    "16:40",
-    "16:45",
-    "16:50",
-    "16:55",
-    "17:00",
-    "17:05",
-    "17:10",
-    "17:15",
-    "17:20",
-    "17:25",
-    "17:30",
-    "17:35",
-    "17:40",
-    "17:45",
-    "17:50",
-    "17:55",
-    "18:00",
-    "18:05",
-    "18:10",
-    "18:15",
-    "18:20",
-    "18:25",
-    "18:30",
-    "18:35",
-    "18:40",
-    "18:45",
-    "18:50",
-    "18:55",
-    "19:00",
-    "19:05",
-    "19:10",
-    "19:15",
-    "19:20",
-    "19:25",
-    "19:30",
-    "19:35",
-    "19:40",
-    "19:45",
-    "19:50",
-    "19:55",
-    "20:00",
-    "20:05",
-    "20:10",
-    "20:15",
-    "20:20",
-    "20:25",
-    "20:30",
-    "20:35",
-    "20:40",
-    "20:45",
-    "20:50",
-    "20:55",
-    "21:00",
-    "21:05",
-    "21:10",
-    "21:15",
-    "21:20",
-    "21:25",
-    "21:30",
-    "21:35",
-    "21:40",
-    "21:45",
-    "21:50",
-    "21:55",
-    "22:00",
-    "22:05",
-    "22:10",
-    "22:15",
-    "22:20",
-    "22:25",
-    "22:30",
-    "22:35",
-    "22:40",
-    "22:45",
-    "22:50",
-    "22:55",
-    "23:00",
-    "23:05",
-    "23:10",
-    "23:15",
-    "23:20",
-    "23:25",
-    "23:30",
-    "23:35",
-    "23:40",
-    "23:45",
-    "23:50",
-    "23:55"
-  ],
-  "taxi-destination": [
-    "none",
-    "do not care",
-    "a and b guest house",
-    "abbey pool and astroturf pitch",
-    "acorn guest house",
-    "adc theatre",
-    "addenbrookes hospital",
-    "alexander bed and breakfast",
-    "ali baba",
-    "all saints church",
-    "allenbell",
-    "alpha milton guest house",
-    "anatolia",
-    "arbury lodge guesthouse",
-    "archway house",
-    "ashley hotel",
-    "ask",
-    "attraction",
-    "autumn house",
-    "avalon",
-    "aylesbray lodge guest house",
-    "backstreet bistro",
-    "ballare",
-    "bangkok city",
-    "bedouin",
-    "birmingham new street train station",
-    "bishops stortford train station",
-    "bloomsbury restaurant",
-    "bridge guest house",
-    "broughton house gallery",
-    "broxbourne train station",
-    "byard art",
-    "cafe jello gallery",
-    "cafe uno",
-    "camboats",
-    "cambridge",
-    "cambridge and county folk museum",
-    "cambridge arts theatre",
-    "cambridge artworks",
-    "cambridge belfry",
-    "cambridge book and print gallery",
-    "cambridge chop house",
-    "cambridge contemporary art",
-    "cambridge county fair next to the city tourist museum",
-    "cambridge lodge restaurant",
-    "cambridge museum of technology",
-    "cambridge punter",
-    "cambridge road church of christ",
-    "cambridge train station",
-    "cambridge university botanic gardens",
-    "carolina bed and breakfast",
-    "castle galleries",
-    "charlie chan",
-    "cherry hinton hall and grounds",
-    "cherry hinton village centre",
-    "cherry hinton water park",
-    "cherry hinton water play",
-    "chiquito restaurant bar",
-    "christ college",
-    "churchills college",
-    "cineworld cinema",
-    "city centre north bed and breakfast",
-    "city stop restaurant",
-    "cityroomz",
-    "clare college",
-    "clare hall",
-    "clowns cafe",
-    "club salsa",
-    "cocum",
-    "copper kettle",
-    "corpus christi",
-    "cote",
-    "cotto",
-    "cow pizza kitchen and bar",
-    "curry garden",
-    "curry king",
-    "curry prince",
-    "da vinci pizzeria",
-    "darrys cookhouse and wine shop",
-    "de luca cucina and bar",
-    "dojo noodle bar",
-    "don pasquale pizzeria",
-    "downing college",
-    "efes restaurant",
-    "el shaddia guesthouse",
-    "ely train station",
-    "emmanuel college",
-    "eraina",
-    "express by holiday inn cambridge",
-    "finches bed and breakfast",
-    "finders corner newmarket road",
-    "fitzbillies restaurant",
-    "fitzwilliam museum",
-    "frankie and bennys",
-    "funky fun house",
-    "galleria",
-    "gallery at 12 a high street",
-    "gastropub",
-    "golden curry",
-    "golden house",
-    "golden wok",
-    "gonville and caius college",
-    "gonville hotel",
-    "good luck",
-    "gourmet burger kitchen",
-    "graffiti",
-    "grafton hotel restaurant",
-    "great saint marys church",
-    "hakka",
-    "hamilton lodge",
-    "hk fusion",
-    "hobsons house",
-    "holy trinity church",
-    "home from home",
-    "hotel du vin and bistro",
-    "hughes hall",
-    "huntingdon marriott hotel",
-    "ian hong",
-    "india house",
-    "j restaurant",
-    "jesus college",
-    "jesus green outdoor pool",
-    "jinling noodle bar",
-    "kambar",
-    "kettles yard",
-    "kings college",
-    "kings hedges learner pool",
-    "kirkwood house",
-    "kohinoor",
-    "kymmoy",
-    "la margherita",
-    "la mimosa",
-    "la raza",
-    "la tasca",
-    "lan hong house",
-    "leicester train station",
-    "lensfield hotel",
-    "limehouse",
-    "little saint marys church",
-    "little seoul",
-    "loch fyne",
-    "london kings cross train station",
-    "london liverpool street train station",
-    "lovell lodge",
-    "lynne strover gallery",
-    "magdalene college",
-    "mahal of cambridge",
-    "maharajah tandoori restaurant",
-    "meghna",
-    "meze bar",
-    "michaelhouse cafe",
-    "midsummer house restaurant",
-    "milton country park",
-    "mumford theatre",
-    "museum of archaelogy and anthropology",
-    "museum of classical archaeology",
-    "nandos",
-    "nandos city centre",
-    "nil",
-    "nirala",
-    "norwich train station",
-    "nusha",
-    "old schools",
-    "panahar",
-    "parkside police station",
-    "parkside pools",
-    "peking restaurant",
-    "pembroke college",
-    "peoples portraits exhibition at girton college",
-    "peterborough train station",
-    "pipasha restaurant",
-    "pizza express",
-    "pizza hut cherry hinton",
-    "pizza hut city centre",
-    "pizza hut fenditton",
-    "prezzo",
-    "primavera",
-    "queens college",
-    "rajmahal",
-    "regency gallery",
-    "restaurant 17",
-    "restaurant 2 two",
-    "restaurant alimentum",
-    "rice boat",
-    "rice house",
-    "riverboat georgina",
-    "riverside brasserie",
-    "rosas bed and breakfast",
-    "royal spice",
-    "royal standard",
-    "ruskin gallery",
-    "saffron brasserie",
-    "saigon city",
-    "saint barnabas",
-    "saint barnabas press gallery",
-    "saint catharines college",
-    "saint johns chop house",
-    "saint johns college",
-    "sala thong",
-    "scott polar museum",
-    "scudamores punting co",
-    "sesame restaurant and bar",
-    "shanghai family restaurant",
-    "sheeps green and lammas land park fen causeway",
-    "shiraz",
-    "sidney sussex college",
-    "sitar tandoori",
-    "sleeperz hotel",
-    "soul tree nightclub",
-    "st johns chop house",
-    "stansted airport train station",
-    "station road",
-    "stazione restaurant and coffee bar",
-    "stevenage train station",
-    "taj tandoori",
-    "tall monument",
-    "tandoori palace",
-    "tang chinese",
-    "tenpin",
-    "thanh binh",
-    "the anatolia",
-    "the cambridge corn exchange",
-    "the cambridge shop",
-    "the fez club",
-    "the gandhi",
-    "the gardenia",
-    "the hotpot",
-    "the junction",
-    "the lucky star",
-    "the man on the moon",
-    "the missing sock",
-    "the oak bistro",
-    "the place",
-    "the regent street city center",
-    "the river bar steakhouse and grill",
-    "the slug and lettuce",
-    "the varsity restaurant",
-    "travellers rest",
-    "trinity college",
-    "ugly duckling",
-    "university arms hotel",
-    "vue cinema",
-    "wagamama",
-    "wandlebury country park",
-    "wankworth hotel",
-    "warkworth house",
-    "whale of a time",
-    "whipple museum of the history of science",
-    "williams art and antiques",
-    "worth house",
-    "yippee noodle bar",
-    "yu garden",
-    "zizzi cambridge",
-    "leverton house",
-    "the cambridge chop house",
-    "saint john's college",
-    "churchill college",
-    "the nirala",
-    "the cow pizza kitchen and bar",
-    "christ's college",
-    "el shaddai",
-    "saint catharine's college",
-    "camb",
-    "the golden curry",
-    "little saint mary's church",
-    "country folk museum",
-    "meze bar restaurant",
-    "the cambridge belfry",
-    "the fitzwilliam museum",
-    "the lensfield hotel",
-    "pizza express fen ditton",
-    "the cambridge punter",
-    "king's college",
-    "the cherry hinton village centre",
-    "shiraz restaurant",
-    "sheep's green and lammas land park fen causeway",
-    "caffe uno",
-    "the ghandi",
-    "the copper kettle",
-    "man on the moon concert hall",
-    "alpha-milton guest house",
-    "queen's college",
-    "restaurant one seven",
-    "restaurant two two",
-    "city centre north b and b",
-    "rosa's bed and breakfast",
-    "the good luck chinese food takeaway",
-    "not museum of archaeology and anthropologymentioned",
-    "tandori in cambridge",
-    "kettle's yard",
-    "megna",
-    "grou",
-    "gallery at twelve a high street",
-    "maharajah tandoori restaurant",
-    "pizza hut fen ditton",
-    "gandhi",
-    "tranh binh",
-    "kambur",
-    "people's portraits exhibition at girton college",
-    "hotel",
-    "restaurant",
-    "the galleria",
-    "queens' college",
-    "great saint mary's church",
-    "theathre",
-    "cambridge artworks",
-    "acorn house",
-    "shiraz",
-    "riverboat georginawd",
-    "mic",
-    "the gallery at twelve",
-    "the soul tree",
-    "finches"
-  ],
-  "taxi-departure": [
-    "none",
-    "do not care",
-    "172 chestertown road",
-    "4455 woodbridge road",
-    "a and b guest house",
-    "abbey pool and astroturf pitch",
-    "acorn guest house",
-    "adc theatre",
-    "addenbrookes hospital",
-    "alexander bed and breakfast",
-    "ali baba",
-    "all saints church",
-    "allenbell",
-    "alpha milton guest house",
-    "alyesbray lodge hotel",
-    "ambridge",
-    "anatolia",
-    "arbury lodge guesthouse",
-    "archway house",
-    "ashley hotel",
-    "ask",
-    "autumn house",
-    "avalon",
-    "aylesbray lodge guest house",
-    "backstreet bistro",
-    "ballare",
-    "bangkok city",
-    "bedouin",
-    "birmingham new street train station",
-    "bishops stortford train station",
-    "bloomsbury restaurant",
-    "bridge guest house",
-    "broughton house gallery",
-    "broxbourne train station",
-    "byard art",
-    "cafe jello gallery",
-    "cafe uno",
-    "caffee uno",
-    "camboats",
-    "cambridge",
-    "cambridge and county folk museum",
-    "cambridge arts theatre",
-    "cambridge artworks",
-    "cambridge belfry",
-    "cambridge book and print gallery",
-    "cambridge chop house",
-    "cambridge contemporary art",
-    "cambridge lodge restaurant",
-    "cambridge museum of technology",
-    "cambridge punter",
-    "cambridge towninfo centre",
-    "cambridge train station",
-    "cambridge university botanic gardens",
-    "carolina bed and breakfast",
-    "castle galleries",
-    "centre of town at my hotel",
-    "charlie chan",
-    "cherry hinton hall and grounds",
-    "cherry hinton village center",
-    "cherry hinton village centre",
-    "cherry hinton water play",
-    "chiquito restaurant bar",
-    "christ college",
-    "churchills college",
-    "cineworld cinema",
-    "citiroomz",
-    "city centre north bed and breakfast",
-    "city stop restaurant",
-    "cityroomz",
-    "clair hall",
-    "clare college",
-    "clare hall",
-    "clowns cafe",
-    "club salsa",
-    "cocum",
-    "copper kettle",
-    "corpus christi",
-    "cote",
-    "cotto",
-    "cow pizza kitchen and bar",
-    "curry garden",
-    "curry king",
-    "curry prince",
-    "curry queen",
-    "da vinci pizzeria",
-    "darrys cookhouse and wine shop",
-    "de luca cucina and bar",
-    "dojo noodle bar",
-    "don pasquale pizzeria",
-    "downing college",
-    "downing street",
-    "el shaddia guesthouse",
-    "ely",
-    "ely train station",
-    "emmanuel college",
-    "eraina",
-    "express by holiday inn cambridge",
-    "finches bed and breakfast",
-    "fitzbillies restaurant",
-    "fitzwilliam museum",
-    "frankie and bennys",
-    "funky fun house",
-    "galleria",
-    "gallery at 12 a high street",
-    "girton college",
-    "golden curry",
-    "golden house",
-    "golden wok",
-    "gonville and caius college",
-    "gonville hotel",
-    "good luck",
-    "gourmet burger kitchen",
-    "graffiti",
-    "grafton hotel restaurant",
-    "great saint marys church",
-    "hakka",
-    "hamilton lodge",
-    "hobsons house",
-    "holy trinity church",
-    "home",
-    "home from home",
-    "hotel",
-    "hotel du vin and bistro",
-    "hughes hall",
-    "huntingdon marriott hotel",
-    "india house",
-    "j restaurant",
-    "jesus college",
-    "jesus green outdoor pool",
-    "jinling noodle bar",
-    "junction theatre",
-    "kambar",
-    "kettles yard",
-    "kings college",
-    "kings hedges learner pool",
-    "kings lynn train station",
-    "kirkwood house",
-    "kohinoor",
-    "kymmoy",
-    "la margherita",
-    "la mimosa",
-    "la raza",
-    "la tasca",
-    "lan hong house",
-    "lensfield hotel",
-    "leverton house",
-    "limehouse",
-    "little saint marys church",
-    "little seoul",
-    "loch fyne",
-    "london kings cross train station",
-    "london liverpool street",
-    "london liverpool street train station",
-    "lovell lodge",
-    "lynne strover gallery",
-    "magdalene college",
-    "mahal of cambridge",
-    "maharajah tandoori restaurant",
-    "meghna",
-    "meze bar",
-    "michaelhouse cafe",
-    "milton country park",
-    "mumford theatre",
-    "museum",
-    "museum of archaelogy and anthropology",
-    "museum of classical archaeology",
-    "nandos",
-    "nandos city centre",
-    "new england",
-    "nirala",
-    "norwich train station",
-    "nstaot mentioned",
-    "nusha",
-    "old schools",
-    "panahar",
-    "parkside police station",
-    "parkside pools",
-    "peking restaurant",
-    "pembroke college",
-    "peoples portraits exhibition at girton college",
-    "peterborough train station",
-    "pizza express",
-    "pizza hut cherry hinton",
-    "pizza hut city centre",
-    "pizza hut fenditton",
-    "prezzo",
-    "primavera",
-    "queens college",
-    "rajmahal",
-    "regency gallery",
-    "restaurant 17",
-    "restaurant 2 two",
-    "restaurant alimentum",
-    "rice boat",
-    "rice house",
-    "riverboat georgina",
-    "riverside brasserie",
-    "rosas bed and breakfast",
-    "royal spice",
-    "royal standard",
-    "ruskin gallery",
-    "saffron brasserie",
-    "saigon city",
-    "saint barnabas press gallery",
-    "saint catharines college",
-    "saint johns chop house",
-    "saint johns college",
-    "sala thong",
-    "scott polar museum",
-    "scudamores punting co",
-    "sesame restaurant and bar",
-    "sheeps green and lammas land park",
-    "sheeps green and lammas land park fen causeway",
-    "shiraz",
-    "sidney sussex college",
-    "sitar tandoori",
-    "soul tree nightclub",
-    "st johns college",
-    "stazione restaurant and coffee bar",
-    "stevenage train station",
-    "taj tandoori",
-    "tandoori palace",
-    "tang chinese",
-    "tenpin",
-    "thanh binh",
-    "the cambridge corn exchange",
-    "the fez club",
-    "the gallery at 12",
-    "the gandhi",
-    "the gardenia",
-    "the hotpot",
-    "the junction",
-    "the lucky star",
-    "the man on the moon",
-    "the missing sock",
-    "the oak bistro",
-    "the place",
-    "the river bar steakhouse and grill",
-    "the slug and lettuce",
-    "the varsity restaurant",
-    "travellers rest",
-    "trinity college",
-    "ugly duckling",
-    "university arms hotel",
-    "vue cinema",
-    "wagamama",
-    "wandlebury country park",
-    "warkworth house",
-    "whale of a time",
-    "whipple museum of the history of science",
-    "williams art and antiques",
-    "worth house",
-    "yippee noodle bar",
-    "yu garden",
-    "zizzi cambridge",
-    "christ's college",
-    "city centre north b and b",
-    "the lensfield hotel",
-    "alpha-milton guest house",
-    "el shaddai",
-    "churchill college",
-    "the cambridge belfry",
-    "king's college",
-    "great saint mary's church",
-    "restaurant two two",
-    "queens' college",
-    "little saint mary's church",
-    "chinese city centre",
-    "kettle's yard",
-    "pizza hut",
-    "the golden curry",
-    "rosa's bed and breakfast",
-    "the cambridge punter",
-    "the byard art museum",
-    "saint catharine's college",
-    "meze bar restaurant",
-    "the good luck chinese food takeaway",
-    "restaurant one seven",
-    "pizza hut fen ditton",
-    "the nirala",
-    "the fitzwilliam museum",
-    "st. john's college",
-    "gallery at twelve a high street",
-    "sheep's green and lammas land park fen causeway",
-    "the cherry hinton village centre",
-    "pizza express fen ditton",
-    "corpus cristi",
-    "cas",
-    "acorn house",
-    "lens",
-    "the cambridge chop house",
-    "the copper kettle",
-    "the avalon",
-    "saint john's college",
-    "aylesbray lodge",
-    "the alexander bed and breakfast",
-    "cambridge belfy",
-    "people's portraits exhibition at girton college",
-    "gonville",
-    "caffe uno",
-    "the cow pizza kitchen and bar",
-    "lovell ldoge",
-    "cinema",
-    "shiraz restaurant",
-    "park",
-    "the allenbell"
-  ],
-  "restaurant-book day": [
-    "none",
-    "do not care",
-    "friday",
-    "monday",
-    "saterday",
-    "sunday",
-    "thursday",
-    "tuesday",
-    "wednesday"
-  ],
-  "restaurant-book people": [
-    "none",
-    "do not care",
-    "1",
-    "10 or more",
-    "2",
-    "3",
-    "4",
-    "5",
-    "6",
-    "7",
-    "8",
-    "9"
-  ],
-  "restaurant-book time": [
-    "none",
-    "do not care",
-    "00:00",
-    "00:05",
-    "00:10",
-    "00:15",
-    "00:20",
-    "00:25",
-    "00:30",
-    "00:35",
-    "00:40",
-    "00:45",
-    "00:50",
-    "00:55",
-    "01:00",
-    "01:05",
-    "01:10",
-    "01:15",
-    "01:20",
-    "01:25",
-    "01:30",
-    "01:35",
-    "01:40",
-    "01:45",
-    "01:50",
-    "01:55",
-    "02:00",
-    "02:05",
-    "02:10",
-    "02:15",
-    "02:20",
-    "02:25",
-    "02:30",
-    "02:35",
-    "02:40",
-    "02:45",
-    "02:50",
-    "02:55",
-    "03:00",
-    "03:05",
-    "03:10",
-    "03:15",
-    "03:20",
-    "03:25",
-    "03:30",
-    "03:35",
-    "03:40",
-    "03:45",
-    "03:50",
-    "03:55",
-    "04:00",
-    "04:05",
-    "04:10",
-    "04:15",
-    "04:20",
-    "04:25",
-    "04:30",
-    "04:35",
-    "04:40",
-    "04:45",
-    "04:50",
-    "04:55",
-    "05:00",
-    "05:05",
-    "05:10",
-    "05:15",
-    "05:20",
-    "05:25",
-    "05:30",
-    "05:35",
-    "05:40",
-    "05:45",
-    "05:50",
-    "05:55",
-    "06:00",
-    "06:05",
-    "06:10",
-    "06:15",
-    "06:20",
-    "06:25",
-    "06:30",
-    "06:35",
-    "06:40",
-    "06:45",
-    "06:50",
-    "06:55",
-    "07:00",
-    "07:05",
-    "07:10",
-    "07:15",
-    "07:20",
-    "07:25",
-    "07:30",
-    "07:35",
-    "07:40",
-    "07:45",
-    "07:50",
-    "07:55",
-    "08:00",
-    "08:05",
-    "08:10",
-    "08:15",
-    "08:20",
-    "08:25",
-    "08:30",
-    "08:35",
-    "08:40",
-    "08:45",
-    "08:50",
-    "08:55",
-    "09:00",
-    "09:05",
-    "09:10",
-    "09:15",
-    "09:20",
-    "09:25",
-    "09:30",
-    "09:35",
-    "09:40",
-    "09:45",
-    "09:50",
-    "09:55",
-    "10:00",
-    "10:05",
-    "10:10",
-    "10:15",
-    "10:20",
-    "10:25",
-    "10:30",
-    "10:35",
-    "10:40",
-    "10:45",
-    "10:50",
-    "10:55",
-    "11:00",
-    "11:05",
-    "11:10",
-    "11:15",
-    "11:20",
-    "11:25",
-    "11:30",
-    "11:35",
-    "11:40",
-    "11:45",
-    "11:50",
-    "11:55",
-    "12:00",
-    "12:05",
-    "12:10",
-    "12:15",
-    "12:20",
-    "12:25",
-    "12:30",
-    "12:35",
-    "12:40",
-    "12:45",
-    "12:50",
-    "12:55",
-    "13:00",
-    "13:05",
-    "13:10",
-    "13:15",
-    "13:20",
-    "13:25",
-    "13:30",
-    "13:35",
-    "13:40",
-    "13:45",
-    "13:50",
-    "13:55",
-    "14:00",
-    "14:05",
-    "14:10",
-    "14:15",
-    "14:20",
-    "14:25",
-    "14:30",
-    "14:35",
-    "14:40",
-    "14:45",
-    "14:50",
-    "14:55",
-    "15:00",
-    "15:05",
-    "15:10",
-    "15:15",
-    "15:20",
-    "15:25",
-    "15:30",
-    "15:35",
-    "15:40",
-    "15:45",
-    "15:50",
-    "15:55",
-    "16:00",
-    "16:05",
-    "16:10",
-    "16:15",
-    "16:20",
-    "16:25",
-    "16:30",
-    "16:35",
-    "16:40",
-    "16:45",
-    "16:50",
-    "16:55",
-    "17:00",
-    "17:05",
-    "17:10",
-    "17:15",
-    "17:20",
-    "17:25",
-    "17:30",
-    "17:35",
-    "17:40",
-    "17:45",
-    "17:50",
-    "17:55",
-    "18:00",
-    "18:05",
-    "18:10",
-    "18:15",
-    "18:20",
-    "18:25",
-    "18:30",
-    "18:35",
-    "18:40",
-    "18:45",
-    "18:50",
-    "18:55",
-    "19:00",
-    "19:05",
-    "19:10",
-    "19:15",
-    "19:20",
-    "19:25",
-    "19:30",
-    "19:35",
-    "19:40",
-    "19:45",
-    "19:50",
-    "19:55",
-    "20:00",
-    "20:05",
-    "20:10",
-    "20:15",
-    "20:20",
-    "20:25",
-    "20:30",
-    "20:35",
-    "20:40",
-    "20:45",
-    "20:50",
-    "20:55",
-    "21:00",
-    "21:05",
-    "21:10",
-    "21:15",
-    "21:20",
-    "21:25",
-    "21:30",
-    "21:35",
-    "21:40",
-    "21:45",
-    "21:50",
-    "21:55",
-    "22:00",
-    "22:05",
-    "22:10",
-    "22:15",
-    "22:20",
-    "22:25",
-    "22:30",
-    "22:35",
-    "22:40",
-    "22:45",
-    "22:50",
-    "22:55",
-    "23:00",
-    "23:05",
-    "23:10",
-    "23:15",
-    "23:20",
-    "23:25",
-    "23:30",
-    "23:35",
-    "23:40",
-    "23:45",
-    "23:50",
-    "23:55"
-  ],
-  "taxi-arrive by": [
-    "none",
-    "do not care",
-    "00:00",
-    "00:05",
-    "00:10",
-    "00:15",
-    "00:20",
-    "00:25",
-    "00:30",
-    "00:35",
-    "00:40",
-    "00:45",
-    "00:50",
-    "00:55",
-    "01:00",
-    "01:05",
-    "01:10",
-    "01:15",
-    "01:20",
-    "01:25",
-    "01:30",
-    "01:35",
-    "01:40",
-    "01:45",
-    "01:50",
-    "01:55",
-    "02:00",
-    "02:05",
-    "02:10",
-    "02:15",
-    "02:20",
-    "02:25",
-    "02:30",
-    "02:35",
-    "02:40",
-    "02:45",
-    "02:50",
-    "02:55",
-    "03:00",
-    "03:05",
-    "03:10",
-    "03:15",
-    "03:20",
-    "03:25",
-    "03:30",
-    "03:35",
-    "03:40",
-    "03:45",
-    "03:50",
-    "03:55",
-    "04:00",
-    "04:05",
-    "04:10",
-    "04:15",
-    "04:20",
-    "04:25",
-    "04:30",
-    "04:35",
-    "04:40",
-    "04:45",
-    "04:50",
-    "04:55",
-    "05:00",
-    "05:05",
-    "05:10",
-    "05:15",
-    "05:20",
-    "05:25",
-    "05:30",
-    "05:35",
-    "05:40",
-    "05:45",
-    "05:50",
-    "05:55",
-    "06:00",
-    "06:05",
-    "06:10",
-    "06:15",
-    "06:20",
-    "06:25",
-    "06:30",
-    "06:35",
-    "06:40",
-    "06:45",
-    "06:50",
-    "06:55",
-    "07:00",
-    "07:05",
-    "07:10",
-    "07:15",
-    "07:20",
-    "07:25",
-    "07:30",
-    "07:35",
-    "07:40",
-    "07:45",
-    "07:50",
-    "07:55",
-    "08:00",
-    "08:05",
-    "08:10",
-    "08:15",
-    "08:20",
-    "08:25",
-    "08:30",
-    "08:35",
-    "08:40",
-    "08:45",
-    "08:50",
-    "08:55",
-    "09:00",
-    "09:05",
-    "09:10",
-    "09:15",
-    "09:20",
-    "09:25",
-    "09:30",
-    "09:35",
-    "09:40",
-    "09:45",
-    "09:50",
-    "09:55",
-    "10:00",
-    "10:05",
-    "10:10",
-    "10:15",
-    "10:20",
-    "10:25",
-    "10:30",
-    "10:35",
-    "10:40",
-    "10:45",
-    "10:50",
-    "10:55",
-    "11:00",
-    "11:05",
-    "11:10",
-    "11:15",
-    "11:20",
-    "11:25",
-    "11:30",
-    "11:35",
-    "11:40",
-    "11:45",
-    "11:50",
-    "11:55",
-    "12:00",
-    "12:05",
-    "12:10",
-    "12:15",
-    "12:20",
-    "12:25",
-    "12:30",
-    "12:35",
-    "12:40",
-    "12:45",
-    "12:50",
-    "12:55",
-    "13:00",
-    "13:05",
-    "13:10",
-    "13:15",
-    "13:20",
-    "13:25",
-    "13:30",
-    "13:35",
-    "13:40",
-    "13:45",
-    "13:50",
-    "13:55",
-    "14:00",
-    "14:05",
-    "14:10",
-    "14:15",
-    "14:20",
-    "14:25",
-    "14:30",
-    "14:35",
-    "14:40",
-    "14:45",
-    "14:50",
-    "14:55",
-    "15:00",
-    "15:05",
-    "15:10",
-    "15:15",
-    "15:20",
-    "15:25",
-    "15:30",
-    "15:35",
-    "15:40",
-    "15:45",
-    "15:50",
-    "15:55",
-    "16:00",
-    "16:05",
-    "16:10",
-    "16:15",
-    "16:20",
-    "16:25",
-    "16:30",
-    "16:35",
-    "16:40",
-    "16:45",
-    "16:50",
-    "16:55",
-    "17:00",
-    "17:05",
-    "17:10",
-    "17:15",
-    "17:20",
-    "17:25",
-    "17:30",
-    "17:35",
-    "17:40",
-    "17:45",
-    "17:50",
-    "17:55",
-    "18:00",
-    "18:05",
-    "18:10",
-    "18:15",
-    "18:20",
-    "18:25",
-    "18:30",
-    "18:35",
-    "18:40",
-    "18:45",
-    "18:50",
-    "18:55",
-    "19:00",
-    "19:05",
-    "19:10",
-    "19:15",
-    "19:20",
-    "19:25",
-    "19:30",
-    "19:35",
-    "19:40",
-    "19:45",
-    "19:50",
-    "19:55",
-    "20:00",
-    "20:05",
-    "20:10",
-    "20:15",
-    "20:20",
-    "20:25",
-    "20:30",
-    "20:35",
-    "20:40",
-    "20:45",
-    "20:50",
-    "20:55",
-    "21:00",
-    "21:05",
-    "21:10",
-    "21:15",
-    "21:20",
-    "21:25",
-    "21:30",
-    "21:35",
-    "21:40",
-    "21:45",
-    "21:50",
-    "21:55",
-    "22:00",
-    "22:05",
-    "22:10",
-    "22:15",
-    "22:20",
-    "22:25",
-    "22:30",
-    "22:35",
-    "22:40",
-    "22:45",
-    "22:50",
-    "22:55",
-    "23:00",
-    "23:05",
-    "23:10",
-    "23:15",
-    "23:20",
-    "23:25",
-    "23:30",
-    "23:35",
-    "23:40",
-    "23:45",
-    "23:50",
-    "23:55"
-  ],
-  "restaurant-area": [
-    "none",
-    "do not care",
-    "centre",
-    "east",
-    "north",
-    "south",
-    "west"
-  ],
-  "hotel-area": [
-    "none",
-    "do not care",
-    "centre",
-    "east",
-    "north",
-    "south",
-    "west"
-  ],
-  "attraction-area": [
-    "none",
-    "do not care",
-    "centre",
-    "east",
-    "north",
-    "south",
-    "west"
-  ]
-}
\ No newline at end of file
diff --git a/convlab/dst/setsumbt/multiwoz/dataset/mwoz21_ont_request.json b/convlab/dst/setsumbt/multiwoz/dataset/mwoz21_ont_request.json
deleted file mode 100644
index b0dd00fdf6dc2b824f1f50a44e776c63ce72f14b..0000000000000000000000000000000000000000
--- a/convlab/dst/setsumbt/multiwoz/dataset/mwoz21_ont_request.json
+++ /dev/null
@@ -1,3128 +0,0 @@
-{
-  "hotel-price range": [
-    "none",
-    "do not care",
-    "cheap",
-    "expensive",
-    "moderate",
-    "request"
-  ],
-  "hotel-type": [
-    "none",
-    "do not care",
-    "bed and breakfast",
-    "guest house",
-    "hotel",
-    "request"
-  ],
-  "hotel-parking": [
-    "none",
-    "do not care",
-    "no",
-    "yes",
-    "request"
-  ],
-  "hotel-book day": [
-    "none",
-    "do not care",
-    "friday",
-    "monday",
-    "saterday",
-    "sunday",
-    "thursday",
-    "tuesday",
-    "wednesday"
-  ],
-  "hotel-book people": [
-    "none",
-    "do not care",
-    "1",
-    "10 or more",
-    "2",
-    "3",
-    "4",
-    "5",
-    "6",
-    "7",
-    "8",
-    "9"
-  ],
-  "hotel-book stay": [
-    "none",
-    "do not care",
-    "1",
-    "10 or more",
-    "2",
-    "3",
-    "4",
-    "5",
-    "6",
-    "7",
-    "8",
-    "9"
-  ],
-  "train-destination": [
-    "none",
-    "do not care",
-    "bishops stortford",
-    "kings lynn",
-    "london liverpool street",
-    "centre",
-    "bishop stortford",
-    "liverpool",
-    "leicester",
-    "broxbourne",
-    "gourmet burger kitchen",
-    "copper kettle",
-    "bournemouth",
-    "stevenage",
-    "liverpool street",
-    "norwich",
-    "huntingdon marriott hotel",
-    "city centre north",
-    "taj tandoori",
-    "the copper kettle",
-    "peterborough",
-    "ely",
-    "lecester",
-    "london",
-    "willi",
-    "stansted airport",
-    "huntington marriott",
-    "cambridge",
-    "gonv",
-    "glastonbury",
-    "hol",
-    "north",
-    "birmingham new street",
-    "norway",
-    "petersborough",
-    "london kings cross",
-    "curry prince",
-    "bishops storford"
-  ],
-  "train-arrive by": [
-    "none",
-    "do not care",
-    "00:00",
-    "00:05",
-    "00:10",
-    "00:15",
-    "00:20",
-    "00:25",
-    "00:30",
-    "00:35",
-    "00:40",
-    "00:45",
-    "00:50",
-    "00:55",
-    "01:00",
-    "01:05",
-    "01:10",
-    "01:15",
-    "01:20",
-    "01:25",
-    "01:30",
-    "01:35",
-    "01:40",
-    "01:45",
-    "01:50",
-    "01:55",
-    "02:00",
-    "02:05",
-    "02:10",
-    "02:15",
-    "02:20",
-    "02:25",
-    "02:30",
-    "02:35",
-    "02:40",
-    "02:45",
-    "02:50",
-    "02:55",
-    "03:00",
-    "03:05",
-    "03:10",
-    "03:15",
-    "03:20",
-    "03:25",
-    "03:30",
-    "03:35",
-    "03:40",
-    "03:45",
-    "03:50",
-    "03:55",
-    "04:00",
-    "04:05",
-    "04:10",
-    "04:15",
-    "04:20",
-    "04:25",
-    "04:30",
-    "04:35",
-    "04:40",
-    "04:45",
-    "04:50",
-    "04:55",
-    "05:00",
-    "05:05",
-    "05:10",
-    "05:15",
-    "05:20",
-    "05:25",
-    "05:30",
-    "05:35",
-    "05:40",
-    "05:45",
-    "05:50",
-    "05:55",
-    "06:00",
-    "06:05",
-    "06:10",
-    "06:15",
-    "06:20",
-    "06:25",
-    "06:30",
-    "06:35",
-    "06:40",
-    "06:45",
-    "06:50",
-    "06:55",
-    "07:00",
-    "07:05",
-    "07:10",
-    "07:15",
-    "07:20",
-    "07:25",
-    "07:30",
-    "07:35",
-    "07:40",
-    "07:45",
-    "07:50",
-    "07:55",
-    "08:00",
-    "08:05",
-    "08:10",
-    "08:15",
-    "08:20",
-    "08:25",
-    "08:30",
-    "08:35",
-    "08:40",
-    "08:45",
-    "08:50",
-    "08:55",
-    "09:00",
-    "09:05",
-    "09:10",
-    "09:15",
-    "09:20",
-    "09:25",
-    "09:30",
-    "09:35",
-    "09:40",
-    "09:45",
-    "09:50",
-    "09:55",
-    "10:00",
-    "10:05",
-    "10:10",
-    "10:15",
-    "10:20",
-    "10:25",
-    "10:30",
-    "10:35",
-    "10:40",
-    "10:45",
-    "10:50",
-    "10:55",
-    "11:00",
-    "11:05",
-    "11:10",
-    "11:15",
-    "11:20",
-    "11:25",
-    "11:30",
-    "11:35",
-    "11:40",
-    "11:45",
-    "11:50",
-    "11:55",
-    "12:00",
-    "12:05",
-    "12:10",
-    "12:15",
-    "12:20",
-    "12:25",
-    "12:30",
-    "12:35",
-    "12:40",
-    "12:45",
-    "12:50",
-    "12:55",
-    "13:00",
-    "13:05",
-    "13:10",
-    "13:15",
-    "13:20",
-    "13:25",
-    "13:30",
-    "13:35",
-    "13:40",
-    "13:45",
-    "13:50",
-    "13:55",
-    "14:00",
-    "14:05",
-    "14:10",
-    "14:15",
-    "14:20",
-    "14:25",
-    "14:30",
-    "14:35",
-    "14:40",
-    "14:45",
-    "14:50",
-    "14:55",
-    "15:00",
-    "15:05",
-    "15:10",
-    "15:15",
-    "15:20",
-    "15:25",
-    "15:30",
-    "15:35",
-    "15:40",
-    "15:45",
-    "15:50",
-    "15:55",
-    "16:00",
-    "16:05",
-    "16:10",
-    "16:15",
-    "16:20",
-    "16:25",
-    "16:30",
-    "16:35",
-    "16:40",
-    "16:45",
-    "16:50",
-    "16:55",
-    "17:00",
-    "17:05",
-    "17:10",
-    "17:15",
-    "17:20",
-    "17:25",
-    "17:30",
-    "17:35",
-    "17:40",
-    "17:45",
-    "17:50",
-    "17:55",
-    "18:00",
-    "18:05",
-    "18:10",
-    "18:15",
-    "18:20",
-    "18:25",
-    "18:30",
-    "18:35",
-    "18:40",
-    "18:45",
-    "18:50",
-    "18:55",
-    "19:00",
-    "19:05",
-    "19:10",
-    "19:15",
-    "19:20",
-    "19:25",
-    "19:30",
-    "19:35",
-    "19:40",
-    "19:45",
-    "19:50",
-    "19:55",
-    "20:00",
-    "20:05",
-    "20:10",
-    "20:15",
-    "20:20",
-    "20:25",
-    "20:30",
-    "20:35",
-    "20:40",
-    "20:45",
-    "20:50",
-    "20:55",
-    "21:00",
-    "21:05",
-    "21:10",
-    "21:15",
-    "21:20",
-    "21:25",
-    "21:30",
-    "21:35",
-    "21:40",
-    "21:45",
-    "21:50",
-    "21:55",
-    "22:00",
-    "22:05",
-    "22:10",
-    "22:15",
-    "22:20",
-    "22:25",
-    "22:30",
-    "22:35",
-    "22:40",
-    "22:45",
-    "22:50",
-    "22:55",
-    "23:00",
-    "23:05",
-    "23:10",
-    "23:15",
-    "23:20",
-    "23:25",
-    "23:30",
-    "23:35",
-    "23:40",
-    "23:45",
-    "23:50",
-    "23:55",
-    "request"
-  ],
-  "train-departure": [
-    "none",
-    "do not care",
-    "bishops stortford",
-    "kings lynn",
-    "brookshite",
-    "london liverpool street",
-    "cam",
-    "liverpool",
-    "bro",
-    "leicester",
-    "broxbourne",
-    "norwhich",
-    "saint johns",
-    "stevenage",
-    "stansted",
-    "london liverpool",
-    "cambrid",
-    "city hall",
-    "rosas bed and breakfast",
-    "alpha-milton",
-    "wandlebury country park",
-    "norwich",
-    "liecester",
-    "stratford",
-    "peterborough",
-    "duxford",
-    "ely",
-    "london",
-    "stansted airport",
-    "lon",
-    "cambridge",
-    "panahar",
-    "cineworld",
-    "leicaster",
-    "birmingham",
-    "cafe uno",
-    "camboats",
-    "huntingdon",
-    "birmingham new street",
-    "arbu",
-    "alpha milton",
-    "east london",
-    "london kings cross",
-    "hamilton lodge",
-    "aylesbray lodge guest",
-    "el shaddai"
-  ],
-  "train-day": [
-    "none",
-    "do not care",
-    "friday",
-    "monday",
-    "saterday",
-    "sunday",
-    "thursday",
-    "tuesday",
-    "wednesday"
-  ],
-  "train-book people": [
-    "none",
-    "do not care",
-    "1",
-    "10 or more",
-    "2",
-    "3",
-    "4",
-    "5",
-    "6",
-    "7",
-    "8",
-    "9"
-  ],
-  "hotel-stars": [
-    "none",
-    "do not care",
-    "0",
-    "1",
-    "2",
-    "3",
-    "4",
-    "5",
-    "request"
-  ],
-  "hotel-internet": [
-    "none",
-    "do not care",
-    "no",
-    "yes",
-    "request"
-  ],
-  "hotel-name": [
-    "none",
-    "do not care",
-    "a and b guest house",
-    "city roomz",
-    "carolina bed and breakfast",
-    "limehouse",
-    "anatolia",
-    "hamilton lodge",
-    "the lensfield hotel",
-    "rosa's bed and breakfast",
-    "gall",
-    "aylesbray lodge",
-    "kirkwood",
-    "cambridge belfry",
-    "warkworth house",
-    "gonville",
-    "belfy hotel",
-    "nus",
-    "alexander",
-    "super 5",
-    "aylesbray lodge guest house",
-    "the gonvile hotel",
-    "allenbell",
-    "nothamilton lodge",
-    "ashley hotel",
-    "autumn house",
-    "hobsons house",
-    "hotel",
-    "ashely hotel",
-    "caridge belfrey",
-    "el shaddia guest house",
-    "avalon",
-    "cote",
-    "city centre north bed and breakfast",
-    "the cambridge belfry",
-    "home from home",
-    "wandlebury coutn",
-    "wankworth house",
-    "city stop rest",
-    "the worth house",
-    "cityroomz",
-    "huntingdon marriottt hotel",
-    "lensfield",
-    "rosas bed and breakfast",
-    "leverton house",
-    "gonville hotel",
-    "holiday inn cambridge",
-    "archway house",
-    "lan hon",
-    "levert",
-    "acorn guest house",
-    "cambridge",
-    "the ashley hotel",
-    "el shaddai",
-    "sleeperz",
-    "alpha milton guest house",
-    "doubletree by hilton cambridge",
-    "tandoori palace",
-    "express by",
-    "express by holiday inn cambridge",
-    "north bed and breakfast",
-    "holiday inn",
-    "arbury lodge guest house",
-    "alexander bed and breakfast",
-    "huntingdon marriott hotel",
-    "royal spice",
-    "sou",
-    "finches bed and breakfast",
-    "the alpha milton",
-    "bridge guest house",
-    "the acorn guest house",
-    "kirkwood house",
-    "eraina",
-    "la margherit",
-    "lensfield hotel",
-    "marriott hotel",
-    "nusha",
-    "city centre bed and breakfast",
-    "the allenbell",
-    "university arms hotel",
-    "clare",
-    "cherr",
-    "wartworth",
-    "acorn place",
-    "lovell lodge",
-    "whale"
-  ],
-  "train-leave at": [
-    "none",
-    "do not care",
-    "00:00",
-    "00:05",
-    "00:10",
-    "00:15",
-    "00:20",
-    "00:25",
-    "00:30",
-    "00:35",
-    "00:40",
-    "00:45",
-    "00:50",
-    "00:55",
-    "01:00",
-    "01:05",
-    "01:10",
-    "01:15",
-    "01:20",
-    "01:25",
-    "01:30",
-    "01:35",
-    "01:40",
-    "01:45",
-    "01:50",
-    "01:55",
-    "02:00",
-    "02:05",
-    "02:10",
-    "02:15",
-    "02:20",
-    "02:25",
-    "02:30",
-    "02:35",
-    "02:40",
-    "02:45",
-    "02:50",
-    "02:55",
-    "03:00",
-    "03:05",
-    "03:10",
-    "03:15",
-    "03:20",
-    "03:25",
-    "03:30",
-    "03:35",
-    "03:40",
-    "03:45",
-    "03:50",
-    "03:55",
-    "04:00",
-    "04:05",
-    "04:10",
-    "04:15",
-    "04:20",
-    "04:25",
-    "04:30",
-    "04:35",
-    "04:40",
-    "04:45",
-    "04:50",
-    "04:55",
-    "05:00",
-    "05:05",
-    "05:10",
-    "05:15",
-    "05:20",
-    "05:25",
-    "05:30",
-    "05:35",
-    "05:40",
-    "05:45",
-    "05:50",
-    "05:55",
-    "06:00",
-    "06:05",
-    "06:10",
-    "06:15",
-    "06:20",
-    "06:25",
-    "06:30",
-    "06:35",
-    "06:40",
-    "06:45",
-    "06:50",
-    "06:55",
-    "07:00",
-    "07:05",
-    "07:10",
-    "07:15",
-    "07:20",
-    "07:25",
-    "07:30",
-    "07:35",
-    "07:40",
-    "07:45",
-    "07:50",
-    "07:55",
-    "08:00",
-    "08:05",
-    "08:10",
-    "08:15",
-    "08:20",
-    "08:25",
-    "08:30",
-    "08:35",
-    "08:40",
-    "08:45",
-    "08:50",
-    "08:55",
-    "09:00",
-    "09:05",
-    "09:10",
-    "09:15",
-    "09:20",
-    "09:25",
-    "09:30",
-    "09:35",
-    "09:40",
-    "09:45",
-    "09:50",
-    "09:55",
-    "10:00",
-    "10:05",
-    "10:10",
-    "10:15",
-    "10:20",
-    "10:25",
-    "10:30",
-    "10:35",
-    "10:40",
-    "10:45",
-    "10:50",
-    "10:55",
-    "11:00",
-    "11:05",
-    "11:10",
-    "11:15",
-    "11:20",
-    "11:25",
-    "11:30",
-    "11:35",
-    "11:40",
-    "11:45",
-    "11:50",
-    "11:55",
-    "12:00",
-    "12:05",
-    "12:10",
-    "12:15",
-    "12:20",
-    "12:25",
-    "12:30",
-    "12:35",
-    "12:40",
-    "12:45",
-    "12:50",
-    "12:55",
-    "13:00",
-    "13:05",
-    "13:10",
-    "13:15",
-    "13:20",
-    "13:25",
-    "13:30",
-    "13:35",
-    "13:40",
-    "13:45",
-    "13:50",
-    "13:55",
-    "14:00",
-    "14:05",
-    "14:10",
-    "14:15",
-    "14:20",
-    "14:25",
-    "14:30",
-    "14:35",
-    "14:40",
-    "14:45",
-    "14:50",
-    "14:55",
-    "15:00",
-    "15:05",
-    "15:10",
-    "15:15",
-    "15:20",
-    "15:25",
-    "15:30",
-    "15:35",
-    "15:40",
-    "15:45",
-    "15:50",
-    "15:55",
-    "16:00",
-    "16:05",
-    "16:10",
-    "16:15",
-    "16:20",
-    "16:25",
-    "16:30",
-    "16:35",
-    "16:40",
-    "16:45",
-    "16:50",
-    "16:55",
-    "17:00",
-    "17:05",
-    "17:10",
-    "17:15",
-    "17:20",
-    "17:25",
-    "17:30",
-    "17:35",
-    "17:40",
-    "17:45",
-    "17:50",
-    "17:55",
-    "18:00",
-    "18:05",
-    "18:10",
-    "18:15",
-    "18:20",
-    "18:25",
-    "18:30",
-    "18:35",
-    "18:40",
-    "18:45",
-    "18:50",
-    "18:55",
-    "19:00",
-    "19:05",
-    "19:10",
-    "19:15",
-    "19:20",
-    "19:25",
-    "19:30",
-    "19:35",
-    "19:40",
-    "19:45",
-    "19:50",
-    "19:55",
-    "20:00",
-    "20:05",
-    "20:10",
-    "20:15",
-    "20:20",
-    "20:25",
-    "20:30",
-    "20:35",
-    "20:40",
-    "20:45",
-    "20:50",
-    "20:55",
-    "21:00",
-    "21:05",
-    "21:10",
-    "21:15",
-    "21:20",
-    "21:25",
-    "21:30",
-    "21:35",
-    "21:40",
-    "21:45",
-    "21:50",
-    "21:55",
-    "22:00",
-    "22:05",
-    "22:10",
-    "22:15",
-    "22:20",
-    "22:25",
-    "22:30",
-    "22:35",
-    "22:40",
-    "22:45",
-    "22:50",
-    "22:55",
-    "23:00",
-    "23:05",
-    "23:10",
-    "23:15",
-    "23:20",
-    "23:25",
-    "23:30",
-    "23:35",
-    "23:40",
-    "23:45",
-    "23:50",
-    "23:55",
-    "request"
-  ],
-  "restaurant-price range": [
-    "none",
-    "do not care",
-    "cheap",
-    "expensive",
-    "moderate",
-    "request"
-  ],
-  "restaurant-food": [
-    "none",
-    "do not care",
-    "british food",
-    "steakhouse",
-    "turkish",
-    "sushi",
-    "north american",
-    "scottish",
-    "french",
-    "austrian",
-    "korean",
-    "eastern european",
-    "swedish",
-    "gastro pub",
-    "modern eclectic",
-    "afternoon tea",
-    "welsh",
-    "christmas",
-    "tuscan",
-    "gastropub",
-    "sri lankan",
-    "molecular gastronomy",
-    "traditional american",
-    "italian",
-    "pizza",
-    "thai",
-    "south african",
-    "creative",
-    "english",
-    "asian",
-    "lebanese",
-    "hungarian",
-    "halal",
-    "portugese",
-    "modern english",
-    "african",
-    "light bites",
-    "malaysian",
-    "venetian",
-    "traditional",
-    "chinese",
-    "vegetarian",
-    "persian",
-    "thai and chinese",
-    "scandinavian",
-    "catalan",
-    "polynesian",
-    "crossover",
-    "canapes",
-    "cantonese",
-    "north african",
-    "seafood",
-    "brazilian",
-    "south indian",
-    "australasian",
-    "belgian",
-    "barbeque",
-    "the americas",
-    "indonesian",
-    "singaporean",
-    "irish",
-    "middle eastern",
-    "dojo noodle bar",
-    "caribbean",
-    "vietnamese",
-    "modern european",
-    "russian",
-    "german",
-    "world",
-    "japanese",
-    "moroccan",
-    "modern global",
-    "indian",
-    "british",
-    "american",
-    "danish",
-    "panasian",
-    "swiss",
-    "basque",
-    "north indian",
-    "modern american",
-    "australian",
-    "european",
-    "corsica",
-    "greek",
-    "northern european",
-    "mediterranean",
-    "portuguese",
-    "romanian",
-    "jamaican",
-    "polish",
-    "international",
-    "unusual",
-    "latin american",
-    "asian oriental",
-    "mexican",
-    "bistro",
-    "cuban",
-    "fusion",
-    "new zealand",
-    "spanish",
-    "eritrean",
-    "afghan",
-    "kosher",
-    "request"
-  ],
-  "attraction-name": [
-    "none",
-    "do not care",
-    "downing college",
-    "fitzwilliam",
-    "clare college",
-    "ruskin gallery",
-    "sidney sussex college",
-    "great saint mary's church",
-    "cherry hinton water play park",
-    "wandlebury country park",
-    "cafe uno",
-    "place",
-    "broughton",
-    "cineworld cinema",
-    "jesus college",
-    "vue cinema",
-    "history of science museum",
-    "mumford theatre",
-    "whale of time",
-    "fitzbillies",
-    "christs church",
-    "churchill college",
-    "museum of classical archaeology",
-    "gonville and caius college",
-    "pizza",
-    "kirkwood",
-    "saint catharines college",
-    "kings college",
-    "parkside",
-    "by",
-    "st catharines college",
-    "saint john's college",
-    "cherry hinton water park",
-    "st christs college",
-    "christ's college",
-    "bangkok city",
-    "scudamores punti co",
-    "free",
-    "great saint marys church",
-    "milton country park",
-    "the fez club",
-    "soultree",
-    "autu",
-    "whipple museum of the history of science",
-    "aylesbray lodge guest house",
-    "broughton house gallery",
-    "peoples portraits exhibition",
-    "primavera",
-    "kettles yard",
-    "all saint's church",
-    "cinema cinema",
-    "regency gallery",
-    "corpus christi",
-    "corn cambridge exchange",
-    "da vinci pizzeria",
-    "school",
-    "hobsons house",
-    "cambride and country folk museum",
-    "north",
-    "da v",
-    "cambridge corn exchange",
-    "soul tree nightclub",
-    "cambridge arts theater",
-    "saint catharine's college",
-    "byard art",
-    "cambridge punter",
-    "cambridge university botanic gardens",
-    "castle galleries",
-    "museum of archaelogy and anthropogy",
-    "no specific location",
-    "cherry hinton hall",
-    "gallery at 12 a high street",
-    "parkside pools",
-    "queen's college",
-    "little saint mary's church",
-    "gallery",
-    "home from home",
-    "tenpin",
-    "the wandlebury",
-    "county folk museum",
-    "swimming pool",
-    "christs college",
-    "cafe jello museum",
-    "scott polar",
-    "christ college",
-    "cambridge museum of technology",
-    "abbey pool and astroturf pitch",
-    "king hedges learner pool",
-    "the cambridge arts theatre",
-    "the castle galleries",
-    "cambridge and country folk museum",
-    "kohinoor",
-    "scudamores punting co",
-    "sidney sussex",
-    "the man on the moon",
-    "little saint marys church",
-    "queens",
-    "the place",
-    "old school",
-    "churchill",
-    "churchills college",
-    "hughes hall",
-    "churchhill college",
-    "riverboat georgina",
-    "belf",
-    "cambridge temporary art",
-    "abc theatre",
-    "cambridge contemporary art museum",
-    "man on the moon",
-    "the junction",
-    "cherry hinton water play",
-    "adc theatre",
-    "gonville hotel",
-    "magdalene college",
-    "peoples portraits exhibition at girton college",
-    "boat",
-    "centre",
-    "sheep's green and lammas land park fen causeway",
-    "the mumford theatre",
-    "archway house",
-    "queens' college",
-    "williams art and antiques",
-    "funky fun house",
-    "cherry hinton village centre",
-    "camboats",
-    "cambridge",
-    "old schools",
-    "kettle's yard",
-    "whale of a time",
-    "the churchill college",
-    "cafe jello gallery",
-    "aut",
-    "salsa",
-    "city",
-    "clare hall",
-    "boating",
-    "pembroke college",
-    "kings hedges learner pool",
-    "caffe uno",
-    "lammas land park",
-    "museum",
-    "the fitzwilliam museum",
-    "the cherry hinton village centre",
-    "the cambridge corn exchange",
-    "fitzwilliam museum",
-    "museum of archaelogy and anthropology",
-    "fez club",
-    "the cambridge punter",
-    "saint johns college",
-    "emmanuel college",
-    "cambridge belf",
-    "scudamore",
-    "lynne strover gallery",
-    "king's college",
-    "whippple museum",
-    "trinity college",
-    "college in the north",
-    "sheep's green",
-    "kambar",
-    "museum of archaelogy",
-    "adc",
-    "garde",
-    "club salsa",
-    "people's portraits exhibition at girton college",
-    "botanic gardens",
-    "carol",
-    "college",
-    "gallery at twelve a high street",
-    "abbey pool and astroturf",
-    "cambridge book and print gallery",
-    "jesus green outdoor pool",
-    "scott polar museum",
-    "saint barnabas press gallery",
-    "cambridge artworks",
-    "older churches",
-    "cambridge contemporary art",
-    "cherry hinton hall and grounds",
-    "univ",
-    "jesus green",
-    "ballare",
-    "abbey pool",
-    "cambridge botanic gardens",
-    "nusha",
-    "worth house",
-    "thanh",
-    "university arms hotel",
-    "cambridge arts theatre",
-    "cafe jello",
-    "cambridge and county folk museum",
-    "the cambridge artworks",
-    "all saints church",
-    "holy trinity church",
-    "contemporary art museum",
-    "architectural churches",
-    "queens college",
-    "trinity street college"
-  ],
-  "restaurant-name": [
-    "none",
-    "do not care",
-    "hotel du vin and bistro",
-    "ask",
-    "gourmet formal kitchen",
-    "the meze bar",
-    "lan hong house",
-    "cow pizza",
-    "one seven",
-    "prezzo",
-    "maharajah tandoori restaurant",
-    "alex",
-    "shanghai",
-    "golden wok",
-    "restaurant",
-    "fitzbillies",
-    "nil",
-    "copper kettle",
-    "meghna",
-    "hk fusion",
-    "bangkok city",
-    "hobsons house",
-    "tang chinese",
-    "anatolia",
-    "ugly duckling",
-    "anatolia and efes restaurant",
-    "sitar tandoori",
-    "city stop",
-    "ashley",
-    "pizza express fen ditton",
-    "molecular gastronomy",
-    "autumn house",
-    "el shaddia guesthouse",
-    "the grafton hotel",
-    "limehouse",
-    "gardenia",
-    "not metioned",
-    "hakka",
-    "michaelhouse cafe",
-    "pipasha",
-    "meze bar",
-    "archway",
-    "molecular gastonomy",
-    "yipee noodle bar",
-    "the peking",
-    "curry prince",
-    "midsummer house restaurant",
-    "pizza hut cherry hinton",
-    "the lucky star",
-    "stazione restaurant and coffee bar",
-    "shanghi family restaurant",
-    "good luck",
-    "j restaurant",
-    "bedouin",
-    "cott",
-    "little seoul",
-    "south",
-    "thanh binh",
-    "el",
-    "efes restaurant",
-    "kohinoor",
-    "clowns",
-    "india",
-    "the slug and lettuce",
-    "shiraz",
-    "barbakan",
-    "zizzi cambridge",
-    "restaurant one seven",
-    "slug and lettuce",
-    "travellers rest",
-    "binh",
-    "worth house",
-    "broughton house gallery",
-    "chiquito",
-    "the river bar steakhouse and grill",
-    "ros",
-    "golden house",
-    "india west",
-    "cam",
-    "panahar",
-    "restaurant 22",
-    "adden",
-    "indian",
-    "hu",
-    "jinling noodle bar",
-    "darrys cookhouse and wine shop",
-    "hobson house",
-    "cambridge be",
-    "el shaddai",
-    "ac",
-    "nandos",
-    "cambridge lodge",
-    "the cow pizza kitchen and bar",
-    "charlie",
-    "rajmahal",
-    "kymmoy",
-    "cambri",
-    "backstreet bistro",
-    "galleria",
-    "restaurant 2 two",
-    "chiquito restaurant bar",
-    "royal standard",
-    "lucky star",
-    "curry king",
-    "grafton hotel restaurant",
-    "mahal of cambridge",
-    "the bedouin",
-    "nus",
-    "the kohinoor",
-    "pizza hut fenditton",
-    "camboats",
-    "the gardenia",
-    "de luca cucina and bar",
-    "nusha",
-    "european",
-    "taj tandoori",
-    "tandoori palace",
-    "golden curry",
-    "efes",
-    "loch fyne",
-    "the maharajah tandoor",
-    "lovel",
-    "restaurant 17",
-    "clowns cafe",
-    "cambridge punter",
-    "bloomsbury restaurant",
-    "la mimosa",
-    "the cambridge chop house",
-    "funky",
-    "cotto",
-    "oak bistro",
-    "restaurant two two",
-    "pipasha restaurant",
-    "river bar steakhouse and grill",
-    "royal spice",
-    "the copper kettle",
-    "graffiti",
-    "nandos city centre",
-    "saffron brasserie",
-    "cambridge chop house",
-    "sitar",
-    "kitchen and bar",
-    "the good luck chinese food takeaway",
-    "clu",
-    "la tasca",
-    "cafe uno",
-    "cote",
-    "the varsity restaurant",
-    "bri",
-    "eraina",
-    "bridge",
-    "fin",
-    "cambridge lodge restaurant",
-    "grafton",
-    "hotpot",
-    "sala thong",
-    "margherita",
-    "wise buddha",
-    "the missing sock",
-    "seasame restaurant and bar",
-    "the dojo noodle bar",
-    "restaurant alimentum",
-    "gastropub",
-    "saigon city",
-    "la margherita",
-    "pizza hut",
-    "curry garden",
-    "ashley hotel",
-    "eraina and michaelhouse cafe",
-    "the golden curry",
-    "curry queen",
-    "cow pizza kitchen and bar",
-    "the peking restaurant:",
-    "hamilton lodge",
-    "alimentum",
-    "yippee noodle bar",
-    "2 two and cote",
-    "shanghai family restaurant",
-    "grafton hotel",
-    "yes",
-    "ali baba",
-    "dif",
-    "fitzbillies restaurant",
-    "peking restaurant",
-    "lev",
-    "nirala",
-    "the alex",
-    "tandoori",
-    "city stop restaurant",
-    "rice house",
-    "cityr",
-    "yu garden",
-    "meze bar restaurant",
-    "the",
-    "don pasquale pizzeria",
-    "rice boat",
-    "the hotpot",
-    "old school",
-    "the oak bistro",
-    "sesame restaurant and bar",
-    "pizza express",
-    "the gandhi",
-    "pizza hut fen ditton",
-    "charlie chan",
-    "da vinci pizzeria",
-    "dojo noodle bar",
-    "gourmet burger kitchen",
-    "the golden house",
-    "india house",
-    "hobso",
-    "missing sock",
-    "pizza hut city centre",
-    "parkside pools",
-    "riverside brasserie",
-    "caffe uno",
-    "primavera",
-    "the nirala",
-    "wagamama",
-    "au",
-    "ian hong house",
-    "frankie and bennys",
-    "4 kings parade city centre",
-    "shiraz restaurant",
-    "scudamores punt",
-    "mahal",
-    "saint johns chop house",
-    "de luca cucina and bar riverside brasserie",
-    "cocum",
-    "la raza"
-  ],
-  "attraction-type": [
-    "none",
-    "do not care",
-    "architecture",
-    "boat",
-    "boating",
-    "camboats",
-    "church",
-    "churchills college",
-    "cinema",
-    "college",
-    "concert",
-    "concerthall",
-    "entertainment",
-    "gallery",
-    "gastropub",
-    "hiking",
-    "hotel",
-    "multiple sports",
-    "museum",
-    "museum kettles yard",
-    "night club",
-    "outdoor",
-    "park",
-    "pool",
-    "special",
-    "sports",
-    "swimming pool",
-    "theater",
-    "theatre",
-    "concert hall",
-    "local site",
-    "nightclub",
-    "hotspot",
-    "request"
-  ],
-  "taxi-leave at": [
-    "none",
-    "do not care",
-    "00:00",
-    "00:05",
-    "00:10",
-    "00:15",
-    "00:20",
-    "00:25",
-    "00:30",
-    "00:35",
-    "00:40",
-    "00:45",
-    "00:50",
-    "00:55",
-    "01:00",
-    "01:05",
-    "01:10",
-    "01:15",
-    "01:20",
-    "01:25",
-    "01:30",
-    "01:35",
-    "01:40",
-    "01:45",
-    "01:50",
-    "01:55",
-    "02:00",
-    "02:05",
-    "02:10",
-    "02:15",
-    "02:20",
-    "02:25",
-    "02:30",
-    "02:35",
-    "02:40",
-    "02:45",
-    "02:50",
-    "02:55",
-    "03:00",
-    "03:05",
-    "03:10",
-    "03:15",
-    "03:20",
-    "03:25",
-    "03:30",
-    "03:35",
-    "03:40",
-    "03:45",
-    "03:50",
-    "03:55",
-    "04:00",
-    "04:05",
-    "04:10",
-    "04:15",
-    "04:20",
-    "04:25",
-    "04:30",
-    "04:35",
-    "04:40",
-    "04:45",
-    "04:50",
-    "04:55",
-    "05:00",
-    "05:05",
-    "05:10",
-    "05:15",
-    "05:20",
-    "05:25",
-    "05:30",
-    "05:35",
-    "05:40",
-    "05:45",
-    "05:50",
-    "05:55",
-    "06:00",
-    "06:05",
-    "06:10",
-    "06:15",
-    "06:20",
-    "06:25",
-    "06:30",
-    "06:35",
-    "06:40",
-    "06:45",
-    "06:50",
-    "06:55",
-    "07:00",
-    "07:05",
-    "07:10",
-    "07:15",
-    "07:20",
-    "07:25",
-    "07:30",
-    "07:35",
-    "07:40",
-    "07:45",
-    "07:50",
-    "07:55",
-    "08:00",
-    "08:05",
-    "08:10",
-    "08:15",
-    "08:20",
-    "08:25",
-    "08:30",
-    "08:35",
-    "08:40",
-    "08:45",
-    "08:50",
-    "08:55",
-    "09:00",
-    "09:05",
-    "09:10",
-    "09:15",
-    "09:20",
-    "09:25",
-    "09:30",
-    "09:35",
-    "09:40",
-    "09:45",
-    "09:50",
-    "09:55",
-    "10:00",
-    "10:05",
-    "10:10",
-    "10:15",
-    "10:20",
-    "10:25",
-    "10:30",
-    "10:35",
-    "10:40",
-    "10:45",
-    "10:50",
-    "10:55",
-    "11:00",
-    "11:05",
-    "11:10",
-    "11:15",
-    "11:20",
-    "11:25",
-    "11:30",
-    "11:35",
-    "11:40",
-    "11:45",
-    "11:50",
-    "11:55",
-    "12:00",
-    "12:05",
-    "12:10",
-    "12:15",
-    "12:20",
-    "12:25",
-    "12:30",
-    "12:35",
-    "12:40",
-    "12:45",
-    "12:50",
-    "12:55",
-    "13:00",
-    "13:05",
-    "13:10",
-    "13:15",
-    "13:20",
-    "13:25",
-    "13:30",
-    "13:35",
-    "13:40",
-    "13:45",
-    "13:50",
-    "13:55",
-    "14:00",
-    "14:05",
-    "14:10",
-    "14:15",
-    "14:20",
-    "14:25",
-    "14:30",
-    "14:35",
-    "14:40",
-    "14:45",
-    "14:50",
-    "14:55",
-    "15:00",
-    "15:05",
-    "15:10",
-    "15:15",
-    "15:20",
-    "15:25",
-    "15:30",
-    "15:35",
-    "15:40",
-    "15:45",
-    "15:50",
-    "15:55",
-    "16:00",
-    "16:05",
-    "16:10",
-    "16:15",
-    "16:20",
-    "16:25",
-    "16:30",
-    "16:35",
-    "16:40",
-    "16:45",
-    "16:50",
-    "16:55",
-    "17:00",
-    "17:05",
-    "17:10",
-    "17:15",
-    "17:20",
-    "17:25",
-    "17:30",
-    "17:35",
-    "17:40",
-    "17:45",
-    "17:50",
-    "17:55",
-    "18:00",
-    "18:05",
-    "18:10",
-    "18:15",
-    "18:20",
-    "18:25",
-    "18:30",
-    "18:35",
-    "18:40",
-    "18:45",
-    "18:50",
-    "18:55",
-    "19:00",
-    "19:05",
-    "19:10",
-    "19:15",
-    "19:20",
-    "19:25",
-    "19:30",
-    "19:35",
-    "19:40",
-    "19:45",
-    "19:50",
-    "19:55",
-    "20:00",
-    "20:05",
-    "20:10",
-    "20:15",
-    "20:20",
-    "20:25",
-    "20:30",
-    "20:35",
-    "20:40",
-    "20:45",
-    "20:50",
-    "20:55",
-    "21:00",
-    "21:05",
-    "21:10",
-    "21:15",
-    "21:20",
-    "21:25",
-    "21:30",
-    "21:35",
-    "21:40",
-    "21:45",
-    "21:50",
-    "21:55",
-    "22:00",
-    "22:05",
-    "22:10",
-    "22:15",
-    "22:20",
-    "22:25",
-    "22:30",
-    "22:35",
-    "22:40",
-    "22:45",
-    "22:50",
-    "22:55",
-    "23:00",
-    "23:05",
-    "23:10",
-    "23:15",
-    "23:20",
-    "23:25",
-    "23:30",
-    "23:35",
-    "23:40",
-    "23:45",
-    "23:50",
-    "23:55",
-    "request"
-  ],
-  "taxi-destination": [
-    "none",
-    "do not care",
-    "a and b guest house",
-    "abbey pool and astroturf pitch",
-    "acorn guest house",
-    "adc theatre",
-    "addenbrookes hospital",
-    "alexander bed and breakfast",
-    "ali baba",
-    "all saints church",
-    "allenbell",
-    "alpha milton guest house",
-    "anatolia",
-    "arbury lodge guesthouse",
-    "archway house",
-    "ashley hotel",
-    "ask",
-    "attraction",
-    "autumn house",
-    "avalon",
-    "aylesbray lodge guest house",
-    "backstreet bistro",
-    "ballare",
-    "bangkok city",
-    "bedouin",
-    "birmingham new street train station",
-    "bishops stortford train station",
-    "bloomsbury restaurant",
-    "bridge guest house",
-    "broughton house gallery",
-    "broxbourne train station",
-    "byard art",
-    "cafe jello gallery",
-    "cafe uno",
-    "camboats",
-    "cambridge",
-    "cambridge and county folk museum",
-    "cambridge arts theatre",
-    "cambridge artworks",
-    "cambridge belfry",
-    "cambridge book and print gallery",
-    "cambridge chop house",
-    "cambridge contemporary art",
-    "cambridge county fair next to the city tourist museum",
-    "cambridge lodge restaurant",
-    "cambridge museum of technology",
-    "cambridge punter",
-    "cambridge road church of christ",
-    "cambridge train station",
-    "cambridge university botanic gardens",
-    "carolina bed and breakfast",
-    "castle galleries",
-    "charlie chan",
-    "cherry hinton hall and grounds",
-    "cherry hinton village centre",
-    "cherry hinton water park",
-    "cherry hinton water play",
-    "chiquito restaurant bar",
-    "christ college",
-    "churchills college",
-    "cineworld cinema",
-    "city centre north bed and breakfast",
-    "city stop restaurant",
-    "cityroomz",
-    "clare college",
-    "clare hall",
-    "clowns cafe",
-    "club salsa",
-    "cocum",
-    "copper kettle",
-    "corpus christi",
-    "cote",
-    "cotto",
-    "cow pizza kitchen and bar",
-    "curry garden",
-    "curry king",
-    "curry prince",
-    "da vinci pizzeria",
-    "darrys cookhouse and wine shop",
-    "de luca cucina and bar",
-    "dojo noodle bar",
-    "don pasquale pizzeria",
-    "downing college",
-    "efes restaurant",
-    "el shaddia guesthouse",
-    "ely train station",
-    "emmanuel college",
-    "eraina",
-    "express by holiday inn cambridge",
-    "finches bed and breakfast",
-    "finders corner newmarket road",
-    "fitzbillies restaurant",
-    "fitzwilliam museum",
-    "frankie and bennys",
-    "funky fun house",
-    "galleria",
-    "gallery at 12 a high street",
-    "gastropub",
-    "golden curry",
-    "golden house",
-    "golden wok",
-    "gonville and caius college",
-    "gonville hotel",
-    "good luck",
-    "gourmet burger kitchen",
-    "graffiti",
-    "grafton hotel restaurant",
-    "great saint marys church",
-    "hakka",
-    "hamilton lodge",
-    "hk fusion",
-    "hobsons house",
-    "holy trinity church",
-    "home from home",
-    "hotel du vin and bistro",
-    "hughes hall",
-    "huntingdon marriott hotel",
-    "ian hong",
-    "india house",
-    "j restaurant",
-    "jesus college",
-    "jesus green outdoor pool",
-    "jinling noodle bar",
-    "kambar",
-    "kettles yard",
-    "kings college",
-    "kings hedges learner pool",
-    "kirkwood house",
-    "kohinoor",
-    "kymmoy",
-    "la margherita",
-    "la mimosa",
-    "la raza",
-    "la tasca",
-    "lan hong house",
-    "leicester train station",
-    "lensfield hotel",
-    "limehouse",
-    "little saint marys church",
-    "little seoul",
-    "loch fyne",
-    "london kings cross train station",
-    "london liverpool street train station",
-    "lovell lodge",
-    "lynne strover gallery",
-    "magdalene college",
-    "mahal of cambridge",
-    "maharajah tandoori restaurant",
-    "meghna",
-    "meze bar",
-    "michaelhouse cafe",
-    "midsummer house restaurant",
-    "milton country park",
-    "mumford theatre",
-    "museum of archaelogy and anthropology",
-    "museum of classical archaeology",
-    "nandos",
-    "nandos city centre",
-    "nil",
-    "nirala",
-    "norwich train station",
-    "nusha",
-    "old schools",
-    "panahar",
-    "parkside police station",
-    "parkside pools",
-    "peking restaurant",
-    "pembroke college",
-    "peoples portraits exhibition at girton college",
-    "peterborough train station",
-    "pipasha restaurant",
-    "pizza express",
-    "pizza hut cherry hinton",
-    "pizza hut city centre",
-    "pizza hut fenditton",
-    "prezzo",
-    "primavera",
-    "queens college",
-    "rajmahal",
-    "regency gallery",
-    "restaurant 17",
-    "restaurant 2 two",
-    "restaurant alimentum",
-    "rice boat",
-    "rice house",
-    "riverboat georgina",
-    "riverside brasserie",
-    "rosas bed and breakfast",
-    "royal spice",
-    "royal standard",
-    "ruskin gallery",
-    "saffron brasserie",
-    "saigon city",
-    "saint barnabas",
-    "saint barnabas press gallery",
-    "saint catharines college",
-    "saint johns chop house",
-    "saint johns college",
-    "sala thong",
-    "scott polar museum",
-    "scudamores punting co",
-    "sesame restaurant and bar",
-    "shanghai family restaurant",
-    "sheeps green and lammas land park fen causeway",
-    "shiraz",
-    "sidney sussex college",
-    "sitar tandoori",
-    "sleeperz hotel",
-    "soul tree nightclub",
-    "st johns chop house",
-    "stansted airport train station",
-    "station road",
-    "stazione restaurant and coffee bar",
-    "stevenage train station",
-    "taj tandoori",
-    "tall monument",
-    "tandoori palace",
-    "tang chinese",
-    "tenpin",
-    "thanh binh",
-    "the anatolia",
-    "the cambridge corn exchange",
-    "the cambridge shop",
-    "the fez club",
-    "the gandhi",
-    "the gardenia",
-    "the hotpot",
-    "the junction",
-    "the lucky star",
-    "the man on the moon",
-    "the missing sock",
-    "the oak bistro",
-    "the place",
-    "the regent street city center",
-    "the river bar steakhouse and grill",
-    "the slug and lettuce",
-    "the varsity restaurant",
-    "travellers rest",
-    "trinity college",
-    "ugly duckling",
-    "university arms hotel",
-    "vue cinema",
-    "wagamama",
-    "wandlebury country park",
-    "wankworth hotel",
-    "warkworth house",
-    "whale of a time",
-    "whipple museum of the history of science",
-    "williams art and antiques",
-    "worth house",
-    "yippee noodle bar",
-    "yu garden",
-    "zizzi cambridge",
-    "leverton house",
-    "the cambridge chop house",
-    "saint john's college",
-    "churchill college",
-    "the nirala",
-    "the cow pizza kitchen and bar",
-    "christ's college",
-    "el shaddai",
-    "saint catharine's college",
-    "camb",
-    "the golden curry",
-    "little saint mary's church",
-    "country folk museum",
-    "meze bar restaurant",
-    "the cambridge belfry",
-    "the fitzwilliam museum",
-    "the lensfield hotel",
-    "pizza express fen ditton",
-    "the cambridge punter",
-    "king's college",
-    "the cherry hinton village centre",
-    "shiraz restaurant",
-    "sheep's green and lammas land park fen causeway",
-    "caffe uno",
-    "the ghandi",
-    "the copper kettle",
-    "man on the moon concert hall",
-    "alpha-milton guest house",
-    "queen's college",
-    "restaurant one seven",
-    "restaurant two two",
-    "city centre north b and b",
-    "rosa's bed and breakfast",
-    "the good luck chinese food takeaway",
-    "not museum of archaeology and anthropologymentioned",
-    "tandori in cambridge",
-    "kettle's yard",
-    "megna",
-    "grou",
-    "gallery at twelve a high street",
-    "maharajah tandoori restaurant",
-    "pizza hut fen ditton",
-    "gandhi",
-    "tranh binh",
-    "kambur",
-    "people's portraits exhibition at girton college",
-    "hotel",
-    "restaurant",
-    "the galleria",
-    "queens' college",
-    "great saint mary's church",
-    "theathre",
-    "cambridge artworks",
-    "acorn house",
-    "shiraz",
-    "riverboat georginawd",
-    "mic",
-    "the gallery at twelve",
-    "the soul tree",
-    "finches"
-  ],
-  "taxi-departure": [
-    "none",
-    "do not care",
-    "172 chestertown road",
-    "4455 woodbridge road",
-    "a and b guest house",
-    "abbey pool and astroturf pitch",
-    "acorn guest house",
-    "adc theatre",
-    "addenbrookes hospital",
-    "alexander bed and breakfast",
-    "ali baba",
-    "all saints church",
-    "allenbell",
-    "alpha milton guest house",
-    "alyesbray lodge hotel",
-    "ambridge",
-    "anatolia",
-    "arbury lodge guesthouse",
-    "archway house",
-    "ashley hotel",
-    "ask",
-    "autumn house",
-    "avalon",
-    "aylesbray lodge guest house",
-    "backstreet bistro",
-    "ballare",
-    "bangkok city",
-    "bedouin",
-    "birmingham new street train station",
-    "bishops stortford train station",
-    "bloomsbury restaurant",
-    "bridge guest house",
-    "broughton house gallery",
-    "broxbourne train station",
-    "byard art",
-    "cafe jello gallery",
-    "cafe uno",
-    "caffee uno",
-    "camboats",
-    "cambridge",
-    "cambridge and county folk museum",
-    "cambridge arts theatre",
-    "cambridge artworks",
-    "cambridge belfry",
-    "cambridge book and print gallery",
-    "cambridge chop house",
-    "cambridge contemporary art",
-    "cambridge lodge restaurant",
-    "cambridge museum of technology",
-    "cambridge punter",
-    "cambridge towninfo centre",
-    "cambridge train station",
-    "cambridge university botanic gardens",
-    "carolina bed and breakfast",
-    "castle galleries",
-    "centre of town at my hotel",
-    "charlie chan",
-    "cherry hinton hall and grounds",
-    "cherry hinton village center",
-    "cherry hinton village centre",
-    "cherry hinton water play",
-    "chiquito restaurant bar",
-    "christ college",
-    "churchills college",
-    "cineworld cinema",
-    "citiroomz",
-    "city centre north bed and breakfast",
-    "city stop restaurant",
-    "cityroomz",
-    "clair hall",
-    "clare college",
-    "clare hall",
-    "clowns cafe",
-    "club salsa",
-    "cocum",
-    "copper kettle",
-    "corpus christi",
-    "cote",
-    "cotto",
-    "cow pizza kitchen and bar",
-    "curry garden",
-    "curry king",
-    "curry prince",
-    "curry queen",
-    "da vinci pizzeria",
-    "darrys cookhouse and wine shop",
-    "de luca cucina and bar",
-    "dojo noodle bar",
-    "don pasquale pizzeria",
-    "downing college",
-    "downing street",
-    "el shaddia guesthouse",
-    "ely",
-    "ely train station",
-    "emmanuel college",
-    "eraina",
-    "express by holiday inn cambridge",
-    "finches bed and breakfast",
-    "fitzbillies restaurant",
-    "fitzwilliam museum",
-    "frankie and bennys",
-    "funky fun house",
-    "galleria",
-    "gallery at 12 a high street",
-    "girton college",
-    "golden curry",
-    "golden house",
-    "golden wok",
-    "gonville and caius college",
-    "gonville hotel",
-    "good luck",
-    "gourmet burger kitchen",
-    "graffiti",
-    "grafton hotel restaurant",
-    "great saint marys church",
-    "hakka",
-    "hamilton lodge",
-    "hobsons house",
-    "holy trinity church",
-    "home",
-    "home from home",
-    "hotel",
-    "hotel du vin and bistro",
-    "hughes hall",
-    "huntingdon marriott hotel",
-    "india house",
-    "j restaurant",
-    "jesus college",
-    "jesus green outdoor pool",
-    "jinling noodle bar",
-    "junction theatre",
-    "kambar",
-    "kettles yard",
-    "kings college",
-    "kings hedges learner pool",
-    "kings lynn train station",
-    "kirkwood house",
-    "kohinoor",
-    "kymmoy",
-    "la margherita",
-    "la mimosa",
-    "la raza",
-    "la tasca",
-    "lan hong house",
-    "lensfield hotel",
-    "leverton house",
-    "limehouse",
-    "little saint marys church",
-    "little seoul",
-    "loch fyne",
-    "london kings cross train station",
-    "london liverpool street",
-    "london liverpool street train station",
-    "lovell lodge",
-    "lynne strover gallery",
-    "magdalene college",
-    "mahal of cambridge",
-    "maharajah tandoori restaurant",
-    "meghna",
-    "meze bar",
-    "michaelhouse cafe",
-    "milton country park",
-    "mumford theatre",
-    "museum",
-    "museum of archaelogy and anthropology",
-    "museum of classical archaeology",
-    "nandos",
-    "nandos city centre",
-    "new england",
-    "nirala",
-    "norwich train station",
-    "nstaot mentioned",
-    "nusha",
-    "old schools",
-    "panahar",
-    "parkside police station",
-    "parkside pools",
-    "peking restaurant",
-    "pembroke college",
-    "peoples portraits exhibition at girton college",
-    "peterborough train station",
-    "pizza express",
-    "pizza hut cherry hinton",
-    "pizza hut city centre",
-    "pizza hut fenditton",
-    "prezzo",
-    "primavera",
-    "queens college",
-    "rajmahal",
-    "regency gallery",
-    "restaurant 17",
-    "restaurant 2 two",
-    "restaurant alimentum",
-    "rice boat",
-    "rice house",
-    "riverboat georgina",
-    "riverside brasserie",
-    "rosas bed and breakfast",
-    "royal spice",
-    "royal standard",
-    "ruskin gallery",
-    "saffron brasserie",
-    "saigon city",
-    "saint barnabas press gallery",
-    "saint catharines college",
-    "saint johns chop house",
-    "saint johns college",
-    "sala thong",
-    "scott polar museum",
-    "scudamores punting co",
-    "sesame restaurant and bar",
-    "sheeps green and lammas land park",
-    "sheeps green and lammas land park fen causeway",
-    "shiraz",
-    "sidney sussex college",
-    "sitar tandoori",
-    "soul tree nightclub",
-    "st johns college",
-    "stazione restaurant and coffee bar",
-    "stevenage train station",
-    "taj tandoori",
-    "tandoori palace",
-    "tang chinese",
-    "tenpin",
-    "thanh binh",
-    "the cambridge corn exchange",
-    "the fez club",
-    "the gallery at 12",
-    "the gandhi",
-    "the gardenia",
-    "the hotpot",
-    "the junction",
-    "the lucky star",
-    "the man on the moon",
-    "the missing sock",
-    "the oak bistro",
-    "the place",
-    "the river bar steakhouse and grill",
-    "the slug and lettuce",
-    "the varsity restaurant",
-    "travellers rest",
-    "trinity college",
-    "ugly duckling",
-    "university arms hotel",
-    "vue cinema",
-    "wagamama",
-    "wandlebury country park",
-    "warkworth house",
-    "whale of a time",
-    "whipple museum of the history of science",
-    "williams art and antiques",
-    "worth house",
-    "yippee noodle bar",
-    "yu garden",
-    "zizzi cambridge",
-    "christ's college",
-    "city centre north b and b",
-    "the lensfield hotel",
-    "alpha-milton guest house",
-    "el shaddai",
-    "churchill college",
-    "the cambridge belfry",
-    "king's college",
-    "great saint mary's church",
-    "restaurant two two",
-    "queens' college",
-    "little saint mary's church",
-    "chinese city centre",
-    "kettle's yard",
-    "pizza hut",
-    "the golden curry",
-    "rosa's bed and breakfast",
-    "the cambridge punter",
-    "the byard art museum",
-    "saint catharine's college",
-    "meze bar restaurant",
-    "the good luck chinese food takeaway",
-    "restaurant one seven",
-    "pizza hut fen ditton",
-    "the nirala",
-    "the fitzwilliam museum",
-    "st. john's college",
-    "gallery at twelve a high street",
-    "sheep's green and lammas land park fen causeway",
-    "the cherry hinton village centre",
-    "pizza express fen ditton",
-    "corpus cristi",
-    "cas",
-    "acorn house",
-    "lens",
-    "the cambridge chop house",
-    "the copper kettle",
-    "the avalon",
-    "saint john's college",
-    "aylesbray lodge",
-    "the alexander bed and breakfast",
-    "cambridge belfy",
-    "people's portraits exhibition at girton college",
-    "gonville",
-    "caffe uno",
-    "the cow pizza kitchen and bar",
-    "lovell ldoge",
-    "cinema",
-    "shiraz restaurant",
-    "park",
-    "the allenbell"
-  ],
-  "restaurant-book day": [
-    "none",
-    "do not care",
-    "friday",
-    "monday",
-    "saterday",
-    "sunday",
-    "thursday",
-    "tuesday",
-    "wednesday"
-  ],
-  "restaurant-book people": [
-    "none",
-    "do not care",
-    "1",
-    "10 or more",
-    "2",
-    "3",
-    "4",
-    "5",
-    "6",
-    "7",
-    "8",
-    "9"
-  ],
-  "restaurant-book time": [
-    "none",
-    "do not care",
-    "00:00",
-    "00:05",
-    "00:10",
-    "00:15",
-    "00:20",
-    "00:25",
-    "00:30",
-    "00:35",
-    "00:40",
-    "00:45",
-    "00:50",
-    "00:55",
-    "01:00",
-    "01:05",
-    "01:10",
-    "01:15",
-    "01:20",
-    "01:25",
-    "01:30",
-    "01:35",
-    "01:40",
-    "01:45",
-    "01:50",
-    "01:55",
-    "02:00",
-    "02:05",
-    "02:10",
-    "02:15",
-    "02:20",
-    "02:25",
-    "02:30",
-    "02:35",
-    "02:40",
-    "02:45",
-    "02:50",
-    "02:55",
-    "03:00",
-    "03:05",
-    "03:10",
-    "03:15",
-    "03:20",
-    "03:25",
-    "03:30",
-    "03:35",
-    "03:40",
-    "03:45",
-    "03:50",
-    "03:55",
-    "04:00",
-    "04:05",
-    "04:10",
-    "04:15",
-    "04:20",
-    "04:25",
-    "04:30",
-    "04:35",
-    "04:40",
-    "04:45",
-    "04:50",
-    "04:55",
-    "05:00",
-    "05:05",
-    "05:10",
-    "05:15",
-    "05:20",
-    "05:25",
-    "05:30",
-    "05:35",
-    "05:40",
-    "05:45",
-    "05:50",
-    "05:55",
-    "06:00",
-    "06:05",
-    "06:10",
-    "06:15",
-    "06:20",
-    "06:25",
-    "06:30",
-    "06:35",
-    "06:40",
-    "06:45",
-    "06:50",
-    "06:55",
-    "07:00",
-    "07:05",
-    "07:10",
-    "07:15",
-    "07:20",
-    "07:25",
-    "07:30",
-    "07:35",
-    "07:40",
-    "07:45",
-    "07:50",
-    "07:55",
-    "08:00",
-    "08:05",
-    "08:10",
-    "08:15",
-    "08:20",
-    "08:25",
-    "08:30",
-    "08:35",
-    "08:40",
-    "08:45",
-    "08:50",
-    "08:55",
-    "09:00",
-    "09:05",
-    "09:10",
-    "09:15",
-    "09:20",
-    "09:25",
-    "09:30",
-    "09:35",
-    "09:40",
-    "09:45",
-    "09:50",
-    "09:55",
-    "10:00",
-    "10:05",
-    "10:10",
-    "10:15",
-    "10:20",
-    "10:25",
-    "10:30",
-    "10:35",
-    "10:40",
-    "10:45",
-    "10:50",
-    "10:55",
-    "11:00",
-    "11:05",
-    "11:10",
-    "11:15",
-    "11:20",
-    "11:25",
-    "11:30",
-    "11:35",
-    "11:40",
-    "11:45",
-    "11:50",
-    "11:55",
-    "12:00",
-    "12:05",
-    "12:10",
-    "12:15",
-    "12:20",
-    "12:25",
-    "12:30",
-    "12:35",
-    "12:40",
-    "12:45",
-    "12:50",
-    "12:55",
-    "13:00",
-    "13:05",
-    "13:10",
-    "13:15",
-    "13:20",
-    "13:25",
-    "13:30",
-    "13:35",
-    "13:40",
-    "13:45",
-    "13:50",
-    "13:55",
-    "14:00",
-    "14:05",
-    "14:10",
-    "14:15",
-    "14:20",
-    "14:25",
-    "14:30",
-    "14:35",
-    "14:40",
-    "14:45",
-    "14:50",
-    "14:55",
-    "15:00",
-    "15:05",
-    "15:10",
-    "15:15",
-    "15:20",
-    "15:25",
-    "15:30",
-    "15:35",
-    "15:40",
-    "15:45",
-    "15:50",
-    "15:55",
-    "16:00",
-    "16:05",
-    "16:10",
-    "16:15",
-    "16:20",
-    "16:25",
-    "16:30",
-    "16:35",
-    "16:40",
-    "16:45",
-    "16:50",
-    "16:55",
-    "17:00",
-    "17:05",
-    "17:10",
-    "17:15",
-    "17:20",
-    "17:25",
-    "17:30",
-    "17:35",
-    "17:40",
-    "17:45",
-    "17:50",
-    "17:55",
-    "18:00",
-    "18:05",
-    "18:10",
-    "18:15",
-    "18:20",
-    "18:25",
-    "18:30",
-    "18:35",
-    "18:40",
-    "18:45",
-    "18:50",
-    "18:55",
-    "19:00",
-    "19:05",
-    "19:10",
-    "19:15",
-    "19:20",
-    "19:25",
-    "19:30",
-    "19:35",
-    "19:40",
-    "19:45",
-    "19:50",
-    "19:55",
-    "20:00",
-    "20:05",
-    "20:10",
-    "20:15",
-    "20:20",
-    "20:25",
-    "20:30",
-    "20:35",
-    "20:40",
-    "20:45",
-    "20:50",
-    "20:55",
-    "21:00",
-    "21:05",
-    "21:10",
-    "21:15",
-    "21:20",
-    "21:25",
-    "21:30",
-    "21:35",
-    "21:40",
-    "21:45",
-    "21:50",
-    "21:55",
-    "22:00",
-    "22:05",
-    "22:10",
-    "22:15",
-    "22:20",
-    "22:25",
-    "22:30",
-    "22:35",
-    "22:40",
-    "22:45",
-    "22:50",
-    "22:55",
-    "23:00",
-    "23:05",
-    "23:10",
-    "23:15",
-    "23:20",
-    "23:25",
-    "23:30",
-    "23:35",
-    "23:40",
-    "23:45",
-    "23:50",
-    "23:55"
-  ],
-  "taxi-arrive by": [
-    "none",
-    "do not care",
-    "00:00",
-    "00:05",
-    "00:10",
-    "00:15",
-    "00:20",
-    "00:25",
-    "00:30",
-    "00:35",
-    "00:40",
-    "00:45",
-    "00:50",
-    "00:55",
-    "01:00",
-    "01:05",
-    "01:10",
-    "01:15",
-    "01:20",
-    "01:25",
-    "01:30",
-    "01:35",
-    "01:40",
-    "01:45",
-    "01:50",
-    "01:55",
-    "02:00",
-    "02:05",
-    "02:10",
-    "02:15",
-    "02:20",
-    "02:25",
-    "02:30",
-    "02:35",
-    "02:40",
-    "02:45",
-    "02:50",
-    "02:55",
-    "03:00",
-    "03:05",
-    "03:10",
-    "03:15",
-    "03:20",
-    "03:25",
-    "03:30",
-    "03:35",
-    "03:40",
-    "03:45",
-    "03:50",
-    "03:55",
-    "04:00",
-    "04:05",
-    "04:10",
-    "04:15",
-    "04:20",
-    "04:25",
-    "04:30",
-    "04:35",
-    "04:40",
-    "04:45",
-    "04:50",
-    "04:55",
-    "05:00",
-    "05:05",
-    "05:10",
-    "05:15",
-    "05:20",
-    "05:25",
-    "05:30",
-    "05:35",
-    "05:40",
-    "05:45",
-    "05:50",
-    "05:55",
-    "06:00",
-    "06:05",
-    "06:10",
-    "06:15",
-    "06:20",
-    "06:25",
-    "06:30",
-    "06:35",
-    "06:40",
-    "06:45",
-    "06:50",
-    "06:55",
-    "07:00",
-    "07:05",
-    "07:10",
-    "07:15",
-    "07:20",
-    "07:25",
-    "07:30",
-    "07:35",
-    "07:40",
-    "07:45",
-    "07:50",
-    "07:55",
-    "08:00",
-    "08:05",
-    "08:10",
-    "08:15",
-    "08:20",
-    "08:25",
-    "08:30",
-    "08:35",
-    "08:40",
-    "08:45",
-    "08:50",
-    "08:55",
-    "09:00",
-    "09:05",
-    "09:10",
-    "09:15",
-    "09:20",
-    "09:25",
-    "09:30",
-    "09:35",
-    "09:40",
-    "09:45",
-    "09:50",
-    "09:55",
-    "10:00",
-    "10:05",
-    "10:10",
-    "10:15",
-    "10:20",
-    "10:25",
-    "10:30",
-    "10:35",
-    "10:40",
-    "10:45",
-    "10:50",
-    "10:55",
-    "11:00",
-    "11:05",
-    "11:10",
-    "11:15",
-    "11:20",
-    "11:25",
-    "11:30",
-    "11:35",
-    "11:40",
-    "11:45",
-    "11:50",
-    "11:55",
-    "12:00",
-    "12:05",
-    "12:10",
-    "12:15",
-    "12:20",
-    "12:25",
-    "12:30",
-    "12:35",
-    "12:40",
-    "12:45",
-    "12:50",
-    "12:55",
-    "13:00",
-    "13:05",
-    "13:10",
-    "13:15",
-    "13:20",
-    "13:25",
-    "13:30",
-    "13:35",
-    "13:40",
-    "13:45",
-    "13:50",
-    "13:55",
-    "14:00",
-    "14:05",
-    "14:10",
-    "14:15",
-    "14:20",
-    "14:25",
-    "14:30",
-    "14:35",
-    "14:40",
-    "14:45",
-    "14:50",
-    "14:55",
-    "15:00",
-    "15:05",
-    "15:10",
-    "15:15",
-    "15:20",
-    "15:25",
-    "15:30",
-    "15:35",
-    "15:40",
-    "15:45",
-    "15:50",
-    "15:55",
-    "16:00",
-    "16:05",
-    "16:10",
-    "16:15",
-    "16:20",
-    "16:25",
-    "16:30",
-    "16:35",
-    "16:40",
-    "16:45",
-    "16:50",
-    "16:55",
-    "17:00",
-    "17:05",
-    "17:10",
-    "17:15",
-    "17:20",
-    "17:25",
-    "17:30",
-    "17:35",
-    "17:40",
-    "17:45",
-    "17:50",
-    "17:55",
-    "18:00",
-    "18:05",
-    "18:10",
-    "18:15",
-    "18:20",
-    "18:25",
-    "18:30",
-    "18:35",
-    "18:40",
-    "18:45",
-    "18:50",
-    "18:55",
-    "19:00",
-    "19:05",
-    "19:10",
-    "19:15",
-    "19:20",
-    "19:25",
-    "19:30",
-    "19:35",
-    "19:40",
-    "19:45",
-    "19:50",
-    "19:55",
-    "20:00",
-    "20:05",
-    "20:10",
-    "20:15",
-    "20:20",
-    "20:25",
-    "20:30",
-    "20:35",
-    "20:40",
-    "20:45",
-    "20:50",
-    "20:55",
-    "21:00",
-    "21:05",
-    "21:10",
-    "21:15",
-    "21:20",
-    "21:25",
-    "21:30",
-    "21:35",
-    "21:40",
-    "21:45",
-    "21:50",
-    "21:55",
-    "22:00",
-    "22:05",
-    "22:10",
-    "22:15",
-    "22:20",
-    "22:25",
-    "22:30",
-    "22:35",
-    "22:40",
-    "22:45",
-    "22:50",
-    "22:55",
-    "23:00",
-    "23:05",
-    "23:10",
-    "23:15",
-    "23:20",
-    "23:25",
-    "23:30",
-    "23:35",
-    "23:40",
-    "23:45",
-    "23:50",
-    "23:55",
-    "request"
-  ],
-  "restaurant-area": [
-    "none",
-    "do not care",
-    "centre",
-    "east",
-    "north",
-    "south",
-    "west",
-    "request"
-  ],
-  "hotel-area": [
-    "none",
-    "do not care",
-    "centre",
-    "east",
-    "north",
-    "south",
-    "west",
-    "request"
-  ],
-  "attraction-area": [
-    "none",
-    "do not care",
-    "centre",
-    "east",
-    "north",
-    "south",
-    "west",
-    "request"
-  ],
-  "hospital-department": [
-    "none",
-    "do not care",
-    "acute medical assessment unit",
-    "acute medicine for the elderly",
-    "antenatal",
-    "cambridge eye unit",
-    "cardiology",
-    "cardiology and coronary care unit",
-    "childrens oncology and haematology",
-    "childrens surgical and medicine",
-    "clinical decisions unit",
-    "clinical research facility",
-    "coronary care unit",
-    "diabetes and endocrinology",
-    "emergency department",
-    "gastroenterology",
-    "gynaecology",
-    "haematology",
-    "haematology and haematological oncology",
-    "haematology day unit",
-    "hepatobillary and gastrointestinal surgery regional referral centre",
-    "hepatology",
-    "infectious diseases",
-    "infusion services",
-    "inpatient occupational therapy",
-    "intermediate dependancy area",
-    "john farman intensive care unit",
-    "medical decisions unit",
-    "medicine for the elderly",
-    "neonatal unit",
-    "neurology",
-    "neurology neurosurgery",
-    "neurosciences",
-    "neurosciences critical care unit",
-    "oncology",
-    "oral and maxillofacial surgery and ent",
-    "paediatric clinic",
-    "paediatric day unit",
-    "paediatric intensive care unit",
-    "plastic and vascular surgery plastics",
-    "psychiatry",
-    "respiratory medicine",
-    "surgery",
-    "teenage cancer trust unit",
-    "transitional care",
-    "transplant high dependency unit",
-    "trauma and orthopaedics",
-    "trauma high dependency unit",
-    "urology"
-  ],
-  "police-postcode": [
-    "request"
-  ],
-  "restaurant-postcode": [
-    "request"
-  ],
-  "train-duration": [
-    "request"
-  ],
-  "train-trainid": [
-    "request"
-  ],
-  "hospital-address": [
-    "request"
-  ],
-  "restaurant-phone": [
-    "request"
-  ],
-  "hotel-phone": [
-    "request"
-  ],
-  "restaurant-address": [
-    "request"
-  ],
-  "hotel-postcode": [
-    "request"
-  ],
-  "attraction-phone": [
-    "request"
-  ],
-  "attraction-entrance fee": [
-    "request"
-  ],
-  "hotel-reference": [
-    "request"
-  ],
-  "taxi-taxi types": [
-    "request"
-  ],
-  "attraction-address": [
-    "request"
-  ],
-  "hospital-phone": [
-    "request"
-  ],
-  "attraction-postcode": [
-    "request"
-  ],
-  "police-address": [
-    "request"
-  ],
-  "taxi-taxi phone": [
-    "request"
-  ],
-  "train-price": [
-    "request"
-  ],
-  "hospital-postcode": [
-    "request"
-  ],
-  "police-phone": [
-    "request"
-  ],
-  "hotel-address": [
-    "request"
-  ],
-  "restaurant-reference": [
-    "request"
-  ],
-  "train-reference": [
-    "request"
-  ]
-}
\ No newline at end of file
diff --git a/convlab/dst/setsumbt/multiwoz/dataset/mwoz21_slot_descriptions.json b/convlab/dst/setsumbt/multiwoz/dataset/mwoz21_slot_descriptions.json
deleted file mode 100644
index 87e315363ad3dfd8cadd5bd10cfd7e5047450160..0000000000000000000000000000000000000000
--- a/convlab/dst/setsumbt/multiwoz/dataset/mwoz21_slot_descriptions.json
+++ /dev/null
@@ -1,57 +0,0 @@
-{
-  "hotel-price range": "preferred cost or price of the hotel",
-  "hotel-type": "what is the type of the hotel",
-  "hotel-parking": "does the hotel have parking",
-  "hotel-book stay": "number of nights for the hotel reservation",
-  "hotel-book day": "starting day of the hotel booking",
-  "hotel-book people": "number of people for the hotel booking",
-  "hotel-area": "area or place of the hotel",
-  "hotel-stars": "star rating of the hotel",
-  "hotel-internet": "does the hotel have internet or wifi",
-  "hotel-name": "name of the hotel",
-  "hotel-phone": "phone number of the hotel",
-  "hotel-postcode": "postcode of the hotel",
-  "hotel-reference": "booking reference of the hotel booking",
-  "hotel-address": "street address of the hotel",
-  "train-destination": "train station you want to travel to",
-  "train-day": "day of the train booking",
-  "train-departure": "train station you want to leave from",
-  "train-arrive by": "arrival time of the train",
-  "train-book people": "number of people for the train booking",
-  "train-leave at": "departure time for the train",
-  "train-duration": "duration of the train journey",
-  "train-trainid": "train identifier or number",
-  "train-price": "how much does the train trip cost",
-  "train-reference": "booking reference of the train booking",
-  "attraction-type": "type of attraction or point of interest",
-  "attraction-area": "area or place of the attraction",
-  "attraction-name": "name of the attraction",
-  "attraction-phone": "phone number of the attraction",
-  "attraction-entrance fee": "entrace fee at the attraction",
-  "attraction-address": "street address of the attraction",
-  "attraction-postcode": "postcode of the attraction",
-  "restaurant-book people": "number of people for the restaurant booking",
-  "restaurant-book day": "weekday for the restaurant booking",
-  "restaurant-book time": "time of the restaurant booking",
-  "restaurant-food": "type of food served at the restaurant",
-  "restaurant-price range": "preferred cost or price of the restaurant",
-  "restaurant-name": "name of the restaurant",
-  "restaurant-area": "area or place of the restaurant",
-  "restaurant-postcode": "postcode of the restaurant",
-  "restaurant-phone": "phone number of the restaurant",
-  "restaurant-address": "street address of the restaurant",
-  "restaurant-reference": "booking reference of the hotel booking",
-  "taxi-leave at": "what time you want the taxi to leave by",
-  "taxi-destination": "where you want the taxi to drop you off",
-  "taxi-departure": "where you want the taxi to pick you up",
-  "taxi-arrive by": "what time you to arrive at your destination",
-  "taxi-taxi types": "vehicle type of the taxi",
-  "taxi-taxi phone": "phone number of the taxi",
-  "hospital-department": "name of hospital department",
-  "hospital-address": "street address of the hospital",
-  "hospital-phone": "phone number of the hospital",
-  "hospital-postcode": "postcode of the hospital",
-  "police-postcode": "postcode of the police station",
-  "police-address": "street address of the police station",
-  "police-phone": "phone number of the police station"
-}
\ No newline at end of file
diff --git a/convlab/dst/setsumbt/multiwoz/dataset/ontology.py b/convlab/dst/setsumbt/multiwoz/dataset/ontology.py
deleted file mode 100644
index c6b9c3365764eb8f1c1cab5d68674dd85b39ce2a..0000000000000000000000000000000000000000
--- a/convlab/dst/setsumbt/multiwoz/dataset/ontology.py
+++ /dev/null
@@ -1,168 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2020 DSML Group, Heinrich Heine University, Düsseldorf
-# Authors: Carel van Niekerk (niekerk@hhu.de)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Create Ontology Embeddings"""
-
-import json
-import os
-import random
-
-import torch
-import numpy as np
-
-
-# Slot mapping table for description extractions
-# SLOT_NAME_MAPPINGS = {
-#     'arrive at': 'arriveAt',
-#     'arrive by': 'arriveBy',
-#     'leave at': 'leaveAt',
-#     'leave by': 'leaveBy',
-#     'arriveby': 'arriveBy',
-#     'arriveat': 'arriveAt',
-#     'leaveat': 'leaveAt',
-#     'leaveby': 'leaveBy',
-#     'price range': 'pricerange'
-# }
-
-# Set up global data directory
-def set_datadir(dir):
-    global DATA_DIR
-    DATA_DIR = dir
-
-
-# Set seeds
-def set_seed(args):
-    random.seed(args.seed)
-    np.random.seed(args.seed)
-    torch.manual_seed(args.seed)
-    if args.n_gpu > 0:
-        torch.cuda.manual_seed_all(args.seed)
-
-
-# Get embeddings for slots and candidates
-def get_slot_candidate_embeddings(set_type, args, tokenizer, embedding_model, save_to_file=True):
-    # Get set alots and candidates
-    reader = open(os.path.join(DATA_DIR, 'ontology_%s.json' % set_type), 'r')
-    ontology = json.load(reader)
-    reader.close()
-
-    reader = open(os.path.join(DATA_DIR, 'slot_descriptions.json'), 'r')
-    slot_descriptions = json.load(reader)
-    reader.close()
-
-    embedding_model.eval()
-
-    slots = dict()
-    for slot in ontology:
-        if args.use_descriptions:
-            # d, s = slot.split('-', 1)
-            # s = SLOT_NAME_MAPPINGS[s] if s in SLOT_NAME_MAPPINGS else s
-            # s = d + '-' + s
-            # if slot in slot_descriptions:
-            desc = slot_descriptions[slot]
-            # elif slot.lower() in slot_descriptions:
-            #     desc = slot_descriptions[s.lower()]
-            # else:
-            #     desc = slot.replace('-', ' ')
-        else:
-            desc = slot
-
-        # Tokenize slot and get embeddings
-        feats = tokenizer.encode_plus(desc, add_special_tokens = True,
-                                            max_length = args.max_slot_len, padding='max_length',
-                                            truncation = 'longest_first')
-
-        with torch.no_grad():
-            input_ids = torch.tensor([feats['input_ids']]).to(embedding_model.device) # [1, max_slot_len]
-            if 'token_type_ids' in feats:
-                token_type_ids = torch.tensor([feats['token_type_ids']]).to(embedding_model.device) # [1, max_slot_len]
-                if 'attention_mask' in feats:
-                    attention_mask = torch.tensor([feats['attention_mask']]).to(embedding_model.device) # [1, max_slot_len]
-                    feats, pooled_feats = embedding_model(input_ids=input_ids, token_type_ids=token_type_ids,
-                                            attention_mask=attention_mask)
-                    attention_mask = attention_mask.unsqueeze(-1).repeat((1, 1, feats.size(-1)))
-                    feats = feats * attention_mask # [1, max_slot_len, hidden_dim]
-                else:
-                    feats, pooled_feats = embedding_model(input_ids=input_ids, token_type_ids=token_type_ids)
-            else:
-                if 'attention_mask' in feats:
-                    attention_mask = torch.tensor([feats['attention_mask']]).to(embedding_model.device)
-                    feats, pooled_feats = embedding_model(input_ids=input_ids, attention_mask=attention_mask)
-                    attention_mask = attention_mask.unsqueeze(-1).repeat((1, 1, feats.size(-1)))
-                    feats = feats * attention_mask # [1, max_slot_len, hidden_dim]
-                else:
-                    feats, pooled_feats = embedding_model(input_ids=input_ids) # [1, max_slot_len, hidden_dim]
-        
-        if args.set_similarity:
-            slot_emb = feats[0, :, :].detach().cpu() # [seq_len, hidden_dim]
-        else:
-            if args.candidate_pooling == 'cls' and pooled_feats is not None:
-                slot_emb = pooled_feats[0, :].detach().cpu() # [hidden_dim]
-            elif args.candidate_pooling == 'mean':
-                feats = feats.sum(1)
-                feats = torch.nn.functional.layer_norm(feats, feats.size())
-                slot_emb = feats[0, :].detach().cpu() # [hidden_dim]
-
-        # Tokenize value candidates and get embeddings
-        values = ontology[slot]
-        is_requestable = False
-        if 'request' in values:
-            is_requestable = True
-            values.remove('request')
-        if values:
-            feats = [tokenizer.encode_plus(val, add_special_tokens = True,
-                                                max_length = args.max_candidate_len, padding='max_length',
-                                                truncation = 'longest_first')
-                    for val in values]
-            with torch.no_grad():
-                input_ids = torch.tensor([f['input_ids'] for f in feats]).to(embedding_model.device) # [num_candidates, max_candidate_len]
-                if 'token_type_ids' in feats[0]:
-                    token_type_ids = torch.tensor([f['token_type_ids'] for f in feats]).to(embedding_model.device) # [num_candidates, max_candidate_len]
-                    if 'attention_mask' in feats[0]:
-                        attention_mask = torch.tensor([f['attention_mask'] for f in feats]).to(embedding_model.device) # [num_candidates, max_candidate_len]
-                        feats, pooled_feats = embedding_model(input_ids=input_ids, token_type_ids=token_type_ids,
-                                                attention_mask=attention_mask)
-                        attention_mask = attention_mask.unsqueeze(-1).repeat((1, 1, feats.size(-1)))
-                        feats = feats * attention_mask # [num_candidates, max_candidate_len, hidden_dim]
-                    else:
-                        feats, pooled_feats = embedding_model(input_ids=input_ids, token_type_ids=token_type_ids) # [num_candidates, max_candidate_len, hidden_dim]
-                else:
-                    if 'attention_mask' in feats[0]:
-                        attention_mask = torch.tensor([f['attention_mask'] for f in feats]).to(embedding_model.device)
-                        feats, pooled_feats = embedding_model(input_ids=input_ids, attention_mask=attention_mask)
-                        attention_mask = attention_mask.unsqueeze(-1).repeat((1, 1, feats.size(-1)))
-                        feats = feats * attention_mask # [num_candidates, max_candidate_len, hidden_dim]
-                    else:
-                        feats, pooled_feats = embedding_model(input_ids=input_ids) # [num_candidates, max_candidate_len, hidden_dim]
-            
-            if args.set_similarity:
-                feats = feats.detach().cpu() # [num_candidates, max_candidate_len, hidden_dim]
-            else:
-                if args.candidate_pooling == 'cls' and pooled_feats is not None:
-                    feats = pooled_feats.detach().cpu()
-                elif args.candidate_pooling == "mean":
-                    feats = feats.sum(1)
-                    feats = torch.nn.functional.layer_norm(feats, feats.size())
-                    feats = feats.detach().cpu()
-        else:
-            feats = None
-        slots[slot] = (slot_emb, feats, is_requestable)
-
-    # Dump tensors for use in training
-    if save_to_file:
-        writer = os.path.join(args.output_dir, 'database', '%s.db' % set_type)
-        torch.save(slots, writer)
-    
-    return slots
diff --git a/convlab/dst/setsumbt/multiwoz/dataset/utils.py b/convlab/dst/setsumbt/multiwoz/dataset/utils.py
deleted file mode 100644
index 485dee643148ad1d37069064ebc4e2e4553b3dac..0000000000000000000000000000000000000000
--- a/convlab/dst/setsumbt/multiwoz/dataset/utils.py
+++ /dev/null
@@ -1,446 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2020 DSML Group, Heinrich Heine University, Düsseldorf
-# Code adapted from the TRADE preprocessing code (https://github.com/jasonwu0731/trade-dst)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""MultiWOZ2.1/3 data processing utilities"""
-
-import re
-import os
-
-from convlab.util.multiwoz.multiwoz_slot_trans import REF_SYS_DA
-from convlab.dst.rule.multiwoz import normalize_value
-
-# ACTIVE_DOMAINS = ['attraction', 'hotel', 'restaurant', 'taxi', 'train']
-ACTIVE_DOMAINS = ['attraction', 'hotel', 'restaurant', 'taxi', 'train', 'hospital', 'police']
-def set_util_domains(domains):
-    global ACTIVE_DOMAINS
-    ACTIVE_DOMAINS = [d for d in domains if d in ACTIVE_DOMAINS]
-
-MAPPING_PATH = os.path.abspath(__file__).replace('utils.py', 'mapping.pair')
-# Read replacement pairs from the mapping.pair file
-REPLACEMENTS = []
-for line in open(MAPPING_PATH).readlines():
-    tok_from, tok_to = line.replace('\n', '').split('\t')
-    REPLACEMENTS.append((' ' + tok_from + ' ', ' ' + tok_to + ' '))
-
-# Extract belief state from mturk annotations
-def build_dialoguestate(metadata, get_domains=False):
-    domains_list = [dom for dom in ACTIVE_DOMAINS if dom in metadata]
-    dialogue_state, domains = [], []
-    for domain in domains_list:
-        active = False
-        # Extract booking information
-        booking = []
-        for slot in sorted(metadata[domain]['book'].keys()):
-            if slot != 'booked':
-                if metadata[domain]['book'][slot] == 'not mentioned':
-                    continue
-                if metadata[domain]['book'][slot] != '':
-                    val = ['%s-book %s' % (domain, slot.strip().lower()), clean_text(metadata[domain]['book'][slot])]
-                    dialogue_state.append(val)
-                    active = True
-
-        for slot in metadata[domain]['semi']:
-            if metadata[domain]['semi'][slot] == 'not mentioned':
-                continue
-            elif metadata[domain]['semi'][slot] in ['dont care', 'dontcare', "don't care", 'don not care',
-                                                    'do not care', 'does not care']:
-                dialogue_state.append(['%s-%s' % (domain, slot.strip().lower()), 'do not care'])
-                active = True
-            elif metadata[domain]['semi'][slot]:
-                dialogue_state.append(['%s-%s' % (domain, slot.strip().lower()), clean_text(metadata[domain]['semi'][slot])])
-                active = True
-
-        if active:
-            domains.append(domain)
-
-    if get_domains:
-        return domains
-    return clean_dialoguestate(dialogue_state)
-
-
-PRICERANGE = ['do not care', 'cheap', 'moderate', 'expensive']
-BOOLEAN = ['do not care', 'yes', 'no']
-DAYS = ['do not care', 'monday', 'tuesday', 'wednesday', 'thursday',
-        'friday', 'saterday', 'sunday']
-QUANTITIES = ['do not care', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10 or more']
-TIME = [[(i, j) for i in range(24)] for j in range(0, 60, 5)]
-TIME = ['do not care'] + ['%02i:%02i' % t for l in TIME for t in l]
-
-VALUE_MAP = {'guesthouse': 'guest house', 'belfry': 'belfray', '-': ' ', '&': 'and', 'b and b': 'bed and breakfast',
-            'cityroomz': 'city roomz', '  ': ' ', 'acorn house': 'acorn guest house', 'marriot': 'marriott',
-            'worth house': 'the worth house', 'alesbray lodge guest house': 'aylesbray lodge',
-            'huntingdon hotel': 'huntingdon marriott hotel', 'huntingd': 'huntingdon marriott hotel',
-            'jamaicanchinese': 'chinese', 'barbequemodern european': 'modern european',
-            'north americanindian': 'north american', 'caribbeanindian': 'indian', 'sheeps': "sheep's"}
-
-def map_values(value):
-    for old, new in VALUE_MAP.items():
-        value = value.replace(old, new)
-    return value
-
-def clean_dialoguestate(states, is_acts=False):
-    # path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))))
-    # path = os.path.join(path, 'data/multiwoz/value_dict.json')
-    # value_dict = json.load(open(path))
-    clean_state = []
-    for slot, value in states:
-        if 'pricerange' in slot:
-            d, s = slot.split('-', 1)
-            s = 'price range'
-            slot = f'{d}-{s}'
-            if value in PRICERANGE:
-                clean_state.append([slot, value])
-            elif True in [v in value for v in PRICERANGE]:
-                value = [v for v in PRICERANGE if v in value][0]
-                clean_state.append([slot, value])
-            elif value == '?' and is_acts:
-                clean_state.append([slot, value])
-            else:
-                continue
-        elif 'parking' in slot or 'internet' in slot:
-            if value in BOOLEAN:
-                clean_state.append([slot, value])
-            if value == 'free':
-                value = 'yes'
-                clean_state.append([slot, value])
-            elif True in [v in value for v in BOOLEAN]:
-                value = [v for v in BOOLEAN if v in value][0]
-                clean_state.append([slot, value])
-            elif value == '?' and is_acts:
-                clean_state.append([slot, value])
-            else:
-                continue
-        elif 'day' in slot:
-            if value in DAYS:
-                clean_state.append([slot, value])
-            elif True in [v in value for v in DAYS]:
-                value = [v for v in DAYS if v in value][0]
-                clean_state.append([slot, value])
-            else:
-                continue
-        elif 'people' in slot or 'duration' in slot or 'stay' in slot:
-            if value in QUANTITIES:
-                clean_state.append([slot, value])
-            elif True in [v in value for v in QUANTITIES]:
-                value = [v for v in QUANTITIES if v in value][0]
-                clean_state.append([slot, value])
-            elif value == '?' and is_acts:
-                clean_state.append([slot, value])
-            else:
-                try:
-                    value = int(value)
-                    if value >= 10:
-                        value = '10 or more'
-                        clean_state.append([slot, value])
-                    else:
-                        continue
-                except:
-                    continue
-        elif 'time' in slot or 'leaveat' in slot or 'arriveby' in slot:
-            if 'leaveat' in slot:
-                d, s = slot.split('-', 1)
-                s = 'leave at'
-                slot = f'{d}-{s}'
-            if 'arriveby' in slot:
-                d, s = slot.split('-', 1)
-                s = 'arrive by'
-                slot = f'{d}-{s}'
-            if value in TIME:
-                if value == 'do not care':
-                    clean_state.append([slot, value])
-                else:
-                    h, m = value.split(':')
-                    if int(m) % 5 == 0:
-                        clean_state.append([slot, value])
-                    else:
-                        m = round(int(m) / 5) * 5
-                        h = int(h)
-                        if m == 60:
-                            m = 0
-                            h += 1
-                        if h >= 24:
-                            h -= 24
-                        value = '%02i:%02i' % (h, m)
-                        clean_state.append([slot, value])
-            elif True in [v in value for v in TIME]:
-                value = [v for v in TIME if v in value][0]
-                h, m = value.split(':')
-                if int(m) % 5 == 0:
-                    clean_state.append([slot, value])
-                else:
-                    m = round(int(m) / 5) * 5
-                    h = int(h)
-                    if m == 60:
-                        m = 0
-                        h += 1
-                    if h >= 24:
-                        h -= 24
-                    value = '%02i:%02i' % (h, m)
-                    clean_state.append([slot, value])
-            elif value == '?' and is_acts:
-                clean_state.append([slot, value])
-            else:
-                continue
-        elif 'stars' in slot:
-            if len(value) == 1 or value == 'do not care':
-                clean_state.append([slot, value])
-            elif value == '?' and is_acts:
-                clean_state.append([slot, value])
-            elif len(value) > 1:
-                try:
-                    value = int(value[0])
-                    value = str(value)
-                    clean_state.append([slot, value])
-                except:
-                    continue
-        elif 'area' in slot:
-            if '|' in value:
-                value = value.split('|', 1)[0]
-            clean_state.append([slot, value])
-        else:
-            if '|' in value:
-                value = value.split('|', 1)[0]
-                value = map_values(value)
-                # d, s = slot.split('-', 1)
-                # value = normalize_value(value_dict, d, s, value)
-            clean_state.append([slot, value])
-    
-    return clean_state
-
-
-# Module to process a dialogue and check its validity
-def process_dialogue(dialogue, max_utt_len=128):
-    if len(dialogue['log']) % 2 != 0:
-        return None
-
-    # Extract user and system utterances
-    usr_utts, sys_utts = [], []
-    avg_len = sum(len(utt['text'].split(' ')) for utt in dialogue['log'])
-    avg_len = avg_len / len(dialogue['log'])
-    if avg_len > max_utt_len:
-        return None
-
-    # If the first term is a system turn then ignore dialogue
-    if dialogue['log'][0]['metadata']:
-        return None
-
-    usr, sys = None, None
-    for turn in dialogue['log']:
-        if not is_ascii(turn['text']):
-            return None
-
-        if not usr or not sys:
-            if len(turn['metadata']) == 0:
-                usr = turn
-            else:
-                sys = turn
-        
-        if usr and sys:
-            states = build_dialoguestate(sys['metadata'], get_domains = False)
-            sys['dialogue_states'] = states
-
-            usr_utts.append(usr)
-            sys_utts.append(sys)
-            usr, sys = None, None
-
-    dial_clean = dict()
-    dial_clean['usr_log'] = usr_utts
-    dial_clean['sys_log'] = sys_utts
-    return dial_clean
-
-
-# Get new domains
-def get_act_domains(prev, crnt):
-    diff = {}
-    if not prev or not crnt:
-        return diff
-
-    for ((prev_dom, prev_val), (crnt_dom, crnt_val)) in zip(prev.items(), crnt.items()):
-        assert prev_dom == crnt_dom
-        if prev_val != crnt_val:
-            diff[crnt_dom] = crnt_val
-    return diff
-
-
-# Get current domains
-def get_domains(dial_log, turn_id, prev_domain):
-    if turn_id == 1:
-        active = build_dialoguestate(dial_log[turn_id]['metadata'], get_domains=True)
-        acts = format_acts(dial_log[turn_id].get('dialog_act', {})) if not active else []
-        acts = [domain for intent, domain, slot, value in acts if domain not in ['', 'general']]
-        active += acts
-        crnt = active[0] if active else ''
-    else:
-        active = get_act_domains(dial_log[turn_id - 2]['metadata'], dial_log[turn_id]['metadata'])
-        active = list(active.keys())
-        acts = format_acts(dial_log[turn_id].get('dialog_act', {})) if not active else []
-        acts = [domain for intent, domain, slot, value in acts if domain not in ['', 'general']]
-        active += acts
-        crnt = [prev_domain] if not active else active
-        crnt = crnt[0]
-
-    return crnt
-
-
-# Function to extract dialogue info from data
-def extract_dialogue(dialogue, max_utt_len=50):
-    dialogue = process_dialogue(dialogue, max_utt_len)
-    if not dialogue:
-        return None
-
-    usr_utts = [turn['text'] for turn in dialogue['usr_log']]
-    sys_utts = [turn['text'] for turn in dialogue['sys_log']]
-    # sys_acts = [format_acts(turn['dialog_act']) if 'dialog_act' in turn else [] for turn in dialogue['sys_log']]
-    usr_acts = [format_acts(turn['dialog_act']) if 'dialog_act' in turn else [] for turn in dialogue['usr_log']]
-    dialogue_states = [turn['dialogue_states'] for turn in dialogue['sys_log']]
-    domains = [turn['domain'] for turn in dialogue['usr_log']]
-
-    # dial = [{'usr': u,'sys': s, 'usr_a': ua, 'sys_a': a, 'domain': d, 'ds': v}
-    #         for u, s, ua, a, d, v in zip(usr_utts, sys_utts, usr_acts, sys_acts, domains, dialogue_states)]
-    dial = [{'usr': u,'sys': s, 'usr_a': ua, 'domain': d, 'ds': v}
-            for u, s, ua, d, v in zip(usr_utts, sys_utts, usr_acts, domains, dialogue_states)]    
-    return dial
-
-
-def format_acts(acts):
-    new_acts = []
-    for key, item in acts.items():
-        domain, intent = key.split('-', 1)
-        if domain.lower() in ACTIVE_DOMAINS + ['general']:
-            state = []
-            for slot, value in item:
-                slot = str(REF_SYS_DA[domain].get(slot, slot)).lower() if domain in REF_SYS_DA else slot
-                value = clean_text(value)
-                slot = slot.replace('_', ' ').replace('ref', 'reference')
-                state.append([f'{domain.lower()}-{slot}', value])
-            state = clean_dialoguestate(state, is_acts=True)
-            if domain == 'general':
-                if intent in ['thank', 'bye']:
-                    state = [['general-none', 'none']]
-                else:
-                    state = []
-            for slot, value in state:
-                if slot not in ['train-people']:
-                    slot = slot.split('-', 1)[-1]
-                    new_acts.append([intent.lower(), domain.lower(), slot, value])
-    
-    return new_acts
-                
-
-# Fix act labels
-def fix_delexicalisation(turn):
-    if 'dialog_act' in turn:
-        for dom, act in turn['dialog_act'].items():
-            if 'Attraction' in dom:
-                if 'restaurant_' in turn['text']:
-                    turn['text'] = turn['text'].replace("restaurant", "attraction")
-                if 'hotel_' in turn['text']:
-                    turn['text'] = turn['text'].replace("hotel", "attraction")
-            if 'Hotel' in dom:
-                if 'attraction_' in turn['text']:
-                    turn['text'] = turn['text'].replace("attraction", "hotel")
-                if 'restaurant_' in turn['text']:
-                    turn['text'] = turn['text'].replace("restaurant", "hotel")
-            if 'Restaurant' in dom:
-                if 'attraction_' in turn['text']:
-                    turn['text'] = turn['text'].replace("attraction", "restaurant")
-                if 'hotel_' in turn['text']:
-                    turn['text'] = turn['text'].replace("hotel", "restaurant")
-
-    return turn
-
-
-# Check if a character is an ascii character
-def is_ascii(s):
-    return all(ord(c) < 128 for c in s)
-
-
-# Insert white space
-def separate_token(token, text):
-    sidx = 0
-    while True:
-        # Find next instance of token
-        sidx = text.find(token, sidx)
-        if sidx == -1:
-            break
-        # If the token is already seperated continue to next
-        if sidx + 1 < len(text) and re.match('[0-9]', text[sidx - 1]) and \
-                re.match('[0-9]', text[sidx + 1]):
-            sidx += 1
-            continue
-        # Create white space separation around token
-        if text[sidx - 1] != ' ':
-            text = text[:sidx] + ' ' + text[sidx:]
-            sidx += 1
-        if sidx + len(token) < len(text) and text[sidx + len(token)] != ' ':
-            text = text[:sidx + 1] + ' ' + text[sidx + 1:]
-        sidx += 1
-    return text
-
-
-def clean_text(text):
-    # Replace white spaces in front and end
-    text = re.sub(r'^\s*|\s*$', '', text.strip().lower())
-
-    # Replace b&v or 'b and b' with 'bed and breakfast'
-    text = re.sub(r"b&b", "bed and breakfast", text)
-    text = re.sub(r"b and b", "bed and breakfast", text)
-
-    # Fix apostrophies
-    text = re.sub(u"(\u2018|\u2019)", "'", text)
-
-    # Correct punctuation
-    text = text.replace(';', ',')
-    text = re.sub('$\/', '', text)
-    text = text.replace('/', ' and ')
-
-    # Replace special characters
-    text = text.replace('-', ' ')
-    text = re.sub('[\"\<>@\(\)]', '', text)
-
-    # Insert white space around special tokens:
-    for token in ['?', '.', ',', '!']:
-        text = separate_token(token, text)
-
-    # insert white space for 's
-    text = separate_token('\'s', text)
-
-    # replace it's, does't, you'd ... etc
-    text = re.sub('^\'', '', text)
-    text = re.sub('\'$', '', text)
-    text = re.sub('\'\s', ' ', text)
-    text = re.sub('\s\'', ' ', text)
-
-    # Perform pair replacements listed in the mapping.pair file
-    for fromx, tox in REPLACEMENTS:
-        text = ' ' + text + ' '
-        text = text.replace(fromx, tox)[1:-1]
-
-    # Remove multiple spaces
-    text = re.sub(' +', ' ', text)
-
-    # Concatenate numbers eg '1 3' -> '13'
-    tokens = text.split()
-    i = 1
-    while i < len(tokens):
-        if re.match(u'^\d+$', tokens[i]) and \
-                re.match(u'\d+$', tokens[i - 1]):
-            tokens[i - 1] += tokens[i]
-            del tokens[i]
-        else:
-            i += 1
-    text = ' '.join(tokens)
-
-    return text
diff --git a/convlab/dst/setsumbt/predict_user_actions.py b/convlab/dst/setsumbt/predict_user_actions.py
new file mode 100644
index 0000000000000000000000000000000000000000..2c304a569cb5e29920332ed21c8f862dd00c1e48
--- /dev/null
+++ b/convlab/dst/setsumbt/predict_user_actions.py
@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
+# Authors: Carel van Niekerk (niekerk@hhu.de)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Predict dataset user action using SetSUMBT Model"""
+
+from copy import deepcopy
+import os
+import json
+from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
+
+from convlab.util.custom_util import flatten_acts as flatten
+from convlab.util import load_dataset, load_policy_data
+from convlab.dst.setsumbt import SetSUMBTTracker
+
+
+def flatten_acts(acts: dict) -> list:
+    """
+    Flatten dictionary actions.
+
+    Args:
+        acts: Dictionary acts
+
+    Returns:
+        flat_acts: Flattened actions
+    """
+    acts = flatten(acts)
+    flat_acts = []
+    for intent, domain, slot, value in acts:
+        flat_acts.append([intent,
+                          domain,
+                          slot if slot != 'none' else '',
+                          value.lower() if value != 'none' else ''])
+
+    return flat_acts
+
+
+def get_user_actions(context: list, system_acts: list) -> list:
+    """
+    Extract user actions from the data.
+
+    Args:
+        context: Previous dialogue turns.
+        system_acts: List of flattened system actions.
+
+    Returns:
+        user_acts: List of flattened user actions.
+    """
+    user_acts = context[-1]['dialogue_acts']
+    user_acts = flatten_acts(user_acts)
+    if len(context) == 3:
+        prev_state = context[-3]['state']
+        cur_state = context[-1]['state']
+        for domain, substate in cur_state.items():
+            for slot, value in substate.items():
+                if prev_state[domain][slot] != value:
+                    act = ['inform', domain, slot, value]
+                    if act not in user_acts and act not in system_acts:
+                        user_acts.append(act)
+
+    return user_acts
+
+
+def extract_dataset(dataset: str = 'multiwoz21') -> list:
+    """
+    Extract acts and utterances from the dataset.
+
+    Args:
+        dataset: Dataset name
+
+    Returns:
+        data: Extracted data
+    """
+    data = load_dataset(dataset_name=dataset)
+    raw_data = load_policy_data(data, data_split='test', context_window_size=3)['test']
+
+    dialogue = list()
+    data = list()
+    for turn in raw_data:
+        state = dict()
+        state['system_utterance'] = turn['context'][-2]['utterance'] if len(turn['context']) > 1 else ''
+        state['utterance'] = turn['context'][-1]['utterance']
+        state['system_actions'] = turn['context'][-2]['dialogue_acts'] if len(turn['context']) > 1 else {}
+        state['system_actions'] = flatten_acts(state['system_actions'])
+        state['user_actions'] = get_user_actions(turn['context'], state['system_actions'])
+        dialogue.append(state)
+        if turn['terminated']:
+            data.append(dialogue)
+            dialogue = list()
+
+    return data
+
+
+def unflatten_acts(acts: list) -> dict:
+    """
+    Convert acts from flat list format to dict format.
+
+    Args:
+        acts: List of flat actions.
+
+    Returns:
+        unflat_acts: Dictionary of acts.
+    """
+    binary_acts = []
+    cat_acts = []
+    for intent, domain, slot, value in acts:
+        include = True if (domain == 'general') or (slot != 'none') else False
+        if include and (value == '' or value == 'none' or intent == 'request'):
+            binary_acts.append({'intent': intent,
+                                'domain': domain,
+                                'slot': slot if slot != 'none' else ''})
+        elif include:
+            cat_acts.append({'intent': intent,
+                             'domain': domain,
+                             'slot': slot if slot != 'none' else '',
+                             'value': value})
+
+    unflat_acts = {'categorical': cat_acts, 'binary': binary_acts, 'non-categorical': list()}
+
+    return unflat_acts
+
+
+def predict_user_acts(data: list, tracker: SetSUMBTTracker) -> list:
+    """
+    Predict the user actions using the SetSUMBT Tracker.
+
+    Args:
+        data: List of dialogues.
+        tracker: SetSUMBT Tracker
+
+    Returns:
+        predict_result: List of turns containing predictions and true user actions.
+    """
+    tracker.init_session()
+    predict_result = []
+    for dial_idx, dialogue in enumerate(data):
+        for turn_idx, state in enumerate(dialogue):
+            sample = {'dial_idx': dial_idx, 'turn_idx': turn_idx}
+
+            tracker.state['history'].append(['sys', state['system_utterance']])
+            predicted_state = deepcopy(tracker.update(state['utterance']))
+            tracker.state['history'].append(['usr', state['utterance']])
+            tracker.state['system_action'] = state['system_actions']
+
+            sample['predictions'] = {'dialogue_acts': unflatten_acts(predicted_state['user_action'])}
+            sample['dialogue_acts'] = unflatten_acts(state['user_actions'])
+
+            predict_result.append(sample)
+
+        tracker.init_session()
+
+    return predict_result
+
+
+if __name__ =="__main__":
+    parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
+    parser.add_argument('--dataset_name', type=str, help='Name of dataset', default="multiwoz21")
+    parser.add_argument('--model_path', type=str, help='Path to model dir')
+    args = parser.parse_args()
+
+    dataset = extract_dataset(args.dataset_name)
+    tracker = SetSUMBTTracker(args.model_path)
+    predict_results = predict_user_acts(dataset, tracker)
+
+    with open(os.path.join(args.model_path, 'predictions', 'test_nlu.json'), 'w') as writer:
+        json.dump(predict_results, writer, indent=2)
+        writer.close()
diff --git a/convlab/dst/setsumbt/process_mwoz_data.py b/convlab/dst/setsumbt/process_mwoz_data.py
deleted file mode 100755
index 701a523613961d83a5188fa9ab0cf786b19a5a7e..0000000000000000000000000000000000000000
--- a/convlab/dst/setsumbt/process_mwoz_data.py
+++ /dev/null
@@ -1,99 +0,0 @@
-import os
-import json
-from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
-
-import torch
-from tqdm import tqdm
-
-from convlab.dst.setsumbt.multiwoz.Tracker import SetSUMBTTracker
-from convlab.util.multiwoz.lexicalize import deflat_da, flat_da
-
-
-def load_data(path):
-    with open(path, 'r') as reader:
-        data = json.load(reader)
-        reader.close()
-    
-    return data
-
-
-def load_tracker(model_checkpoint):
-    model = SetSUMBTTracker(model_path=model_checkpoint)
-    model.init_session()
-
-    return model
-
-
-def process_dialogue(dial, model, get_full_belief_state):
-    model.store_full_belief_state = get_full_belief_state
-    model.init_session()
-
-    model.state['history'].append(['sys', ''])
-    processed_dial = []
-    belief_state = {}
-    for turn in dial:
-        if not turn['metadata']:
-            state = model.update(turn['text'])
-            model.state['history'].append(['usr', turn['text']])
-            
-            acts = model.state['user_action']
-            acts = [[val.replace('-', ' ') for val in act] for act in acts]
-            acts = flat_da(acts)
-            acts = deflat_da(acts)
-            turn['dialog_act'] = acts
-        else:
-            model.state['history'].append(['sys', turn['text']])
-            turn['metadata'] = model.state['belief_state']
-        
-        if get_full_belief_state:
-            for slot, probs in model.full_belief_state.items():
-                if slot not in belief_state:
-                    belief_state[slot] = [probs[0]]
-                else:
-                    belief_state[slot].append(probs[0])
-        
-        processed_dial.append(turn)
-    
-    if get_full_belief_state:
-        belief_state = {slot: torch.cat(probs, 0).cpu() for slot, probs in belief_state.items()}
-
-    return processed_dial, belief_state
-
-
-def process_dialogues(data, model, get_full_belief_state=False):
-    processed_data = {}
-    belief_states = {}
-    for dial_id, dial in tqdm(data.items()):
-        dial['log'], bs = process_dialogue(dial['log'], model, get_full_belief_state)
-        processed_data[dial_id] = dial
-        if get_full_belief_state:
-            belief_states[dial_id] = bs
-
-    return processed_data, belief_states
-
-
-def get_arguments():
-    parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
-
-    parser.add_argument('--model_path')
-    parser.add_argument('--data_path')
-    parser.add_argument('--get_full_belief_state', action='store_true')
-
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    args = get_arguments()
-
-    print('Loading data and model...')
-    data = load_data(os.path.join(args.data_path, 'data.json'))
-    model = load_tracker(args.model_path)
-
-    print('Processing data...\n')
-    data, belief_states = process_dialogues(data, model, get_full_belief_state=args.get_full_belief_state)
-    
-    print('Saving results...\n')
-    torch.save(belief_states, os.path.join(args.data_path, 'setsumbt_belief_states.bin'))
-    with open(os.path.join(args.data_path, 'setsumbt_data.json'), 'w') as writer:
-        json.dump(data, writer, indent=2)
-        writer.close()
diff --git a/convlab/dst/setsumbt/run.py b/convlab/dst/setsumbt/run.py
index b9c9a75b86d47cd5db733b4755d6af11f08b827d..e45bf129f0c9f2c5c1fba01d4b5eb80e29a5a1f0 100644
--- a/convlab/dst/setsumbt/run.py
+++ b/convlab/dst/setsumbt/run.py
@@ -33,8 +33,8 @@ def main():
     if args.run_nbt:
         from convlab.dst.setsumbt.do.nbt import main
         main(args, config)
-    if args.run_calibration:
-        from convlab.dst.setsumbt.do.calibration import main
+    if args.run_evaluation:
+        from convlab.dst.setsumbt.do.evaluate import main
         main(args, config)
 
 
diff --git a/convlab/dst/setsumbt/tracker.py b/convlab/dst/setsumbt/tracker.py
new file mode 100644
index 0000000000000000000000000000000000000000..eca7f1749369f9569d6b923312a93cd317e0701c
--- /dev/null
+++ b/convlab/dst/setsumbt/tracker.py
@@ -0,0 +1,446 @@
+import os
+import json
+import copy
+import logging
+
+import torch
+import transformers
+from transformers import BertModel, BertConfig, BertTokenizer, RobertaModel, RobertaConfig, RobertaTokenizer
+
+from convlab.dst.setsumbt.modeling import RobertaSetSUMBT, BertSetSUMBT
+from convlab.dst.setsumbt.modeling.training import set_ontology_embeddings
+from convlab.dst.dst import DST
+from convlab.util.custom_util import model_downloader
+
+USE_CUDA = torch.cuda.is_available()
+transformers.logging.set_verbosity_error()
+
+
+class SetSUMBTTracker(DST):
+    """SetSUMBT Tracker object for Convlab dialogue system"""
+
+    def __init__(self,
+                 model_path: str = "",
+                 model_type: str = "roberta",
+                 return_turn_pooled_representation: bool = False,
+                 return_confidence_scores: bool = False,
+                 confidence_threshold='auto',
+                 return_belief_state_entropy: bool = False,
+                 return_belief_state_mutual_info: bool = False,
+                 store_full_belief_state: bool = False):
+        """
+        Args:
+            model_path: Model path or download URL
+            model_type: Transformer type (roberta/bert)
+            return_turn_pooled_representation: If true a turn level pooled representation is returned
+            return_confidence_scores: If true act confidence scores are included in the state
+            confidence_threshold: Confidence threshold value for constraints or option auto
+            return_belief_state_entropy: If true belief state distribution entropies are included in the state
+            return_belief_state_mutual_info: If true belief state distribution mutual infos are included in the state
+            store_full_belief_state: If true full belief state is stored within tracker object
+        """
+        super(SetSUMBTTracker, self).__init__()
+
+        self.model_type = model_type
+        self.model_path = model_path
+        self.return_turn_pooled_representation = return_turn_pooled_representation
+        self.return_confidence_scores = return_confidence_scores
+        self.confidence_threshold = confidence_threshold
+        self.return_belief_state_entropy = return_belief_state_entropy
+        self.return_belief_state_mutual_info = return_belief_state_mutual_info
+        self.store_full_belief_state = store_full_belief_state
+        if self.store_full_belief_state:
+            self.full_belief_state = {}
+        self.info_dict = {}
+
+        # Download model if needed
+        if not os.path.exists(self.model_path):
+            # Get path /.../convlab/dst/setsumbt/multiwoz/models
+            download_path = os.path.dirname(os.path.abspath(__file__))
+            download_path = os.path.join(download_path, 'models')
+            if not os.path.exists(download_path):
+                os.mkdir(download_path)
+            model_downloader(download_path, self.model_path)
+            # Downloadable model path format http://.../model_name.zip
+            self.model_path = self.model_path.split('/')[-1].replace('.zip', '')
+            self.model_path = os.path.join(download_path, self.model_path)
+
+        # Select model type based on the encoder
+        if model_type == "roberta":
+            self.config = RobertaConfig.from_pretrained(self.model_path)
+            self.tokenizer = RobertaTokenizer
+            self.model = RobertaSetSUMBT
+        elif model_type == "bert":
+            self.config = BertConfig.from_pretrained(self.model_path)
+            self.tokenizer = BertTokenizer
+            self.model = BertSetSUMBT
+        else:
+            logging.debug("Name Error: Not Implemented")
+
+        self.device = torch.device('cuda') if USE_CUDA else torch.device('cpu')
+
+        self.load_weights()
+
+    def load_weights(self):
+        """Load model weights and model ontology"""
+        logging.info('Loading SetSUMBT pretrained model.')
+        self.tokenizer = self.tokenizer.from_pretrained(self.config.tokenizer_name)
+        logging.info(f'Model tokenizer loaded from {self.config.tokenizer_name}.')
+        self.model = self.model.from_pretrained(self.model_path, config=self.config)
+        logging.info(f'Model loaded from {self.model_path}.')
+
+        # Transfer model to compute device and setup eval environment
+        self.model = self.model.to(self.device)
+        self.model.eval()
+        logging.info(f'Model transferred to device: {self.device}')
+
+        logging.info('Loading model ontology')
+        f = open(os.path.join(self.model_path, 'database', 'test.json'), 'r')
+        self.ontology = json.load(f)
+        f.close()
+
+        db = torch.load(os.path.join(self.model_path, 'database', 'test.db'))
+        set_ontology_embeddings(self.model, db)
+
+        if self.return_confidence_scores:
+            logging.info('Model returns user action and belief state confidence scores.')
+            self.get_thresholds(self.confidence_threshold)
+            logging.info('Uncertain Querying set up and thresholds set up at:')
+            logging.info(self.confidence_thresholds)
+        if self.return_belief_state_entropy:
+            logging.info('Model returns belief state distribution entropy scores (Total uncertainty).')
+        if self.return_belief_state_mutual_info:
+            logging.info('Model returns belief state distribution mutual information scores (Knowledge uncertainty).')
+        logging.info('Ontology loaded successfully.')
+
+    def get_thresholds(self, threshold='auto') -> dict:
+        """
+        Setup dictionary of domain specific confidence thresholds
+
+        Args:
+            threshold: Threshold value or option auto
+
+        Returns:
+            confidence_thresholds: Domain specific confidence thresholds
+        """
+        self.confidence_thresholds = dict()
+        for domain, substate in self.ontology.items():
+            for slot, slot_info in substate.items():
+                # Auto thresholds are set based on the number of value candidates per slot
+                if domain not in self.confidence_thresholds:
+                    self.confidence_thresholds[domain] = dict()
+                if threshold == 'auto':
+                    thres = 1.0 / (float(len(slot_info['possible_values'])) - 2.1)
+                    self.confidence_thresholds[domain][slot] = max(0.05, thres)
+                else:
+                    self.confidence_thresholds[domain][slot] = max(0.05, threshold)
+
+        return self.confidence_thresholds
+
+    def init_session(self):
+        self.state = dict()
+        self.state['belief_state'] = dict()
+        self.state['booked'] = dict()
+        for domain, substate in self.ontology.items():
+            self.state['belief_state'][domain] = dict()
+            for slot, slot_info in substate.items():
+                if slot_info['possible_values'] and slot_info['possible_values'] != ['?']:
+                    self.state['belief_state'][domain][slot] = ''
+            self.state['booked'][domain] = list()
+        self.state['history'] = []
+        self.state['system_action'] = []
+        self.state['user_action'] = []
+        self.state['terminated'] = False
+        self.active_domains = {}
+        self.hidden_states = None
+        self.info_dict = {}
+
+    def update(self, user_act: str = '') -> dict:
+        """
+        Update user actions and dialogue and belief states.
+
+        Args:
+            user_act:
+
+        Returns:
+
+        """
+        prev_state = self.state
+        _output = self.predict(self.get_features(user_act))
+
+        # Format state entropy
+        if _output[5] is not None:
+            state_entropy = dict()
+            for slot, e in _output[5].items():
+                domain, slot = slot.split('-', 1)
+                if domain not in state_entropy:
+                    state_entropy[domain] = dict()
+                state_entropy[domain][slot] = e
+        else:
+            state_entropy = None
+
+        # Format state mutual information
+        if _output[6] is not None:
+            state_mutual_info = dict()
+            for slot, mi in _output[6].items():
+                domain, slot = slot.split('-', 1)
+                if domain not in state_mutual_info:
+                    state_mutual_info[domain] = dict()
+                state_mutual_info[domain][slot] = mi[0, 0]
+        else:
+            state_mutual_info = None
+
+        # Format all confidence scores
+        belief_state_confidence = None
+        if _output[4] is not None:
+            belief_state_confidence = dict()
+            belief_state_conf, request_probs, active_domain_probs, general_act_probs = _output[4]
+            for slot, p in belief_state_conf.items():
+                domain, slot = slot.split('-', 1)
+                if domain not in belief_state_confidence:
+                    belief_state_confidence[domain] = dict()
+                if slot not in belief_state_confidence[domain]:
+                    belief_state_confidence[domain][slot] = dict()
+                belief_state_confidence[domain][slot]['inform'] = p
+
+            for slot, p in request_probs.items():
+                domain, slot = slot.split('-', 1)
+                if domain not in belief_state_confidence:
+                    belief_state_confidence[domain] = dict()
+                if slot not in belief_state_confidence[domain]:
+                    belief_state_confidence[domain][slot] = dict()
+                belief_state_confidence[domain][slot]['request'] = p
+
+            for domain, p in active_domain_probs.items():
+                if domain not in belief_state_confidence:
+                    belief_state_confidence[domain] = dict()
+                belief_state_confidence[domain]['none'] = {'inform': p}
+
+            if 'general' not in belief_state_confidence:
+                belief_state_confidence['general'] = dict()
+            belief_state_confidence['general']['none'] = general_act_probs
+
+        # Get new domain activation actions
+        new_domains = [d for d, active in _output[1].items() if active]
+        new_domains = [d for d in new_domains if not self.active_domains.get(d, False)]
+        self.active_domains = _output[1]
+
+        user_acts = _output[2]
+        for domain in new_domains:
+            user_acts.append(['inform', domain, 'none', 'none'])
+
+        new_belief_state = copy.deepcopy(prev_state['belief_state'])
+        for domain, substate in _output[0].items():
+            for slot, value in substate.items():
+                value = '' if value == 'none' else value
+                value = 'dontcare' if value == 'do not care' else value
+                value = 'guesthouse' if value == 'guest house' else value
+
+                if domain not in new_belief_state:
+                    if domain == 'bus':
+                        continue
+                    else:
+                        logging.debug('Error: domain <{}> not in belief state'.format(domain))
+
+                # Uncertainty clipping of state
+                if belief_state_confidence is not None:
+                    threshold = self.confidence_thresholds[domain][slot]
+                    if belief_state_confidence[domain][slot].get('inform', 1.0) < threshold:
+                        value = ''
+
+                new_belief_state[domain][slot] = value
+                if prev_state['belief_state'][domain][slot] != value:
+                    user_acts.append(['inform', domain, slot, value])
+                else:
+                    bug = f'Unknown slot name <{slot}> with value <{value}> of domain <{domain}>'
+                    logging.debug(bug)
+
+        new_state = copy.deepcopy(dict(prev_state))
+        new_state['belief_state'] = new_belief_state
+        new_state['active_domains'] = self.active_domains
+        if belief_state_confidence is not None:
+            new_state['belief_state_probs'] = belief_state_confidence
+        if state_entropy is not None:
+            new_state['entropy'] = state_entropy
+        if state_mutual_info is not None:
+            new_state['mutual_information'] = state_mutual_info
+
+        user_acts = [act for act in user_acts if act not in new_state['system_action']]
+        new_state['user_action'] = user_acts
+
+        if _output[3] is not None:
+            new_state['turn_pooled_representation'] = _output[3]
+
+        self.state = new_state
+        self.info_dict = copy.deepcopy(dict(new_state))
+
+        return self.state
+
+    def predict(self, features: dict) -> tuple:
+        """
+        Model forward pass and prediction post processing.
+
+        Args:
+            features: Dictionary of model input features
+
+        Returns:
+            out: Model predictions and uncertainty features
+        """
+        state_mutual_info = None
+        with torch.no_grad():
+            turn_pooled_representation = None
+            if self.return_turn_pooled_representation:
+                _outputs = self.model(input_ids=features['input_ids'], token_type_ids=features['token_type_ids'],
+                                      attention_mask=features['attention_mask'], hidden_state=self.hidden_states,
+                                      get_turn_pooled_representation=True)
+                belief_state = _outputs[0]
+                request_probs = _outputs[1]
+                active_domain_probs = _outputs[2]
+                general_act_probs = _outputs[3]
+                self.hidden_states = _outputs[4]
+                turn_pooled_representation = _outputs[5]
+            elif self.return_belief_state_mutual_info:
+                _outputs = self.model(input_ids=features['input_ids'], token_type_ids=features['token_type_ids'],
+                                      attention_mask=features['attention_mask'], hidden_state=self.hidden_states,
+                                      get_turn_pooled_representation=True, calculate_state_mutual_info=True)
+                belief_state = _outputs[0]
+                request_probs = _outputs[1]
+                active_domain_probs = _outputs[2]
+                general_act_probs = _outputs[3]
+                self.hidden_states = _outputs[4]
+                state_mutual_info = _outputs[5]
+            else:
+                _outputs = self.model(input_ids=features['input_ids'], token_type_ids=features['token_type_ids'],
+                                      attention_mask=features['attention_mask'], hidden_state=self.hidden_states,
+                                      get_turn_pooled_representation=False)
+                belief_state, request_probs, active_domain_probs, general_act_probs, self.hidden_states = _outputs
+
+        # Convert belief state into dialog state
+        dialogue_state = dict()
+        for slot, probs in belief_state.items():
+            dom, slot = slot.split('-', 1)
+            if dom not in dialogue_state:
+                dialogue_state[dom] = dict()
+            val = self.ontology[dom][slot]['possible_values'][probs[0, 0, :].argmax().item()]
+            if val != 'none':
+                dialogue_state[dom][slot] = val
+
+        if self.store_full_belief_state:
+            self.full_belief_state = belief_state
+
+        # Obtain model output probabilities
+        if self.return_confidence_scores:
+            state_entropy = None
+            if self.return_belief_state_entropy:
+                state_entropy = {slot: probs[0, 0, :] for slot, probs in belief_state.items()}
+                state_entropy = {slot: self.relative_entropy(p).item() for slot, p in state_entropy.items()}
+
+            # Confidence score is the max probability across all not "none" values candidates.
+            belief_state_conf = {slot: probs[0, 0, 1:].max().item() for slot, probs in belief_state.items()}
+            _request_probs = {slot: p[0, 0].item() for slot, p in request_probs.items()}
+            _active_domain_probs = {domain: p[0, 0].item() for domain, p in active_domain_probs.items()}
+            _general_act_probs = {'bye': general_act_probs[0, 0, 1].item(), 'thank': general_act_probs[0, 0, 2].item()}
+            confidence_scores = (belief_state_conf, _request_probs, _active_domain_probs, _general_act_probs)
+        else:
+            confidence_scores = None
+            state_entropy = None
+
+        # Construct request action prediction
+        request_acts = [slot for slot, p in request_probs.items() if p[0, 0].item() > 0.5]
+        request_acts = [slot.split('-', 1) for slot in request_acts]
+        request_acts = [['request', domain, slot, '?'] for domain, slot in request_acts]
+
+        # Construct active domain set
+        active_domains = {domain: p[0, 0].item() > 0.5 for domain, p in active_domain_probs.items()}
+
+        # Construct general domain action
+        general_acts = general_act_probs[0, 0, :].argmax(-1).item()
+        general_acts = [[], ['bye'], ['thank']][general_acts]
+        general_acts = [[act, 'general', 'none', 'none'] for act in general_acts]
+
+        user_acts = request_acts + general_acts
+
+        out = (dialogue_state, active_domains, user_acts, turn_pooled_representation, confidence_scores)
+        out += (state_entropy, state_mutual_info)
+        return out
+
+    def relative_entropy(self, probs: torch.Tensor) -> torch.Tensor:
+        """
+        Compute relative entrop for a probability distribution
+
+        Args:
+            probs: Probability distributions
+
+        Returns:
+            entropy: Relative entropy
+        """
+        entropy = probs * torch.log(probs + 1e-8)
+        entropy = -entropy.sum()
+        # Maximum entropy of a K dimentional distribution is ln(K)
+        entropy /= torch.log(torch.tensor(probs.size(-1)).float())
+
+        return entropy
+
+    def get_features(self, user_act: str) -> dict:
+        """
+        Tokenize utterances and construct model input features
+
+        Args:
+            user_act: User action string
+
+        Returns:
+            features: Model input features
+        """
+        # Extract system utterance from dialog history
+        context = self.state['history']
+        if context:
+            if context[-1][0] != 'sys':
+                system_act = ''
+            else:
+                system_act = context[-1][-1]
+        else:
+            system_act = ''
+
+        # Tokenize dialog
+        features = self.tokenizer.encode_plus(user_act, system_act, add_special_tokens=True,
+                                              max_length=self.config.max_turn_len, padding='max_length',
+                                              truncation='longest_first')
+
+        input_ids = torch.tensor(features['input_ids']).reshape(
+            1, 1, -1).to(self.device) if 'input_ids' in features else None
+        token_type_ids = torch.tensor(features['token_type_ids']).reshape(
+            1, 1, -1).to(self.device) if 'token_type_ids' in features else None
+        attention_mask = torch.tensor(features['attention_mask']).reshape(
+            1, 1, -1).to(self.device) if 'attention_mask' in features else None
+        features = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': attention_mask}
+
+        return features
+
+
+# if __name__ == "__main__":
+#     from convlab.policy.vector.vector_uncertainty import VectorUncertainty
+#     # from convlab.policy.vector.vector_binary import VectorBinary
+#     tracker = SetSUMBTTracker(model_path='/gpfs/project/niekerk/src/SetSUMBT/models/SetSUMBT+ActPrediction-multiwoz21-roberta-gru-cosine-labelsmoothing-Seed0-10-08-22-12-42',
+#                               return_confidence_scores=True, confidence_threshold='auto',
+#                               return_belief_state_entropy=True)
+#     vector = VectorUncertainty(use_state_total_uncertainty=True, confidence_thresholds=tracker.confidence_thresholds,
+#                                use_masking=True)
+#     # vector = VectorBinary()
+#     tracker.init_session()
+#
+#     state = tracker.update('hey. I need a cheap restaurant.')
+#     tracker.state['history'].append(['usr', 'hey. I need a cheap restaurant.'])
+#     tracker.state['history'].append(['sys', 'There are many cheap places, which food do you like?'])
+#     state = tracker.update('If you have something Asian that would be great.')
+#     tracker.state['history'].append(['usr', 'If you have something Asian that would be great.'])
+#     tracker.state['history'].append(['sys', 'The Golden Wok is a nice cheap chinese restaurant.'])
+#     tracker.state['system_action'] = [['inform', 'restaurant', 'food', 'chinese'],
+#                                       ['inform', 'restaurant', 'name', 'the golden wok']]
+#     state = tracker.update('Great. Where are they located?')
+#     tracker.state['history'].append(['usr', 'Great. Where are they located?'])
+#     state = tracker.state
+#     state['terminated'] = False
+#     state['booked'] = {}
+#
+#     print(state)
+#     print(vector.state_vectorize(state))
diff --git a/convlab/dst/setsumbt/utils.py b/convlab/dst/setsumbt/utils.py
index 75a6a1febd7510f1a6d152676b82d709abf177f0..ff374116a3f8e88e6219fdc8b134d40b0bee7caf 100644
--- a/convlab/dst/setsumbt/utils.py
+++ b/convlab/dst/setsumbt/utils.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2021 DSML Group, Heinrich Heine University, Düsseldorf
+# Copyright 2022 DSML Group, Heinrich Heine University, Düsseldorf
 # Authors: Carel van Niekerk (niekerk@hhu.de)
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,151 +15,124 @@
 # limitations under the License.
 """SetSUMBT utils"""
 
-import re
 import os
+import json
 import shutil
 from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
-from glob import glob
 from datetime import datetime
 
-from google.cloud import storage
+from git import Repo
 
 
-def get_args(MODELS):
+def get_args(base_models: dict):
     # Get arguments
     parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
 
+    # Config file usage
+    parser.add_argument('--starting_config_name', default=None, type=str)
+
     # Optional
-    parser.add_argument('--tensorboard_path',
-                        help='Path to tensorboard', default='')
+    parser.add_argument('--tensorboard_path', help='Path to tensorboard', default='')
     parser.add_argument('--logging_path', help='Path for log file', default='')
-    parser.add_argument(
-        '--seed', help='Seed value for reproducability', default=0, type=int)
+    parser.add_argument('--seed', help='Seed value for reproducibility', default=0, type=int)
 
     # DATASET (Optional)
-    parser.add_argument(
-        '--dataset', help='Dataset Name: multiwoz21/simr', default='multiwoz21')
-    parser.add_argument('--shrink_active_domains', help='Shrink active domains to only well represented test set domains',
-                        action='store_true')
-    parser.add_argument(
-        '--data_dir', help='Data storage directory', default=None)
-    parser.add_argument(
-        '--max_dialogue_len', help='Maximum number of turns per dialogue', default=12, type=int)
-    parser.add_argument(
-        '--max_turn_len', help='Maximum number of tokens per turn', default=64, type=int)
-    parser.add_argument(
-        '--max_slot_len', help='Maximum number of tokens per slot description', default=12, type=int)
-    parser.add_argument('--max_candidate_len',
-                        help='Maximum number of tokens per value candidate', default=12, type=int)
-    parser.add_argument('--force_processing', action='store_true',
-                        help='Force preprocessing of data.')
-    parser.add_argument('--data_sampling_size',
-                        help='Resampled dataset size', default=-1, type=int)
-    parser.add_argument('--use_descriptions', help='Use slot descriptions rather than slot names for embeddings',
+    parser.add_argument('--dataset', help='Dataset Name (See Convlab 3 unified format for possible datasets',
+                        default='multiwoz21')
+    parser.add_argument('--dataset_train_ratio', help='Fraction of training set to use in training', default=1.0,
+                        type=float)
+    parser.add_argument('--max_dialogue_len', help='Maximum number of turns per dialogue', default=12, type=int)
+    parser.add_argument('--max_turn_len', help='Maximum number of tokens per turn', default=64, type=int)
+    parser.add_argument('--max_slot_len', help='Maximum number of tokens per slot description', default=12, type=int)
+    parser.add_argument('--max_candidate_len', help='Maximum number of tokens per value candidate', default=12,
+                        type=int)
+    parser.add_argument('--force_processing', action='store_true', help='Force preprocessing of data.')
+    parser.add_argument('--data_sampling_size', help='Resampled dataset size', default=-1, type=int)
+    parser.add_argument('--no_descriptions', help='Do not use slot descriptions rather than slot names for embeddings',
                         action='store_true')
 
     # MODEL
     # Environment
-    parser.add_argument(
-        '--output_dir', help='Output storage directory', default=None)
-    parser.add_argument('--model_type', help='Encoder Model Type: bert/roberta',
-                        default='roberta')
-    parser.add_argument('--model_name_or_path', help='Name or path of the pretrained model.',
-                        default=None)
+    parser.add_argument('--output_dir', help='Output storage directory', default=None)
+    parser.add_argument('--model_type', help='Encoder Model Type: bert/roberta', default='roberta')
+    parser.add_argument('--model_name_or_path', help='Name or path of the pretrained model.', default=None)
     parser.add_argument('--candidate_embedding_model_name', default=None,
                         help='Name of the pretrained candidate embedding model.')
+    parser.add_argument('--transformers_local_files_only', help='Use local files only for huggingface transformers',
+                        action='store_true')
 
     # Architecture
     parser.add_argument('--freeze_encoder', help='No training performed on the turn encoder Bert Model',
                         action='store_true')
     parser.add_argument('--slot_attention_heads', help='Number of attention heads for slot conditioning',
                         default=12, type=int)
-    parser.add_argument('--dropout_rate', help='Dropout Rate',
-                        default=0.3, type=float)
-    parser.add_argument(
-        '--nbt_type', help='Belief Tracker type: gru/lstm', default='gru')
+    parser.add_argument('--dropout_rate', help='Dropout Rate', default=0.3, type=float)
+    parser.add_argument('--nbt_type', help='Belief Tracker type: gru/lstm', default='gru')
     parser.add_argument('--nbt_hidden_size', help='Hidden embedding size for the Neural Belief Tracker',
                         default=300, type=int)
-    parser.add_argument(
-        '--nbt_layers', help='Number of RNN layers in the NBT', default=1, type=int)
-    parser.add_argument(
-        '--rnn_zero_init', help='Zero Initialise RNN hidden states', action='store_true')
+    parser.add_argument('--nbt_layers', help='Number of RNN layers in the NBT', default=1, type=int)
+    parser.add_argument('--rnn_zero_init', help='Zero Initialise RNN hidden states', action='store_true')
     parser.add_argument('--distance_measure', default='cosine',
                         help='Similarity measure for candidate scoring: cosine/euclidean')
-    parser.add_argument(
-        '--ensemble_size', help='Number of models in ensemble', default=-1, type=int)
-    parser.add_argument('--set_similarity', action='store_true',
-                        help='Set True to not use set similarity (Model tracks latent belief state as sequence and performs semantic similarity of sets)')
-    parser.add_argument('--set_pooling', help='Set pooling method for set similarity model using single embedding distances',
+    parser.add_argument('--ensemble_size', help='Number of models in ensemble', default=-1, type=int)
+    parser.add_argument('--no_set_similarity', action='store_true', help='Set True to not use set similarity')
+    parser.add_argument('--set_pooling',
+                        help='Set pooling method for set similarity model using single embedding distances',
                         default='cnn')
-    parser.add_argument('--candidate_pooling', help='Pooling approach for non set based candidate representations: cls/mean',
+    parser.add_argument('--candidate_pooling',
+                        help='Pooling approach for non set based candidate representations: cls/mean',
                         default='mean')
-    parser.add_argument('--predict_actions', help='Model predicts user actions and active domain',
+    parser.add_argument('--no_action_prediction', help='Model does not predicts user actions and active domain',
                         action='store_true')
 
     # Loss
-    parser.add_argument('--loss_function', help='Loss Function for training: crossentropy/bayesianmatching/labelsmoothing/distillation/distribution_distillation',
+    parser.add_argument('--loss_function',
+                        help='Loss Function for training: crossentropy/bayesianmatching/labelsmoothing/...',
                         default='labelsmoothing')
     parser.add_argument('--kl_scaling_factor', help='Scaling factor for KL divergence in bayesian matching loss',
                         type=float)
     parser.add_argument('--prior_constant', help='Constant parameter for prior in bayesian matching loss',
                         type=float)
-    parser.add_argument('--ensemble_smoothing',
-                        help='Ensemble distribution smoothing constant', type=float)
-    parser.add_argument('--annealing_base_temp', help='Ensemble Distribution destillation temp annealing base temp',
+    parser.add_argument('--ensemble_smoothing', help='Ensemble distribution smoothing constant', type=float)
+    parser.add_argument('--annealing_base_temp', help='Ensemble Distribution distillation temp annealing base temp',
+                        type=float)
+    parser.add_argument('--annealing_cycle_len', help='Ensemble Distribution distillation temp annealing cycle length',
                         type=float)
-    parser.add_argument('--annealing_cycle_len', help='Ensemble Distribution destillation temp annealing cycle length',
+    parser.add_argument('--label_smoothing', help='Label smoothing coefficient.', type=float)
+    parser.add_argument('--user_goal_loss_weight', help='Weight of the user goal prediction loss. 0.0<weight<=1.0',
                         type=float)
-    parser.add_argument('--inhibiting_factor',
-                        help='Inhibiting factor for Inhibited Softmax CE', type=float)
-    parser.add_argument('--label_smoothing',
-                        help='Label smoothing coefficient.', type=float)
-    parser.add_argument(
-        '--user_goal_loss_weight', help='Weight of the user goal prediction loss. 0.0<weight<=1.0', type=float)
-    parser.add_argument(
-        '--user_request_loss_weight', help='Weight of the user request prediction loss. 0.0<weight<=1.0', type=float)
-    parser.add_argument(
-        '--user_general_act_loss_weight', help='Weight of the user general act prediction loss. 0.0<weight<=1.0', type=float)
-    parser.add_argument(
-        '--active_domain_loss_weight', help='Weight of the active domain prediction loss. 0.0<weight<=1.0', type=float)
+    parser.add_argument('--user_request_loss_weight',
+                        help='Weight of the user request prediction loss. 0.0<weight<=1.0', type=float)
+    parser.add_argument('--user_general_act_loss_weight',
+                        help='Weight of the user general act prediction loss. 0.0<weight<=1.0', type=float)
+    parser.add_argument('--active_domain_loss_weight',
+                        help='Weight of the active domain prediction loss. 0.0<weight<=1.0', type=float)
 
     # TRAINING
-    parser.add_argument('--train_batch_size',
-                        help='Training Set Batch Size', default=4, type=int)
-    parser.add_argument('--max_training_steps', help='Maximum number of training update steps',
-                        default=-1, type=int)
+    parser.add_argument('--train_batch_size', help='Training Set Batch Size', default=8, type=int)
+    parser.add_argument('--max_training_steps', help='Maximum number of training update steps', default=-1, type=int)
     parser.add_argument('--gradient_accumulation_steps', default=1, type=int,
                         help='Number of batches accumulated for one update step')
-    parser.add_argument('--num_train_epochs',
-                        help='Number of training epochs', default=50, type=int)
+    parser.add_argument('--num_train_epochs', help='Number of training epochs', default=50, type=int)
     parser.add_argument('--patience', help='Number of training steps without improving model before stopping.',
-                        default=25, type=int)
-    parser.add_argument(
-        '--weight_decay', help='Weight decay rate', default=0.01, type=float)
-    parser.add_argument('--learning_rate',
-                        help='Initial Learning Rate', default=5e-5, type=float)
-    parser.add_argument('--warmup_proportion', help='Warmup proportion for linear scheduler',
-                        default=0.2, type=float)
-    parser.add_argument(
-        '--max_grad_norm', help='Maximum norm of the loss gradients', default=1.0, type=float)
-    parser.add_argument(
-        '--save_steps', help='Number of update steps between saving model', default=-1, type=int)
-    parser.add_argument(
-        '--keep_models', help='How many model checkpoints should be kept during training', default=1, type=int)
+                        default=20, type=int)
+    parser.add_argument('--weight_decay', help='Weight decay rate', default=0.01, type=float)
+    parser.add_argument('--learning_rate', help='Initial Learning Rate', default=5e-5, type=float)
+    parser.add_argument('--warmup_proportion', help='Warmup proportion for linear scheduler', default=0.2, type=float)
+    parser.add_argument('--max_grad_norm', help='Maximum norm of the loss gradients', default=1.0, type=float)
+    parser.add_argument('--save_steps', help='Number of update steps between saving model', default=-1, type=int)
+    parser.add_argument('--keep_models', help='How many model checkpoints should be kept during training',
+                        default=1, type=int)
 
     # CALIBRATION
-    parser.add_argument(
-        '--temp_scaling', help='Temperature scaling coefficient', default=1.0, type=float)
+    parser.add_argument('--temp_scaling', help='Temperature scaling coefficient', default=1.0, type=float)
 
     # EVALUATION
-    parser.add_argument('--dev_batch_size',
-                        help='Dev Set Batch Size', default=16, type=int)
-    parser.add_argument('--test_batch_size',
-                        help='Test Set Batch Size', default=16, type=int)
+    parser.add_argument('--dev_batch_size', help='Dev Set Batch Size', default=16, type=int)
+    parser.add_argument('--test_batch_size', help='Test Set Batch Size', default=16, type=int)
 
     # COMPUTING
-    parser.add_argument(
-        '--n_gpu', help='Number of GPUs to use', default=1, type=int)
+    parser.add_argument('--n_gpu', help='Number of GPUs to use', default=1, type=int)
     parser.add_argument('--fp16', action='store_true',
                         help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit")
     parser.add_argument('--fp16_opt_level', type=str, default='O1',
@@ -167,32 +140,35 @@ def get_args(MODELS):
                              "See details at https://nvidia.github.io/apex/amp.html")
 
     # ACTIONS
-    parser.add_argument('--run_nbt', help='Run NBT script',
-                        action='store_true')
-    parser.add_argument('--run_calibration',
-                        help='Run calibration', action='store_true')
+    parser.add_argument('--run_nbt', help='Run NBT script', action='store_true')
+    parser.add_argument('--run_evaluation', help='Run evaluation script', action='store_true')
 
     # RUN_NBT ACTIONS
-    parser.add_argument(
-        '--do_train', help='Perform training', action='store_true')
-    parser.add_argument(
-        '--do_eval', help='Perform model evaluation during training', action='store_true')
-    parser.add_argument(
-        '--do_test', help='Evaulate model on test data', action='store_true')
+    parser.add_argument('--do_train', help='Perform training', action='store_true')
+    parser.add_argument('--do_eval', help='Perform model evaluation during training', action='store_true')
+    parser.add_argument('--do_test', help='Evaluate model on test data', action='store_true')
     args = parser.parse_args()
 
-    # Setup default directories
-    if not args.data_dir:
-        args.data_dir = os.path.dirname(os.path.abspath(__file__))
-        args.data_dir = os.path.join(args.data_dir, 'data')
-        os.makedirs(args.data_dir, exist_ok=True)
+    if args.starting_config_name:
+        args = get_starting_config(args)
+
+    if args.do_train:
+        args.do_eval = True
+
+    # Simplify args
+    args.set_similarity = not args.no_set_similarity
+    args.use_descriptions = not args.no_descriptions
+    args.predict_actions = not args.no_action_prediction
 
+    # Setup default directories
     if not args.output_dir:
         args.output_dir = os.path.dirname(os.path.abspath(__file__))
         args.output_dir = os.path.join(args.output_dir, 'models')
 
-        name = 'SetSUMBT'
-        name += '-Acts' if args.predict_actions else ''
+        name = 'SetSUMBT' if args.set_similarity else 'SUMBT'
+        name += '+ActPrediction' if args.predict_actions else ''
+        name += '-' + args.dataset
+        name += '-' + str(round(args.dataset_train_ratio*100)) + '%' if args.dataset_train_ratio != 1.0 else ''
         name += '-' + args.model_type
         name += '-' + args.nbt_type
         name += '-' + args.distance_measure
@@ -208,9 +184,6 @@ def get_args(MODELS):
             args.kl_scaling_factor = 0.001
         if not args.prior_constant:
             args.prior_constant = 1.0
-    if args.loss_function == 'inhibitedce':
-        if not args.inhibiting_factor:
-            args.inhibiting_factor = 1.0
     if args.loss_function == 'labelsmoothing':
         if not args.label_smoothing:
             args.label_smoothing = 0.05
@@ -233,10 +206,8 @@ def get_args(MODELS):
         if not args.active_domain_loss_weight:
             args.active_domain_loss_weight = 0.2
 
-    args.tensorboard_path = args.tensorboard_path if args.tensorboard_path else os.path.join(
-        args.output_dir, 'tb_logs')
-    args.logging_path = args.logging_path if args.logging_path else os.path.join(
-        args.output_dir, 'run.log')
+    args.tensorboard_path = args.tensorboard_path if args.tensorboard_path else os.path.join(args.output_dir, 'tb_logs')
+    args.logging_path = args.logging_path if args.logging_path else os.path.join(args.output_dir, 'run.log')
 
     # Default model_name's
     if not args.model_name_or_path:
@@ -250,30 +221,62 @@ def get_args(MODELS):
     if not args.candidate_embedding_model_name:
         args.candidate_embedding_model_name = args.model_name_or_path
 
-    if args.model_type in MODELS:
-        configClass = MODELS[args.model_type][-2]
+    if args.model_type in base_models:
+        config_class = base_models[args.model_type][-2]
     else:
         raise NameError('NotImplemented')
-    config = build_config(configClass, args)
+    config = build_config(config_class, args)
     return args, config
 
 
-def build_config(configClass, args):
-    if args.model_type == 'fasttext':
-        config = configClass.from_pretrained('bert-base-uncased')
-        config.model_type == 'fasttext'
-        config.fasttext_path = args.model_name_or_path
-        config.vocab_size = None
-    elif not os.path.exists(args.model_name_or_path):
-        config = configClass.from_pretrained(args.model_name_or_path)
+def get_starting_config(args):
+    path = os.path.dirname(os.path.realpath(__file__))
+    path = os.path.join(path, 'configs', f"{args.starting_config_name}.json")
+    reader = open(path, 'r')
+    config = json.load(reader)
+    reader.close()
+
+    if "model_type" in config:
+        if config["model_type"].lower() == 'setsumbt':
+            config["model_type"] = 'roberta'
+            config["no_set_similarity"] = False
+            config["no_descriptions"] = False
+        elif config["model_type"].lower() == 'sumbt':
+            config["model_type"] = 'bert'
+            config["no_set_similarity"] = True
+            config["no_descriptions"] = False
+
+    variables = vars(args).keys()
+    for key, value in config.items():
+        if key in variables:
+            setattr(args, key, value)
+
+    return args
+
+
+def get_git_info():
+    repo = Repo(os.path.dirname(os.path.realpath(__file__)), search_parent_directories=True)
+    branch_name = repo.active_branch.name
+    commit_hex = repo.head.object.hexsha
+
+    info = f"{branch_name}/{commit_hex}"
+    return info
+
+
+def build_config(config_class, args):
+    config = config_class.from_pretrained(args.model_name_or_path)
+    config.code_version = get_git_info()
+    if not os.path.exists(args.model_name_or_path):
         config.tokenizer_name = args.model_name_or_path
-    elif 'tod-bert' in args.model_name_or_path.lower():
-        config = configClass.from_pretrained(args.model_name_or_path)
+    try:
+        config.tokenizer_name = config.tokenizer_name
+    except AttributeError:
         config.tokenizer_name = args.model_name_or_path
-    else:
-        config = configClass.from_pretrained(args.model_name_or_path)
-    if args.candidate_embedding_model_name:
-        config.candidate_embedding_model_name = args.candidate_embedding_model_name
+    try:
+        config.candidate_embedding_model_name = config.candidate_embedding_model_name
+    except:
+        if args.candidate_embedding_model_name:
+            config.candidate_embedding_model_name = args.candidate_embedding_model_name
     config.max_dialogue_len = args.max_dialogue_len
     config.max_turn_len = args.max_turn_len
     config.max_slot_len = args.max_slot_len
diff --git a/convlab/dst/trippy/README.md b/convlab/dst/trippy/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..178c578287152c894c8727156a8db6e3c255b549
--- /dev/null
+++ b/convlab/dst/trippy/README.md
@@ -0,0 +1,59 @@
+# Introduction
+
+This is the TripPy DST module for ConvLab-3.
+
+## Supported encoders
+
+* RoBERTa
+* BERT (full support w.i.p.)
+* ELECTRA (full support w.i.p.)
+
+## Supported datasets
+
+* MultiWOZ 2.X
+* Unified Data Format
+
+## Requirements
+
+transformers (tested: 4.18.0)
+torch (tested: 1.8.0)
+
+# Parameters
+
+```
+model_type # Default: "roberta", Type of the model (Supported: "roberta", "bert", "electra")
+model_name # Default: "roberta-base", Name of the model (Use -h to print a list of names)
+model_path # Path to a model checkpoint
+dataset_name # Default: "multiwoz21", Name of the dataset the model was trained on and/or is being applied to
+local_files_only # Default: False, Set to True to load local files only. Useful for offline systems 
+nlu_usr_config # Path to a NLU config file. Only needed for internal evaluation
+nlu_sys_config # Path to a NLU config file. Only needed for internal evaluation
+nlu_usr_path # Path to a NLU model file. Only needed for internal evaluation
+nlu_sys_path # Path to a NLU model file. Only needed for internal evaluation
+no_eval # Default: True, Set to True if internal evaluation should be conducted
+no_history # Default: False, Set to True if dialogue history should be omitted during inference
+```
+
+# Training
+
+TripPy can easily be trained for the abovementioned supported datasets using the original code in the official [TripPy repository](https://gitlab.cs.uni-duesseldorf.de/general/dsml/trippy-public). Simply clone the code and run the appropriate DO.* script to train a TripPy DST. After training, set model_path to the preferred checkpoint to use TripPy in ConvLab-3.
+
+# Training and evaluation with PPO policy
+
+Switch to the directory:
+```
+cd ../../policy/ppo
+```
+
+Edit trippy_config.json and trippy_config_eval.json accordingly, e.g., edit paths to model checkpoints.
+
+For training, run
+```
+train.py --path trippy_config.json
+```
+
+For evaluation, set training epochs to 0.
+
+# Paper
+
+[TripPy: A Triple Copy Strategy for Value Independent Neural Dialog State Tracking](https://aclanthology.org/2020.sigdial-1.4/)
diff --git a/convlab/dst/trippy/__init__.py b/convlab/dst/trippy/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..bb36114aeabe0859fdae1c4230c147a7a40305ff
--- /dev/null
+++ b/convlab/dst/trippy/__init__.py
@@ -0,0 +1 @@
+from convlab.dst.trippy.tracker import TRIPPY
diff --git a/convlab/dst/trippy/dataset_interfacer.py b/convlab/dst/trippy/dataset_interfacer.py
new file mode 100644
index 0000000000000000000000000000000000000000..51a692bbaa50349a70053f05933a0cea1f8246c3
--- /dev/null
+++ b/convlab/dst/trippy/dataset_interfacer.py
@@ -0,0 +1,174 @@
+# coding=utf-8
+#
+# Copyright 2020-2022 Heinrich Heine University Duesseldorf
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import logging
+
+
+class DatasetInterfacer(object):
+    _domain_map_trippy_to_udf = {}
+    _slot_map_trippy_to_udf = {}
+    _generic_referral = {}
+
+    def __init__(self):
+        pass
+
+    def map_trippy_to_udf(self, domain, slot):
+        d = self._domain_map_trippy_to_udf.get(domain, domain)
+        s = slot
+        if d in self._slot_map_trippy_to_udf:
+            s = self._slot_map_trippy_to_udf[d].get(slot, slot)
+        return d, s
+
+    def get_generic_referral(self, domain, slot):
+        d, s = self.map_trippy_to_udf(domain, slot)
+        ref = "the %s %s" % (d, s)
+        if d in self._generic_referral:
+            ref = self._generic_referral[d].get(s, s)
+        return ref
+
+    def normalize_values(self, text):
+        return text
+
+    def normalize_text(self, text):
+        return text
+
+    def normalize_prediction(self, domain, slot, value, predictions=None, config=None):
+        return value
+
+
+class MultiwozInterfacer(DatasetInterfacer):
+    _slot_map_trippy_to_udf = {
+        'hotel': {
+            'pricerange': 'price range',
+            'book_stay': 'book stay',
+            'book_day': 'book day',
+            'book_people': 'book people',
+            'addr': 'address',
+            'post': 'postcode',
+            'price': 'price range',
+            'people': 'book people'
+        },
+        'restaurant': {
+            'pricerange': 'price range',
+            'book_time': 'book time',
+            'book_day': 'book day',
+            'book_people': 'book people',
+            'addr': 'address',
+            'post': 'postcode',
+            'price': 'price range',
+            'people': 'book people'
+        },
+        'taxi': {
+            'arriveBy': 'arrive by',
+            'leaveAt': 'leave at',
+            'arrive': 'arrive by',
+            'leave': 'leave at',
+            'car': 'type',
+            'car type': 'type',
+            'depart': 'departure',
+            'dest': 'destination'
+        },
+        'train': {
+            'arriveBy': 'arrive by',
+            'leaveAt': 'leave at',
+            'book_people': 'book people',
+            'arrive': 'arrive by',
+            'leave': 'leave at',
+            'depart': 'departure',
+            'dest': 'destination',
+            'id': 'train id',
+            'people': 'book people',
+            'time': 'duration',
+            'ticket': 'price',
+            'trainid': 'train id'
+        },
+        'attraction': {
+            'post': 'postcode',
+            'addr': 'address',
+            'fee': 'entrance fee',
+            'price': 'entrance fee'
+        },
+        'general': {},
+        'hospital': {
+            'post': 'postcode',
+            'addr': 'address'
+        },
+        'police': {
+            'post': 'postcode',
+            'addr': 'address'
+        }
+    }
+
+    _generic_referral = {
+        'hotel': {
+            'name': 'the hotel',
+            'area': 'same area as the hotel',
+            'price range': 'in the same price range as the hotel'
+        },
+        'restaurant': {
+            'name': 'the restaurant',
+            'area': 'same area as the restaurant',
+            'price range': 'in the same price range as the restaurant'
+        },
+        'attraction': {
+            'name': 'the attraction',
+            'area': 'same area as the attraction'
+        }
+    }
+    
+    def normalize_values(self, text):
+        text = text.lower()
+        text_to_num = {"zero": "0", "one": "1", "me": "1", "two": "2", "three": "3", "four": "4", "five": "5", "six": "6", "seven": "7"}
+        text = re.sub("\s*(\W)\s*", r"\1" , text) # Re-attach special characters
+        text = re.sub("s'([^s])", r"s' \1", text) # Add space after plural genitive apostrophe
+        if text in text_to_num:
+            text = text_to_num[text]
+        return text
+
+    def normalize_text(self, text):
+        norm_text = text.lower()
+        #norm_text = re.sub("n't", " not", norm_text) # Does not make much of a difference
+        norm_text = ' '.join([tok for tok in map(str.strip, re.split("(\W+)", norm_text)) if len(tok) > 0])
+        return norm_text
+
+    def normalize_prediction(self, domain, slot, value, predictions=None, class_predictions=None, config=None):
+        v = value
+        if domain == 'hotel' and slot == 'type':
+            # Map Boolean predictions to regular predictions.
+            v = "hotel" if value == "yes" else value
+            v = "guesthouse" if value == "no" else value
+            # HOTFIX: Avoid overprediction of hotel type caused by ambiguous rule based user simulator NLG.
+            if predictions['hotel-name'] != 'none':
+                v = 'none'
+            if config.dst_class_types[class_predictions['hotel-none']] == 'request':
+                v = 'none'
+        return v
+
+
+DATASET_INTERFACERS = {
+    'multiwoz21': MultiwozInterfacer()
+}
+
+
+def create_dataset_interfacer(dataset_name="multiwoz21"):
+    if dataset_name in DATASET_INTERFACERS:
+        return DATASET_INTERFACERS[dataset_name]
+    else:
+        logging.warn("You attempt to create a dataset interfacer for an unknown dataset '%s'. Creating generic dataset interfacer." % (dataset_name))
+        return DatasetInterfacer()
+
+
diff --git a/convlab/dst/trippy/modeling_dst.py b/convlab/dst/trippy/modeling_dst.py
new file mode 100644
index 0000000000000000000000000000000000000000..2828d17ed1e97ebb60b74ab999c28efb1e7bfa88
--- /dev/null
+++ b/convlab/dst/trippy/modeling_dst.py
@@ -0,0 +1,218 @@
+# coding=utf-8
+#
+# Copyright 2020-2022 Heinrich Heine University Duesseldorf
+#
+# Part of this code is based on the source code of BERT-DST
+# (arXiv:1907.03040)
+# Part of this code is based on the source code of Transformers
+# (arXiv:1910.03771)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+from torch import nn
+from torch.nn import CrossEntropyLoss
+
+from transformers import (BertModel, BertPreTrainedModel,
+                          RobertaModel, RobertaPreTrainedModel,
+                          ElectraModel, ElectraPreTrainedModel)
+
+PARENT_CLASSES = {
+    'bert': BertPreTrainedModel,
+    'roberta': RobertaPreTrainedModel,
+    'electra': ElectraPreTrainedModel
+}
+
+MODEL_CLASSES = {
+    BertPreTrainedModel: BertModel,
+    RobertaPreTrainedModel: RobertaModel,
+    ElectraPreTrainedModel: ElectraModel
+}
+
+
+class ElectraPooler(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
+        self.activation = nn.Tanh()
+        
+    def forward(self, hidden_states):
+        # We "pool" the model by simply taking the hidden state corresponding
+        # to the first token.
+        first_token_tensor = hidden_states[:, 0]
+        pooled_output = self.dense(first_token_tensor)
+        pooled_output = self.activation(pooled_output)
+        return pooled_output
+
+
+def TransformerForDST(parent_name):
+    if parent_name not in PARENT_CLASSES:
+        raise ValueError("Unknown model %s" % (parent_name))
+
+    class TransformerForDST(PARENT_CLASSES[parent_name]):
+        def __init__(self, config):
+            assert config.model_type in PARENT_CLASSES
+            assert self.__class__.__bases__[0] in MODEL_CLASSES
+            super(TransformerForDST, self).__init__(config)
+            self.model_type = config.model_type
+            self.slot_list = config.dst_slot_list
+            self.class_types = config.dst_class_types
+            self.class_labels = config.dst_class_labels
+            self.token_loss_for_nonpointable = config.dst_token_loss_for_nonpointable
+            self.refer_loss_for_nonpointable = config.dst_refer_loss_for_nonpointable
+            self.stack_token_logits = config.dst_stack_token_logits
+            self.class_aux_feats_inform = config.dst_class_aux_feats_inform
+            self.class_aux_feats_ds = config.dst_class_aux_feats_ds
+            self.class_loss_ratio = config.dst_class_loss_ratio
+
+            # Only use refer loss if refer class is present in dataset.
+            if 'refer' in self.class_types:
+                self.refer_index = self.class_types.index('refer')
+            else:
+                self.refer_index = -1
+
+            # Make sure this module has the same name as in the pretrained checkpoint you want to load!
+            self.add_module(self.model_type, MODEL_CLASSES[self.__class__.__bases__[0]](config))
+            if self.model_type == "electra":
+                self.pooler = ElectraPooler(config)
+            
+            self.dropout = nn.Dropout(config.dst_dropout_rate)
+            self.dropout_heads = nn.Dropout(config.dst_heads_dropout_rate)
+
+            if self.class_aux_feats_inform:
+                self.add_module("inform_projection", nn.Linear(len(self.slot_list), len(self.slot_list)))
+            if self.class_aux_feats_ds:
+                self.add_module("ds_projection", nn.Linear(len(self.slot_list), len(self.slot_list)))
+
+            aux_dims = len(self.slot_list) * (self.class_aux_feats_inform + self.class_aux_feats_ds) # second term is 0, 1 or 2
+
+            for slot in self.slot_list:
+                self.add_module("class_" + slot, nn.Linear(config.hidden_size + aux_dims, self.class_labels))
+                self.add_module("token_" + slot, nn.Linear(config.hidden_size, 2))
+                self.add_module("refer_" + slot, nn.Linear(config.hidden_size + aux_dims, len(self.slot_list) + 1))
+
+            self.init_weights()
+
+        def forward(self,
+                    input_ids,
+                    input_mask=None,
+                    segment_ids=None,
+                    position_ids=None,
+                    head_mask=None,
+                    start_pos=None,
+                    end_pos=None,
+                    inform_slot_id=None,
+                    refer_id=None,
+                    class_label_id=None,
+                    diag_state=None):
+            outputs = getattr(self, self.model_type)(
+                input_ids,
+                attention_mask=input_mask,
+                token_type_ids=segment_ids,
+                position_ids=position_ids,
+                head_mask=head_mask
+            )
+
+            sequence_output = outputs[0]
+            if self.model_type == "electra":
+                pooled_output = self.pooler(sequence_output)
+            else:
+                pooled_output = outputs[1]
+
+            sequence_output = self.dropout(sequence_output)
+            pooled_output = self.dropout(pooled_output)
+
+            if inform_slot_id is not None:
+                inform_labels = torch.stack(list(inform_slot_id.values()), 1).float()
+            if diag_state is not None:
+                diag_state_labels = torch.clamp(torch.stack(list(diag_state.values()), 1).float(), 0.0, 1.0)
+
+            total_loss = 0
+            per_slot_per_example_loss = {}
+            per_slot_class_logits = {}
+            per_slot_start_logits = {}
+            per_slot_end_logits = {}
+            per_slot_refer_logits = {}
+            for slot in self.slot_list:
+                if self.class_aux_feats_inform and self.class_aux_feats_ds:
+                    pooled_output_aux = torch.cat((pooled_output, self.inform_projection(inform_labels), self.ds_projection(diag_state_labels)), 1)
+                elif self.class_aux_feats_inform:
+                    pooled_output_aux = torch.cat((pooled_output, self.inform_projection(inform_labels)), 1)
+                elif self.class_aux_feats_ds:
+                    pooled_output_aux = torch.cat((pooled_output, self.ds_projection(diag_state_labels)), 1)
+                else:
+                    pooled_output_aux = pooled_output
+                class_logits = self.dropout_heads(getattr(self, 'class_' + slot)(pooled_output_aux))
+
+                token_logits = self.dropout_heads(getattr(self, 'token_' + slot)(sequence_output))
+                start_logits, end_logits = token_logits.split(1, dim=-1)
+                start_logits = start_logits.squeeze(-1)
+                end_logits = end_logits.squeeze(-1)
+
+                refer_logits = self.dropout_heads(getattr(self, 'refer_' + slot)(pooled_output_aux))
+
+                per_slot_class_logits[slot] = class_logits
+                per_slot_start_logits[slot] = start_logits
+                per_slot_end_logits[slot] = end_logits
+                per_slot_refer_logits[slot] = refer_logits
+
+                # If there are no labels, don't compute loss
+                if class_label_id is not None and start_pos is not None and end_pos is not None and refer_id is not None:
+                    # If we are on multi-GPU, split add a dimension
+                    if len(start_pos[slot].size()) > 1:
+                        start_pos[slot] = start_pos[slot].squeeze(-1)
+                    if len(end_pos[slot].size()) > 1:
+                        end_pos[slot] = end_pos[slot].squeeze(-1)
+                    # sometimes the start/end positions are outside our model inputs, we ignore these terms
+                    ignored_index = start_logits.size(1) # This is a single index
+                    start_pos[slot].clamp_(0, ignored_index)
+                    end_pos[slot].clamp_(0, ignored_index)
+
+                    class_loss_fct = CrossEntropyLoss(reduction='none')
+                    token_loss_fct = CrossEntropyLoss(reduction='none', ignore_index=ignored_index)
+                    refer_loss_fct = CrossEntropyLoss(reduction='none')
+
+                    if not self.stack_token_logits:
+                        start_loss = token_loss_fct(start_logits, start_pos[slot])
+                        end_loss = token_loss_fct(end_logits, end_pos[slot])
+                    else:
+                        start_loss = token_loss_fct(torch.cat((start_logits, end_logits), 1), start_pos[slot])
+                        end_loss = token_loss_fct(torch.cat((end_logits, start_logits), 1), end_pos[slot])
+
+                    token_loss = (start_loss + end_loss) / 2.0
+
+                    token_is_pointable = (start_pos[slot] > 0).float()
+                    if not self.token_loss_for_nonpointable:
+                        token_loss *= token_is_pointable
+
+                    refer_loss = refer_loss_fct(refer_logits, refer_id[slot])
+                    token_is_referrable = torch.eq(class_label_id[slot], self.refer_index).float()
+                    if not self.refer_loss_for_nonpointable:
+                        refer_loss *= token_is_referrable
+
+                    class_loss = class_loss_fct(class_logits, class_label_id[slot])
+
+                    if self.refer_index > -1:
+                        per_example_loss = (self.class_loss_ratio) * class_loss + ((1 - self.class_loss_ratio) / 2) * token_loss + ((1 - self.class_loss_ratio) / 2) * refer_loss
+                    else:
+                        per_example_loss = self.class_loss_ratio * class_loss + (1 - self.class_loss_ratio) * token_loss
+
+                    total_loss += per_example_loss.sum()
+                    per_slot_per_example_loss[slot] = per_example_loss
+
+            # add hidden states and attention if they are here
+            outputs = (total_loss,) + (per_slot_per_example_loss, per_slot_class_logits, per_slot_start_logits, per_slot_end_logits, per_slot_refer_logits,) + outputs[2:]
+
+            return outputs
+
+    return TransformerForDST
diff --git a/convlab/dst/trippy/tracker.py b/convlab/dst/trippy/tracker.py
new file mode 100644
index 0000000000000000000000000000000000000000..b0470266b2ce2c7d5cfba29d0270972b0c7cfa78
--- /dev/null
+++ b/convlab/dst/trippy/tracker.py
@@ -0,0 +1,544 @@
+# coding=utf-8
+#
+# Copyright 2020-2022 Heinrich Heine University Duesseldorf
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import json
+import copy
+import logging
+
+import torch
+from transformers import (BertConfig, BertTokenizer,
+                          RobertaConfig, RobertaTokenizer,
+                          ElectraConfig, ElectraTokenizer)
+
+from convlab.dst.dst import DST
+from convlab.dst.trippy.modeling_dst import (TransformerForDST)
+from convlab.dst.trippy.dataset_interfacer import (create_dataset_interfacer)
+from convlab.util import relative_import_module_from_unified_datasets
+
+MODEL_CLASSES = {
+    'bert': (BertConfig, TransformerForDST('bert'), BertTokenizer),
+    'roberta': (RobertaConfig, TransformerForDST('roberta'), RobertaTokenizer),
+    'electra': (ElectraConfig, TransformerForDST('electra'), ElectraTokenizer),
+}
+
+
+class TRIPPY(DST):
+    def print_header(self):
+        logging.info(" _________  ________  ___  ________  ________  ___    ___ ")
+        logging.info("|\___   ___\\\   __  \|\  \|\   __  \|\   __  \|\  \  /  /|")
+        logging.info("\|___ \  \_\ \  \|\  \ \  \ \  \|\  \ \  \|\  \ \  \/  / /")
+        logging.info("     \ \  \ \ \   _  _\ \  \ \   ____\ \   ____\ \    / / ")
+        logging.info("      \ \  \ \ \  \\\  \\\ \  \ \  \___|\ \  \___|\/  /  /  ")
+        logging.info("       \ \__\ \ \__\\\ _\\\ \__\ \__\    \ \__\ __/  / /    ")
+        logging.info("        \|__|  \|__|\|__|\|__|\|__|     \|__||\___/ /     ")
+        logging.info("          (c) 2022 Heinrich Heine University \|___|/      ")
+        logging.info("")
+
+    def print_dialog(self, hst):
+        logging.info("Dialogue %s, turn %s:" % (self.global_diag_cnt, self.global_turn_cnt))
+        for utt in hst[:-2]:
+            logging.info("  \033[92m%s\033[0m" % (utt))
+        if len(hst) > 1:
+            logging.info(" %s" % (hst[-2]))
+            logging.info(" %s" % (hst[-1]))
+
+    def print_inform_memory(self, inform_mem):
+        logging.info("Inform memory:")
+        is_all_none = True
+        for s in inform_mem:
+            if inform_mem[s] != 'none':
+                logging.info("  %s = %s" % (s, inform_mem[s]))
+                is_all_none = False
+        if is_all_none:
+            logging.info("  -")
+
+    def eval_user_acts(self, user_act, user_acts):
+        logging.info("User acts:")
+        for ua in user_acts:
+            if ua not in user_act:
+                logging.info("  \033[33m%s\033[0m" % (ua))
+            else:
+                logging.info("  \033[92m%s\033[0m" % (ua))
+        for ua in user_act:
+            if ua not in user_acts:
+                logging.info("  \033[91m%s\033[0m" % (ua))
+
+    def eval_dialog_state(self, state_updates, new_belief_state):
+        logging.info("Dialogue state:")
+        for d in self.gt_belief_state:
+            logging.info("  %s:" % (d))
+            for s in new_belief_state[d]:
+                is_printed = False
+                is_updated = False
+                if state_updates[d][s] > 0:
+                    is_updated = True
+                log_str = ""
+                if is_updated:
+                    log_str += "\033[3m"
+                if new_belief_state[d][s] != self.gt_belief_state[d][s]:
+                    self.global_eval_stats[d][s]['FP'] += 1
+                    if self.gt_belief_state[d][s] == '':
+                        log_str += "    \033[33m%s: %s\033[0m" % (s, new_belief_state[d][s])
+                    else:
+                        log_str += "    \033[91m%s: %s\033[0m (label: %s)" % (s, new_belief_state[d][s] if new_belief_state[d][s] != '' else 'none', self.gt_belief_state[d][s])
+                        self.global_eval_stats[d][s]['FN'] += 1
+                    is_printed = True
+                elif new_belief_state[d][s] != '':
+                    log_str += "    \033[92m%s: %s\033[0m" % (s, new_belief_state[d][s])
+                    self.global_eval_stats[d][s]['TP'] += 1
+                    is_printed = True
+                if is_updated:
+                    log_str += " (%s)" % (self.config.dst_class_types[state_updates[d][s]])
+                    logging.info(log_str)
+                elif is_printed:
+                    logging.info(log_str)
+
+    def eval_print_stats(self):
+        logging.info("Statistics:")
+        for d in self.global_eval_stats:
+            for s in self.global_eval_stats[d]:
+                TP = self.global_eval_stats[d][s]['TP']
+                FP = self.global_eval_stats[d][s]['FP']
+                FN = self.global_eval_stats[d][s]['FN']
+                prec = TP / ( TP + FP + 1e-8)
+                rec = TP / ( TP + FN + 1e-8)
+                f1 = 2 * ((prec * rec) / (prec + rec + 1e-8))
+                logging.info("  %s %s Recall: %.2f, Precision: %.2f, F1: %.2f" % (d, s, rec, prec, f1))
+
+    def __init__(self, model_type="roberta",
+                 model_name="roberta-base",
+                 model_path="",
+                 dataset_name="multiwoz21",
+                 local_files_only=False,
+                 nlu_usr_config="",
+                 nlu_sys_config="",
+                 nlu_usr_path="",
+                 nlu_sys_path="",
+                 no_eval=True,
+                 no_history=False):
+        super(TRIPPY, self).__init__()
+
+        self.print_header()
+
+        self.model_type = model_type.lower()
+        self.model_name = model_name.lower()
+        self.model_path = model_path
+        self.local_files_only = local_files_only
+        self.nlu_usr_config = nlu_usr_config
+        self.nlu_sys_config = nlu_sys_config
+        self.nlu_usr_path = nlu_usr_path
+        self.nlu_sys_path = nlu_sys_path
+        self.dataset_name = dataset_name
+        self.no_eval = no_eval
+        self.no_history = no_history
+
+        assert self.model_type in ['roberta'] # TODO: ensure proper behavior for 'bert', 'electra'
+        assert self.dataset_name in ['multiwoz21', 'multiwoz22', 'multiwoz23'] # TODO: ensure proper behavior for other datasets
+
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+        _ontology = relative_import_module_from_unified_datasets(self.dataset_name, 'preprocess.py', 'ontology')
+        self.template_state = _ontology['state']
+        
+        self.config_class, self.model_class, self.tokenizer_class = MODEL_CLASSES[self.model_type]
+        self.config = self.config_class.from_pretrained(self.model_path, local_files_only=self.local_files_only)
+
+        self.dataset_interfacer = create_dataset_interfacer(dataset_name)
+
+        # For internal evaluation only
+        self.nlu_usr = None
+        self.nlu_sys = None
+        self.global_eval_stats = copy.deepcopy(self.template_state)
+        for d in self.global_eval_stats:
+            for s in self.global_eval_stats[d]:
+                self.global_eval_stats[d][s] = {'TP': 0, 'FP': 0, 'FN': 0}
+        self.global_diag_cnt = -3
+        self.global_turn_cnt = -1
+        if not self.no_eval:
+            global BERTNLU
+            from convlab.nlu.jointBERT.unified_datasets import BERTNLU
+            self.load_nlu()
+
+        # For semantic action pipelines only
+        self.nlg_usr = None
+        self.nlg_sys = None
+
+        logging.info("DST INIT VARS: %s" % (vars(self)))
+
+        self.load_weights()
+    
+    def load_weights(self):
+        self.tokenizer = self.tokenizer_class.from_pretrained(self.model_name, local_files_only=self.local_files_only) # TODO: do_lower_case?
+        self.model = self.model_class.from_pretrained(self.model_path, config=self.config, local_files_only=self.local_files_only)
+        self.model.to(self.device)
+        self.model.eval()
+        logging.info("DST model weights loaded from %s" % (self.model_path))
+
+    def load_nlu(self):
+        """ Loads NLUs for internal evaluation """
+        # NLU for system utterances is used in case the policy does or can not provide semantic actions.
+        # The sole purpose of this is to fill the inform memory.
+        # NLU for user utterances is used in case the user simulator does or can not provide semantic actions.
+        # The sole purpose of this is to enable internal DST evaluation.
+        if self.nlu_usr_config == self.nlu_sys_config and \
+           self.nlu_usr_path == self.nlu_sys_path:
+            self.nlu_usr = BERTNLU(mode="all", config_file=self.nlu_usr_config, model_file=self.nlu_usr_path)
+            self.nlu_sys = self.nlu_usr
+        else:
+            self.nlu_usr = BERTNLU(mode="user", config_file=self.nlu_usr_config, model_file=self.nlu_usr_path)
+            self.nlu_sys = BERTNLU(mode="sys", config_file=self.nlu_sys_config, model_file=self.nlu_sys_path)
+        logging.info("DST user NLU model weights loaded from %s" % (self.nlu_usr_path))
+        logging.info("DST sys NLU model weights loaded from %s" % (self.nlu_sys_path))
+
+    def load_nlg(self):
+        if self.dataset_name in ['multiwoz21', 'multiwoz22', 'multiwoz23']:
+            from convlab.nlg.template.multiwoz import TemplateNLG
+            self.nlg_usr = TemplateNLG(is_user=True)
+            self.nlg_sys = TemplateNLG(is_user=False)
+            logging.info("DST template NLG loaded for dataset %s" % (self.dataset_name))
+        else:
+            raise Exception("DST no NLG for dataset %s available." % (self.dataset_name))
+            
+    def init_session(self):
+        # Initialise empty state
+        self.state = {'user_action': [],
+                      'system_action': [],
+                      'belief_state': {},
+                      'booked': {},
+                      'request_state': {},
+                      'terminated': False,
+                      'history': []}
+        self.state['belief_state'] = copy.deepcopy(self.template_state)
+        self.history = []
+        self.ds_aux = {slot: torch.tensor([0]).to(self.device) for slot in self.config.dst_slot_list}
+        self.gt_belief_state = copy.deepcopy(self.template_state)
+        self.global_diag_cnt += 1
+        self.global_turn_cnt = -1
+
+    def update_gt_belief_state(self, user_act):
+        for intent, domain, slot, value in user_act:
+            if domain == 'police':
+                continue
+            if intent == 'inform':
+                if slot == 'none' or slot == '':
+                    continue
+                if slot in self.gt_belief_state[domain]:
+                    self.gt_belief_state[domain][slot] = value
+
+    def update(self, user_act=''):
+        prev_state = self.state
+
+        if not self.no_eval:
+            logging.info("-" * 40)
+
+        if self.no_history:
+            self.history = []
+        self.history.append(['sys', self.get_text(prev_state['history'][-2][1], is_user=False, normalize=True)])
+        self.history.append(['user', self.get_text(prev_state['history'][-1][1], is_user=True, normalize=True)])
+
+        self.global_turn_cnt += 1
+        if not self.no_eval:
+            self.print_dialog(self.history)
+
+        # --- Get inform memory and auxiliary features ---
+
+        # system_action is a list of semantic system actions.
+        # TripPy uses system actions to fill the inform memory.
+        # End-to-end policies like Lava produce plain text instead.
+        # If system_action is plain text, get acts using NLU.
+        if isinstance(prev_state['system_action'], str):
+            s_acts = self.get_acts(prev_state['system_action'])
+        elif isinstance(prev_state['system_action'], list):
+            s_acts = prev_state['system_action']
+        else:
+            raise Exception('Unknown format for system action:', prev_state['system_action'])
+
+        if not self.no_eval:
+            # user_action is a list of semantic user actions if no NLG is used
+            # in the pipeline, otherwise user_action is plain text.
+            # TripPy uses user actions to perform internal DST evaluation.
+            # If user_action is plain text, get acts using NLU.
+            if isinstance(prev_state['user_action'], str):
+                u_acts = self.get_acts(prev_state['user_action'], is_user=True)
+            elif isinstance(prev_state['user_action'], list):
+                u_acts = prev_state['user_action'] # This is the same as user_act
+            else:
+                raise Exception('Unknown format for user action:', prev_state['user_action'])
+
+        # Fill the inform memory.
+        inform_aux, inform_mem = self.get_inform_aux(s_acts)
+        if not self.no_eval:
+            self.print_inform_memory(inform_mem)
+
+        # --- Tokenize dialogue context and feed DST model ---
+
+        used_ds_aux = None if not self.config.dst_class_aux_feats_ds else self.ds_aux
+        used_inform_aux = None if not self.config.dst_class_aux_feats_inform else inform_aux
+        features = self.get_features(self.history, ds_aux=used_ds_aux, inform_aux=used_inform_aux)
+        pred_states, pred_classes = self.predict(features, inform_mem)
+
+        # --- Update ConvLab-style dialogue state ---
+
+        new_belief_state = copy.deepcopy(prev_state['belief_state'])
+        user_acts = []
+        for state, value in pred_states.items():
+            value = self.dataset_interfacer.normalize_values(value)
+            domain, slot = state.split('-', 1)
+            value = self.dataset_interfacer.normalize_prediction(domain, slot, value,
+                                                                 predictions=pred_states,
+                                                                 class_predictions=pred_classes,
+                                                                 config=self.config)
+            if value == 'none':
+                continue
+            if slot in new_belief_state[domain]:
+                new_belief_state[domain][slot] = value
+                user_acts.append(['inform', domain, slot, value])
+            else:
+                raise Exception('Unknown slot name <{}> with value <{}> of domain <{}>'.format(slot, value, domain))
+
+        if not self.no_eval:
+            self.update_gt_belief_state(u_acts) # For evaluation
+
+        # BELIEF STATE UPDATE
+        new_state = copy.deepcopy(dict(prev_state))
+        new_state['belief_state'] = new_belief_state # TripPy
+
+        state_updates = {}
+        for cl in pred_classes:
+            cl_d, cl_s = cl.split('-')
+            # Some reformatting for the evaluation further down
+            if cl_d not in state_updates:
+                state_updates[cl_d] = {}
+            state_updates[cl_d][cl_s] = pred_classes[cl]
+            # We care only about the requestable slots here
+            if self.config.dst_class_types[pred_classes[cl]] != 'request':
+                continue
+            if cl_d != 'general' and cl_s == 'none':
+                user_acts.append(['inform', cl_d, '', ''])
+            elif cl_d == 'general':
+                user_acts.append([cl_s, 'general', '', ''])
+            else:
+                user_acts.append(['request', cl_d, cl_s, ''])
+
+        # USER ACTS UPDATE
+        new_state['user_action'] = user_acts # TripPy
+
+        if not self.no_eval:
+            self.eval_user_acts(u_acts, user_acts)
+            self.eval_dialog_state(state_updates, new_belief_state)
+
+        self.state = new_state
+
+        # Print eval statistics
+        if self.state['terminated'] and not self.no_eval:
+            logging.info("Booked: %s" % self.state['booked'])
+            self.eval_print_stats()
+            logging.info("=" * 10 + "End of the dialogue" + "=" * 10)
+        self.ds_aux = self.update_ds_aux(self.state['belief_state'], pred_states)
+
+        return self.state
+    
+    def predict(self, features, inform_mem):
+        with torch.no_grad():
+            outputs = self.model(input_ids=features['input_ids'],
+                                 input_mask=features['attention_mask'],
+                                 inform_slot_id=features['inform_slot_id'],
+                                 diag_state=features['diag_state'])
+
+        input_tokens = self.tokenizer.convert_ids_to_tokens(features['input_ids'][0]) # unmasked!
+
+        per_slot_class_logits = outputs[2]
+        per_slot_start_logits = outputs[3]
+        per_slot_end_logits = outputs[4]
+        per_slot_refer_logits = outputs[5]
+
+        # TODO: maybe add assert to check that batch=1
+        
+        predictions = {}
+        class_predictions = {}
+
+        for slot in self.config.dst_slot_list:
+            d, s = slot.split('-')
+            slot_udf = "%s-%s" % (self.dataset_interfacer.map_trippy_to_udf(d, s))
+
+            predictions[slot_udf] = 'none'
+            class_predictions[slot_udf] = 0
+
+            class_logits = per_slot_class_logits[slot][0]
+            start_logits = per_slot_start_logits[slot][0]
+            end_logits = per_slot_end_logits[slot][0]
+            refer_logits = per_slot_refer_logits[slot][0]
+
+            class_prediction = int(class_logits.argmax())
+            start_prediction = int(start_logits.argmax())
+            end_prediction = int(end_logits.argmax())
+            refer_prediction = int(refer_logits.argmax())
+
+            if class_prediction == self.config.dst_class_types.index('dontcare'):
+                predictions[slot_udf] = 'dontcare'
+            elif class_prediction == self.config.dst_class_types.index('copy_value'):
+                predictions[slot_udf] = ' '.join(input_tokens[start_prediction:end_prediction + 1])
+                predictions[slot_udf] = re.sub("(^| )##", "", predictions[slot_udf])
+                if "\u0120" in predictions[slot_udf]:
+                    predictions[slot_udf] = re.sub(" ", "", predictions[slot_udf])
+                    predictions[slot_udf] = re.sub("\u0120", " ", predictions[slot_udf])
+                    predictions[slot_udf] = predictions[slot_udf].strip()
+            elif 'true' in self.config.dst_class_types and class_prediction == self.config.dst_class_types.index('true'):
+                predictions[slot_udf] = "yes" # 'true'
+            elif 'false' in self.config.dst_class_types and class_prediction == self.config.dst_class_types.index('false'):
+                predictions[slot_udf] = "no" # 'false'
+            elif class_prediction == self.config.dst_class_types.index('inform'):
+                predictions[slot_udf] = inform_mem[slot_udf]
+            # Referral case is handled below
+
+        # Referral case. All other slot values need to be seen first in order
+        # to be able to do this correctly.
+        for slot in self.config.dst_slot_list:
+            d, s = slot.split('-')
+            slot_udf = "%s-%s" % (self.dataset_interfacer.map_trippy_to_udf(d, s))
+
+            class_logits = per_slot_class_logits[slot][0]
+            refer_logits = per_slot_refer_logits[slot][0]
+
+            class_prediction = int(class_logits.argmax())
+            refer_prediction = int(refer_logits.argmax())
+
+            if 'refer' in self.config.dst_class_types and class_prediction == self.config.dst_class_types.index('refer'):
+                # Only slots that have been mentioned before can be referred to.
+                # First try to resolve a reference within the same turn. (One can think of a situation
+                # where one slot is referred to in the same utterance. This phenomenon is however
+                # currently not properly covered in the training data label generation process)
+                # Then try to resolve a reference given the current dialogue state.
+                referred_slot = self.config.dst_slot_list[refer_prediction - 1]
+                referred_slot_d, referred_slot_s = referred_slot.split('-')
+                referred_slot_d, referred_slot_s = self.dataset_interfacer.map_trippy_to_udf(referred_slot_d, referred_slot_s)
+                referred_slot_udf = "%s-%s" % (referred_slot_d, referred_slot_s)
+                predictions[slot_udf] = predictions[referred_slot_udf]
+                if predictions[slot_udf] == 'none':
+                    if self.state['belief_state'][referred_slot_d][referred_slot_s] != '':
+                        predictions[slot_udf] = self.state['belief_state'][referred_slot_d][referred_slot_s]
+                if predictions[slot_udf] == 'none':
+                    ref_slot = self.config.dst_slot_list[refer_prediction - 1]
+                    ref_slot_d, ref_slot_s = ref_slot.split('-')
+                    generic_ref = self.dataset_interfacer.get_generic_referral(ref_slot_d, ref_slot_s)
+                    predictions[slot_udf] = generic_ref
+
+            class_predictions[slot_udf] = class_prediction
+
+        return predictions, class_predictions
+
+    def get_features(self, context, ds_aux=None, inform_aux=None):
+        assert(self.model_type == "roberta") # TODO: generalize to other BERT-like models
+        input_tokens = ['<s>'] # TODO: use tokenizer token names rather than strings
+        e_itr = 0
+        for e_itr, e in enumerate(reversed(context)):
+            if e[1] not in ['null', '']:
+                input_tokens.append(e[1])
+            if e_itr < 2:
+                input_tokens.append('</s> </s>')
+        if e_itr == 0:
+            input_tokens.append('</s> </s>')
+        input_tokens.append('</s>')
+        input_tokens = ' '.join(input_tokens)
+
+        # TODO: delex sys utt currently not supported
+        features = self.tokenizer.encode_plus(input_tokens, add_special_tokens=False, max_length=self.config.dst_max_seq_length)
+
+        input_ids = torch.tensor(features['input_ids']).reshape(1,-1).to(self.device)
+        attention_mask = torch.tensor(features['attention_mask']).reshape(1,-1).to(self.device)
+        features = {'input_ids': input_ids,
+                    'attention_mask': attention_mask,
+                    'inform_slot_id': inform_aux,
+                    'diag_state': ds_aux}
+
+        return features
+
+    def update_ds_aux(self, state, pred_states, terminated=False):
+        ds_aux = copy.deepcopy(self.ds_aux)
+        for slot in self.config.dst_slot_list:
+            d, s = slot.split('-')
+            d_udf, s_udf = self.dataset_interfacer.map_trippy_to_udf(d, s)
+            slot_udf = "%s-%s" % (d_udf, s_udf)
+            if d_udf in state and s_udf in state[d_udf]:
+                ds_aux[slot][0] = int(state[d_udf][s_udf] != '')
+            else:
+                # Requestable slots are not found in the DS
+                ds_aux[slot][0] = int(pred_states[slot_udf] != 'none')
+        return ds_aux
+
+    def get_inform_aux(self, state):
+        # Initialise auxiliary variables.
+        # For inform_aux, only the proper order of slots
+        # as defined in dst_slot_list is relevant, but not
+        # the actual slot names (as inform_aux will be
+        # converted into a simple binary list in the model)
+        inform_aux = {}
+        inform_mem = {}
+        for slot in self.config.dst_slot_list:
+            d, s = slot.split('-')
+            d_udf, s_udf = self.dataset_interfacer.map_trippy_to_udf(d, s)
+            inform_aux["%s-%s" % (d_udf, s_udf)] = torch.tensor([0]).to(self.device)
+            inform_mem["%s-%s" % (d_udf, s_udf)] = 'none'
+        for e in state:
+            a, d, s, v = e
+            # TODO: offerbook needed? booked needed?
+            if a in ['inform', 'recommend', 'select', 'book', 'offerbook']:
+                slot = "%s-%s" % (d, s)
+                if slot in inform_aux:
+                    inform_aux[slot][0] = 1
+                    inform_mem[slot] = self.dataset_interfacer.normalize_values(v)
+        return inform_aux, inform_mem
+
+    def get_acts(self, act, is_user=False):
+        if isinstance(act, list):
+            return act
+        context = self.state['history']
+        if context[-1][0] not in ['user', 'usr']:
+            raise Exception("Wrong order of utterances, check your input.")
+        system_context = [self.get_text(t) for s,t in context[:-2]]
+        user_context = [self.get_text(t, is_user=True) for s,t in context[:-1]]
+        if is_user:
+            if self.nlu_usr is None:
+                raise Exception("You attempt to convert semantic user actions into text, but no NLU module is loaded.")
+            acts = self.nlu_usr.predict(act, context=user_context)
+        else:
+            if self.nlu_sys is None:
+                raise Exception("You attempt to convert semantic system actions into text, but no NLU module is loaded.")
+            acts = self.nlu_sys.predict(act, context=system_context)
+        for act_itr in range(len(acts)):
+            acts[act_itr][-1] = self.dataset_interfacer.normalize_values(acts[act_itr][-1])
+        return acts
+
+    def get_text(self, act, is_user=False, normalize=False):
+        if act == 'null':
+            return 'null'
+        if not isinstance(act, list):
+            result = act
+        else:
+            if self.nlg_usr is None or self.nlg_sys is None:
+                logging.warn("You attempt to input semantic actions into TripPy, which expects text.")
+                logging.warn("Attempting to load NLG modules in order to convert actions into text.")
+                self.load_nlg()
+            if is_user:
+                result = self.nlg_usr.generate(act)
+            else:
+                result = self.nlg_sys.generate(act)
+        if normalize:
+            return self.dataset_interfacer.normalize_text(result)
+        else:
+            return result
diff --git a/convlab/e2e/soloist/READEME.md b/convlab/e2e/soloist/READEME.md
new file mode 100644
index 0000000000000000000000000000000000000000..12367222d48670054855f5592f369fad391f1be8
--- /dev/null
+++ b/convlab/e2e/soloist/READEME.md
@@ -0,0 +1,95 @@
+# SOLOIST
+
+On top of the pre-trained LMs, SOLOIST subsumes different components of task-oriented dialogs into a single model and emplies a pre-training then fine-tuning schema to build task bots.
+
+## Usage
+
+Follow the instruction under each dataset's directory to prepare data training and evaluation.
+
+#### Dataset Creation
+Create datasets of three settings. 
+```sh
+$ cd multiwoz
+$ python script/create_dataset.py joint
+$ python script/create_dataset.py transfer
+$ python script/create_dataset.py single
+```
+
+#### Train a model
+
+```sh
+$ python train.py --model_name_or_path t5-base --dataset_name e2e_dataloader.py --output_dir ./model --per_device_train_batch_size=2 --per_device_eval_batch_size=2 --max_target_length 128 --max_length 512 --num_train_epochs 50 --save_steps 10000 --preprocessing_num_workers 1 --num_beams 5 --learning_rate 5e-5 --dataset_config_name SINGLE --logging_steps 100
+```
+
+The model (`pytorch_model.bin`) will be saved under the `output_dir` of the config file. The script will save predictions for validation/test every epoch.
+
+#### Test a model
+
+The result will be saved under the `output_dir` of the config file. For evaluation, a 3rd party package is used. Please follow the instructions at https://github.com/Tomiinek/MultiWOZ_Evaluation
+
+
+## Performance on unified format datasets of different settings
+
+ Note that we use almost the same hyper-parameters for different settings, which may not be optimal.
+
+<table>
+<thead>
+  <tr>
+    <th></th>
+    <th colspan=2>MultiWOZ 2.1</th>
+    <th colspan=2>SGD</th>
+    <th colspan=2>Taskmaster-1</th>
+  </tr>
+</thead>
+<thead>
+  <tr>
+    <th>Model</th>
+    <th>Combined</th><th>BLEU</th>
+    <th>Slot F1</th><th>BLEU</th>
+    <th>Slot F1</th><th>BLEU</th>
+  </tr>
+</thead>
+<tbody>
+  <tr>
+    <td>SOLOIST w/o pre-training</td>
+    <td>67.0</td><td>16.8</td>
+    <td>56.9</td><td>11.2</td>
+    <td>8.5</td><td>28.0</td>    
+  </tr>
+  <tr>
+    <td>SOLOIST </td>
+    <td>71.4</td><td>17.1</td>
+    <td>69.7</td><td>23.1</td>
+    <td>9.2</td><td>29.2</td>
+
+  </tr>
+</tbody>
+</table>
+
+- Slot F1: F1 measure of the delexicalized slot predictions over the corpus.
+
+## References
+
+```
+@article{peng2021soloist,
+  title={Soloist: Buildingtask bots at scale with transfer learning and machine teaching},
+  author={Peng, Baolin and Li, Chunyuan and Li, Jinchao and Shayandeh, Shahin and Liden, Lars and Gao, Jianfeng},
+  journal={Transactions of the Association for Computational Linguistics},
+  volume={9},
+  pages={807--824},
+  year={2021},
+  publisher={MIT Press}
+}
+@article{nekvinda2021shades,
+  title={Shades of BLEU, flavours of success: The case of MultiWOZ},
+  author={Nekvinda, Tom{\'a}{\v{s}} and Du{\v{s}}ek, Ond{\v{r}}ej},
+  journal={arXiv preprint arXiv:2106.05555},
+  year={2021}
+}
+@article{peng2022godel,
+  title={GODEL: Large-Scale Pre-Training for Goal-Directed Dialog},
+  author={Peng, Baolin and Galley, Michel and He, Pengcheng and Brockett, Chris and Liden, Lars and Nouri, Elnaz and Yu, Zhou and Dolan, Bill and Gao, Jianfeng},
+  journal={arXiv preprint arXiv:2206.11309},
+  year={2022}
+}
+```
\ No newline at end of file
diff --git a/convlab/e2e/soloist/e2e_dataloader.py b/convlab/e2e/soloist/e2e_dataloader.py
new file mode 100644
index 0000000000000000000000000000000000000000..ac7be4d2e3d13f79e10f824505c8e1ab33ff4f35
--- /dev/null
+++ b/convlab/e2e/soloist/e2e_dataloader.py
@@ -0,0 +1,124 @@
+import datasets
+import jsonlines
+import random
+
+# coding=utf-8
+# Copyright 2020 HuggingFace Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+"""Corpus for E2E Dialog Modeling"""
+
+
+import csv
+
+import datasets
+
+
+_DESCRIPTION = """\
+E2E Dialog Modeling
+"""
+
+_CITATION = """\
+E2E Dialog Modeling
+"""
+
+_DOWNLOAD_URL = ""
+_WEBPAGE = ""
+
+class UnifiedDialogConfig(datasets.BuilderConfig):
+    """BuilderConfig for SuperGLUE."""
+
+    def __init__(self, data_name, **kwargs):
+        """BuilderConfig for SuperGLUE.
+        Args:
+          features: `list[string]`, list of the features that will appear in the
+            feature dict. Should not include "label".
+          data_url: `string`, url to download the zip file from.
+          citation: `string`, citation for the data set.
+          url: `string`, url for information about the data set.
+          label_classes: `list[string]`, the list of classes for the label if the
+            label is present as a string. Non-string labels will be cast to either
+            'False' or 'True'.
+          **kwargs: keyword arguments forwarded to super.
+        """
+        # Version history:
+        # 1.0.2: Fixed non-nondeterminism in ReCoRD.
+        # 1.0.1: Change from the pre-release trial version of SuperGLUE (v1.9) to
+        #        the full release (v2.0).
+        # 1.0.0: S3 (new shuffling, sharding and slicing mechanism).
+        # 0.0.2: Initial version.
+        super(UnifiedDialogConfig, self).__init__(version=datasets.Version("1.0.2"), **kwargs)
+        self.data_name = data_name
+        
+
+
+class Summarization(datasets.GeneratorBasedBuilder):
+    """Summarization"""
+
+    BUILDER_CONFIGS = [
+         UnifiedDialogConfig(name='JOINT',data_name='joint'),
+         UnifiedDialogConfig(name='TRANSFER',data_name='transfer'),
+         UnifiedDialogConfig(name='SINGLE',data_name='single'),
+     ]
+    
+    
+    random.seed(2022)
+
+    def _info(self):
+        return datasets.DatasetInfo(
+            description=_DESCRIPTION,
+            features=datasets.Features(
+                {
+                    "Context": datasets.Value("string"),
+                    "Knowledge": datasets.Value("string"),
+                    "Response": datasets.Value("string"),
+                    "Dataset": datasets.Value("string"),
+                }
+            ),
+            homepage=_WEBPAGE,
+            citation=_CITATION,
+        )
+
+    def _split_generators(self, dl_manager):        
+        
+        data_name = self.config.data_name
+
+        if data_name == 'joint':
+            train_path = f'./multiwoz/data/joint_train.jsonl'
+            validation_path = f'./multiwoz/data/single_validation.jsonl'
+            test_path = f'./multiwoz/data/single_test.jsonl'
+        elif data_name == 'transfer':
+            train_path = f'./multiwoz/data/transfer_train.jsonl'
+            validation_path = f'./multiwoz/data/single_validation.jsonl'
+            test_path = f'./multiwoz/data/single_test.jsonl'
+        elif data_name == 'single':
+            train_path = f'./multiwoz/data/single_train.jsonl'
+            validation_path = f'./multiwoz/data/single_validation.jsonl'
+            test_path = f'./multiwoz/data/single_test.jsonl'
+        else:
+            raise('Please specific dataset config.')
+
+        return [
+            datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepath": train_path}),
+            datasets.SplitGenerator(name=datasets.Split.VALIDATION, gen_kwargs={"filepath": validation_path}),
+            datasets.SplitGenerator(name=datasets.Split.TEST, gen_kwargs={"filepath": test_path}),
+        ]
+    def _generate_examples(self, filepath):
+        
+        with open(filepath, "r", encoding="utf-8") as reader:
+            key = 0
+            for item in jsonlines.Reader(reader):
+                yield key, item
+                key += 1
\ No newline at end of file
diff --git a/convlab/e2e/soloist/multiwoz/script/create_dataset.py b/convlab/e2e/soloist/multiwoz/script/create_dataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..2426af9f6100c07caee4a67d89231cb70481bf8a
--- /dev/null
+++ b/convlab/e2e/soloist/multiwoz/script/create_dataset.py
@@ -0,0 +1,74 @@
+import jsonlines
+import copy
+import fire
+
+from convlab.util.unified_datasets_util import create_delex_data, load_dataset
+from convlab.util import load_e2e_data
+
+def state_to_string(state):
+    domain_str = []
+    for domain,svs in state.items():
+        svs_str = []
+        for s,v in svs.items():
+            if v != '':
+                svs_str.append(f'{s} is {v}')
+        svs_str = ' ; '.join(svs_str)
+        if svs_str != '':
+            domain_str.append(f'{domain} {svs_str}')
+    domain_str = ' | '.join(domain_str)
+    return domain_str
+
+def context_to_string(context):
+    response = ' EOS '.join(i['utterance'].strip() for i in context)
+    return response
+
+def delex_function(d,s,v):
+    s = s.replace(' ','')
+    str_ = f'[{d}_{s}]'
+    return str_
+
+def create_dataset(mode='joint'):
+    dataset_list = {
+        'joint': ['tm1','sgd','multiwoz21'],
+        'transfer': ['tm1','sgd'],
+        'single': ['multiwoz21']
+    }
+
+    examples = []
+    for _data in dataset_list[mode]:
+        
+        dataset = load_dataset(_data)
+        dataset, delex_vocab = create_delex_data(dataset, delex_func=delex_function)
+        e2e_data = load_e2e_data(dataset, delex_utterance = True)
+        
+        split_list = ['train','validation','test'] if mode == 'single' else ['train']
+        
+        for split in split_list:
+            data = e2e_data[split]
+            for i in data:
+                response = i['delex_utterance'].strip()
+                context = i['context']
+                context = context_to_string(context)
+                
+                example = {}
+                example['Context'] = context
+                try:
+                    knowledge = state_to_string(i['context'][-1]['state'])
+                except Exception:
+                    knowledge = ''
+                example['Knowledge'] = knowledge
+                example['Response'] = 'Agent: ' + response.strip()
+                example['Dataset'] = f'{_data}'
+                examples.append(copy.copy(example))
+            if mode == 'single':
+                with jsonlines.open(f'./data/{mode}_{split}.jsonl', "w") as writer:
+                    for item in examples:
+                        writer.write(item)
+                examples = []
+    if mode != 'single':  
+        with jsonlines.open(f'./data/{mode}_train.jsonl', "w") as writer:
+            for item in examples:
+                writer.write(item)
+            
+if __name__ == '__main__':
+    fire.Fire(create_dataset)
\ No newline at end of file
diff --git a/convlab/e2e/soloist/multiwoz/script/create_mwoz_e2e_json.py b/convlab/e2e/soloist/multiwoz/script/create_mwoz_e2e_json.py
deleted file mode 100644
index 5953c19f50e410c8c37002309904de264ae01602..0000000000000000000000000000000000000000
--- a/convlab/e2e/soloist/multiwoz/script/create_mwoz_e2e_json.py
+++ /dev/null
@@ -1,136 +0,0 @@
-import jsonlines
-import json,copy
-fidx = open('test.idx.txt','w')
-
-data = json.load(open('data/test.json'))
-examples = []
-for i in data:
-    name = i['file'].lower()   
-    history = [] 
-    for turn in i['info']:
-        history.append(turn['user_orig'])
-
-        bs = turn['BS']
-        bs_str = []
-        for domain, states in bs.items():
-            domain_str = []
-            for state in states:
-                domain_str.append(f'{state[0]} = {state[1]}')
-            domain_str = ' ; '.join(domain_str)
-            bs_str.append(domain + ' ' + domain_str)
-        bs_str = ' | '.join(bs_str)
-
-        db_str = 'kb '
-        db = turn['KB']
-        if db == 0:
-            db_str += 'zero'
-        elif db_str == 1:
-            db_str += 'one'
-        elif db_str == 2:
-            db_str += 'two'
-        else:
-            db_str += 'more than two'
-
-        act_seq = ' '.join(turn['act'].keys())
-        example = {}
-        example['Context'] = ' EOS '.join(history[:])
-        example['Knowledge'] = ''
-        example['Response'] = 'belief : ' + bs_str + ' EOS ' + turn['sys'].strip()
-
-        history.append(turn['sys'].strip())
-        examples.append(copy.copy(example))
-        fidx.write(name + '\n')
-
-writer =  jsonlines.open('multiwoz_test_e2e.jsonl', mode='w')
-for i in examples:    
-    writer.write(i)
-
-
-data = json.load(open('data/val.json'))
-examples = []
-for i in data:
-    name = i['file'].lower()   
-    history = [] 
-    for turn in i['info']:
-        history.append(turn['user_orig'])
-
-
-        bs = turn['BS']
-        bs_str = []
-        for domain, states in bs.items():
-            domain_str = []
-            for state in states:
-                domain_str.append(f'{state[0]} = {state[1]}')
-            domain_str = ' ; '.join(domain_str)
-            bs_str.append(domain + ' ' + domain_str)
-        bs_str = ' | '.join(bs_str)
-
-        db_str = 'kb '
-        db = turn['KB']
-        if db == 0:
-            db_str += 'zero'
-        elif db_str == 1:
-            db_str += 'one'
-        elif db_str == 2:
-            db_str += 'two'
-        else:
-            db_str += 'more than two'
-
-        act_seq = ' '.join(turn['act'].keys())
-        example = {}
-        example['Context'] = ' EOS '.join(history[:])
-        example['Knowledge'] = ''
-        example['Response'] = 'belief : ' + bs_str + ' EOS ' + turn['sys'].strip()
-
-        history.append(turn['sys'].strip())
-        examples.append(copy.copy(example))
-        # fidx.write(name + '\n')
-
-writer =  jsonlines.open('multiwoz_valid_e2e.jsonl', mode='w')
-for i in examples:    
-    writer.write(i)
-
-
-data = json.load(open('data/train.json'))
-examples = []
-for i in data:
-    name = i['file'].lower()   
-    history = [] 
-    for turn in i['info']:
-        history.append(turn['user_orig'])
-
-
-        bs = turn['BS']
-        bs_str = []
-        for domain, states in bs.items():
-            domain_str = []
-            for state in states:
-                domain_str.append(f'{state[0]} = {state[1]}')
-            domain_str = ' ; '.join(domain_str)
-            bs_str.append(domain + ' ' + domain_str)
-        bs_str = ' | '.join(bs_str)
-
-        db_str = 'kb '
-        db = turn['KB']
-        if db == 0:
-            db_str += 'zero'
-        elif db_str == 1:
-            db_str += 'one'
-        elif db_str == 2:
-            db_str += 'two'
-        else:
-            db_str += 'more than two'
-
-        act_seq = ' '.join(turn['act'].keys())
-        example = {}
-        example['Context'] = ' EOS '.join(history[:])
-        example['Knowledge'] = ''
-        example['Response'] = 'belief : ' + bs_str + ' EOS ' + turn['sys'].strip()
-
-        history.append(turn['sys'].strip())
-        examples.append(copy.copy(example))
-        # fidx.write(name + '\n')
-
-writer =  jsonlines.open('multiwoz_train_e2e.jsonl', mode='w')
-for i in examples:    
-    writer.write(i)
diff --git a/convlab/e2e/soloist/multiwoz/soloist_net.py b/convlab/e2e/soloist/multiwoz/soloist_net.py
deleted file mode 100644
index 45f98200bc7eaf5387f796c8d29d3bb0555ac9e0..0000000000000000000000000000000000000000
--- a/convlab/e2e/soloist/multiwoz/soloist_net.py
+++ /dev/null
@@ -1,277 +0,0 @@
-import argparse
-import logging
-import math
-import os
-import random
-
-import datasets
-import nltk
-import numpy as np
-import torch
-from datasets import load_dataset, load_metric
-from torch.utils.data.dataloader import DataLoader
-from tqdm.auto import tqdm
-
-import transformers
-from accelerate import Accelerator
-from filelock import FileLock
-from transformers import (
-    CONFIG_MAPPING,
-    MODEL_MAPPING,
-    AdamW,
-    AutoConfig,
-    AutoModelForSeq2SeqLM,
-    AutoTokenizer,
-    DataCollatorForSeq2Seq,
-    SchedulerType,
-    get_scheduler,
-    set_seed,
-)
-from transformers.file_utils import is_offline_mode
-from transformers.utils.versions import require_version
-
-import copy, operator
-from queue import PriorityQueue
-import numpy as np
-import torch
-import torch.nn.functional as F
-from torch import nn
-from torch.autograd import Variable
-from torch.distributions import Categorical
-from convlab.e2e.soloist.multiwoz.config import global_config as cfg
-
-logger = logging.getLogger(__name__)
-logging.basicConfig(
-        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
-        datefmt="%m/%d/%Y %H:%M:%S",
-        level=logging.INFO,
-    )
-
-def cuda_(var):
-    return var.cuda() if cfg.cuda and torch.cuda.is_available() else var
-
-
-def tensor(var):
-    return cuda_(torch.tensor(var))
-
-class SOLOIST:
-
-    def __init__(self) -> None:
-        
-        self.config = AutoConfig.from_pretrained(cfg.model_name_or_path)
-        self.model = AutoModelForSeq2SeqLM.from_pretrained(cfg.model_name_or_path,config=self.config)
-        self.tokenizer = AutoTokenizer.from_pretrained('t5-base')
-        print('model loaded!')
-
-        self.model = self.model.cuda() if torch.cuda.is_available() else self.model
-
-    def generate(self, inputs):
-
-        self.model.eval()
-        inputs = self.tokenizer([inputs])
-        input_ids = tensor(inputs['input_ids'])
-        # generated_tokens = self.model.generate(input_ids = input_ids, max_length = cfg.max_length, num_beams = cfg.num_beams)
-        generated_tokens = self.model.generate(input_ids = input_ids, max_length = cfg.max_length, top_p=cfg.top_p)
-        decoded_preds = self.tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
-
-        return decoded_preds[0]
-
-    
-    def train_loop(self):
-
-        def preprocess_function(examples):
-            contextes = examples['Context']
-            responses = examples['Response']
-            belief = examples['Belief']
-            responses_labels = []
-            inputs = []
-
-            for context, response, kb in zip(contextes, responses, belief):
-                if cfg.no_kb:
-                    inputs.append(context + ' => ')
-                else:
-
-                    if cfg.format_version == 'e2e':
-                        context = ' EOS '.join(context.split(' EOS ')[-10:])
-                        _input = context
-                    
-                    if cfg.format_version == 'e2e+lm':
-                        context = ' EOS '.join(context.split(' EOS ')[-10:])
-                        inputs.append('[E2E] ' + context)
-                        responses_labels.append(response)
-                        inputs.append('[LM] ' + context )
-                        responses_labels.append(response.split(' EOS ')[1])
-                        continue
-                    
-                    if cfg.format_version == 'v2':
-                        _input = kb + context
-                    
-                    if cfg.format_version == 'v3':
-                        _input = ''
-                        context = context.split(' EOS ')
-                        for idx, turn in enumerate(context):
-                            if idx % 2 == 0:
-                                _input += 'user : ' + turn.strip()
-                            else:
-                                _input += ' system : ' + turn.strip()
-                        _input = _input + ' <|Knowledge|> ' + kb
-
-                    if cfg.format_version == 'v4':
-                        _input = ''
-                        context = context.split(' EOS ')
-                        for idx, turn in enumerate(context):
-                            if idx % 2 == 0:
-                                _input += 'user : ' + turn.strip()
-                            else:
-                                _input += ' system : ' + turn.strip()
-                        _input = kb + _input
-                    
-                    inputs.append(_input)
-                    responses_labels.append(response)
-            model_inputs = self.tokenizer(inputs, max_length=cfg.max_length, padding="max_length", truncation=True)
-
-            
-            with self.tokenizer.as_target_tokenizer():
-                labels = self.tokenizer(responses_labels, max_length=cfg.max_target_length, padding="max_length", truncation=True)
-
-            
-            if cfg.ignore_pad_token_for_loss:
-                labels["labels"] = [
-                    [(l if l != self.tokenizer.pad_token_id else -100) for l in label] for label in labels["input_ids"]
-                ]
-
-            model_inputs["labels"] = labels["labels"]
-            return model_inputs
-
-        raw_datasets = load_dataset(cfg.dataset_name)
-        column_names = ['Context','Response','Belief']
-        lm_datasets = raw_datasets.map(
-            preprocess_function,
-            batched=True,
-            remove_columns=column_names,
-            num_proc=cfg.preprocessing_num_workers,
-            load_from_cache_file=False,
-            desc=f"Processing dataset",
-        )
-
-        train_dataset = lm_datasets["test"]
-        # train_dataset = lm_datasets["validation"]
-        eval_dataset = lm_datasets["test"]
-        test_dataset = lm_datasets["test"]
-        for index in random.sample(range(len(train_dataset)), 1):
-            logger.info(f"Sample {index} of the training set: {train_dataset[index]}.") 
-
-        label_pad_token_id = -100 if cfg.ignore_pad_token_for_loss else self.tokenizer.pad_token_id
-
-        accelerator = Accelerator()
-        logger.info(accelerator.state)
-        data_collator = DataCollatorForSeq2Seq(
-            self.tokenizer,
-            model=self.model,
-            label_pad_token_id=label_pad_token_id,
-            pad_to_multiple_of=8 if accelerator.use_fp16 else None,
-        )
-
-
-        train_dataloader = DataLoader(
-        train_dataset, shuffle=True, collate_fn=data_collator, batch_size=cfg.per_device_train_batch_size
-        )
-        eval_dataloader = DataLoader(eval_dataset, collate_fn=data_collator, batch_size=cfg.per_device_eval_batch_size)
-        test_dataloader = DataLoader(test_dataset, collate_fn=data_collator, batch_size=cfg.per_device_eval_batch_size)
-
-        # Optimizer
-        # Split weights in two groups, one with weight decay and the other not.
-        no_decay = ["bias", "LayerNorm.weight"]
-        optimizer_grouped_parameters = [
-            {
-                "params": [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay)],
-                "weight_decay": cfg.weight_decay,
-            },
-            {
-                "params": [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay)],
-                "weight_decay": 0.0,
-            },
-        ]
-        optimizer = AdamW(optimizer_grouped_parameters, lr=cfg.learning_rate)
-
-        # Prepare everything with our `accelerator`.
-        self.model, optimizer, train_dataloader, eval_dataloader, test_dataloader = accelerator.prepare(
-            self.model, optimizer, train_dataloader, eval_dataloader, test_dataloader
-        )
-
-        # Note -> the training dataloader needs to be prepared before we grab his length below (cause its length will be
-        # shorter in multiprocess)
-
-        # Scheduler and math around the number of training steps.
-        num_update_steps_per_epoch = math.ceil(len(train_dataloader) / cfg.gradient_accumulation_steps)
-        if cfg.max_train_steps is None:
-            cfg.max_train_steps = cfg.num_train_epochs * num_update_steps_per_epoch
-        else:
-            cfg.num_train_epochs = math.ceil(cfg.max_train_steps / num_update_steps_per_epoch)
-
-        lr_scheduler = get_scheduler(
-            name=cfg.lr_scheduler_type,
-            optimizer=optimizer,
-            num_warmup_steps=cfg.num_warmup_steps,
-            num_training_steps=cfg.max_train_steps,
-        )
-
-        # Metric
-
-        # Train!
-        total_batch_size = cfg.per_device_train_batch_size * accelerator.num_processes * cfg.gradient_accumulation_steps
-
-        logger.info("***** Running training *****")
-        logger.info(f"  Num examples = {len(train_dataset)}")
-        logger.info(f"  Num Epochs = {cfg.num_train_epochs}")
-        logger.info(f"  Instantaneous batch size per device = {cfg.per_device_train_batch_size}")
-        logger.info(f"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}")
-        logger.info(f"  Gradient Accumulation steps = {cfg.gradient_accumulation_steps}")
-        logger.info(f"  Total optimization steps = {cfg.max_train_steps}")
-        # Only show the progress bar once on each machine.
-        progress_bar = tqdm(range(cfg.max_train_steps), disable=not accelerator.is_local_main_process)
-        completed_steps = 0
-        global_steps = 0
-        tr_loss, logging_loss = 0.0, 0.0
-        for epoch in range(cfg.num_train_epochs):
-            self.model.train()
-            # for step, batch in enumerate(train_dataloader):
-            for step, batch in enumerate(train_dataloader):
-                global_steps += 1            
-                outputs = self.model(**batch)
-                loss = outputs.loss
-                loss = loss / cfg.gradient_accumulation_steps
-                tr_loss += loss.item()
-                accelerator.backward(loss)
-                
-                if step % cfg.gradient_accumulation_steps == 0 or step == len(train_dataloader) - 1:
-                    optimizer.step()
-                    lr_scheduler.step()
-                    optimizer.zero_grad()
-                    completed_steps += 1
-
-                if completed_steps >= cfg.max_train_steps:
-                    break
-
-                if step % cfg.logging_steps == 0:
-                    logger.info(f"  EVALERR:  {(tr_loss - logging_loss)/float(cfg.logging_steps)}")
-                    logging_loss = tr_loss
-                    progress_bar.update(cfg.logging_steps)
-
-                if cfg.output_dir is not None and global_steps % cfg.save_steps == 0 and global_steps > 0:
-                    
-                    accelerator.wait_for_everyone()
-                    if accelerator.is_local_main_process:               
-                        checkpoint_prefix = 'checkpoint'
-                        output_dir = os.path.join(cfg.output_dir, '{}-{}'.format(checkpoint_prefix, global_steps))
-                        if not os.path.exists(output_dir):
-                            os.makedirs(output_dir)
-                        unwrapped_model = accelerator.unwrap_model(self.model)
-                        unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
-
-                        self.tokenizer.save_pretrained(output_dir)
-                        torch.save(cfg, os.path.join(output_dir, 'training_args.bin'))
-                        logger.info("Saving model checkpoint to %s", output_dir)
-
-
-    
\ No newline at end of file
diff --git a/convlab/e2e/soloist/train.py b/convlab/e2e/soloist/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..1c2078954173bcf20fe2f21691b720eebf4d74c9
--- /dev/null
+++ b/convlab/e2e/soloist/train.py
@@ -0,0 +1,836 @@
+#!/usr/bin/env python
+# coding=utf-8
+# Copyright The HuggingFace Team and The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Fine-tuning a 🤗 Transformers model on summarization.
+"""
+# You can also adapt this script on your own summarization task. Pointers for this are left as comments.
+
+import argparse
+import logging
+import math
+import os
+import random
+import json
+
+import datasets
+import nltk
+import numpy as np
+import torch
+from datasets import load_dataset, load_metric
+from torch.utils.data.dataloader import DataLoader
+from tqdm.auto import tqdm
+
+import transformers
+from accelerate import Accelerator
+from filelock import FileLock
+from transformers import (
+    CONFIG_MAPPING,
+    MODEL_MAPPING,
+    AdamW,
+    AutoConfig,
+    AutoModelForSeq2SeqLM,
+    AutoTokenizer,
+    DataCollatorForSeq2Seq,
+    SchedulerType,
+    get_scheduler,
+    set_seed,
+)
+from transformers.file_utils import is_offline_mode
+from transformers.utils.versions import require_version
+
+from nltk.tokenize import TweetTokenizer
+import re
+re_art = re.compile(r'\b(a|an|the)\b')
+re_punc = re.compile(r'[!"#$%&()*+,-./:;<=>?@\[\]\\^`{|}~_\']')
+
+
+def normalize_answer(s):
+    return s
+
+def clean_str(txt):
+	#print("in=[%s]" % txt)
+	txt = txt.lower()
+	txt = re.sub('^',' ', txt)
+	txt = re.sub('$',' ', txt)
+
+	# url and tag
+	words = []
+	for word in txt.split():
+		i = word.find('http') 
+		if i >= 0:
+			word = word[:i] + ' ' + '__url__'
+		words.append(word.strip())
+	txt = ' '.join(words)
+
+	# remove markdown URL
+	txt = re.sub(r'\[([^\]]*)\] \( *__url__ *\)', r'\1', txt)
+
+	# remove illegal char
+	txt = re.sub('__url__','URL',txt)
+	txt = re.sub(r"[^A-Za-z0-9():,.!?\"\']", " ", txt)
+	txt = re.sub('URL','__url__',txt)	
+
+	# contraction
+	add_space = ["'s", "'m", "'re", "n't", "'ll","'ve","'d","'em"]
+	tokenizer = TweetTokenizer(preserve_case=False)
+	txt = ' ' + ' '.join(tokenizer.tokenize(txt)) + ' '
+	txt = txt.replace(" won't ", " will n't ")
+	txt = txt.replace(" can't ", " can n't ")
+	for a in add_space:
+		txt = txt.replace(a+' ', ' '+a+' ')
+
+	txt = re.sub(r'^\s+', '', txt)
+	txt = re.sub(r'\s+$', '', txt)
+	txt = re.sub(r'\s+', ' ', txt) # remove extra spaces
+	
+	#print("out=[%s]" % txt)
+	return txt
+
+logger = logging.getLogger(__name__)
+require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/summarization/requirements.txt")
+
+
+import os
+from dotenv import load_dotenv
+load_dotenv()
+if os.getenv('WANDB_API_KEY') is None:
+    USE_WANDB = False
+else:
+    USE_WANDB = True
+    wandb_key = os.getenv('WANDB_API_KEY')
+
+# You should update this to your particular problem to have better documentation of `model_type`
+MODEL_CONFIG_CLASSES = list(MODEL_MAPPING.keys())
+MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)
+
+try:
+    nltk.data.find("tokenizers/punkt")
+except (LookupError, OSError):
+    if is_offline_mode():
+        raise LookupError(
+            "Offline mode: run this script without TRANSFORMERS_OFFLINE first to download nltk data files"
+        )
+    with FileLock(".lock") as lock:
+        nltk.download("punkt", quiet=True)
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Finetune a transformers model on a text classification task")
+    parser.add_argument(
+        "--dataset_name",
+        type=str,
+        default=None,
+        help="The name of the dataset to use (via the datasets library).",
+    )
+    parser.add_argument(
+        "--dataset_config_name",
+        type=str,
+        default=None,
+        help="The configuration name of the dataset to use (via the datasets library).",
+    )
+    parser.add_argument(
+        "--train_file", type=str, default=None, help="A csv or a json file containing the training data."
+    )
+    parser.add_argument(
+        "--validation_file", type=str, default=None, help="A csv or a json file containing the validation data."
+    )
+    
+    parser.add_argument(
+        "--max_source_length",
+        type=int,
+        default=1024,
+        help="The maximum total input sequence length after "
+        "tokenization.Sequences longer than this will be truncated, sequences shorter will be padded.",
+    )
+    parser.add_argument(
+        "--source_prefix",
+        type=str,
+        default=None,
+        help="A prefix to add before every source text " "(useful for T5 models).",
+    )
+    parser.add_argument(
+        "--preprocessing_num_workers",
+        type=int,
+        default=None,
+        help="The number of processes to use for the preprocessing.",
+    )
+
+    parser.add_argument(
+        "--max_target_length",
+        type=int,
+        default=64,
+        help="The maximum total sequence length for target text after "
+        "tokenization. Sequences longer than this will be truncated, sequences shorter will be padded."
+        "during ``evaluate`` and ``predict``.",
+    )
+    parser.add_argument(
+        "--val_max_target_length",
+        type=int,
+        default=None,
+        help="The maximum total sequence length for validation "
+        "target text after tokenization.Sequences longer than this will be truncated, sequences shorter will be "
+        "padded. Will default to `max_target_length`.This argument is also used to override the ``max_length`` "
+        "param of ``model.generate``, which is used during ``evaluate`` and ``predict``.",
+    )
+    parser.add_argument(
+        "--max_length",
+        type=int,
+        default=128,
+        help=(
+            "The maximum total input sequence length after tokenization. Sequences longer than this will be truncated,"
+            " sequences shorter will be padded if `--pad_to_max_lengh` is passed."
+        ),
+    )
+    parser.add_argument(
+        "--num_beams",
+        type=int,
+        default=None,
+        help="Number of beams to use for evaluation. This argument will be "
+        "passed to ``model.generate``, which is used during ``evaluate`` and ``predict``.",
+    )
+    parser.add_argument(
+        "--model_name_or_path",
+        type=str,
+        help="Path to pretrained model or model identifier from huggingface.co/models.",
+        required=True,
+    )
+    parser.add_argument(
+        "--config_name",
+        type=str,
+        default=None,
+        help="Pretrained config name or path if not the same as model_name",
+    )
+    parser.add_argument(
+        "--tokenizer_name",
+        type=str,
+        default=None,
+        help="Pretrained tokenizer name or path if not the same as model_name",
+    )
+    parser.add_argument(
+        "--text_column",
+        type=str,
+        default=None,
+        help="The name of the column in the datasets containing the full texts (for summarization).",
+    )
+    parser.add_argument(
+        "--summary_column",
+        type=str,
+        default=None,
+        help="The name of the column in the datasets containing the summaries (for summarization).",
+    )
+    parser.add_argument(
+        "--use_slow_tokenizer",
+        action="store_true",
+        help="If passed, will use a slow tokenizer (not backed by the 🤗 Tokenizers library).",
+    )
+    parser.add_argument(
+        "--per_device_train_batch_size",
+        type=int,
+        default=8,
+        help="Batch size (per device) for the training dataloader.",
+    )
+    parser.add_argument(
+        "--per_device_eval_batch_size",
+        type=int,
+        default=8,
+        help="Batch size (per device) for the evaluation dataloader.",
+    )
+    parser.add_argument(
+        "--learning_rate",
+        type=float,
+        default=5e-5,
+        help="Initial learning rate (after the potential warmup period) to use.",
+    )
+    parser.add_argument("--weight_decay", type=float, default=0.0, help="Weight decay to use.")
+    parser.add_argument("--num_train_epochs", type=int, default=3, help="Total number of training epochs to perform.")
+    parser.add_argument(
+        "--max_train_steps",
+        type=int,
+        default=None,
+        help="Total number of training steps to perform. If provided, overrides num_train_epochs.",
+    )
+    parser.add_argument(
+        "--gradient_accumulation_steps",
+        type=int,
+        default=1,
+        help="Number of updates steps to accumulate before performing a backward/update pass.",
+    )
+    parser.add_argument(
+        "--lr_scheduler_type",
+        type=SchedulerType,
+        default="linear",
+        help="The scheduler type to use.",
+        choices=["linear", "cosine", "cosine_with_restarts", "polynomial", "constant", "constant_with_warmup"],
+    )
+    parser.add_argument(
+        "--num_warmup_steps", type=int, default=0, help="Number of steps for the warmup in the lr scheduler."
+    )
+    parser.add_argument("--output_dir", type=str, default=None, help="Where to store the final model.")
+    parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.")
+    parser.add_argument(
+        "--model_type",
+        type=str,
+        default=None,
+        help="Model type to use if training from scratch.",
+        choices=MODEL_TYPES,
+    )
+
+
+    parser.add_argument(
+        "--overwrite_cache", type=bool, default=False, help="Overwrite the cached training and evaluation sets"
+    )
+
+    parser.add_argument(
+        "--pad_to_max_length", type=bool, default=True, help="do pading"
+    )
+
+    parser.add_argument(
+        "--ignore_pad_token_for_loss", type=bool, default=True, help="do pading"
+    )
+
+    parser.add_argument(
+        "--logging_steps", type=int, default=500, help="do pading"
+    )
+
+    parser.add_argument(
+        "--save_steps", type=int, default=5000, help="do pading"
+    )
+
+    parser.add_argument(
+        "--save_every_checkpoint", action="store_true"
+    )
+
+    parser.add_argument(
+        "--max_grad_norm", type=float, default=1.0, help="max_grad_norm"
+    )
+
+    parser.add_argument(
+        "--exp_name",
+        type=str,
+        help="Description to the experiment",
+        default='multiwoz',
+    )
+
+    parser.add_argument(
+        "--use_special_token",
+        action="store_true",
+        help="add special token or not"
+    )
+
+    parser.add_argument(
+        "--format_version",
+        type=str, default='v1',
+        help="format version"
+    )
+
+    parser.add_argument(
+        "--wandb_exp_name",
+        type=str,
+        default='multiwoz',
+        help="Description to the experiment worksheet name",
+    )
+
+
+    args = parser.parse_args()
+
+    # Sanity checks
+    if args.dataset_name is None and args.train_file is None and args.validation_file is None:
+        raise ValueError("Need either a dataset name or a training/validation file.")
+    else:
+        if args.train_file is not None:
+            extension = args.train_file.split(".")[-1]
+            assert extension in ["csv", "json"], "`train_file` should be a csv or a json file."
+        if args.validation_file is not None:
+            extension = args.validation_file.split(".")[-1]
+            assert extension in ["csv", "json"], "`validation_file` should be a csv or a json file."
+
+    if args.output_dir is not None:
+        os.makedirs(args.output_dir, exist_ok=True)
+
+    return args
+
+
+def main():
+    args = parse_args()
+
+    if args.source_prefix is None and args.model_name_or_path in [
+        "t5-small",
+        "t5-base",
+        "t5-large",
+        "t5-3b",
+        "t5-11b",
+    ]:
+        logger.warning(
+            "You're running a t5 model but didn't provide a source prefix, which is the expected, e.g. with "
+            "`--source_prefix 'summarize: ' `"
+        )
+    # Initialize the accelerator. We will let the accelerator handle device placement for us in this example.
+    accelerator = Accelerator()
+    # Make one log on every process with the configuration for debugging.
+    logging.basicConfig(
+        format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+        datefmt="%m/%d/%Y %H:%M:%S",
+        level=logging.INFO,
+    )
+    logger.info(accelerator.state)
+
+    # Setup logging, we only want one process per machine to log things on the screen.
+    # accelerator.is_local_main_process is only True for one process per machine.
+    logger.setLevel(logging.INFO if accelerator.is_local_main_process else logging.ERROR)
+    if accelerator.is_local_main_process:
+        datasets.utils.logging.set_verbosity_warning()
+        transformers.utils.logging.set_verbosity_info()
+    else:
+        datasets.utils.logging.set_verbosity_error()
+        transformers.utils.logging.set_verbosity_error()
+
+    # If passed along, set the training seed now.
+    if args.seed is not None:
+        set_seed(args.seed)
+
+    if accelerator.is_local_main_process and USE_WANDB:
+        config = dict(
+        dataset_id = "",
+        infra = "",
+        )
+        import wandb
+        wandb.init(
+        project=args.wandb_exp_name,
+        notes="Finetuning",
+        tags=["multiwoz"],
+        config=config,
+        entity= 'Convlab3')
+
+        wandb.run.name = args.exp_name
+
+
+    # Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
+    # or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
+    # (the dataset will be downloaded automatically from the datasets Hub).
+    #
+    # For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
+    # 'text' is found. You can easily tweak this behavior (see below).
+    #
+    # In distributed training, the load_dataset function guarantee that only one local process can concurrently
+    # download the dataset.
+    if args.dataset_name is not None:
+        # Downloading and loading a dataset from the hub.
+        raw_datasets = load_dataset(args.dataset_name, args.dataset_config_name)
+    else:
+        data_files = {}
+        if args.train_file is not None:
+            data_files["train"] = args.train_file
+        if args.validation_file is not None:
+            data_files["validation"] = args.validation_file
+        extension = args.train_file.split(".")[-1]
+        raw_datasets = load_dataset(extension, data_files=data_files)
+    # See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
+    # https://huggingface.co/docs/datasets/loading_datasets.html.
+
+    # Load pretrained model and tokenizer
+    #
+    # In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently
+    # download model & vocab.
+    if args.config_name:
+        config = AutoConfig.from_pretrained(args.config_name)
+    elif args.model_name_or_path:
+        config = AutoConfig.from_pretrained(args.model_name_or_path)
+    else:
+        config = CONFIG_MAPPING[args.model_type]()
+        logger.warning("You are instantiating a new config instance from scratch.")
+
+    if args.tokenizer_name:
+        tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name, use_fast=not args.use_slow_tokenizer)
+    elif args.model_name_or_path:
+        tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path, use_fast=not args.use_slow_tokenizer)
+    else:
+        raise ValueError(
+            "You are instantiating a new tokenizer from scratch. This is not supported by this script."
+            "You can do it from another script, save it, and load it from here, using --tokenizer_name."
+        )
+
+    if args.model_name_or_path:
+        model = AutoModelForSeq2SeqLM.from_pretrained(
+            args.model_name_or_path,
+            from_tf=bool(".ckpt" in args.model_name_or_path),
+            config=config,
+        )
+    else:
+        logger.info("Training new model from scratch")
+        model = AutoModelForSeq2SeqLM.from_config(config)
+
+    if 'blender' in args.model_name_or_path.lower():
+        model.model.encoder.embed_positions.weight = torch.nn.Parameter(model.model.encoder.embed_positions.weight.repeat(4,1))
+    tokenizer.add_special_tokens({'pad_token': '[PAD]'})
+
+    if args.use_special_token:
+        special_tokens = [i.strip() for i in open('special_tokens.txt')]
+        tokenizer.add_tokens(special_tokens)
+
+    model.resize_token_embeddings(len(tokenizer))
+    if model.config.decoder_start_token_id is None:
+        raise ValueError("Make sure that `config.decoder_start_token_id` is correctly defined")
+
+    prefix = args.source_prefix if args.source_prefix is not None else ""
+    max_length = args.max_length
+    padding = "max_length" if args.pad_to_max_length else False
+    max_target_length = args.max_target_length
+    def preprocess_function(examples):
+        contextes = examples['Context']
+        responses = examples['Response']
+        kbs = examples['Knowledge']
+
+        responses_labels = []
+        inputs = []
+
+        for context, response, kb in zip(contextes, responses, kbs):
+            if args.format_version == 'v1':
+                _input = ' EOS '.join(context.split(' EOS ')[-10:])
+                _response = 'Belief: ' + kb + response
+                inputs.append(_input)
+                responses_labels.append(_response)
+
+        model_inputs = tokenizer(inputs, max_length=args.max_length, padding=padding, truncation=True)
+
+        # labels = model_inputs
+        # Setup the tokenizer for targets
+        with tokenizer.as_target_tokenizer():
+            labels = tokenizer(responses_labels, max_length=max_target_length, padding=padding, truncation=True)
+
+        # If we are padding here, replace all tokenizer.pad_token_id in the labels by -100 when we want to ignore
+        # padding in the loss.
+        if padding == "max_length" and args.ignore_pad_token_for_loss:
+            labels["labels"] = [
+                [(l if l != tokenizer.pad_token_id else -100) for l in label] for label in labels["input_ids"]
+            ]
+
+        model_inputs["labels"] = labels["labels"]
+        return model_inputs
+
+    # Note that with `batched=True`, this map processes 1,000 texts together, so group_texts throws away a remainder
+    # for each of those groups of 1,000 texts. You can adjust that batch_size here but a higher value might be slower
+    # to preprocess.
+    #
+    # To speed up this part, we use multiprocessing. See the documentation of the map method for more information:
+    # https://huggingface.co/docs/datasets/package_reference/main_classes.html#datasets.Dataset.map
+    
+    # del raw_datasets['train']
+    # del raw_datasets['test']
+    column_names = ['Context','Response','Knowledge','Dataset']
+    # # column_names = ['text']
+    # raw_datasets = load_dataset(args.dataset_name, args.dataset_config_name)
+    lm_datasets = raw_datasets.map(
+        preprocess_function,
+        batched=True,
+        remove_columns=column_names,
+        num_proc=args.preprocessing_num_workers,
+        load_from_cache_file=False,
+        desc=f"Processing dataset",
+    )
+
+    train_dataset = lm_datasets["test"]
+    eval_dataset = lm_datasets["validation"]
+    test_dataset = lm_datasets["test"]
+
+    # Log a few random samples from the training set:
+    for index in random.sample(range(len(train_dataset)), 1):
+        logger.info(f"Sample {index} of the training set: {train_dataset[index]}.")
+
+
+    label_pad_token_id = -100 if args.ignore_pad_token_for_loss else tokenizer.pad_token_id
+    data_collator = DataCollatorForSeq2Seq(
+        tokenizer,
+        model=model,
+        label_pad_token_id=label_pad_token_id,
+        pad_to_multiple_of=8 if accelerator.use_fp16 else None,
+    )
+
+    def postprocess_text(preds, labels):
+        preds = [normalize_answer(pred.strip().replace('Agent :','')) for pred in preds]
+        labels = [normalize_answer(label.strip().replace('Agent :','')) for label in labels]
+
+        # rougeLSum expects newline after each sentence
+        # preds = ["\n".join(nltk.sent_tokenize(pred)) for pred in preds]
+        # labels = ["\n".join(nltk.sent_tokenize(label)) for label in labels]
+
+        return preds, labels
+
+    train_dataloader = DataLoader(
+        train_dataset, shuffle=True, collate_fn=data_collator, batch_size=args.per_device_train_batch_size
+    )
+    eval_dataloader = DataLoader(eval_dataset, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size)
+    test_dataloader = DataLoader(test_dataset, collate_fn=data_collator, batch_size=args.per_device_eval_batch_size)
+
+    # Optimizer
+    # Split weights in two groups, one with weight decay and the other not.
+    no_decay = ["bias", "LayerNorm.weight"]
+    optimizer_grouped_parameters = [
+        {
+            "params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
+            "weight_decay": args.weight_decay,
+        },
+        {
+            "params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],
+            "weight_decay": 0.0,
+        },
+    ]
+    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate)
+
+    # Prepare everything with our `accelerator`.
+    model, optimizer, train_dataloader, eval_dataloader, test_dataloader = accelerator.prepare(
+        model, optimizer, train_dataloader, eval_dataloader, test_dataloader
+    )
+
+    # Note -> the training dataloader needs to be prepared before we grab his length below (cause its length will be
+    # shorter in multiprocess)
+
+    # Scheduler and math around the number of training steps.
+    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps)
+    if args.max_train_steps is None:
+        args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch
+    else:
+        args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch)
+
+    lr_scheduler = get_scheduler(
+        name=args.lr_scheduler_type,
+        optimizer=optimizer,
+        num_warmup_steps=args.num_warmup_steps,
+        num_training_steps=args.max_train_steps,
+    )
+
+    # Metric
+
+    # Train!
+    total_batch_size = args.per_device_train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps
+
+    logger.info("***** Running training *****")
+    logger.info(f"  Num examples = {len(train_dataset)}")
+    logger.info(f"  Num Epochs = {args.num_train_epochs}")
+    logger.info(f"  Instantaneous batch size per device = {args.per_device_train_batch_size}")
+    logger.info(f"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}")
+    logger.info(f"  Gradient Accumulation steps = {args.gradient_accumulation_steps}")
+    logger.info(f"  Total optimization steps = {args.max_train_steps}")
+    # Only show the progress bar once on each machine.
+    progress_bar = tqdm(range(args.max_train_steps), disable=not accelerator.is_local_main_process)
+    completed_steps = 0
+    global_steps = 0
+    tr_loss, logging_loss = 0.0, 0.0
+    for epoch in range(args.num_train_epochs):
+        model.train()
+        
+        for step, batch in enumerate(train_dataloader):
+            
+            global_steps += 1            
+            outputs = model(**batch)
+            loss = outputs.loss
+            loss = loss / args.gradient_accumulation_steps
+            tr_loss += loss.item()
+            accelerator.backward(loss)            
+            
+            if step % args.gradient_accumulation_steps == 0 or step == len(train_dataloader) - 1:
+                optimizer.step()
+                lr_scheduler.step()
+                optimizer.zero_grad()
+                completed_steps += 1
+
+            if completed_steps >= args.max_train_steps:
+                break
+
+            if step % args.logging_steps == 0:
+                logger.info(f"  EVALERR:  {(tr_loss - logging_loss)/float(args.logging_steps)}")
+                if accelerator.is_local_main_process and USE_WANDB:
+                    wandb.log({'loss': tr_loss - logging_loss})
+                logging_loss = tr_loss
+                progress_bar.update(args.logging_steps)
+
+            if args.output_dir is not None and global_steps % args.save_steps == 0 and global_steps > 0:
+                print('hit store')
+                accelerator.wait_for_everyone()
+                if accelerator.is_local_main_process:               
+                    checkpoint_prefix = 'checkpoint'
+                    output_dir = os.path.join(args.output_dir, '{}-{}'.format(checkpoint_prefix, global_steps))
+                    if not os.path.exists(output_dir):
+                        os.makedirs(output_dir)
+                    unwrapped_model = accelerator.unwrap_model(model)
+                    unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
+
+                    tokenizer.save_pretrained(output_dir)
+                    torch.save(args, os.path.join(output_dir, 'training_args.bin'))
+                    logger.info("Saving model checkpoint to %s", output_dir)
+
+        model.eval()
+        if args.val_max_target_length is None:
+            args.val_max_target_length = args.max_target_length
+
+        gen_kwargs = {
+            "max_length": args.val_max_target_length if args is not None else config.max_length,
+            "num_beams": args.num_beams,
+        }
+
+        def chunks(lst, n):
+            for i in range(0, len(lst), n):
+                yield lst[i:i + n]
+
+        metric = load_metric("./rouge_metric.py")
+        metric_bleu = load_metric("./bleu_metric.py")
+        decoded_preds_all = []
+        for step, batch in enumerate(eval_dataloader):
+            with torch.no_grad():
+                generated_tokens = accelerator.unwrap_model(model).generate(
+                    batch["input_ids"],
+                    attention_mask=batch["attention_mask"],
+                    **gen_kwargs,
+                )
+
+                generated_tokens = accelerator.pad_across_processes(
+                    generated_tokens, dim=1, pad_index=tokenizer.pad_token_id
+                )
+                labels = batch["labels"]
+                if not args.pad_to_max_length:
+                    # If we did not pad to max length, we need to pad the labels too
+                    labels = accelerator.pad_across_processes(batch["labels"], dim=1, pad_index=tokenizer.pad_token_id)
+
+                generated_tokens = accelerator.gather(generated_tokens).cpu().numpy()
+                labels = accelerator.gather(labels).cpu().numpy()
+
+                if args.ignore_pad_token_for_loss:
+                    # Replace -100 in the labels as we can't decode them.
+                    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
+                if isinstance(generated_tokens, tuple):
+                    generated_tokens = generated_tokens[0]
+                decoded_preds = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
+                decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
+
+                metric.add_batch(predictions=decoded_preds, references=decoded_labels)
+                _decoded_preds = [i.split() for i in decoded_preds]
+                _decoded_labels = [[i.split()] for i in decoded_labels]
+                decoded_preds_all.extend(decoded_preds)
+                metric_bleu.add_batch(predictions=_decoded_preds, references=_decoded_labels)
+            
+                
+        result = metric.compute(use_stemmer=True)
+        # Extract a few results from ROUGE
+        result = {key: value.mid.fmeasure * 100 for key, value in result.items()}
+
+        result = {k: round(v, 4) for k, v in result.items()}
+
+        logger.info(result)
+
+        result_bleu = metric_bleu.compute()
+        logger.info(result_bleu)
+
+        accelerator.wait_for_everyone()
+        if accelerator.is_local_main_process and USE_WANDB:
+            wandb.log({'valid_bleu': result_bleu['bleu']})
+            wandb.log({'valid_rouge': result['rougeL']})
+
+        
+        if args.output_dir is not None:
+            accelerator.wait_for_everyone()
+            if accelerator.is_local_main_process:               
+                if not os.path.exists(args.output_dir):
+                    os.makedirs(args.output_dir)
+                output_dir_file_name = os.path.join(args.output_dir, 'valid-step-{}'.format(completed_steps))
+                print(output_dir_file_name)
+                json.dump(decoded_preds_all, open(output_dir_file_name,'w'), indent=2)
+                logger.info("Saving model outputs to %s", output_dir_file_name)
+        
+        metric = load_metric("rouge")
+        metric_bleu = load_metric("bleu")
+
+        gen_kwargs = {
+            "max_length": args.val_max_target_length if args is not None else config.max_length,
+            "num_beams": args.num_beams,
+        }
+        decoded_preds_all = []
+        for step, batch in enumerate(test_dataloader):
+            with torch.no_grad():
+                generated_tokens = accelerator.unwrap_model(model).generate(
+                    batch["input_ids"],
+                    attention_mask=batch["attention_mask"],
+                    **gen_kwargs,
+                )
+
+                generated_tokens = accelerator.pad_across_processes(
+                    generated_tokens, dim=1, pad_index=tokenizer.pad_token_id
+                )
+                labels = batch["labels"]
+                if not args.pad_to_max_length:
+                    # If we did not pad to max length, we need to pad the labels too
+                    labels = accelerator.pad_across_processes(batch["labels"], dim=1, pad_index=tokenizer.pad_token_id)
+
+                generated_tokens = accelerator.gather(generated_tokens).cpu().numpy()
+                labels = accelerator.gather(labels).cpu().numpy()
+
+                if args.ignore_pad_token_for_loss:
+                    # Replace -100 in the labels as we can't decode them.
+                    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
+                if isinstance(generated_tokens, tuple):
+                    generated_tokens = generated_tokens[0]
+                decoded_preds = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
+                decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
+
+                decoded_preds, decoded_labels = postprocess_text(decoded_preds, decoded_labels)
+                metric.add_batch(predictions=decoded_preds, references=decoded_labels)
+                _decoded_preds = [i.split() for i in decoded_preds]
+                _decoded_labels = [[i.split()] for i in decoded_labels]
+                decoded_preds_all.extend(_decoded_preds)
+                metric_bleu.add_batch(predictions=_decoded_preds, references=_decoded_labels)
+                
+        result = metric.compute(use_stemmer=True)
+        # Extract a few results from ROUGE
+        result = {key: value.mid.fmeasure * 100 for key, value in result.items()}
+
+        result = {k: round(v, 4) for k, v in result.items()}
+
+        logger.info(result)
+
+        result_bleu = metric_bleu.compute()
+        logger.info(result_bleu)
+
+        accelerator.wait_for_everyone()
+        if accelerator.is_local_main_process and USE_WANDB:
+            wandb.log({'test_bleu': result_bleu['bleu']})
+            wandb.log({'test_rouge': result['rougeL']})
+
+        import json
+        if args.output_dir is not None:
+            accelerator.wait_for_everyone()
+            if accelerator.is_local_main_process:               
+                if not os.path.exists(args.output_dir):
+                    os.makedirs(args.output_dir)
+                output_dir_file_name = os.path.join(args.output_dir, 'test-step-{}'.format(completed_steps))
+                print(output_dir_file_name)
+                json.dump(decoded_preds_all, open(output_dir_file_name,'w'), indent=2)
+                logger.info("Saving model outputs to %s", output_dir_file_name)
+
+        
+        if args.output_dir is not None and args.save_every_checkpoint:
+            accelerator.wait_for_everyone()
+            if accelerator.is_local_main_process:               
+                checkpoint_prefix = 'checkpoint'
+                output_dir = os.path.join(args.output_dir, '{}-epoch-{}'.format(checkpoint_prefix, epoch))
+                if not os.path.exists(output_dir):
+                    os.makedirs(output_dir)
+                unwrapped_model = accelerator.unwrap_model(model)
+                unwrapped_model.save_pretrained(output_dir, save_function=accelerator.save)
+
+                tokenizer.save_pretrained(output_dir)
+                torch.save(args, os.path.join(output_dir, 'training_args.bin'))
+                logger.info("Saving model checkpoint to %s", output_dir)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/convlab/evaluator/multiwoz_eval.py b/convlab/evaluator/multiwoz_eval.py
index b7331479e2ea2ee1a1f5c4bce7aef82546dcbd2e..cb6c8feb73aea6e4481a51fa0eb2466a6a07d1c6 100755
--- a/convlab/evaluator/multiwoz_eval.py
+++ b/convlab/evaluator/multiwoz_eval.py
@@ -3,15 +3,33 @@
 import logging
 import re
 import numpy as np
+import pdb
+
 from copy import deepcopy
+from data.unified_datasets.multiwoz21.preprocess import reverse_da, reverse_da_slot_name_map
+from convlab.util.multiwoz.multiwoz_slot_trans import REF_SYS_DA
 from convlab.evaluator.evaluator import Evaluator
+from data.unified_datasets.multiwoz21.preprocess import reverse_da_slot_name_map
 from convlab.policy.rule.multiwoz.policy_agenda_multiwoz import unified_format, act_dict_to_flat_tuple
 from convlab.util.multiwoz.dbquery import Database
-import os
 from convlab.util import relative_import_module_from_unified_datasets
 
+# import reflect table
+REF_SYS_DA_M = {}
+for dom, ref_slots in REF_SYS_DA.items():
+    dom = dom.lower()
+    REF_SYS_DA_M[dom] = {}
+    for slot_a, slot_b in ref_slots.items():
+        if slot_a == 'Ref':
+            slot_b = 'ref'
+        REF_SYS_DA_M[dom][slot_a.lower()] = slot_b
+    REF_SYS_DA_M[dom]['none'] = 'none'
+REF_SYS_DA_M['taxi']['phone'] = 'phone'
+REF_SYS_DA_M['taxi']['car'] = 'car type'
+
 reverse_da = relative_import_module_from_unified_datasets('multiwoz21', 'preprocess.py', 'reverse_da')
 
+
 requestable = \
     {'attraction': ['post', 'phone', 'addr', 'fee', 'area', 'type'],
      'restaurant': ['addr', 'phone', 'post', 'ref', 'price', 'area', 'food'],
@@ -24,13 +42,13 @@ requestable = \
 belief_domains = requestable.keys()
 
 mapping = {'restaurant': {'addr': 'address', 'area': 'area', 'food': 'food', 'name': 'name', 'phone': 'phone',
-                          'post': 'postcode', 'price': 'pricerange'},
+                          'post': 'postcode', 'price': 'pricerange', 'ref': 'ref'},
            'hotel': {'addr': 'address', 'area': 'area', 'internet': 'internet', 'parking': 'parking', 'name': 'name',
-                     'phone': 'phone', 'post': 'postcode', 'price': 'pricerange', 'stars': 'stars', 'type': 'type'},
+                     'phone': 'phone', 'post': 'postcode', 'price': 'pricerange', 'stars': 'stars', 'type': 'type', 'ref': 'ref'},
            'attraction': {'addr': 'address', 'area': 'area', 'fee': 'entrance fee', 'name': 'name', 'phone': 'phone',
                           'post': 'postcode', 'type': 'type'},
            'train': {'id': 'trainID', 'arrive': 'arriveBy', 'day': 'day', 'depart': 'departure', 'dest': 'destination',
-                     'time': 'duration', 'leave': 'leaveAt', 'ticket': 'price'},
+                     'time': 'duration', 'leave': 'leaveAt', 'ticket': 'price', 'ref': 'ref'},
            'taxi': {'car': 'car type', 'phone': 'phone'},
            'hospital': {'post': 'postcode', 'phone': 'phone', 'addr': 'address', 'department': 'department'},
            'police': {'post': 'postcode', 'phone': 'phone', 'addr': 'address'}}
@@ -39,6 +57,32 @@ mapping = {'restaurant': {'addr': 'address', 'area': 'area', 'food': 'food', 'na
 time_re = re.compile(r'^(([01]\d|2[0-4]):([0-5]\d)|24:00)$')
 NUL_VALUE = ["", "dont care", 'not mentioned',
              "don't care", "dontcare", "do n't care"]
+REF_SYS_DA_M = {}
+for dom, ref_slots in REF_SYS_DA.items():
+    dom = dom.lower()
+    REF_SYS_DA_M[dom] = {}
+    for slot_a, slot_b in ref_slots.items():
+        if slot_a == 'Ref':
+            slot_b = 'ref'
+        REF_SYS_DA_M[dom][slot_a.lower()] = slot_b
+    REF_SYS_DA_M[dom]['none'] = 'none'
+REF_SYS_DA_M['taxi']['phone'] = 'phone'
+REF_SYS_DA_M['taxi']['car'] = 'car type'
+DEF_VAL_UNK = '?'  # Unknown
+DEF_VAL_DNC = 'dontcare'  # Do not care
+DEF_VAL_NUL = 'none'  # for none
+DEF_VAL_BOOKED = 'yes'  # for booked
+DEF_VAL_NOBOOK = 'no'  # for booked
+
+NOT_SURE_VALS = [DEF_VAL_UNK, DEF_VAL_DNC, DEF_VAL_NUL, DEF_VAL_NOBOOK]
+
+# Not sure values in inform
+DEF_VAL_UNK = '?'  # Unknown
+DEF_VAL_DNC = 'dontcare'  # Do not care
+DEF_VAL_NUL = 'none'  # for none
+DEF_VAL_BOOKED = 'yes'  # for booked
+DEF_VAL_NOBOOK = 'no'  # for booked
+NOT_SURE_VALS = [DEF_VAL_UNK, DEF_VAL_DNC, DEF_VAL_NUL, DEF_VAL_NOBOOK]
 
 
 class MultiWozEvaluator(Evaluator):
@@ -56,7 +100,8 @@ class MultiWozEvaluator(Evaluator):
         self.success = 0
         self.success_strict = 0
         self.successful_domains = []
-        logging.info(f"We check booking constraints: {self.check_book_constraints}")
+        logging.info(
+            f"We check booking constraints: {self.check_book_constraints}")
 
     def _init_dict(self):
         dic = {}
@@ -93,12 +138,19 @@ class MultiWozEvaluator(Evaluator):
         """
         self.sys_da_array = []
         self.usr_da_array = []
-        self.goal = goal
+        self.goal = deepcopy(goal)
         self.cur_domain = ''
         self.booked = self._init_dict_booked()
         self.booked_states = self._init_dict_booked()
         self.successful_domains = []
 
+    @staticmethod
+    def _convert_action(act):
+        act = unified_format(act)
+        act = reverse_da(act)
+        act = act_dict_to_flat_tuple(act)
+        return act
+
     def add_sys_da(self, da_turn, belief_state=None):
         """add sys_da into array
 
@@ -107,11 +159,16 @@ class MultiWozEvaluator(Evaluator):
                 list[intent, domain, slot, value]
         """
 
-        sys_dialog_act = da_turn
-        sys_dialog_act = unified_format(sys_dialog_act)
-        sys_dialog_act = reverse_da(sys_dialog_act)
-        sys_dialog_act = act_dict_to_flat_tuple(sys_dialog_act)
-        da_turn = sys_dialog_act
+        new_acts = list()
+        for intent, domain, slot, value in da_turn:
+            if intent.lower() == 'book':
+                ref = [_value for _intent, _domain, _slot, _value in da_turn if _domain == domain and _intent.lower() == 'inform' and _slot.lower() == 'ref']
+                ref = ref[0] if ref else ''
+                value = ref
+            new_acts.append([intent, domain, slot, value])
+        da_turn = new_acts
+
+        da_turn = self._convert_action(da_turn)
 
         for intent, domain, slot, value in da_turn:
             dom_int = '-'.join([domain, intent])
@@ -131,12 +188,15 @@ class MultiWozEvaluator(Evaluator):
                 else:
                     if not self.booked[domain] and re.match(r'^\d{8}$', value) and \
                             len(self.dbs[domain]) > int(value):
-                        self.booked[domain] = self.dbs[domain][int(value)].copy()
+                        self.booked[domain] = self.dbs[domain][int(
+                            value)].copy()
                         self.booked[domain]['Ref'] = value
                         if belief_state is not None:
-                            self.booked_states[domain] = deepcopy(belief_state[domain])
+                            self.booked_states[domain] = deepcopy(
+                                belief_state[domain])
                         else:
                             self.booked_states[domain] = None
+        self.goal = self.update_goal(self.goal, da_turn)
 
     def add_usr_da(self, da_turn):
         """add usr_da into array
@@ -145,6 +205,7 @@ class MultiWozEvaluator(Evaluator):
             da_turn:
                 list[intent, domain, slot, value]
         """
+        da_turn = self._convert_action(da_turn)
         for intent, domain, slot, value in da_turn:
             dom_int = '-'.join([domain, intent])
             domain = dom_int.split('-')[0].lower()
@@ -384,7 +445,9 @@ class MultiWozEvaluator(Evaluator):
                     goal[d]['info'][mapping[d][s]] = v
                 elif i == 'request':
                     goal[d]['reqt'].append(s)
-        TP, FP, FN, _, _, _ = self._inform_F1_goal(goal, self.sys_da_array)
+
+        TP, FP, FN, bad_inform, reqt_not_inform, inform_not_reqt = self._inform_F1_goal(
+            goal, self.sys_da_array)
         if aggregate:
             try:
                 rec = TP / (TP + FN)
@@ -405,22 +468,22 @@ class MultiWozEvaluator(Evaluator):
         """
         booking_done = self.check_booking_done(ref2goal)
         book_sess = self.book_rate(ref2goal)
-        #book_constraint_sess = self.book_rate_constrains(ref2goal)
-        book_constraint_sess = 1
+        book_constraint_sess = self.book_rate_constrains(ref2goal)
         inform_sess = self.inform_F1(ref2goal)
         goal_sess = self.final_goal_analyze()
-        #goal_sess = 1
-        # book rate == 1 & inform recall == 1
+
         if ((book_sess == 1 and inform_sess[1] == 1)
             or (book_sess == 1 and inform_sess[1] is None)
             or (book_sess is None and inform_sess[1] == 1)) \
                 and goal_sess == 1:
             self.complete = 1
             self.success = 1
-            self.success_strict = 1 if (book_constraint_sess == 1 or book_constraint_sess is None) else 0
+            self.success_strict = 1 if (
+                book_constraint_sess == 1 or book_constraint_sess is None) else 0
             return self.success if not self.check_book_constraints else self.success_strict
         else:
-            self.complete = 1 if booking_done and (inform_sess[1] == 1 or inform_sess[1] is None) else 0
+            self.complete = 1 if booking_done and (
+                inform_sess[1] == 1 or inform_sess[1] is None) else 0
             self.success = 0
             self.success_strict = 0
             return 0
@@ -473,13 +536,16 @@ class MultiWozEvaluator(Evaluator):
                 elif i == 'request':
                     goal[d]['reqt'].append(s)
 
-        book_constraints = self._book_goal_constraints(goal, self.booked_states, [domain])
-        book_constraints = np.mean(book_constraints) if book_constraints else None
+        book_constraints = self._book_goal_constraints(
+            goal, self.booked_states, [domain])
+        book_constraints = np.mean(
+            book_constraints) if book_constraints else None
 
         book_rate = self._book_rate_goal(goal, self.booked, [domain])
         book_rate = np.mean(book_rate) if book_rate else None
         match, mismatch = self._final_goal_analyze_domain(domain)
-        goal_sess = 1 if (match == 0 and mismatch == 0) else match / (match + mismatch)
+        goal_sess = 1 if (match == 0 and mismatch ==
+                          0) else match / (match + mismatch)
 
         inform = self._inform_F1_goal(goal, self.sys_da_array, [domain])
         try:
@@ -488,9 +554,10 @@ class MultiWozEvaluator(Evaluator):
             inform_rec = None
 
         if ((book_rate == 1 and inform_rec == 1) or (book_rate == 1 and inform_rec is None) or
-            (book_rate is None and inform_rec == 1)) and goal_sess == 1:
+                (book_rate is None and inform_rec == 1)) and goal_sess == 1:
             domain_success = 1
-            domain_strict_success = 1 if (book_constraints == 1 or book_constraints is None) else 0
+            domain_strict_success = 1 if (
+                book_constraints == 1 or book_constraints is None) else 0
             return domain_success if not self.check_book_constraints else domain_strict_success
         else:
             return 0
@@ -514,7 +581,7 @@ class MultiWozEvaluator(Evaluator):
         else:
             info_constraints = []
         query_result = self.database.query(
-            domain, info_constraints, soft_contraints=reqt_constraints)
+            domain, info_constraints + reqt_constraints)
         if not query_result:
             mismatch += 1
 
@@ -547,7 +614,7 @@ class MultiWozEvaluator(Evaluator):
             else:
                 info_constraints = []
             query_result = self.database.query(
-                domain, info_constraints, soft_contraints=reqt_constraints)
+                domain, info_constraints + reqt_constraints)
             if not query_result:
                 mismatch += 1
                 continue
@@ -593,3 +660,32 @@ class MultiWozEvaluator(Evaluator):
                     self.successful_domains.append(self.cur_domain)
 
         return reward
+
+    def evaluate_dialog(self, goal, user_acts, system_acts, system_states):
+
+        self.add_goal(goal.domain_goals)
+        for sys_act, sys_state, user_act in zip(system_acts, system_states, user_acts):
+            self.add_sys_da(sys_act, sys_state)
+            self.add_usr_da(user_act)
+        self.task_success()
+        return {"complete": self.complete, "success": self.success, "success_strict": self.success_strict}
+
+    def update_goal(self, goal, system_action):
+        for intent, domain, slot, val in system_action:
+            # need to reverse slot to old representation
+            if slot in reverse_da_slot_name_map:
+                slot = reverse_da_slot_name_map[slot]
+            elif domain in reverse_da_slot_name_map and slot in reverse_da_slot_name_map[domain]:
+                slot = reverse_da_slot_name_map[domain][slot]
+            else:
+                slot = slot.capitalize()
+            if intent.lower() in ['inform', 'recommend']:
+                if domain.lower() in goal:
+                    if 'reqt' in goal[domain.lower()]:
+                        if REF_SYS_DA_M.get(domain.lower(), {}).get(slot.lower(), slot.lower()) \
+                                in goal[domain.lower()]['reqt']:
+                            if val in NOT_SURE_VALS:
+                                val = '\"' + val + '\"'
+                            goal[domain.lower()]['reqt'][
+                                REF_SYS_DA_M.get(domain.lower(), {}).get(slot.lower(), slot.lower())] = val
+        return goal
diff --git a/convlab/nlg/template/multiwoz/manual_system_template_nlg.json b/convlab/nlg/template/multiwoz/manual_system_template_nlg.json
index 414f0b80b740d0c33568ba21bab5e13e822d95df..300369b525d3d5c4781ea00587f198e56657a37b 100755
--- a/convlab/nlg/template/multiwoz/manual_system_template_nlg.json
+++ b/convlab/nlg/template/multiwoz/manual_system_template_nlg.json
@@ -225,7 +225,7 @@
             "Do you know the name of it ?",
             "can you give me the name of it ?"
         ],
-        "Price": [
+        "Fee": [
             "any specific price range to help narrow down available options ?",
             "What price range would you like ?",
             "what is your price range for that ?",
@@ -363,42 +363,6 @@
             "I ' m sorry but there is no availability for #BOOKING-NOBOOK-PEOPLE# people ."
         ]
     },
-    "Booking-Request": {
-        "Day": [
-            "What day would you like your booking for ?",
-            "What day would you like that reservation ?",
-            "what day would you like the booking to be made for ?",
-            "What day would you like to book ?",
-            "Ok , what day would you like to make the reservation on ?"
-        ],
-        "Stay": [
-            "How many nights will you be staying ?",
-            "And how many nights ?",
-            "for how many days ?",
-            "And for how many days ?",
-            "how many days would you like to stay ?",
-            "How many nights would you like to book it for ?",
-            "And what nights would you like me to reserve for you ?",
-            "How many nights are you wanting to stay ?",
-            "How many days will you be staying ?"
-        ],
-        "People": [
-            "For how many people ?",
-            "How many people will be ?",
-            "How many people will be with you ?",
-            "How many people is the reservation for ?"
-        ],
-        "Time": [
-            "Do you have a time preference ?",
-            "what time are you looking for a reservation at ?",
-            "For what time ?",
-            "What time would you like me to make your reservation ?",
-            "What time would you like the reservation for ?",
-            "what time should I make the reservation for ?",
-            "What time would you prefer ?",
-            "What time would you like the reservation for ?"
-        ]
-    },
     "Hotel-Inform": {
         "Internet": [
             "it has wifi .",
@@ -697,6 +661,30 @@
             "Do you need free parking ?",
             "Will you need parking while you 're there ?",
             "Will you be needing free parking ?"
+        ],
+        "Day": [
+            "What day would you like your booking for ?",
+            "What day would you like that reservation ?",
+            "what day would you like the booking to be made for ?",
+            "What day would you like to book ?",
+            "Ok , what day would you like to make the reservation on ?"
+        ],
+        "Stay": [
+            "How many nights will you be staying ?",
+            "And how many nights ?",
+            "for how many days ?",
+            "And for how many days ?",
+            "how many days would you like to stay ?",
+            "How many nights would you like to book it for ?",
+            "And what nights would you like me to reserve for you ?",
+            "How many nights are you wanting to stay ?",
+            "How many days will you be staying ?"
+        ],
+        "People": [
+            "For how many people ?",
+            "How many people will be ?",
+            "How many people will be with you ?",
+            "How many people is the reservation for ?"
         ]
     },
     "Restaurant-Inform": {
@@ -918,6 +906,29 @@
             "what is the name of the restaurant you are needing information on ?",
             "Do you know the name of the location ?",
             "Is there a certain restaurant you 're looking for ?"
+        ],
+        "Day": [
+            "What day would you like your booking for ?",
+            "What day would you like that reservation ?",
+            "what day would you like the booking to be made for ?",
+            "What day would you like to book ?",
+            "Ok , what day would you like to make the reservation on ?"
+        ],
+        "People": [
+            "For how many people ?",
+            "How many people will be ?",
+            "How many people will be with you ?",
+            "How many people is the reservation for ?"
+        ],
+        "Time": [
+            "Do you have a time preference ?",
+            "what time are you looking for a reservation at ?",
+            "For what time ?",
+            "What time would you like me to make your reservation ?",
+            "What time would you like the reservation for ?",
+            "what time should I make the reservation for ?",
+            "What time would you prefer ?",
+            "What time would you like the reservation for ?"
         ]
     },
     "Taxi-Inform": {
@@ -1331,6 +1342,77 @@
             "Is there a time you need to arrive by ?"
         ]
     },
+    "Police-Inform": {
+        "Addr": [
+            "it is located in #POLICE-INFORM-ADDR#",
+            "adress is #POLICE-INFORM-ADDR#",
+            "It is on #POLICE-INFORM-ADDR# .",
+            "their address in our system is listed as #POLICE-INFORM-ADDR# .",
+            "The address is #POLICE-INFORM-ADDR# .",
+            "it 's located at #POLICE-INFORM-ADDR# .",
+            "#POLICE-INFORM-ADDR# is the address",
+            "They are located at #POLICE-INFORM-ADDR# ."
+        ],
+        "Post": [
+            "The postcode of the police is #POLICE-INFORM-POST# .",
+            "The post code is #POLICE-INFORM-POST# .",
+            "Its postcode is #POLICE-INFORM-POST# .",
+            "Their postcode is #POLICE-INFORM-POST# ."
+        ],
+        "Name": [
+            "I think a fun place to visit is #POLICE-INFORM-NAME# .",
+            "#POLICE-INFORM-NAME# looks good .",
+            "#POLICE-INFORM-NAME# is available , would that work for you ?",
+            "we have #POLICE-INFORM-NAME# .",
+            "#POLICE-INFORM-NAME# is popular among visitors .",
+            "How about #POLICE-INFORM-NAME# ?",
+            "What about #POLICE-INFORM-NAME# ?",
+            "you might want to try the #POLICE-INFORM-NAME# ."
+        ],
+        "Phone": [
+            "The police phone number is #POLICE-INFORM-PHONE# .",
+            "Here is the police phone number , #POLICE-INFORM-PHONE# ."
+        ]
+    },
+    "Hospital-Inform": {
+        "Addr": [
+            "it is located in #HOSPITAL-INFORM-ADDR#",
+            "adress is #HOSPITAL-INFORM-ADDR#",
+            "It is on #HOSPITAL-INFORM-ADDR# .",
+            "their address in our system is listed as #HOSPITAL-INFORM-ADDR# .",
+            "The address is #HOSPITAL-INFORM-ADDR# .",
+            "it 's located at #HOSPITAL-INFORM-ADDR# .",
+            "#HOSPITAL-INFORM-ADDR# is the address",
+            "They are located at #HOSPITAL-INFORM-ADDR# ."
+        ],
+        "Post": [
+            "The postcode of the hospital is #HOSPITAL-INFORM-POST# .",
+            "The post code is #HOSPITAL-INFORM-POST# .",
+            "Its postcode is #HOSPITAL-INFORM-POST# .",
+            "Their postcode is #HOSPITAL-INFORM-POST# ."
+        ],
+        "Department": [
+            "The department of the hospital is #HOSPITAL-INFORM-POST# .",
+            "The department is #HOSPITAL-INFORM-POST# .",
+            "Its department is #HOSPITAL-INFORM-POST# .",
+            "Their department is #HOSPITAL-INFORM-POST# ."
+	    
+        ],
+        "Phone": [
+            "The hospital phone number is #HOSPITAL-INFORM-PHONE# .",
+            "Here is the hospital phone number , #HOSPITAL-INFORM-PHONE# ."
+        ]
+    },
+    "Hospital-Request": {
+	"Department": [
+            "What is the name of the hospital department ?",
+            "What hospital department are you thinking about ?",
+            "I ' m sorry for the confusion , what hospital department are you interested in ?",
+            "What hospital department were you thinking of ?",
+            "Do you know the department of it ?",
+            "can you give me the department of it ?"
+	]
+    },
     "general-bye": {
         "none": [
             "Thank you for using our services .",
@@ -1378,4 +1460,4 @@
             "You 're welcome . Have a good day !"
         ]
     }
-}
\ No newline at end of file
+}
diff --git a/convlab/nlg/template/multiwoz/nlg.py b/convlab/nlg/template/multiwoz/nlg.py
index f83a6db4e2ad6f77f9bdc154cfda7bf2db0ff2c5..5f362ebbabd68ee0b6fab62c692caf0e8da436e9 100755
--- a/convlab/nlg/template/multiwoz/nlg.py
+++ b/convlab/nlg/template/multiwoz/nlg.py
@@ -31,33 +31,33 @@ def read_json(filename):
 
 # supported slot
 Slot2word = {
-    'Fee': 'fee',
+    'Fee': 'entrance fee',
     'Addr': 'address',
     'Area': 'area',
-    'Stars': 'stars',
-    'Internet': 'Internet',
+    'Stars': 'number of stars',
+    'Internet': 'internet',
     'Department': 'department',
     'Choice': 'choice',
     'Ref': 'reference number',
     'Food': 'food',
     'Type': 'type',
     'Price': 'price range',
-    'Stay': 'stay',
+    'Stay': 'length of the stay',
     'Phone': 'phone number',
     'Post': 'postcode',
     'Day': 'day',
     'Name': 'name',
     'Car': 'car type',
-    'Leave': 'leave',
+    'Leave': 'departure time',
     'Time': 'time',
-    'Arrive': 'arrive',
-    'Ticket': 'ticket',
+    'Arrive': 'arrival time',
+    'Ticket': 'ticket price',
     'Depart': 'departure',
-    'People': 'people',
+    'People': 'number of people',
     'Dest': 'destination',
     'Parking': 'parking',
-    'Open': 'open',
-    'Id': 'Id',
+    'Open': 'opening hours',
+    'Id': 'id',
     # 'TrainID': 'TrainID'
 }
 
@@ -271,6 +271,10 @@ class TemplateNLG(NLG):
             elif 'request' == intent[1]:
                 for slot, value in slot_value_pairs:
                     if dialog_act not in template or slot not in template[dialog_act]:
+                        if dialog_act not in template:
+                            print("WARNING (nlg.py): (User?: %s) dialog_act '%s' not in template!" % (self.is_user, dialog_act))
+                        else:
+                            print("WARNING (nlg.py): (User?: %s) slot '%s' of dialog_act '%s' not in template!" % (self.is_user, slot, dialog_act))
                         sentence = 'What is the {} of {} ? '.format(
                             slot.lower(), dialog_act.split('-')[0].lower())
                         sentences += self._add_random_noise(sentence)
@@ -288,7 +292,7 @@ class TemplateNLG(NLG):
                         value_lower = value.lower()
                     if value in ["do nt care", "do n't care", "dontcare"]:
                         sentence = 'I don\'t care about the {} of the {}'.format(
-                            slot, dialog_act.split('-')[0])
+                            slot2word.get(slot, slot), dialog_act.split('-')[0])
                     elif self.is_user and dialog_act.split('-')[1] == 'inform' and slot == 'choice' and value_lower == 'any':
                         # user have no preference, any choice is ok
                         sentence = random.choice([
diff --git a/convlab/nlu/README.md b/convlab/nlu/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..eb332bdf3f906c42a7cfd465205a386e8a702609
--- /dev/null
+++ b/convlab/nlu/README.md
@@ -0,0 +1,70 @@
+## NLU benchmark for BERTNLU and MILU on multiwoz21, tm1, tm2, tm3
+
+To illustrate that it is easy to use the model for any dataset that in our unified format, we report the performance on several datasets in our unified format. We follow `README.md` and config files in `unified_datasets/` to generate `predictions.json`, then evaluate it using `../evaluate_unified_datasets.py`. Note that we use almost the same hyper-parameters for different datasets, which may not be optimal.
+
+<table>
+<thead>
+  <tr>
+    <th></th>
+    <th colspan=2>MultiWOZ 2.1</th>
+    <th colspan=2>Taskmaster-1</th>
+    <th colspan=2>Taskmaster-2</th>
+    <th colspan=2>Taskmaster-3</th>
+  </tr>
+</thead>
+<thead>
+  <tr>
+    <th>Model</th>
+    <th>Acc</th><th>F1</th>
+    <th>Acc</th><th>F1</th>
+    <th>Acc</th><th>F1</th>
+    <th>Acc</th><th>F1</th>
+  </tr>
+</thead>
+<tbody>
+  <tr>
+    <td>T5-small</td>
+    <td>77.8</td><td>86.5</td>
+    <td>74.0</td><td>52.5</td>
+    <td>80.0</td><td>71.4</td>
+    <td>87.2</td><td>83.1</td>
+  </tr>
+  <tr>
+    <td>T5-small (context=3)</td>
+    <td>82.0</td><td>90.3</td>
+    <td>76.2</td><td>56.2</td>
+    <td>82.4</td><td>74.3</td>
+    <td>89.0</td><td>85.1</td>
+  </tr>
+  <tr>
+    <td>BERTNLU</td>
+    <td>74.5</td><td>85.9</td>
+    <td>72.8</td><td>50.6</td>
+    <td>79.2</td><td>70.6</td>
+    <td>86.1</td><td>81.9</td>
+  </tr>
+  <tr>
+    <td>BERTNLU (context=3)</td>
+    <td>80.6</td><td>90.3</td>
+    <td>74.2</td><td>52.7</td>
+    <td>80.9</td><td>73.3</td>
+    <td>87.8</td><td>83.8</td>
+  </tr>
+  <tr>
+    <td>MILU</td>
+    <td>72.9</td><td>85.2</td>
+    <td>72.9</td><td>49.2</td>
+    <td>79.1</td><td>68.7</td>
+    <td>85.4</td><td>80.3</td>
+  </tr>
+  <tr>
+    <td>MILU (context=3)</td>
+    <td>76.6</td><td>87.9</td>
+    <td>72.4</td><td>48.5</td>
+    <td>78.9</td><td>68.4</td>
+    <td>85.1</td><td>80.1</td>
+  </tr>
+</tbody>
+
+- Acc: whether all dialogue acts of an utterance are correctly predicted
+- F1: F1 measure of the dialogue act predictions over the corpus.
\ No newline at end of file
diff --git a/convlab/nlu/jointBERT/multiwoz/nlu.py b/convlab/nlu/jointBERT/multiwoz/nlu.py
index e25fbad1227c4b1f85ae2ae42a8ac899fa61d7b8..1373919e5861156c87a2ba14d6506d13e0842204 100755
--- a/convlab/nlu/jointBERT/multiwoz/nlu.py
+++ b/convlab/nlu/jointBERT/multiwoz/nlu.py
@@ -74,7 +74,8 @@ class BERTNLU(NLU):
         for token in token_list:
             token = token.strip()
             self.nlp.tokenizer.add_special_case(
-                token, [{ORTH: token, LEMMA: token, POS: u'NOUN'}])
+                #token, [{ORTH: token, LEMMA: token, POS: u'NOUN'}])
+                token, [{ORTH: token}])
         logging.info("BERTNLU loaded")
 
     def predict(self, utterance, context=list()):
diff --git a/convlab/nlu/jointBERT/unified_datasets/configs/multiwoz21_sys_context3.json b/convlab/nlu/jointBERT/unified_datasets/configs/multiwoz21_sys_context3.json
new file mode 100755
index 0000000000000000000000000000000000000000..dfbef5a39963f030ccf989276fcaf7efb8141cc1
--- /dev/null
+++ b/convlab/nlu/jointBERT/unified_datasets/configs/multiwoz21_sys_context3.json
@@ -0,0 +1,27 @@
+{
+  "dataset_name": "multiwoz21",
+  "data_dir": "unified_datasets/data/multiwoz21/system/context_window_size_3",
+  "output_dir": "unified_datasets/output/multiwoz21/system/context_window_size_3",
+  "zipped_model_path": "unified_datasets/output/multiwoz21/system/context_window_size_3/bertnlu_unified_multiwoz21_system_context3.zip",
+  "log_dir": "unified_datasets/output/multiwoz21/system/context_window_size_3/log",
+  "DEVICE": "cuda:0",
+  "seed": 2019,
+  "cut_sen_len": 40,
+  "use_bert_tokenizer": true,
+  "context_window_size": 3,
+  "model": {
+    "finetune": true,
+    "context": true,
+    "context_grad": true,
+    "pretrained_weights": "bert-base-uncased",
+    "check_step": 1000,
+    "max_step": 10000,
+    "batch_size": 128,
+    "learning_rate": 1e-4,
+    "adam_epsilon": 1e-8,
+    "warmup_steps": 0,
+    "weight_decay": 0.0,
+    "dropout": 0.1,
+    "hidden_units": 1536
+  }
+}
diff --git a/convlab/policy/README.md b/convlab/policy/README.md
index 1990cdd6b03a38fe6f5a4f8b4eb8a9708761c590..233ea0eece10d0789791e3a97c5d1aa8311f232a 100755
--- a/convlab/policy/README.md
+++ b/convlab/policy/README.md
@@ -1,36 +1,199 @@
 # Dialog Policy
 
-In the pipeline task-oriented dialog framework, the dialog policy module
-takes as input the dialog state, and chooses the system action bases on
-it.
-
-This directory contains the interface definition of dialog policy
+In the pipeline task-oriented dialog framework, the dialogue policy module
+takes as input the dialog state, and chooses the system action based on
+it. This directory contains the interface definition of dialogue policy
 module for both system side and user simulator side, as well as some
-implementations under different sub-directories.
+implementations under different sub-directories. 
+
+An important additional module for the policy is the vectoriser which translates the dialogue state into a vectorised form that the dialogue policy network expects as input. 
+Moreover, it translates the vectorised act that the policy took back into semantic form. More information can be found in the directory /convlab/policy/vector.
+
+We currently maintain the following policies:
+
+**system policies**: GDPL, MLE, PG, PPO and VTRACE DPT
+
+**user policies**: rule, TUS, GenTUS
+
+
+## Overview
+
+Every policy directory typically has two python scripts, **1) train.py** for running an RL training and **2) a dedicated script** that implements the algorithm and loads the policy network (e.g. **ppo.py** for the ppo policy).
+Moreover, two config files define the environment and the hyper parameters for the algorithm:
+
+- config.json: defines the hyper parameters such as learning rate and algorithm related parameters
+- environment.json: defines the learning environment (MDP) for the policy. This includes the NLU, DST and NLG component for both system and user policy as well as which user policy should be used. It also defines the number of total training dialogues as well as evaluation dialogues and frequency.
+
+An example for the environment.json is the **semantic_level_config.json** in the policy subfolders.
+
+
+
+## Workflow
+
+The workflow can be generally decomposed into three steps that will be explained in more detail below:
+
+1. set up the environment configuration and policy parameters
+2. run a reinforcement learning training with the given configurations
+3. evaluate your trained models
+
+#### Set up the environment 
+
+The necessary step before starting a training is to set up the environment and policy parameters. Information about policy parameters can be found in each policy subfolder. The following example defines an environment for the policy with the rule-based dialogue state tracker, no NLU, no NLG, and the rule-based user simulator:
+
+```
+{
+	"model": {
+		"load_path": "", # specify a loading path to load a pre-trained model, omit the ending .pol.mdl
+		"use_pretrained_initialisation": false, # will download a provided ConvLab-3 model
+		"pretrained_load_path": "",
+		"seed": 0, # the seed for the experiment
+		"eval_frequency": 5, # how often evaluation should take place
+		"process_num": 4, # how many processes the evaluation should use for speed up
+		"sys_semantic_to_usr": false,
+		"num_eval_dialogues": 500 # how many dialogues should be used for evaluation
+	},
+	"vectorizer_sys": {
+		"uncertainty_vector_mul": {
+			"class_path": "convlab.policy.vector.vector_binary.VectorBinary",
+			"ini_params": {
+				"use_masking": true,
+				"manually_add_entity_names": false,
+				"seed": 0
+			}
+		}
+	},
+	"nlu_sys": {},
+	"dst_sys": {
+		"RuleDST": {
+			"class_path": "convlab.dst.rule.multiwoz.dst.RuleDST",
+			"ini_params": {}
+		}
+	},
+	"sys_nlg": {},
+	"nlu_usr": {},
+	"dst_usr": {},
+	"policy_usr": {
+		"RulePolicy": {
+			"class_path": "convlab.policy.rule.multiwoz.RulePolicy",
+			"ini_params": {
+				"character": "usr"
+			}
+		}
+	},
+	"usr_nlg": {}
+}
+```
+
+#### Executing a training
+
+Once you set up your configuration, you are ready to start an experiment by executing
+
+```sh
+$ python convlab/policy/policy_subfolder/train.py --path=your_environment_config --seed=your_seed
+```
+
+You can specify the seed either in the environment config or through the argument parser. If you do not specify an environment config, it will automatically load the default config. 
+
+Once the training started, it will automatically generate an **experiment** folder and a corresponding experiment-TIMESTEP folder in it. Inside of that, there are 4 subfolders configs, logs, save and TB_summary:
+
+- **configs**: containts information about which config was used
+- **logs**: will save information created by a logger during training
+- **save**: a folder for saving model checkpoints
+- **TB_summary**: saves a tensorboard summary that will be later used for plotting graphs
+
+Once the training finished, it will move the experiment-TIMESTAMP folder into the **finished_experiments** folder.
+
+#### Evaluating your models
+
+The evaluation tools can be found in the folder convlab/policy/plot_results. Please have a look in the README for detailed instructions. 
+
+#### Running Evaluation Dialogues
+
+You can run evaluation dialogues with a trained model using 
+
+```sh
+$ python convlab/policy/evaluate.py --model_name=NAME --config_path=PATH --num_dialogues=NUM --verbose
+```
+
+- model_name: specify which model is used, i.e. MLE, PPO, PG, DDPT
+- config_path: specify the config-path that was used during RL training, for instance semantic_level_config.json
+- num_dialogues: number of evaluation dialogues
+- verbose: can be also excluded. If used, it will print the dialogues in the termain consoloe together with its goal. That helps in analysing the behaviour of the policy.
+
+## Adding a new policy
+
+If you would like to add a new policy, start by creating a subfolder for it. Then make sure that you have the four files mentioned in **Overview** section in it.
+
+#### Algorithm script
+
+Here you define your algorithm and policy network. Please ensure that you also load a vectoriser here that is inherited from the vector/vector_base.py class. 
+
+In addition, your policy module is required to have a **predict** method where the skeleton usually looks something like:
+
+    def predict(self, state):
+        """
+        Predict an system action given state.
+        Args:
+            state (dict): Dialog state. Please refer to util/state.py
+        Returns:
+            action : System act, with the form of (act_type, {slot_name_1: value_1, slot_name_2, value_2, ...})
+        """
+        
+        # uses the vector class for vectorisation of the dialogue state and also creates an action mask
+        s, action_mask = self.vector.state_vectorize(state) 
+        s_vec = torch.Tensor(s)
+        mask_vec = torch.Tensor(action_mask)
+        
+        # predict an action using the policy network
+        a = self.policy.select_action(s_vec, mask_vec)
+
+        # map the action indices back to semantic actions using the vectoriser
+        action = self.vector.action_devectorize(a.detach().numpy())
+        return action
+
+#### train.py script
 
-## Interface
+The train.py script is responsible for several different functions. In the following we will provide some code or pointers on how to do these steps. Have a look at the train.py files as well.
 
-The interfaces for dialog policy are defined in policy.Policy:
+1. load the config and set seed
+    ```
+    environment_config = load_config_file(path)
+    conf = get_config(path, args)
+    seed = conf['model']['seed']
+    set_seed(seed)
+   save_config(vars(parser.parse_args()), environment_config, config_save_path)
+    ```
 
-- **predict** takes as input agent state (often the state tracked by DST)
-and outputs the next system action.
+2. saves additional information (through a logger and tensorboard writer)
 
-- **init_session** reset the model variables for a new dialog session.
+    ```
+    logger, tb_writer, current_time, save_path, config_save_path, dir_path, log_save_path = \
+        init_logging(os.path.dirname(os.path.abspath(__file__)), mode)
+   ```   
+   
+   
+3. load the policy module
 
-## Rule based simulator results
+    ```
+    policy_sys = PPO(True, seed=conf['model']['seed'], vectorizer=conf['vectorizer_sys_activated'])
+    ```
+4. load the environment using th environment-config
+    ```
+   env, sess = env_config(conf, policy_sys)
+   ```
 
-| Model | Complete rate | Success rate | Average return | Turns | Average actions |
-|-------|---------------|--------------|----------------|-------|-----------------|
-| MLE   |               |              |                |       |                 |
-| PG    |               |              |                |       |                 |
-| GDPL  |               |              |                |       |                 |
-| PPO   |               |              |                |       |                 |
+5. collect dialogues and execute policy updates: use the update function of policy_sys and implement a create_episodes function.
+6. run evaluation during training and save policy checkpoints
 
-## Transformer based user simulator (TUS) results
+    ```
+    logging.info(f"Evaluating after Dialogues: {num_dialogues} - {time_now}" + '-' * 60)
+    eval_dict = eval_policy(conf, policy_sys, env, sess, save_eval, log_save_path)
+    best_complete_rate, best_success_rate, best_return = \
+        save_best(policy_sys, best_complete_rate, best_success_rate, best_return,
+                  eval_dict["complete_rate"], eval_dict["success_rate_strict"],
+                  eval_dict["avg_return"], save_path)
+    policy_sys.save(save_path, "last")
+    for key in eval_dict:
+        tb_writer.add_scalar(key, eval_dict[key], idx * conf['model']['batchsz'])
+    ```
 
-| Model | Complete rate | Success rate | Average return | Turns | Average actions |
-|-------|---------------|--------------|----------------|-------|-----------------|
-| MLE   |               |              |                |       |                 |
-| PG    |               |              |                |       |                 |
-| GDPL  |               |              |                |       |                 |
-| PPO   |               |              |                |       |                 |
diff --git a/convlab/policy/evaluate.py b/convlab/policy/evaluate.py
index 78682e6ced2ccaa01e769c78c3388ad19c78c0c9..7a692261869f35e587c34a26e425d1489abdcf56 100755
--- a/convlab/policy/evaluate.py
+++ b/convlab/policy/evaluate.py
@@ -12,7 +12,8 @@ from convlab.dialog_agent.session import BiSession
 from convlab.evaluator.multiwoz_eval import MultiWozEvaluator
 from convlab.policy.rule.multiwoz import RulePolicy
 from convlab.task.multiwoz.goal_generator import GoalGenerator
-from convlab.util.custom_util import set_seed, get_config, env_config, create_goals
+from convlab.util.custom_util import set_seed, get_config, env_config, create_goals, data_goals
+from tqdm import tqdm
 
 
 def init_logging(log_dir_path, path_suffix=None):
@@ -36,7 +37,7 @@ def init_logging(log_dir_path, path_suffix=None):
 DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
 
-def evaluate(config_path, model_name, verbose=False):
+def evaluate(config_path, model_name, verbose=False, model_path="", goals_from_data=False, dialogues=500):
     seed = 0
     set_seed(seed)
 
@@ -56,9 +57,15 @@ def evaluate(config_path, model_name, verbose=False):
     elif model_name == "GDPL":
         from convlab.policy.gdpl import GDPL
         policy_sys = GDPL(vectorizer=conf['vectorizer_sys_activated'])
+    elif model_name == "DDPT":
+        from convlab.policy.vtrace_DPT import VTRACE
+        policy_sys = VTRACE(is_train=False, vectorizer=conf['vectorizer_sys_activated'])
 
     try:
-        policy_sys.load(conf['model']['load_path'])
+        if model_path:
+            policy_sys.load(model_path)
+        else:
+            policy_sys.load(conf['model']['load_path'])
     except Exception as e:
         logging.info(f"Could not load a policy: {e}")
 
@@ -68,11 +75,16 @@ def evaluate(config_path, model_name, verbose=False):
     task_success = {'Complete': [], 'Success': [],
                     'Success strict': [], 'total_return': [], 'turns': []}
 
-    dialogues = 500
     goal_generator = GoalGenerator()
-    goals = create_goals(goal_generator, num_goals=dialogues, single_domains=False, allowed_domains=None)
+    if goals_from_data:
+        logging.info("read goals from dataset...")
+        goals = data_goals(dialogues, dataset="multiwoz21", dial_ids_order=0)
+    else:
+        logging.info("create goals from goal_generator...")
+        goals = create_goals(goal_generator, num_goals=dialogues,
+                             single_domains=False, allowed_domains=None)
 
-    for seed in range(1000, 1000 + dialogues):
+    for seed in tqdm(range(1000, 1000 + dialogues)):
         set_seed(seed)
         sess.init_session(goal=goals[seed-1000])
         sys_response = []
@@ -113,7 +125,10 @@ def evaluate(config_path, model_name, verbose=False):
                 task_succ = sess.evaluator.task_success()
                 task_succ = sess.evaluator.success
                 task_succ_strict = sess.evaluator.success_strict
-                complete = sess.evaluator.complete
+                if goals_from_data:
+                    complete = sess.user_agent.policy.policy.goal.task_complete()
+                else:
+                    complete = sess.evaluator.complete
                 break
 
         if verbose:
@@ -139,17 +154,29 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("--model_name", type=str,
                         default="PPO", help="name of model")
-    parser.add_argument("--config_path", type=str,
-                        default='', help="path of model")
-    parser.add_argument("--verbose", action='store_true',
+    parser.add_argument("-C", "--config_path", type=str,
+                        default='', help="config path defining the environment for simulation and system pipeline")
+    parser.add_argument("--model_path", type=str,
+                        default='', help="if this is set, tries to load the model weights from this path"
+                                         ", otherwise from config")
+    parser.add_argument("-N", "--num_dialogues", type=int,
+                        default=500, help="# of evaluation dialogue")
+    parser.add_argument("-V", "--verbose", action='store_true',
                         help="whether to output utterances")
     parser.add_argument("--log_path_suffix", type=str,
                         default="", help="suffix of path of log file")
     parser.add_argument("--log_dir_path", type=str,
                         default="log", help="path of log directory")
+    parser.add_argument("-D", "--goals_from_data", action='store_true',
+                        help="load goal from the dataset")
 
     args = parser.parse_args()
 
     init_logging(log_dir_path=args.log_dir_path,
                  path_suffix=args.log_path_suffix)
-    evaluate(config_path=args.config_path, model_name=args.model_name, verbose=args.verbose)
+    evaluate(config_path=args.config_path,
+             model_name=args.model_name,
+             verbose=args.verbose,
+             model_path=args.model_path,
+             goals_from_data=args.goals_from_data,
+             dialogues=args.num_dialogues)
diff --git a/convlab/policy/evaluate_distributed.py b/convlab/policy/evaluate_distributed.py
index 2b362d865880d21a220db467e1de5e1c20eafb2e..1f7b3ffe93c040e6e18aa0ccd88b8c21fbcd178c 100644
--- a/convlab/policy/evaluate_distributed.py
+++ b/convlab/policy/evaluate_distributed.py
@@ -2,17 +2,13 @@
 
 import random
 import torch
-import sys
-import torch
-from pprint import pprint
-
-import matplotlib.pyplot as plt
 import numpy as np
-from convlab.policy.rlmodule import Memory_evaluator, Transition
+
+from convlab.policy.rlmodule import Memory_evaluator
 from torch import multiprocessing as mp
 
 
-def sampler(pid, queue, evt, sess, seed_range):
+def sampler(pid, queue, evt, sess, seed_range, goals):
     """
     This is a sampler function, and it will be called by multiprocess.Process to sample data from environment by multiple
     processes.
@@ -31,7 +27,8 @@ def sampler(pid, queue, evt, sess, seed_range):
         torch.cuda.manual_seed(seed)
         random.seed(seed)
         np.random.seed(seed)
-        sess.init_session()
+        goal = goals.pop()
+        sess.init_session(goal=goal)
         sys_response = '' if sess.sys_agent.nlg is not None else []
         sys_response = [] if sess.sys_agent.return_semantic_acts else sys_response
         total_return_success = 0.0
@@ -46,6 +43,7 @@ def sampler(pid, queue, evt, sess, seed_range):
         request = 0
         select = 0
         offer = 0
+        recommend = 0
         task_success = {}
 
         for i in range(40):
@@ -70,6 +68,8 @@ def sampler(pid, queue, evt, sess, seed_range):
                     select += 1
                 if intent.lower() == 'offerbook':
                     offer += 1
+                if intent.lower() == 'recommend':
+                    recommend += 1
 
             if session_over is True:
                 success = sess.evaluator.task_success()
@@ -84,7 +84,7 @@ def sampler(pid, queue, evt, sess, seed_range):
             task_success[key].append(success_strict)
 
         buff.push(complete, success, success_strict, total_return_complete, total_return_success, turns, avg_actions / turns,
-                  task_success, book, inform, request, select, offer)
+                  task_success, book, inform, request, select, offer, recommend)
 
     # this is end of sampling all batchsz of items.
     # when sampling is over, push all buff data into queue
@@ -92,7 +92,7 @@ def sampler(pid, queue, evt, sess, seed_range):
     evt.wait()
 
 
-def sample(sess, seedrange, process_num):
+def sample(sess, seedrange, process_num, goals):
     """
     Given batchsz number of task, the batchsz will be splited equally to each processes
     and when processes return, it merge all data and return
@@ -112,7 +112,8 @@ def sample(sess, seedrange, process_num):
     processes = []
     for i in range(process_num):
         process_args = (
-            i, queue, evt, sess, seedrange[i * num_seeds_per_thread: (i+1) * num_seeds_per_thread])
+            i, queue, evt, sess, seedrange[i * num_seeds_per_thread: (i+1) * num_seeds_per_thread],
+            goals[i * num_seeds_per_thread: (i+1) * num_seeds_per_thread])
         processes.append(mp.Process(target=sampler, args=process_args))
     for p in processes:
         # set the process as daemon, and it will be killed once the main process is stoped.
@@ -132,13 +133,13 @@ def sample(sess, seedrange, process_num):
     return buff.get_batch()
 
 
-def evaluate_distributed(sess, seed_range, process_num):
+def evaluate_distributed(sess, seed_range, process_num, goals):
 
-    batch = sample(sess, seed_range, process_num)
-    return np.average(batch.complete), np.average(batch.success), np.average(batch.success_strict), \
-           np.average(batch.total_return_success), np.average(batch.turns), np.average(batch.avg_actions), \
-           batch.task_success, np.average(batch.book_actions), np.average(batch.inform_actions), np.average(batch.request_actions), \
-           np.average(batch.select_actions), np.average(batch.offer_actions)
+    batch = sample(sess, seed_range, process_num, goals)
+    return batch.complete, batch.success, batch.success_strict, batch.total_return_success, batch.turns, \
+           batch.avg_actions, batch.task_success, np.average(batch.book_actions), np.average(batch.inform_actions), \
+           np.average(batch.request_actions), np.average(batch.select_actions), np.average(batch.offer_actions), \
+           np.average(batch.recommend_actions)
 
 
 if __name__ == "__main__":
diff --git a/convlab/policy/gdpl/README.md b/convlab/policy/gdpl/README.md
index e9e62e96a733fa5a7e0e953df6294e97ae4499f7..7f7e4939c8fd9a2cac010885f9306c3601a4de82 100755
--- a/convlab/policy/gdpl/README.md
+++ b/convlab/policy/gdpl/README.md
@@ -1,33 +1,53 @@
-# GDPL
+# Guided Dialogue Policy Learning (GDPL)
 
-A join policy optimization and reward estimation method using adversarial inverse reinforcement learning that learns a dialog policy and builds a reward estimator simultaneously. The reward estimator evaluates the state-action pairs to guide the dialog policy at each dialog turn.
+GDPL uses the PPO algorithm to optimize the policy. The difference to vanilla PPO is that it is not using the extrinsic reward for optimization but leverages inverse reinforcement learning to train a reward estimator. This reward estimator provides the reward that should be optimized.
 
-## Train
+## Supervised pre-training
 
-Run `train.py` in the `gdpl` directory:
+If you want to obtain a supervised model for pre-training, please have a look in the MLE policy folder.
 
-```bash
-python train.py
+## RL training
+
+Starting a RL training is as easy as executing
+
+```sh
+$ python train.py --path=your_environment_config --seed=SEED
 ```
 
-For better performance, we can do immitating learning before reinforcement learning. The immitating learning is implemented in the `mle` directory.
+One example for the environment-config is **semantic_level_config.json**, where parameters for the training are specified, for instance
 
-For example, if the trained model of immitating learning is saved at FOLDER_OF_MODEL/best_mle.pol.mdl, then you can run
+- load_path: provide a path to initialise the model with a pre-trained model, skip the ending .pol.mdl
+- process_num: the number of processes to use during evaluation to speed it up
+- num_eval_dialogues: how many evaluation dialogues should be used
+- epoch: how many training epochs to run. One epoch consists of collecting dialogues + performing an update
+- eval_frequency: after how many epochs perform an evaluation
+- batchsz: the number of training dialogues collected before doing an update
 
-```bash
-python train.py --load_path FOLDER_OF_MODEL/best_mle
-```
+Moreover, you can specify the full dialogue pipeline here, such as the user policy, NLU for system and user, etc.
+
+Parameters that are tied to the RL algorithm and the model architecture can be changed in config.json.
+
+
+## Evaluation
 
-Note that the *.pol.mdl* suffix should not appear in the --load_path argument.
+For creating evaluation plots and running evaluation dialogues, please have a look in the README of the policy folder.
 
-## Reference
+## References
 
 ```
-@inproceedings{takanobu2019guided,
-  title={Guided Dialog Policy Learning: Reward Estimation for Multi-Domain Task-Oriented Dialog},
-  author={Takanobu, Ryuichi and Zhu, Hanlin and Huang, Minlie},
-  booktitle={EMNLP-IJCNLP},
-  pages={100--110},
-  year={2019}
+@inproceedings{takanobu-etal-2019-guided,
+    title = "Guided Dialog Policy Learning: Reward Estimation for Multi-Domain Task-Oriented Dialog",
+    author = "Takanobu, Ryuichi  and
+      Zhu, Hanlin  and
+      Huang, Minlie",
+    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing and the 9th International Joint Conference on Natural Language Processing (EMNLP-IJCNLP)",
+    month = nov,
+    year = "2019",
+    address = "Hong Kong, China",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/D19-1010",
+    doi = "10.18653/v1/D19-1010",
+    pages = "100--110",
+    abstract = "Dialog policy decides what and how a task-oriented dialog system will respond, and plays a vital role in delivering effective conversations. Many studies apply Reinforcement Learning to learn a dialog policy with the reward function which requires elaborate design and pre-specified user goals. With the growing needs to handle complex goals across multiple domains, such manually designed reward functions are not affordable to deal with the complexity of real-world tasks. To this end, we propose Guided Dialog Policy Learning, a novel algorithm based on Adversarial Inverse Reinforcement Learning for joint reward estimation and policy optimization in multi-domain task-oriented dialog. The proposed approach estimates the reward signal and infers the user goal in the dialog sessions. The reward estimator evaluates the state-action pairs so that it can guide the dialog policy at each dialog turn. Extensive experiments on a multi-domain dialog dataset show that the dialog policy guided by the learned reward function achieves remarkably higher task success than state-of-the-art baselines.",
 }
 ```
\ No newline at end of file
diff --git a/convlab/policy/gdpl/semantic_level_config.json b/convlab/policy/gdpl/semantic_level_config.json
index e64159c7d9eb5ff9e5c910f9cd27786cfa2d16be..0dcd46662620371c7647b05d60df0402e3737f0a 100644
--- a/convlab/policy/gdpl/semantic_level_config.json
+++ b/convlab/policy/gdpl/semantic_level_config.json
@@ -5,7 +5,7 @@
 		"pretrained_load_path": "",
 		"batchsz": 1000,
 		"seed": 0,
-		"epoch": 50,
+		"epoch": 10,
 		"eval_frequency": 5,
 		"process_num": 4,
 		"sys_semantic_to_usr": false,
diff --git a/convlab/policy/gdpl/train.py b/convlab/policy/gdpl/train.py
index 3d560b5650d5eb87e02334a42d62b4bdfd097279..15e9b7d08812a53c917b7d9f2f9e6aa60bfa97c7 100755
--- a/convlab/policy/gdpl/train.py
+++ b/convlab/policy/gdpl/train.py
@@ -186,7 +186,7 @@ if __name__ == '__main__':
     parser = ArgumentParser()
     parser.add_argument("--path", type=str, default='convlab/policy/gdpl/semantic_level_config.json',
                         help="Load path for config file")
-    parser.add_argument("--seed", type=int, default=0,
+    parser.add_argument("--seed", type=int, default=None,
                         help="Seed for the policy parameter initialization")
     parser.add_argument("--pretrain", action='store_true', help="whether to pretrain the reward estimator")
     parser.add_argument("--mode", type=str, default='info',
@@ -202,7 +202,7 @@ if __name__ == '__main__':
     logger, tb_writer, current_time, save_path, config_save_path, dir_path, log_save_path = \
         init_logging(os.path.dirname(os.path.abspath(__file__)), mode)
 
-    args = [('model', 'seed', seed)]
+    args = [('model', 'seed', seed)] if seed is not None else list()
 
     environment_config = load_config_file(path)
     save_config(vars(parser.parse_args()), environment_config, config_save_path)
@@ -265,7 +265,7 @@ if __name__ == '__main__':
 
         if idx % conf['model']['eval_frequency'] == 0 and idx != 0:
             time_now = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
-            logging.info(f"Evaluating at Epoch: {idx} - {time_now}" + '-'*60)
+            logging.info(f"Evaluating after Dialogues: {idx * conf['model']['batchsz']} - {time_now}" + '-' * 60)
 
             eval_dict = eval_policy(conf, policy_sys, env, sess, save_eval, log_save_path)
 
diff --git a/convlab/policy/genTUS/README.md b/convlab/policy/genTUS/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..f1a3687e6d7cbf2c2ae95666ec0300a597b91ce5
--- /dev/null
+++ b/convlab/policy/genTUS/README.md
@@ -0,0 +1,72 @@
+**GenTUS** is a data-driven user simulator with transformers, which can generate semantic actions and utterence. It is able to trasfer to a new ontology in a zero-shot fashion.
+
+## Introduction
+We propose a generative transform-based user simulator (GenTUS) in this work. GenTUS consists of an encoder-decoder structure, which can optimise both the user policy and natural language generation jointly. GenTUS generates semantic actions and natural language utterances, preserving interpretability and enhancing language variation. 
+
+The code of TUS is in `convlab/policy/genTUS`.
+
+## Usage
+### Train GenTUS from scratch
+You need to generate the input files by `build_data.py`, then train the model by `train_model.py`.
+```
+python3 convlab/policy/genTUS/unify/build_data.py --dataset $dataset --add-history --dial-ids-order $dial_ids_order --split2ratio $split2ratio
+python3 convlab/policy/genTUS/train_model.py --data-name $dataset --dial-ids-order $dial_ids_order --split2ratio $split2ratio --batch-size 8
+```
+
+`dataset` can be `multiwoz21`, `sgd`, `tm`, `sgd+tm`, or `all`.
+`dial_ids_order` can be 0, 1 or 2
+`split2ratio` can be 0.01, 0.1 or 1
+
+The `build_data.py` will generate three files, `train.json`, `validation.json`, and `test.json`, under the folder `convlab/policy/genTUS/unify/data/${dataset}_${dial_ids_order}_${split2ration}`. 
+We trained GenTUS on A100 or RTX6000.
+
+### Evaluate TUS
+```
+python3 convlab/policy/genTUS/evaluate.py --model-checkpoint $model_checkpoint --input-file $in_file --dataset $dataset --do-nlg
+```
+The `in_file` is the file generated by `build_data.py`.
+
+### Train a dialogue policy with GenTUS
+You can use it as a normal user simulator by `PipelineAgent`. For example,
+```python
+from convlab.dialog_agent import PipelineAgent
+from convlab.util.custom_util import set_seed
+
+model_checkpoint = 'convlab/policy/genTUS/unify/experiments/multiwoz21-exp'
+usr_policy = UserPolicy(model_checkpoint, mode="semantic")
+simulator = PipelineAgent(None, None, usr_policy, None, 'user')
+```
+then you can train your system with this simulator.
+
+You can also change the `mode` to `"language"`, then GenTUS will response in natural language instead of semantic actions.
+
+
+<!---citation--->
+## Citing
+
+```
+@inproceedings{lin-etal-2022-gentus,
+    title = "{G}en{TUS}: Simulating User Behaviour and Language in Task-oriented Dialogues with Generative Transformers",
+    author = "Lin, Hsien-chin  and
+      Geishauser, Christian  and
+      Feng, Shutong  and
+      Lubis, Nurul  and
+      van Niekerk, Carel  and
+      Heck, Michael  and
+      Gasic, Milica",
+    booktitle = "Proceedings of the 23rd Annual Meeting of the Special Interest Group on Discourse and Dialogue",
+    month = sep,
+    year = "2022",
+    address = "Edinburgh, UK",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2022.sigdial-1.28",
+    pages = "270--282",
+    abstract = "User simulators (USs) are commonly used to train task-oriented dialogue systems via reinforcement learning. The interactions often take place on semantic level for efficiency, but there is still a gap from semantic actions to natural language, which causes a mismatch between training and deployment environment. Incorporating a natural language generation (NLG) module with USs during training can partly deal with this problem. However, since the policy and NLG of USs are optimised separately, these simulated user utterances may not be natural enough in a given context. In this work, we propose a generative transformer-based user simulator (GenTUS). GenTUS consists of an encoder-decoder structure, which means it can optimise both the user policy and natural language generation jointly. GenTUS generates both semantic actions and natural language utterances, preserving interpretability and enhancing language variation. In addition, by representing the inputs and outputs as word sequences and by using a large pre-trained language model we can achieve generalisability in feature representation. We evaluate GenTUS with automatic metrics and human evaluation. Our results show that GenTUS generates more natural language and is able to transfer to an unseen ontology in a zero-shot fashion. In addition, its behaviour can be further shaped with reinforcement learning opening the door to training specialised user simulators.",
+}
+
+
+```
+
+## License
+
+Apache License 2.0
diff --git a/convlab/policy/genTUS/evaluate.py b/convlab/policy/genTUS/evaluate.py
new file mode 100644
index 0000000000000000000000000000000000000000..87de854970d2701900ba180d2bf15736071e0c1a
--- /dev/null
+++ b/convlab/policy/genTUS/evaluate.py
@@ -0,0 +1,257 @@
+import json
+import os
+import sys
+from argparse import ArgumentParser
+from pprint import pprint
+
+import torch
+from convlab.nlg.evaluate import fine_SER
+from datasets import load_metric
+
+# from convlab.policy.genTUS.pg.stepGenTUSagent import \
+#     stepGenTUSPG as UserPolicy
+from convlab.policy.genTUS.stepGenTUS import UserActionPolicy
+from tqdm import tqdm
+
+sys.path.append(os.path.dirname(os.path.dirname(
+    os.path.dirname(os.path.abspath(__file__)))))
+
+
+def arg_parser():
+    parser = ArgumentParser()
+    parser.add_argument("--model-checkpoint", type=str, help="the model path")
+    parser.add_argument("--model-weight", type=str,
+                        help="the model weight", default="")
+    parser.add_argument("--input-file", type=str, help="the testing input file",
+                        default="")
+    parser.add_argument("--generated-file", type=str, help="the generated results",
+                        default="")
+    parser.add_argument("--only-action", action="store_true")
+    parser.add_argument("--dataset", default="multiwoz")
+    parser.add_argument("--do-semantic", action="store_true",
+                        help="do semantic evaluation")
+    parser.add_argument("--do-nlg", action="store_true",
+                        help="do nlg generation")
+    parser.add_argument("--do-golden-nlg", action="store_true",
+                        help="do golden nlg generation")
+    return parser.parse_args()
+
+
+class Evaluator:
+    def __init__(self, model_checkpoint, dataset, model_weight=None, only_action=False):
+        self.dataset = dataset
+        self.model_checkpoint = model_checkpoint
+        self.model_weight = model_weight
+        # if model_weight:
+        #     self.usr_policy = UserPolicy(
+        #         self.model_checkpoint, only_action=only_action)
+        #     self.usr_policy.load(model_weight)
+        #     self.usr = self.usr_policy.usr
+        # else:
+        self.usr = UserActionPolicy(
+            model_checkpoint, only_action=only_action, dataset=self.dataset)
+        self.usr.load(os.path.join(model_checkpoint, "pytorch_model.bin"))
+
+    def generate_results(self, f_eval, golden=False):
+        in_file = json.load(open(f_eval))
+        r = {
+            "input": [],
+            "golden_acts": [],
+            "golden_utts": [],
+            "gen_acts": [],
+            "gen_utts": []
+        }
+        for dialog in tqdm(in_file['dialog']):
+            inputs = dialog["in"]
+            labels = self.usr._parse_output(dialog["out"])
+            if golden:
+                usr_act = labels["action"]
+                usr_utt = self.usr.generate_text_from_give_semantic(
+                    inputs, usr_act)
+
+            else:
+                output = self.usr._parse_output(
+                    self.usr._generate_action(inputs))
+                usr_act = self.usr._remove_illegal_action(output["action"])
+                usr_utt = output["text"]
+            r["input"].append(inputs)
+            r["golden_acts"].append(labels["action"])
+            r["golden_utts"].append(labels["text"])
+            r["gen_acts"].append(usr_act)
+            r["gen_utts"].append(usr_utt)
+
+        return r
+
+    def read_generated_result(self, f_eval):
+        in_file = json.load(open(f_eval))
+        r = {
+            "input": [],
+            "golden_acts": [],
+            "golden_utts": [],
+            "gen_acts": [],
+            "gen_utts": []
+        }
+        for dialog in tqdm(in_file['dialog']):
+            for x in dialog:
+                r[x].append(dialog[x])
+
+        return r
+
+    def nlg_evaluation(self, input_file=None, generated_file=None, golden=False):
+        if input_file:
+            print("Force generation")
+            gen_r = self.generate_results(input_file, golden)
+
+        elif generated_file:
+            gen_r = self.read_generated_result(generated_file)
+        else:
+            print("You must specify the input_file or the generated_file")
+
+        nlg_eval = {
+            "golden": golden,
+            "metrics": {},
+            "dialog": []
+        }
+        for input, golden_act, golden_utt, gen_act, gen_utt in zip(gen_r["input"], gen_r["golden_acts"], gen_r["golden_utts"], gen_r["gen_acts"], gen_r["gen_utts"]):
+            nlg_eval["dialog"].append({
+                "input": input,
+                "golden_acts": golden_act,
+                "golden_utts": golden_utt,
+                "gen_acts": gen_act,
+                "gen_utts": gen_utt
+            })
+
+        if golden:
+            print("Calculate BLEU")
+            bleu_metric = load_metric("sacrebleu")
+            labels = [[utt] for utt in gen_r["golden_utts"]]
+
+            bleu_score = bleu_metric.compute(predictions=gen_r["gen_utts"],
+                                             references=labels,
+                                             force=True)
+            print("bleu_metric", bleu_score)
+            nlg_eval["metrics"]["bleu"] = bleu_score
+
+        else:
+            print("Calculate SER")
+            missing, hallucinate, total, hallucination_dialogs, missing_dialogs = fine_SER(
+                gen_r["gen_acts"], gen_r["gen_utts"])
+
+            print("{} Missing acts: {}, Total acts: {}, Hallucinations {}, SER {}".format(
+                "genTUSNLG", missing, total, hallucinate, missing/total))
+            nlg_eval["metrics"]["SER"] = missing/total
+
+        dir_name = self.model_checkpoint
+        json.dump(nlg_eval,
+                  open(os.path.join(dir_name, "nlg_eval.json"), 'w'),
+                  indent=2)
+        return os.path.join(dir_name, "nlg_eval.json")
+
+    def evaluation(self, input_file=None, generated_file=None):
+        force_prediction = True
+        if generated_file:
+            gen_file = json.load(open(generated_file))
+            force_prediction = False
+            if gen_file["golden"]:
+                force_prediction = True
+
+        if force_prediction:
+            in_file = json.load(open(input_file))
+            dialog_result = []
+            gen_acts, golden_acts = [], []
+            # scores = {"precision": [], "recall": [], "f1": [], "turn_acc": []}
+            for dialog in tqdm(in_file['dialog']):
+                inputs = dialog["in"]
+                labels = self.usr._parse_output(dialog["out"])
+                ans_action = self.usr._remove_illegal_action(labels["action"])
+                preds = self.usr._generate_action(inputs)
+                preds = self.usr._parse_output(preds)
+                usr_action = self.usr._remove_illegal_action(preds["action"])
+
+                gen_acts.append(usr_action)
+                golden_acts.append(ans_action)
+
+                d = {"input": inputs,
+                     "golden_acts": ans_action,
+                     "gen_acts": usr_action}
+                if "text" in preds:
+                    d["golden_utts"] = labels["text"]
+                    d["gen_utts"] = preds["text"]
+                    # print("pred text", preds["text"])
+
+                dialog_result.append(d)
+        else:
+            gen_acts, golden_acts = [], []
+            for dialog in gen_file['dialog']:
+                gen_acts.append(dialog["gen_acts"])
+                golden_acts.append(dialog["golden_acts"])
+            dialog_result = gen_file['dialog']
+
+        scores = {"precision": [], "recall": [], "f1": [], "turn_acc": []}
+
+        for gen_act, golden_act in zip(gen_acts, golden_acts):
+            s = f1_measure(preds=gen_act, labels=golden_act)
+            for metric in scores:
+                scores[metric].append(s[metric])
+
+        result = {}
+        for metric in scores:
+            result[metric] = sum(scores[metric])/len(scores[metric])
+            print(f"{metric}: {result[metric]}")
+
+        result["dialog"] = dialog_result
+        basename = "semantic_evaluation_result"
+        json.dump(result, open(os.path.join(
+            self.model_checkpoint, f"{self.dataset}-{basename}.json"), 'w'))
+        # if self.model_weight:
+        #     json.dump(result, open(os.path.join(
+        #         'results', f"{basename}.json"), 'w'))
+        # else:
+        #     json.dump(result, open(os.path.join(
+        #         self.model_checkpoint, f"{self.dataset}-{basename}.json"), 'w'))
+
+
+def f1_measure(preds, labels):
+    tp = 0
+    score = {"precision": 0, "recall": 0, "f1": 0, "turn_acc": 0}
+    for p in preds:
+        if p in labels:
+            tp += 1.0
+    if preds:
+        score["precision"] = tp/len(preds)
+    if labels:
+        score["recall"] = tp/len(labels)
+    if (score["precision"] + score["recall"]) > 0:
+        score["f1"] = 2*(score["precision"]*score["recall"]) / \
+            (score["precision"]+score["recall"])
+    if tp == len(preds) and tp == len(labels):
+        score["turn_acc"] = 1
+    return score
+
+
+def main():
+    args = arg_parser()
+    eval = Evaluator(args.model_checkpoint,
+                     args.dataset,
+                     args.model_weight,
+                     args.only_action)
+    print("model checkpoint", args.model_checkpoint)
+    print("generated_file", args.generated_file)
+    print("input_file", args.input_file)
+    with torch.no_grad():
+        if args.do_semantic:
+            eval.evaluation(args.input_file)
+        if args.do_nlg:
+            nlg_result = eval.nlg_evaluation(input_file=args.input_file,
+                                             generated_file=args.generated_file,
+                                             golden=args.do_golden_nlg)
+            if args.generated_file:
+                generated_file = args.generated_file
+            else:
+                generated_file = nlg_result
+            eval.evaluation(args.input_file,
+                            generated_file)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/convlab/policy/genTUS/ppo/vector.py b/convlab/policy/genTUS/ppo/vector.py
new file mode 100644
index 0000000000000000000000000000000000000000..4c502a46f87582008ff49219f8a14844378b9ed2
--- /dev/null
+++ b/convlab/policy/genTUS/ppo/vector.py
@@ -0,0 +1,148 @@
+import json
+
+import torch
+from convlab.policy.genTUS.unify.knowledge_graph import KnowledgeGraph
+from convlab.policy.genTUS.token_map import tokenMap
+from convlab.policy.tus.unify.Goal import Goal
+from transformers import BartTokenizer
+
+
+class stepGenTUSVector:
+    def __init__(self, model_checkpoint, max_in_len=400, max_out_len=80, allow_general_intent=True):
+        self.tokenizer = BartTokenizer.from_pretrained(model_checkpoint)
+        self.vocab = len(self.tokenizer)
+        self.max_in_len = max_in_len
+        self.max_out_len = max_out_len
+        self.token_map = tokenMap(tokenizer=self.tokenizer)
+        self.token_map.default(only_action=True)
+        self.kg = KnowledgeGraph(self.tokenizer)
+        self.mentioned_domain = []
+        self.allow_general_intent = allow_general_intent
+        self.candidate_num = 5
+        if self.allow_general_intent:
+            print("---> allow_general_intent")
+
+    def init_session(self, goal: Goal):
+        self.goal = goal
+        self.mentioned_domain = []
+
+    def encode(self, raw_inputs, max_length, return_tensors="pt", truncation=True):
+        model_input = self.tokenizer(raw_inputs,
+                                     max_length=max_length,
+                                     return_tensors=return_tensors,
+                                     truncation=truncation,
+                                     padding="max_length")
+        return model_input
+
+    def decode(self, generated_so_far, skip_special_tokens=True):
+        output = self.tokenizer.decode(
+            generated_so_far, skip_special_tokens=skip_special_tokens)
+        return output
+
+    def state_vectorize(self, action, history, turn):
+        self.goal.update_user_goal(action=action)
+        inputs = json.dumps({"system": action,
+                             "goal": self.goal.get_goal_list(),
+                             "history": history,
+                             "turn": str(turn)})
+        inputs = self.encode(inputs, self.max_in_len)
+        s_vec, action_mask = inputs["input_ids"][0], inputs["attention_mask"][0]
+
+        return s_vec, action_mask
+
+    def action_vectorize(self, action, s=None):
+        # action:  [[intent, domain, slot, value], ...]
+        vec = {"vector": torch.tensor([]), "mask": torch.tensor([])}
+        if s is not None:
+            raw_inputs = self.decode(s[0])
+            self.kg.parse_input(raw_inputs)
+
+        self._append(vec, self._get_id("<s>"))
+        self._append(vec, self.token_map.get_id('start_json'))
+        self._append(vec, self.token_map.get_id('start_act'))
+
+        act_len = len(action)
+        for i, (intent, domain, slot, value) in enumerate(action):
+            if value == '?':
+                value = '<?>'
+            c_idx = {x: None for x in ["intent", "domain", "slot", "value"]}
+
+            if s is not None:
+                c_idx["intent"] = self._candidate_id(self.kg.candidate(
+                    "intent", allow_general_intent=self.allow_general_intent))
+                c_idx["domain"] = self._candidate_id(self.kg.candidate(
+                    "domain", intent=intent))
+                c_idx["slot"] = self._candidate_id(self.kg.candidate(
+                    "slot", intent=intent, domain=domain, is_mentioned=self.is_mentioned(domain)))
+                c_idx["value"] = self._candidate_id(self.kg.candidate(
+                    "value", intent=intent, domain=domain, slot=slot))
+
+            self._append(vec, self._get_id(intent), c_idx["intent"])
+            self._append(vec, self.token_map.get_id('sep_token'))
+            self._append(vec, self._get_id(domain), c_idx["domain"])
+            self._append(vec, self.token_map.get_id('sep_token'))
+            self._append(vec, self._get_id(slot), c_idx["slot"])
+            self._append(vec, self.token_map.get_id('sep_token'))
+            self._append(vec, self._get_id(value), c_idx["value"])
+
+            c_idx = [0]*self.candidate_num
+            c_idx[0] = self.token_map.get_id('end_act')[0]
+            c_idx[1] = self.token_map.get_id('sep_act')[0]
+            if i == act_len - 1:
+                x = self.token_map.get_id('end_act')
+            else:
+                x = self.token_map.get_id('sep_act')
+
+            self._append(vec, x, c_idx)
+
+        self._append(vec, self._get_id("</s>"))
+
+        # pad
+        if len(vec["vector"]) < self.max_out_len:
+            pad_len = self.max_out_len-len(vec["vector"])
+            self._append(vec, x=torch.tensor([1]*pad_len))
+        for vec_type in vec:
+            vec[vec_type] = vec[vec_type].to(torch.int64)
+
+        return vec
+
+    def _append(self, vec, x, candidate=None):
+        if type(x) is list:
+            x = torch.tensor(x)
+        mask = self._mask(x, candidate)
+        vec["vector"] = torch.cat((vec["vector"], x), dim=-1)
+        vec["mask"] = torch.cat((vec["mask"], mask), dim=0)
+
+    def _mask(self, idx, c_idx=None):
+        mask = torch.zeros(len(idx), self.candidate_num)
+        mask[:, 0] = idx
+        if c_idx is not None and len(c_idx) > 1:
+            mask[0, :] = torch.tensor(c_idx)
+
+        return mask
+
+    def _candidate_id(self, candidate):
+        if len(candidate) > self.candidate_num:
+            print(f"too many candidates. Max = {self.candidate_num}")
+        c_idx = [0]*self.candidate_num
+        for i, idx in enumerate([self._get_id(c)[0] for c in candidate[:self.candidate_num]]):
+            c_idx[i] = idx
+        return c_idx
+
+    def _get_id(self, value):
+        token_id = self.tokenizer(value, add_special_tokens=False)
+        return token_id["input_ids"]
+
+    def action_devectorize(self, action_id):
+        return self.decode(action_id)
+
+    def update_mentioned_domain(self, semantic_act):
+        for act in semantic_act:
+            domain = act[1]
+            if domain not in self.mentioned_domain:
+                self.mentioned_domain.append(domain)
+
+    def is_mentioned(self, domain):
+        if domain in self.mentioned_domain:
+            return True
+        return False
diff --git a/convlab/policy/genTUS/stepGenTUS.py b/convlab/policy/genTUS/stepGenTUS.py
new file mode 100644
index 0000000000000000000000000000000000000000..0b5af9f3315e9db1cc98e4618e1800155d97e670
--- /dev/null
+++ b/convlab/policy/genTUS/stepGenTUS.py
@@ -0,0 +1,655 @@
+import json
+import os
+
+import torch
+from transformers import BartTokenizer
+
+from convlab.policy.genTUS.ppo.vector import stepGenTUSVector
+from convlab.policy.genTUS.stepGenTUSmodel import stepGenTUSmodel
+from convlab.policy.genTUS.token_map import tokenMap
+from convlab.policy.genTUS.unify.Goal import Goal
+from convlab.policy.genTUS.unify.knowledge_graph import KnowledgeGraph
+from convlab.policy.policy import Policy
+from convlab.task.multiwoz.goal_generator import GoalGenerator
+from convlab.util.custom_util import model_downloader
+
+
+DEBUG = False
+
+
+class UserActionPolicy(Policy):
+    def __init__(self, model_checkpoint, mode="semantic", only_action=True, max_turn=40, **kwargs):
+        self.mode = mode
+        # if mode == "semantic" and only_action:
+        #     # only generate semantic action in prediction
+        print("model_checkpoint", model_checkpoint)
+        self.only_action = only_action
+        if self.only_action:
+            print("change mode to semantic because only_action=True")
+            self.mode = "semantic"
+        self.max_in_len = 500
+        self.max_out_len = 100 if only_action else 200
+        max_act_len = kwargs.get("max_act_len", 2)
+        print("max_act_len", max_act_len)
+        self.max_action_len = max_act_len
+        if "max_act_len" in kwargs:
+            self.max_out_len = 30 * self.max_action_len
+            print("max_act_len", self.max_out_len)
+        self.max_turn = max_turn
+        if mode not in ["semantic", "language"]:
+            print("Unknown user mode")
+
+        self.reward = {"success":  self.max_turn*2,
+                       "fail": self.max_turn*-1}
+        self.tokenizer = BartTokenizer.from_pretrained(model_checkpoint)
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        train_whole_model = kwargs.get("whole_model", True)
+        self.model = stepGenTUSmodel(
+            model_checkpoint, train_whole_model=train_whole_model)
+        self.model.eval()
+        self.model.to(self.device)
+        self.model.share_memory()
+
+        self.turn_level_reward = kwargs.get("turn_level_reward", True)
+        self.cooperative = kwargs.get("cooperative", True)
+
+        dataset = kwargs.get("dataset", "")
+        self.kg = KnowledgeGraph(
+            tokenizer=self.tokenizer,
+            dataset=dataset)
+
+        self.goal_gen = GoalGenerator()
+
+        self.vector = stepGenTUSVector(
+            model_checkpoint, self.max_in_len, self.max_out_len)
+        self.norm_reward = False
+
+        self.action_penalty = kwargs.get("action_penalty", False)
+        self.usr_act_penalize = kwargs.get("usr_act_penalize", 0)
+        self.goal_list_type = kwargs.get("goal_list_type", "normal")
+        self.update_mode = kwargs.get("update_mode", "normal")
+        self.max_history = kwargs.get("max_history", 3)
+        self.init_session()
+
+    def _update_seq(self, sub_seq: list, pos: int):
+        for x in sub_seq:
+            self.seq[0, pos] = x
+            pos += 1
+
+        return pos
+
+    def _generate_action(self, raw_inputs, mode="max", allow_general_intent=True):
+        # TODO no duplicate
+        self.kg.parse_input(raw_inputs)
+        model_input = self.vector.encode(raw_inputs, self.max_in_len)
+        # start token
+        self.seq = torch.zeros(1, self.max_out_len, device=self.device).long()
+        pos = self._update_seq([0], 0)
+        pos = self._update_seq(self.token_map.get_id('start_json'), pos)
+        pos = self._update_seq(self.token_map.get_id('start_act'), pos)
+
+        # get semantic actions
+        for act_len in range(self.max_action_len):
+            pos = self._get_semantic_action(
+                model_input, pos, mode, allow_general_intent)
+
+            terminate, token_name = self._stop_semantic(
+                model_input, pos, act_len)
+            pos = self._update_seq(self.token_map.get_id(token_name), pos)
+
+            if terminate:
+                break
+
+        if self.only_action:
+            # return semantic action. Don't need to generate text
+            return self.vector.decode(self.seq[0, :pos])
+
+        # TODO remove illegal action here?
+
+        # get text output
+        pos = self._update_seq(self.token_map.get_id("start_text"), pos)
+
+        text = self._get_text(model_input, pos)
+
+        return text
+
+    def generate_text_from_give_semantic(self, raw_inputs, semantic_action):
+        self.kg.parse_input(raw_inputs)
+        model_input = self.vector.encode(raw_inputs, self.max_in_len)
+        self.seq = torch.zeros(1, self.max_out_len, device=self.device).long()
+        pos = self._update_seq([0], 0)
+        pos = self._update_seq(self.token_map.get_id('start_json'), pos)
+        pos = self._update_seq(self.token_map.get_id('start_act'), pos)
+
+        if len(semantic_action) == 0:
+            pos = self._update_seq(self.token_map.get_id("end_act"), pos)
+
+        for act_id, (intent, domain, slot, value) in enumerate(semantic_action):
+            pos = self._update_seq(self.kg._get_token_id(intent), pos)
+            pos = self._update_seq(self.token_map.get_id('sep_token'), pos)
+            pos = self._update_seq(self.kg._get_token_id(domain), pos)
+            pos = self._update_seq(self.token_map.get_id('sep_token'), pos)
+            pos = self._update_seq(self.kg._get_token_id(slot), pos)
+            pos = self._update_seq(self.token_map.get_id('sep_token'), pos)
+            pos = self._update_seq(self.kg._get_token_id(value), pos)
+
+            if act_id == len(semantic_action) - 1:
+                token_name = "end_act"
+            else:
+                token_name = "sep_act"
+            pos = self._update_seq(self.token_map.get_id(token_name), pos)
+        pos = self._update_seq(self.token_map.get_id("start_text"), pos)
+
+        raw_output = self._get_text(model_input, pos)
+        return self._parse_output(raw_output)["text"]
+
+    def _get_text(self, model_input, pos):
+        s_pos = pos
+        for i in range(s_pos, self.max_out_len):
+            next_token_logits = self.model.get_next_token_logits(
+                model_input, self.seq[:1, :pos])
+            next_token = torch.argmax(next_token_logits, dim=-1)
+
+            if self._stop_text(next_token):
+                # text = self.vector.decode(self.seq[0, s_pos:pos])
+                # text = self._norm_str(text)
+                # return self.vector.decode(self.seq[0, :s_pos]) + text + '"}'
+                break
+
+            pos = self._update_seq([next_token], pos)
+        text = self.vector.decode(self.seq[0, s_pos:pos])
+        text = self._norm_str(text)
+        return self.vector.decode(self.seq[0, :s_pos]) + text + '"}'
+        # TODO return None
+
+    def _stop_text(self, next_token):
+        if next_token == self.token_map.get_id("end_json")[0]:
+            return True
+        elif next_token == self.token_map.get_id("end_json_2")[0]:
+            return True
+
+        return False
+
+    @staticmethod
+    def _norm_str(text: str):
+        text = text.strip('"')
+        text = text.replace('"', "'")
+        text = text.replace('\\', "")
+        return text
+
+    def _stop_semantic(self, model_input, pos, act_length=0):
+
+        outputs = self.model.get_next_token_logits(
+            model_input, self.seq[:1, :pos])
+        tokens = {}
+        for token_name in ['sep_act', 'end_act']:
+            tokens[token_name] = {
+                "token_id": self.token_map.get_id(token_name)}
+            hash_id = tokens[token_name]["token_id"][0]
+            tokens[token_name]["score"] = outputs[:, hash_id].item()
+
+        if tokens['end_act']["score"] > tokens['sep_act']["score"]:
+            terminate = True
+        else:
+            terminate = False
+
+        if act_length >= self.max_action_len - 1:
+            terminate = True
+
+        token_name = "end_act" if terminate else "sep_act"
+
+        return terminate, token_name
+
+    def _get_semantic_action(self, model_input, pos, mode="max", allow_general_intent=True):
+
+        intent = self._get_intent(
+            model_input, self.seq[:1, :pos], mode, allow_general_intent)
+        pos = self._update_seq(intent["token_id"], pos)
+        pos = self._update_seq(self.token_map.get_id('sep_token'), pos)
+
+        # get domain
+        domain = self._get_domain(
+            model_input, self.seq[:1, :pos], intent["token_name"], mode)
+        pos = self._update_seq(domain["token_id"], pos)
+        pos = self._update_seq(self.token_map.get_id('sep_token'), pos)
+
+        # get slot
+        slot = self._get_slot(
+            model_input, self.seq[:1, :pos], intent["token_name"], domain["token_name"], mode)
+        pos = self._update_seq(slot["token_id"], pos)
+        pos = self._update_seq(self.token_map.get_id('sep_token'), pos)
+
+        # get value
+
+        value = self._get_value(
+            model_input, self.seq[:1, :pos], intent["token_name"], domain["token_name"], slot["token_name"], mode)
+        pos = self._update_seq(value["token_id"], pos)
+
+        return pos
+
+    def _get_intent(self, model_input, generated_so_far, mode="max", allow_general_intent=True):
+        next_token_logits = self.model.get_next_token_logits(
+            model_input, generated_so_far)
+
+        return self.kg.get_intent(next_token_logits, mode, allow_general_intent)
+
+    def _get_domain(self, model_input, generated_so_far, intent, mode="max"):
+        next_token_logits = self.model.get_next_token_logits(
+            model_input, generated_so_far)
+
+        return self.kg.get_domain(next_token_logits, intent, mode)
+
+    def _get_slot(self, model_input, generated_so_far, intent, domain, mode="max"):
+        next_token_logits = self.model.get_next_token_logits(
+            model_input, generated_so_far)
+        is_mentioned = self.vector.is_mentioned(domain)
+        return self.kg.get_slot(next_token_logits, intent, domain, mode, is_mentioned)
+
+    def _get_value(self, model_input, generated_so_far, intent, domain, slot, mode="max"):
+        next_token_logits = self.model.get_next_token_logits(
+            model_input, generated_so_far)
+
+        return self.kg.get_value(next_token_logits, intent, domain, slot, mode)
+
+    def _remove_illegal_action(self, action):
+        # Transform illegal action to legal action
+        new_action = []
+        for act in action:
+            if len(act) == 4:
+                if "<?>" in act[-1]:
+                    act = [act[0], act[1], act[2], "?"]
+                if act not in new_action:
+                    new_action.append(act)
+            else:
+                print("illegal action:", action)
+        return new_action
+
+    def _parse_output(self, in_str):
+        in_str = str(in_str)
+        in_str = in_str.replace('<s>', '').replace(
+            '<\\s>', '').replace('o"clock', "o'clock")
+        action = {"action": [], "text": ""}
+        try:
+            action = json.loads(in_str)
+        except:
+            print("invalid action:", in_str)
+            print("-"*20)
+        return action
+
+    def predict(self, sys_act, mode="max", allow_general_intent=True):
+        # raw_sys_act = sys_act
+        # sys_act = sys_act[:5]
+        # update goal
+        # TODO
+        allow_general_intent = False
+        self.model.eval()
+
+        if not self.add_sys_from_reward:
+            self.goal.update_user_goal(action=sys_act, char="sys")
+            self.sys_acts.append(sys_act)  # for terminate conversation
+
+        # update constraint
+        self.time_step += 2
+
+        history = []
+        if self.usr_acts:
+            if self.max_history == 1:
+                history = self.usr_acts[-1]
+            else:
+                history = self.usr_acts[-1*self.max_history:]
+        inputs = json.dumps({"system": sys_act,
+                             "goal": self.goal.get_goal_list(),
+                             "history": history,
+                             "turn": str(int(self.time_step/2))})
+        with torch.no_grad():
+            raw_output = self._generate_action(
+                raw_inputs=inputs, mode=mode, allow_general_intent=allow_general_intent)
+        output = self._parse_output(raw_output)
+        self.semantic_action = self._remove_illegal_action(output["action"])
+        if not self.only_action:
+            self.utterance = output["text"]
+
+        # TODO
+        if self.is_finish():
+            self.semantic_action, self.utterance = self._good_bye()
+
+        # if self.is_finish():
+        #     print("terminated")
+
+        # if self.is_finish():
+        #     good_bye = self._good_bye()
+        #     self.goal.add_usr_da(good_bye)
+        #     return good_bye
+
+        self.goal.update_user_goal(action=self.semantic_action, char="usr")
+        self.vector.update_mentioned_domain(self.semantic_action)
+        self.usr_acts.append(self.semantic_action)
+
+        # if self._usr_terminate(usr_action):
+        #     print("terminated by user")
+        #     self.terminated = True
+
+        del inputs
+
+        if self.mode == "language":
+            # print("in", sys_act)
+            # print("out", self.utterance)
+            return self.utterance
+        else:
+            return self.semantic_action
+
+    def init_session(self, goal=None):
+        self.token_map = tokenMap(tokenizer=self.tokenizer)
+        self.token_map.default(only_action=self.only_action)
+        self.time_step = 0
+        remove_domain = "police"  # remove police domain in inference
+
+        if not goal:
+            self._new_goal(remove_domain=remove_domain)
+        else:
+            self._read_goal(goal)
+
+        self.vector.init_session(goal=self.goal)
+
+        self.terminated = False
+        self.add_sys_from_reward = False
+        self.sys_acts = []
+        self.usr_acts = []
+        self.semantic_action = []
+        self.utterance = ""
+
+    def _read_goal(self, data_goal):
+        self.goal = Goal(goal=data_goal)
+
+    def _new_goal(self, remove_domain="police", domain_len=None):
+        self.goal = Goal(goal_generator=self.goal_gen)
+        # keep_generate_goal = True
+        # # domain_len = 1
+        # while keep_generate_goal:
+        #     self.goal = Goal(goal_generator=self.goal_gen,
+        #                      goal_list_type=self.goal_list_type,
+        #                      update_mode=self.update_mode)
+        #     if (domain_len and len(self.goal.domains) != domain_len) or \
+        #             (remove_domain and remove_domain in self.goal.domains):
+        #         keep_generate_goal = True
+        #     else:
+        #         keep_generate_goal = False
+
+    def load(self, model_path):
+        self.model.load_state_dict(torch.load(
+            model_path, map_location=self.device))
+        # self.model = BartForConditionalGeneration.from_pretrained(
+        #     model_checkpoint)
+
+    def get_goal(self):
+        if self.goal.raw_goal is not None:
+            return self.goal.raw_goal
+        goal = {}
+        for domain in self.goal.domain_goals:
+            if domain not in goal:
+                goal[domain] = {}
+            for intent in self.goal.domain_goals[domain]:
+                if intent == "inform":
+                    slot_type = "info"
+                elif intent == "request":
+                    slot_type = "reqt"
+                elif intent == "book":
+                    slot_type = "book"
+                else:
+                    print("unknown slot type")
+                if slot_type not in goal[domain]:
+                    goal[domain][slot_type] = {}
+                for slot, value in self.goal.domain_goals[domain][intent].items():
+                    goal[domain][slot_type][slot] = value
+        return goal
+
+    def get_reward(self, sys_response=None):
+        self.add_sys_from_reward = False if sys_response is None else True
+
+        if self.add_sys_from_reward:
+            self.goal.update_user_goal(action=sys_response, char="sys")
+            self.goal.add_sys_da(sys_response)  # for evaluation
+            self.sys_acts.append(sys_response)  # for terminate conversation
+
+        if self.is_finish():
+            if self.is_success():
+                reward = self.reward["success"]
+                self.success = True
+            else:
+                reward = self.reward["fail"]
+                self.success = False
+
+        else:
+            reward = -1
+            if self.turn_level_reward:
+                reward += self.turn_reward()
+
+            self.success = None
+            # if self.action_penalty:
+            #     reward += self._system_action_penalty()
+
+        if self.norm_reward:
+            reward = (reward - 20)/60
+        return reward
+
+    def _system_action_penalty(self):
+        free_action_len = 3
+        if len(self.sys_acts) < 1:
+            return 0
+        # TODO only penalize the slots not in user goal
+        # else:
+        #     penlaty = 0
+        #     for i in range(len(self.sys_acts[-1])):
+        #         penlaty += -1*i
+        #     return penlaty
+        if len(self.sys_acts[-1]) > 3:
+            return -1*(len(self.sys_acts[-1])-free_action_len)
+        return 0
+
+    def turn_reward(self):
+        r = 0
+        r += self._new_act_reward()
+        r += self._reply_reward()
+        r += self._usr_act_len()
+        return r
+
+    def _usr_act_len(self):
+        last_act = self.usr_acts[-1]
+        penalty = 0
+        if len(last_act) > 2:
+            penalty = (2-len(last_act))*self.usr_act_penalize
+        return penalty
+
+    def _new_act_reward(self):
+        last_act = self.usr_acts[-1]
+        if last_act != self.semantic_action:
+            print(f"---> why? last {last_act} usr {self.semantic_action}")
+        new_act = []
+        for act in last_act:
+            if len(self.usr_acts) < 2:
+                break
+            if act[1].lower() == "general":
+                new_act.append(0)
+            elif act in self.usr_acts[-2]:
+                new_act.append(-1)
+            elif act not in self.usr_acts[-2]:
+                new_act.append(1)
+
+        return sum(new_act)
+
+    def _reply_reward(self):
+        if self.cooperative:
+            return self._cooperative_reply_reward()
+        else:
+            return self._non_cooperative_reply_reward()
+
+    def _non_cooperative_reply_reward(self):
+        r = []
+        reqts = []
+        infos = []
+        reply_len = 0
+        max_len = 1
+        for act in self.sys_acts[-1]:
+            if act[0] == "request":
+                reqts.append([act[1], act[2]])
+        for act in self.usr_acts[-1]:
+            if act[0] == "inform":
+                infos.append([act[1], act[2]])
+        for req in reqts:
+            if req in infos:
+                if reply_len < max_len:
+                    r.append(1)
+                elif reply_len == max_len:
+                    r.append(0)
+                else:
+                    r.append(-5)
+
+        if r:
+            return sum(r)
+        return 0
+
+    def _cooperative_reply_reward(self):
+        r = []
+        reqts = []
+        infos = []
+        for act in self.sys_acts[-1]:
+            if act[0] == "request":
+                reqts.append([act[1], act[2]])
+        for act in self.usr_acts[-1]:
+            if act[0] == "inform":
+                infos.append([act[1], act[2]])
+        for req in reqts:
+            if req in infos:
+                r.append(1)
+            else:
+                r.append(-1)
+        if r:
+            return sum(r)
+        return 0
+
+    def _usr_terminate(self):
+        for act in self.semantic_action:
+            if act[0] in ['thank', 'bye']:
+                return True
+        return False
+
+    def is_finish(self):
+        # stop by model generation?
+        if self._finish_conversation_rule():
+            self.terminated = True
+            return True
+        elif self._usr_terminate():
+            self.terminated = True
+            return True
+        self.terminated = False
+        return False
+
+    def is_success(self):
+        task_complete = self.goal.task_complete()
+        # goal_status = self.goal.all_mentioned()
+        # should mentioned all slots
+        if task_complete:  # and goal_status["complete"] > 0.6:
+            return True
+        return False
+
+    def _good_bye(self):
+        if self.is_success():
+            return [['thank', 'general', 'none', 'none']], "thank you. bye"
+            # if self.mode == "semantic":
+            #     return [['thank', 'general', 'none', 'none']]
+            # else:
+            #     return "bye"
+        else:
+            return [["bye", "general", "None", "None"]], "bye"
+            if self.mode == "semantic":
+                return [["bye", "general", "None", "None"]]
+            return "bye"
+
+    def _finish_conversation_rule(self):
+        if self.is_success():
+            return True
+
+        if self.time_step > self.max_turn:
+            return True
+
+        if (len(self.sys_acts) > 4) and (self.sys_acts[-1] == self.sys_acts[-2]) and (self.sys_acts[-2] == self.sys_acts[-3]):
+            return True
+        return False
+
+    def is_terminated(self):
+        # Is there any action to say?
+        self.is_finish()
+        return self.terminated
+
+
+class UserPolicy(Policy):
+    def __init__(self,
+                 model_checkpoint,
+                 mode="semantic",
+                 only_action=True,
+                 sample=False,
+                 action_penalty=False,
+                 **kwargs):
+        # self.config = config
+        if not os.path.exists(os.path.dirname(model_checkpoint)):
+            os.mkdir(os.path.dirname(model_checkpoint))
+            model_downloader(os.path.dirname(model_checkpoint),
+                             "https://zenodo.org/record/7372442/files/multiwoz21-exp.zip")
+
+        self.policy = UserActionPolicy(
+            model_checkpoint,
+            mode=mode,
+            only_action=only_action,
+            action_penalty=action_penalty,
+            **kwargs)
+        self.policy.load(os.path.join(
+            model_checkpoint, "pytorch_model.bin"))
+        self.sample = sample
+
+    def predict(self, sys_act, mode="max"):
+        if self.sample:
+            mode = "sample"
+        else:
+            mode = "max"
+        response = self.policy.predict(sys_act, mode)
+        return response
+
+    def init_session(self, goal=None):
+        self.policy.init_session(goal)
+
+    def is_terminated(self):
+        return self.policy.is_terminated()
+
+    def get_reward(self, sys_response=None):
+        return self.policy.get_reward(sys_response)
+
+    def get_goal(self):
+        if hasattr(self.policy, 'get_goal'):
+            return self.policy.get_goal()
+        return None
+
+
+if __name__ == "__main__":
+    import os
+
+    from convlab.dialog_agent import PipelineAgent
+    # from convlab.nlu.jointBERT.multiwoz import BERTNLU
+    from convlab.util.custom_util import set_seed
+
+    set_seed(20220220)
+    # Test semantic level behaviour
+    model_checkpoint = 'convlab/policy/genTUS/unify/experiments/multiwoz21-exp'
+    usr_policy = UserPolicy(
+        model_checkpoint,
+        mode="semantic")
+    # usr_policy.policy.load(os.path.join(model_checkpoint, "pytorch_model.bin"))
+    usr_nlu = None  # BERTNLU()
+    usr = PipelineAgent(usr_nlu, None, usr_policy, None, name='user')
+    print(usr.policy.get_goal())
+
+    print(usr.response([]))
+    print(usr.policy.policy.goal.status)
+    print(usr.response([["request", "attraction", "area", "?"]]))
+    print(usr.policy.policy.goal.status)
+    print(usr.response([["request", "attraction", "area", "?"]]))
+    print(usr.policy.policy.goal.status)
diff --git a/convlab/policy/genTUS/stepGenTUSmodel.py b/convlab/policy/genTUS/stepGenTUSmodel.py
new file mode 100644
index 0000000000000000000000000000000000000000..e2eaf7bc5064262808120aac4a9cbe2eb007d863
--- /dev/null
+++ b/convlab/policy/genTUS/stepGenTUSmodel.py
@@ -0,0 +1,114 @@
+
+import json
+
+import torch
+from torch.nn.functional import softmax, one_hot, cross_entropy
+
+from convlab.policy.genTUS.unify.knowledge_graph import KnowledgeGraph
+from convlab.policy.genTUS.token_map import tokenMap
+from convlab.policy.genTUS.utils import append_tokens
+from transformers import (BartConfig, BartForConditionalGeneration,
+                          BartTokenizer)
+
+
+class stepGenTUSmodel(BartForConditionalGeneration):
+    def __init__(self, model_checkpoint, train_whole_model=True, **kwargs):
+        config = BartConfig.from_pretrained(model_checkpoint)
+        super().__init__(config, **kwargs)
+
+        self.tokenizer = BartTokenizer.from_pretrained(model_checkpoint)
+        self.vocab = len(self.tokenizer)
+        self.kg = KnowledgeGraph(self.tokenizer)
+        self.action_kg = KnowledgeGraph(self.tokenizer)
+        self.token_map = tokenMap(self.tokenizer)
+        # only_action doesn't matter. it is only used for get_log_prob
+        self.token_map.default(only_action=True)
+
+        if not train_whole_model:
+            for param in self.parameters():
+                param.requires_grad = False
+
+            for param in self.model.decoder.layers[-1].fc1.parameters():
+                param.requires_grad = True
+            for param in self.model.decoder.layers[-1].fc2.parameters():
+                param.requires_grad = True
+
+    def get_trainable_param(self):
+
+        return filter(
+            lambda p: p.requires_grad, self.parameters())
+
+    def get_next_token_logits(self, model_input, generated_so_far):
+        input_ids = model_input["input_ids"].to(self.device)
+        attention_mask = model_input["attention_mask"].to(self.device)
+        outputs = self.forward(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            decoder_input_ids=generated_so_far,
+            return_dict=True)
+        return outputs.logits[:, -1, :]
+
+    def get_log_prob(self, s, a, action_mask, prob_mask):
+        output = self.forward(input_ids=s,
+                              attention_mask=action_mask,
+                              decoder_input_ids=a)
+        prob = self._norm_prob(a[:, 1:].long(),
+                               output.logits[:, :-1, :],
+                               prob_mask[:, 1:, :].long())
+        return prob
+
+    def _norm_prob(self, a, prob, mask):
+        prob = softmax(prob, -1)
+        base = self._base(prob, mask).to(self.device)  # [b, seq_len]
+        prob = (prob*one_hot(a, num_classes=self.vocab)).sum(-1)
+        prob = torch.log(prob / base)
+        pad_mask = a != 1
+        prob = prob*pad_mask.float()
+        return prob.sum(-1)
+
+    @staticmethod
+    def _base(prob, mask):
+        batch_size, seq_len, dim = prob.shape
+        base = torch.zeros(batch_size, seq_len)
+        for b in range(batch_size):
+            for s in range(seq_len):
+                temp = [prob[b, s, c] for c in mask[b, s, :] if c > 0]
+                base[b, s] = torch.sum(torch.tensor(temp))
+        return base
+
+
+if __name__ == "__main__":
+    import os
+    from convlab.util.custom_util import set_seed
+    from convlab.policy.genTUS.stepGenTUS import UserActionPolicy
+    set_seed(0)
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    model_checkpoint = 'results/genTUS-22-01-31-09-21/'
+    usr = UserActionPolicy(model_checkpoint=model_checkpoint)
+    usr.model.load_state_dict(torch.load(
+        os.path.join(model_checkpoint, "pytorch_model.bin"), map_location=device))
+    usr.model.eval()
+
+    test_file = "convlab/policy/genTUS/data/goal_status_validation_v1.json"
+    data = json.load(open(test_file))
+    test_id = 20
+    inputs = usr.tokenizer(data["dialog"][test_id]["in"],
+                           max_length=400,
+                           return_tensors="pt",
+                           truncation=True)
+
+    actions = [data["dialog"][test_id]["out"],
+               data["dialog"][test_id+100]["out"]]
+
+    for action in actions:
+        action = json.loads(action)
+        vec = usr.vector.action_vectorize(
+            action["action"], s=inputs["input_ids"])
+
+        print({"action": action["action"]})
+        print("get_log_prob", usr.model.get_log_prob(
+            inputs["input_ids"],
+            torch.unsqueeze(vec["vector"], 0),
+            inputs["attention_mask"],
+            torch.unsqueeze(vec["mask"], 0)))
diff --git a/convlab/policy/genTUS/token_map.py b/convlab/policy/genTUS/token_map.py
new file mode 100644
index 0000000000000000000000000000000000000000..7825c2880928c40f68284b0c3199932cd1cfc477
--- /dev/null
+++ b/convlab/policy/genTUS/token_map.py
@@ -0,0 +1,64 @@
+import json
+
+
+class tokenMap:
+    def __init__(self, tokenizer):
+        self.tokenizer = tokenizer
+        self.token_name = {}
+        self.hash_map = {}
+        self.debug = False
+        self.default()
+
+    def default(self, only_action=False):
+        self.format_tokens = {
+            'start_json': '{"action": [',   # 49643, 10845, 7862, 646
+            'start_act': '["',              # 49329
+            'sep_token': '", "',            # 1297('",'), 22
+            'sep_act': '"], ["',               # 49177
+            'end_act': '"]], "',            # 42248, 7479, 22
+            'start_text': 'text": "',       # 29015, 7862, 22
+            'end_json': '}',                 # 24303
+            'end_json_2': '"}'                 # 48805
+        }
+        if only_action:
+            self.format_tokens['end_act'] = '"]]}'
+        for token_name in self.format_tokens:
+            self.add_token(
+                token_name, self.format_tokens[token_name])
+
+    def add_token(self, token_name, value):
+        if token_name in self.token_name and self.debug:
+            print(f"---> duplicate token: {token_name}({value})!!!!!!!")
+
+        token_id = self.tokenizer(str(value), add_special_tokens=False)[
+            "input_ids"]
+        self.token_name[token_name] = {"value": value, "token_id": token_id}
+        # print(token_id)
+        hash_id = token_id[0]
+        if hash_id in self.hash_map and self.debug:
+            print(
+                f"---> conflict hash number {hash_id}: {self.hash_map[hash_id]['name']} and {token_name}")
+        self.hash_map[hash_id] = {
+            "name": token_name, "value": value, "token_id": token_id}
+
+    def get_info(self, hash_id):
+        return self.hash_map[hash_id]
+
+    def get_id(self, token_name):
+        # workaround
+        # if token_name not in self.token_name[token_name]:
+        #     self.add_token(token_name, token_name)
+        return self.token_name[token_name]["token_id"]
+
+    def get_token_value(self, token_name):
+        return self.token_name[token_name]["value"]
+
+    def token_name_is_in(self, token_name):
+        if token_name in self.token_name:
+            return True
+        return False
+
+    def hash_id_is_in(self, hash_id):
+        if hash_id in self.hash_map:
+            return True
+        return False
diff --git a/convlab/policy/genTUS/train_model.py b/convlab/policy/genTUS/train_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..2162417461d692514e3b27742dbfd477491fc24e
--- /dev/null
+++ b/convlab/policy/genTUS/train_model.py
@@ -0,0 +1,258 @@
+import json
+import os
+import sys
+from argparse import ArgumentParser
+from datetime import datetime
+from pprint import pprint
+import numpy as np
+import torch
+import transformers
+from datasets import Dataset, load_metric
+from tqdm import tqdm
+from transformers import (AutoModelForSeq2SeqLM, AutoTokenizer,
+                          BartForConditionalGeneration, BartTokenizer,
+                          DataCollatorForSeq2Seq, Seq2SeqTrainer,
+                          Seq2SeqTrainingArguments)
+
+sys.path.append(os.path.dirname(os.path.dirname(
+    os.path.dirname(os.path.abspath(__file__)))))
+
+os.environ["WANDB_DISABLED"] = "true"
+
+METRIC = load_metric("sacrebleu")
+TOKENIZER = BartTokenizer.from_pretrained("facebook/bart-base")
+TOKENIZER.add_tokens(["<?>"])
+MAX_IN_LEN = 500
+MAX_OUT_LEN = 500
+
+
+def arg_parser():
+    parser = ArgumentParser()
+    # data_name, dial_ids_order, split2ratio
+    parser.add_argument("--model-type", type=str, default="unify",
+                        help="unify or multiwoz")
+    parser.add_argument("--data-name", type=str, default="multiwoz21",
+                        help="multiwoz21, sgd, tm1, tm2, tm3, sgd+tm, or all")
+    parser.add_argument("--dial-ids-order", type=int, default=0)
+    parser.add_argument("--split2ratio", type=float, default=1)
+    parser.add_argument("--batch-size", type=int, default=16)
+    parser.add_argument("--model-checkpoint", type=str,
+                        default="facebook/bart-base")
+    return parser.parse_args()
+
+
+def gentus_compute_metrics(eval_preds):
+    preds, labels = eval_preds
+    if isinstance(preds, tuple):
+        preds = preds[0]
+    decoded_preds = TOKENIZER.batch_decode(
+        preds, skip_special_tokens=True, max_length=MAX_OUT_LEN)
+
+    # Replace -100 in the labels as we can't decode them.
+    labels = np.where(labels != -100, labels, TOKENIZER.pad_token_id)
+    decoded_labels = TOKENIZER.batch_decode(
+        labels, skip_special_tokens=True, max_length=MAX_OUT_LEN)
+
+    act, text = postprocess_text(decoded_preds, decoded_labels)
+
+    result = METRIC.compute(
+        # predictions=decoded_preds, references=decoded_labels)
+        predictions=text["preds"], references=text["labels"])
+    result = {"bleu": result["score"]}
+    f1_scores = f1_measure(pred_acts=act["preds"], label_acts=act["labels"])
+    for s in f1_scores:
+        result[s] = f1_scores[s]
+
+    result = {k: round(v, 4) for k, v in result.items()}
+    return result
+
+
+def postprocess_text(preds, labels):
+    act = {"preds": [], "labels": []}
+    text = {"preds": [], "labels": []}
+
+    for pred, label in zip(preds, labels):
+        model_output = parse_output(pred.strip())
+        label_output = parse_output(label.strip())
+        if len(label_output["text"]) < 1:
+            continue
+        act["preds"].append(model_output.get("action", []))
+        text["preds"].append(model_output.get("text", pred.strip()))
+        act["labels"].append(label_output["action"])
+        text["labels"].append([label_output["text"]])
+
+    return act, text
+
+
+def parse_output(in_str):
+    in_str = in_str.replace('<s>', '').replace('<\\s>', '')
+    try:
+        output = json.loads(in_str)
+    except:
+        # print(f"invalid action {in_str}")
+        output = {"action": [], "text": ""}
+    return output
+
+
+def f1_measure(pred_acts, label_acts):
+    result = {"precision": [], "recall": [], "f1": []}
+    for pred, label in zip(pred_acts, label_acts):
+        r = tp_fn_fp(pred, label)
+        for m in result:
+            result[m].append(r[m])
+    for m in result:
+        result[m] = sum(result[m])/len(result[m])
+
+    return result
+
+
+def tp_fn_fp(pred, label):
+    tp, fn, fp = 0.0, 0.0, 0.0
+    precision, recall, f1 = 0, 0, 0
+    for p in pred:
+        if p in label:
+            tp += 1
+        else:
+            fp += 1
+    for l in label:
+        if l not in pred:
+            fn += 1
+    if (tp+fp) > 0:
+        precision = tp / (tp+fp)
+    if (tp+fn) > 0:
+        recall = tp/(tp+fn)
+    if (precision + recall) > 0:
+        f1 = (2*precision*recall)/(precision+recall)
+
+    return {"precision": precision, "recall": recall, "f1": f1}
+
+
+class TrainerHelper:
+    def __init__(self, tokenizer, max_input_length=500, max_target_length=500):
+        print("transformers version is: ", transformers.__version__)
+        self.tokenizer = tokenizer
+        self.max_input_length = max_input_length
+        self.max_target_length = max_target_length
+        self.base_name = "convlab/policy/genTUS"
+        self.dir_name = ""
+
+    def _get_data_folder(self, model_type, data_name, dial_ids_order=0, split2ratio=1):
+        # base_name = "convlab/policy/genTUS/unify/data"
+        if model_type not in ["unify", "multiwoz"]:
+            print("Unknown model type. Currently only support unify and multiwoz")
+        self.dir_name = f"{data_name}_{dial_ids_order}_{split2ratio}"
+        return os.path.join(self.base_name, model_type, 'data', self.dir_name)
+
+    def get_model_folder(self, model_type):
+        folder_name = os.path.join(
+            self.base_name, model_type, "experiments", self.dir_name)
+        if not os.path.exists(folder_name):
+            os.makedirs(folder_name)
+        return folder_name
+
+    def parse_data(self, model_type, data_name, dial_ids_order=0, split2ratio=1):
+        data_folder = self._get_data_folder(
+            model_type, data_name, dial_ids_order, split2ratio)
+
+        raw_data = {}
+        for d_type in ["train", "validation", "test"]:
+            f_name = os.path.join(data_folder, f"{d_type}.json")
+            raw_data[d_type] = json.load(open(f_name))
+
+        tokenized_datasets = {}
+        for data_type, data in raw_data.items():
+            tokenized_datasets[data_type] = Dataset.from_dict(
+                self._preprocess(data["dialog"]))
+
+        return tokenized_datasets
+
+    def _preprocess(self, examples):
+        model_inputs = {"input_ids": [], "attention_mask": [], "labels": []}
+        if isinstance(examples, dict):
+            examples = [examples]
+        for example in tqdm(examples):
+            inputs = self.tokenizer(example["in"],
+                                    max_length=self.max_input_length,
+                                    truncation=True)
+
+            # Setup the tokenizer for targets
+            with self.tokenizer.as_target_tokenizer():
+                labels = self.tokenizer(example["out"],
+                                        max_length=self.max_target_length,
+                                        truncation=True)
+            for key in ["input_ids", "attention_mask"]:
+                model_inputs[key].append(inputs[key])
+            model_inputs["labels"].append(labels["input_ids"])
+
+        return model_inputs
+
+
+def train(model_type, data_name, dial_ids_order, split2ratio, batch_size=16, max_input_length=500, max_target_length=500, model_checkpoint="facebook/bart-base"):
+    tokenizer = TOKENIZER
+
+    train_helper = TrainerHelper(
+        tokenizer=tokenizer, max_input_length=max_input_length, max_target_length=max_target_length)
+    data = train_helper.parse_data(model_type=model_type,
+                                   data_name=data_name,
+                                   dial_ids_order=dial_ids_order,
+                                   split2ratio=split2ratio)
+
+    model = BartForConditionalGeneration.from_pretrained(model_checkpoint)
+    model.resize_token_embeddings(len(tokenizer))
+    fp16 = False
+    if torch.cuda.is_available():
+        fp16 = True
+
+    model_dir = os.path.join(
+        train_helper.get_model_folder(model_type),
+        f"{datetime.now().strftime('%y-%m-%d-%H-%M')}")
+
+    args = Seq2SeqTrainingArguments(
+        model_dir,
+        evaluation_strategy="epoch",
+        learning_rate=2e-5,
+        per_device_train_batch_size=batch_size,
+        per_device_eval_batch_size=batch_size,
+        weight_decay=0.01,
+        save_total_limit=2,
+        num_train_epochs=5,
+        predict_with_generate=True,
+        fp16=fp16,
+        push_to_hub=False,
+        generation_max_length=max_target_length,
+        logging_dir=os.path.join(model_dir, 'log')
+    )
+    data_collator = DataCollatorForSeq2Seq(
+        tokenizer, model=model, padding=True)
+
+    # customize this trainer
+    trainer = Seq2SeqTrainer(
+        model=model,
+        args=args,
+        train_dataset=data["train"],
+        eval_dataset=data["test"],
+        data_collator=data_collator,
+        tokenizer=tokenizer,
+        compute_metrics=gentus_compute_metrics)
+    print("start training...")
+    trainer.train()
+    print("saving model...")
+    trainer.save_model()
+
+
+def main():
+    args = arg_parser()
+    print("---> data_name", args.data_name)
+    train(model_type=args.model_type,
+          data_name=args.data_name,
+          dial_ids_order=args.dial_ids_order,
+          split2ratio=args.split2ratio,
+          batch_size=args.batch_size,
+          max_input_length=MAX_IN_LEN,
+          max_target_length=MAX_OUT_LEN,
+          model_checkpoint=args.model_checkpoint)
+
+
+if __name__ == "__main__":
+    main()
+    # sgd+tm: 46000
diff --git a/convlab/policy/genTUS/unify/Goal.py b/convlab/policy/genTUS/unify/Goal.py
new file mode 100644
index 0000000000000000000000000000000000000000..6a77b090a7266d07f653e707bd4b749b6a6114bb
--- /dev/null
+++ b/convlab/policy/genTUS/unify/Goal.py
@@ -0,0 +1,233 @@
+"""
+The user goal for unify data format
+"""
+import json
+from convlab.policy.tus.unify.Goal import old_goal2list
+from convlab.task.multiwoz.goal_generator import GoalGenerator
+from convlab.policy.rule.multiwoz.policy_agenda_multiwoz import Goal as ABUS_Goal
+from convlab.util.custom_util import slot_mapping
+DEF_VAL_UNK = '?'  # Unknown
+DEF_VAL_DNC = 'dontcare'  # Do not care
+DEF_VAL_NUL = 'none'  # for none
+NOT_SURE_VALS = [DEF_VAL_UNK, DEF_VAL_DNC, DEF_VAL_NUL, ""]
+
+NOT_MENTIONED = "not mentioned"
+FULFILLED = "fulfilled"
+REQUESTED = "requested"
+CONFLICT = "conflict"
+
+
+class Goal:
+    """ User Goal Model Class. """
+
+    def __init__(self, goal=None, goal_generator=None):
+        """
+        create new Goal from a dialog or from goal_generator
+        Args:
+            goal: can be a list (create from a dialog), an abus goal, or none
+        """
+        self.domains = []
+        self.domain_goals = {}
+        self.status = {}
+        self.invert_slot_mapping = {v: k for k, v in slot_mapping.items()}
+        self.raw_goal = None
+
+        self._init_goal_from_data(goal, goal_generator)
+        self._init_status()
+
+    def __str__(self):
+        return '-----Goal-----\n' + \
+               json.dumps(self.domain_goals, indent=4) + \
+               '\n-----Goal-----'
+
+    def _init_goal_from_data(self, goal=None, goal_generator=None):
+        if not goal and goal_generator:
+            goal = ABUS_Goal(goal_generator)
+            self.raw_goal = goal.domain_goals
+            goal = old_goal2list(goal.domain_goals)
+
+        elif isinstance(goal, dict):
+            self.raw_goal = goal
+            goal = old_goal2list(goal)
+
+        elif isinstance(goal, ABUS_Goal):
+            self.raw_goal = goal.domain_goals
+            goal = old_goal2list(goal.domain_goals)
+
+        # else:
+        #     print("unknow goal")
+
+        # be careful of this order
+        for domain, intent, slot, value in goal:
+            if domain == "none":
+                continue
+            if domain not in self.domains:
+                self.domains.append(domain)
+                self.domain_goals[domain] = {}
+            if intent not in self.domain_goals[domain]:
+                self.domain_goals[domain][intent] = {}
+
+            if not value:
+                if intent == "request":
+                    self.domain_goals[domain][intent][slot] = DEF_VAL_UNK
+                else:
+                    print(
+                        f"unknown no value intent {domain}, {intent}, {slot}")
+            else:
+                self.domain_goals[domain][intent][slot] = value
+
+    def _init_status(self):
+        for domain, domain_goal in self.domain_goals.items():
+            if domain not in self.status:
+                self.status[domain] = {}
+            for slot_type, sub_domain_goal in domain_goal.items():
+                if slot_type not in self.status[domain]:
+                    self.status[domain][slot_type] = {}
+                for slot in sub_domain_goal:
+                    if slot not in self.status[domain][slot_type]:
+                        self.status[domain][slot_type][slot] = {}
+                    self.status[domain][slot_type][slot] = {
+                        "value": str(sub_domain_goal[slot]),
+                        "status": NOT_MENTIONED}
+
+    def get_goal_list(self, data_goal=None):
+        goal_list = []
+        if data_goal:
+            # make sure the order!!!
+            for domain, intent, slot, _ in data_goal:
+                status = self._get_status(domain, intent, slot)
+                value = self.domain_goals[domain][intent][slot]
+                goal_list.append([intent, domain, slot, value, status])
+            return goal_list
+        else:
+            for domain, domain_goal in self.domain_goals.items():
+                for intent, sub_goal in domain_goal.items():
+                    for slot, value in sub_goal.items():
+                        status = self._get_status(domain, intent, slot)
+                        goal_list.append([intent, domain, slot, value, status])
+
+        return goal_list
+
+    def _get_status(self, domain, intent, slot):
+        if domain not in self.status:
+            return NOT_MENTIONED
+        if intent not in self.status[domain]:
+            return NOT_MENTIONED
+        if slot not in self.status[domain][intent]:
+            return NOT_MENTIONED
+        return self.status[domain][intent][slot]["status"]
+
+    def task_complete(self):
+        """
+        Check that all requests have been met
+        Returns:
+            (boolean): True to accomplish.
+        """
+        for domain, domain_goal in self.status.items():
+            if domain not in self.domain_goals:
+                continue
+            for slot_type, sub_domain_goal in domain_goal.items():
+                if slot_type not in self.domain_goals[domain]:
+                    continue
+                for slot, status in sub_domain_goal.items():
+                    if slot not in self.domain_goals[domain][slot_type]:
+                        continue
+                    # for strict success, turn this on
+                    if status["status"] in [NOT_MENTIONED, CONFLICT]:
+                        if status["status"] == CONFLICT and slot in ["arrive by", "leave at"]:
+                            continue
+                        return False
+                    if "?" in status["value"]:
+                        return False
+
+        return True
+
+    # TODO change to update()?
+    def update_user_goal(self, action, char="usr"):
+        # update request and booked
+        if char == "usr":
+            self._user_action_update(action)
+        elif char == "sys":
+            self._system_action_update(action)
+        else:
+            print("!!!UNKNOWN CHAR!!!")
+
+    def _user_action_update(self, action):
+        # no need to update user goal
+        for intent, domain, slot, _ in action:
+            goal_intent = self._check_slot_and_intent(domain, slot)
+            if not goal_intent:
+                continue
+            # fulfilled by user
+            if is_inform(intent):
+                self._set_status(goal_intent, domain, slot, FULFILLED)
+            # requested by user
+            if is_request(intent):
+                self._set_status(goal_intent, domain, slot, REQUESTED)
+
+    def _system_action_update(self, action):
+        for intent, domain, slot, value in action:
+            goal_intent = self._check_slot_and_intent(domain, slot)
+            if not goal_intent:
+                continue
+            # fulfill request by system
+            if is_inform(intent) and is_request(goal_intent):
+                self._set_status(goal_intent, domain, slot, FULFILLED)
+                self._set_goal(goal_intent, domain, slot, value)
+
+            if is_inform(intent) and is_inform(goal_intent):
+                # fulfill infrom by system
+                if value == self.domain_goals[domain][goal_intent][slot]:
+                    self._set_status(goal_intent, domain, slot, FULFILLED)
+                # conflict system inform
+                else:
+                    self._set_status(goal_intent, domain, slot, CONFLICT)
+            # requested by system
+            if is_request(intent) and is_inform(goal_intent):
+                self._set_status(goal_intent, domain, slot, REQUESTED)
+
+    def _set_status(self, intent, domain, slot, status):
+        self.status[domain][intent][slot]["status"] = status
+
+    def _set_goal(self, intent, domain, slot, value):
+        # old_value = self.domain_goals[domain][intent][slot]
+        self.domain_goals[domain][intent][slot] = value
+        self.status[domain][intent][slot]["value"] = value
+        # print(
+        #     f"updating user goal {intent}-{domain}-{slot} {old_value}-> {value}")
+
+    def _check_slot_and_intent(self, domain, slot):
+        not_found = ""
+        if domain not in self.domain_goals:
+            return not_found
+        for intent in self.domain_goals[domain]:
+            if slot in self.domain_goals[domain][intent]:
+                return intent
+        return not_found
+
+
+def is_inform(intent):
+    if "inform" in intent:
+        return True
+    return False
+
+
+def is_request(intent):
+    if "request" in intent:
+        return True
+    return False
+
+
+def transform_data_act(data_action):
+    action_list = []
+    for _, dialog_act in data_action.items():
+        for act in dialog_act:
+            value = act.get("value", "")
+            if not value:
+                if "request" in act["intent"]:
+                    value = "?"
+                else:
+                    value = "none"
+            action_list.append(
+                [act["intent"], act["domain"], act["slot"], value])
+    return action_list
diff --git a/convlab/policy/genTUS/unify/build_data.py b/convlab/policy/genTUS/unify/build_data.py
new file mode 100644
index 0000000000000000000000000000000000000000..50873a1d4b6ffaa8ee49c84a4b088ae56ad13554
--- /dev/null
+++ b/convlab/policy/genTUS/unify/build_data.py
@@ -0,0 +1,211 @@
+import json
+import os
+import sys
+from argparse import ArgumentParser
+
+from tqdm import tqdm
+
+from convlab.policy.genTUS.unify.Goal import Goal, transform_data_act
+from convlab.policy.tus.unify.util import create_goal, load_experiment_dataset
+
+
+sys.path.append(os.path.dirname(os.path.dirname(
+    os.path.dirname(os.path.abspath(__file__)))))
+
+
+def arg_parser():
+    parser = ArgumentParser()
+    parser.add_argument("--dataset", type=str, default="multiwoz21",
+                        help="the dataset, such as multiwoz21, sgd, tm1, tm2, and tm3.")
+    parser.add_argument("--dial-ids-order", type=int, default=0)
+    parser.add_argument("--split2ratio", type=float, default=1)
+    parser.add_argument("--random-order", action="store_true")
+    parser.add_argument("--no-status", action="store_true")
+    parser.add_argument("--add-history",  action="store_true")
+    parser.add_argument("--remove-domain", type=str, default="")
+
+    return parser.parse_args()
+
+class DataBuilder:
+    def __init__(self, dataset='multiwoz21'):
+        self.dataset = dataset
+
+    def setup_data(self,
+                   raw_data,
+                   random_order=False,
+                   no_status=False,
+                   add_history=False,
+                   remove_domain=None):
+        examples = {data_split: {"dialog": []} for data_split in raw_data}
+
+        for data_split, dialogs in raw_data.items():
+            for dialog in tqdm(dialogs, ascii=True):
+                example = self._one_dialog(dialog=dialog,
+                                           add_history=add_history,
+                                           random_order=random_order,
+                                           no_status=no_status)
+                examples[data_split]["dialog"] += example
+
+        return examples
+
+    def _one_dialog(self, dialog, add_history=True, random_order=False, no_status=False):
+        example = []
+        history = []
+
+        data_goal = self.norm_domain_goal(create_goal(dialog))
+        if not data_goal:
+            return example
+        user_goal = Goal(goal=data_goal)
+
+        for turn_id in range(0, len(dialog["turns"]), 2):
+            sys_act = self._get_sys_act(dialog, turn_id)
+
+            user_goal.update_user_goal(action=sys_act, char="sys")
+            usr_goal_str = self._user_goal_str(user_goal, data_goal, random_order, no_status)
+
+            usr_act = self.norm_domain(transform_data_act(
+                dialog["turns"][turn_id]["dialogue_acts"]))
+            user_goal.update_user_goal(action=usr_act, char="usr")
+
+            # change value "?" to "<?>"
+            usr_act = self._modify_act(usr_act)
+
+            in_str = self._dump_in_str(sys_act, usr_goal_str, history, turn_id, add_history)
+            out_str = self._dump_out_str(usr_act, dialog["turns"][turn_id]["utterance"])
+
+            history.append(usr_act)
+            if usr_act:
+                example.append({"in": in_str, "out": out_str})
+
+        return example
+
+    def _get_sys_act(self, dialog, turn_id):
+        sys_act = []
+        if turn_id > 0:
+            sys_act = self.norm_domain(transform_data_act(
+                dialog["turns"][turn_id - 1]["dialogue_acts"]))
+        return sys_act
+
+    def _user_goal_str(self, user_goal, data_goal, random_order, no_status):
+        if random_order:
+            usr_goal_str = user_goal.get_goal_list()
+        else:
+            usr_goal_str = user_goal.get_goal_list(data_goal=data_goal)
+
+        if no_status:
+            usr_goal_str = self._remove_status(usr_goal_str)
+        return usr_goal_str
+
+    def _dump_in_str(self, sys_act, usr_goal_str, history, turn_id, add_history):
+        in_str = {}
+        in_str["system"] = self._modify_act(sys_act)
+        in_str["goal"] = usr_goal_str
+        if add_history:
+            h = []
+            if history:
+                h = history[-3:]
+            in_str["history"] = h
+            in_str["turn"] = str(int(turn_id/2))
+
+        return json.dumps(in_str)
+
+    def _dump_out_str(self, usr_act, text):
+        out_str = {"action": usr_act, "text": text}
+        return json.dumps(out_str)
+
+    @staticmethod
+    def _norm_intent(intent):
+        if intent in ["inform_intent", "negate_intent", "affirm_intent", "request_alts"]:
+            return f"_{intent}"
+        return intent
+
+    def norm_domain(self, x):
+        if not x:
+            return x
+        norm_result = []
+        # print(x)
+        for intent, domain, slot, value in x:
+            if "_" in domain:
+                domain = domain.split('_')[0]
+            if not domain:
+                domain = "none"
+            if not slot:
+                slot = "none"
+            if not value:
+                if intent == "request":
+                    value = "<?>"
+                else:
+                    value = "none"
+            norm_result.append([self._norm_intent(intent), domain, slot, value])
+        return norm_result
+
+    def norm_domain_goal(self, x):
+        if not x:
+            return x
+        norm_result = []
+        # take care of the order!
+        for domain, intent, slot, value in x:
+            if "_" in domain:
+                domain = domain.split('_')[0]
+            if not domain:
+                domain = "none"
+            if not slot:
+                slot = "none"
+            if not value:
+                if intent == "request":
+                    value = "<?>"
+                else:
+                    value = "none"
+            norm_result.append([domain, self._norm_intent(intent), slot, value])
+        return norm_result
+
+    @staticmethod
+    def _remove_status(goal_list):
+        new_list = [[goal[0], goal[1], goal[2], goal[3]]
+                    for goal in goal_list]
+        return new_list
+
+    @staticmethod
+    def _modify_act(act):
+        new_act = []
+        for i, d, s, value in act:
+            if value == "?":
+                new_act.append([i, d, s, "<?>"])
+            else:
+                new_act.append([i, d, s, value])
+        return new_act
+
+
+if __name__ == "__main__":
+    args = arg_parser()
+
+    base_name = "convlab/policy/genTUS/unify/data"
+    dir_name = f"{args.dataset}_{args.dial_ids_order}_{args.split2ratio}"
+    folder_name = os.path.join(base_name, dir_name)
+    remove_domain = args.remove_domain
+
+    if not os.path.exists(folder_name):
+        os.makedirs(folder_name)
+
+    dataset = load_experiment_dataset(
+        data_name=args.dataset,
+        dial_ids_order=args.dial_ids_order,
+        split2ratio=args.split2ratio)
+    data_builder = DataBuilder(dataset=args.dataset)
+    data = data_builder.setup_data(
+        raw_data=dataset,
+        random_order=args.random_order,
+        no_status=args.no_status,
+        add_history=args.add_history,
+        remove_domain=remove_domain)
+
+    for data_type in data:
+        if remove_domain:
+            file_name = os.path.join(
+                folder_name,
+                f"no{remove_domain}_{data_type}.json")
+        else:
+            file_name = os.path.join(
+                folder_name,
+                f"{data_type}.json")
+        json.dump(data[data_type], open(file_name, 'w'), indent=2)
diff --git a/convlab/policy/genTUS/unify/knowledge_graph.py b/convlab/policy/genTUS/unify/knowledge_graph.py
new file mode 100644
index 0000000000000000000000000000000000000000..68af13e481fe4799dfc2a6f3763b526611eabd9c
--- /dev/null
+++ b/convlab/policy/genTUS/unify/knowledge_graph.py
@@ -0,0 +1,252 @@
+import json
+from random import choices
+
+from convlab.policy.genTUS.token_map import tokenMap
+
+from transformers import BartTokenizer
+
+DEBUG = False
+DATASET = "unify"
+
+
+class KnowledgeGraph:
+    def __init__(self, tokenizer: BartTokenizer, ontology_file=None, dataset="multiwoz21"):
+        print("dataset", dataset)
+        self.debug = DEBUG
+        self.tokenizer = tokenizer
+
+        if "multiwoz" in dataset:
+            self.domain_intent = ["inform", "request"]
+            self.general_intent = ["thank", "bye"]
+        # use sgd dataset intents as default
+        else:
+            self.domain_intent = ["_inform_intent",
+                                  "_negate_intent",
+                                  "_affirm_intent",
+                                  "inform",
+                                  "request",
+                                  "affirm",
+                                  "negate",
+                                  "select",
+                                  "_request_alts"]
+            self.general_intent = ["thank_you", "goodbye"]
+
+        self.general_domain = "none"
+        self.kg_map = {"intent": tokenMap(tokenizer=self.tokenizer)}
+
+        for intent in self.domain_intent + self.general_intent:
+            self.kg_map["intent"].add_token(intent, intent)
+
+        self.init()
+
+    def init(self):
+        for map_type in ["domain", "slot", "value"]:
+            self.kg_map[map_type] = tokenMap(tokenizer=self.tokenizer)
+        self.add_token("<?>", "value")
+
+    def parse_input(self, in_str):
+        self.init()
+        inputs = json.loads(in_str)
+        self.sys_act = inputs["system"]
+        self.user_goal = {}
+        self._add_none_domain()
+        for intent, domain, slot, value, _ in inputs["goal"]:
+            self._update_user_goal(intent, domain, slot, value, source="goal")
+
+        for intent, domain, slot, value in self.sys_act:
+            self._update_user_goal(intent, domain, slot, value, source="sys")
+
+    def _add_none_domain(self):
+        self.user_goal["none"] = {"none": "none"}
+        # add slot
+        self.add_token("none", "domain")
+        self.add_token("none", "slot")
+        self.add_token("none", "value")
+
+    def _update_user_goal(self, intent, domain, slot, value, source="goal"):
+
+        if value == "?":
+            value = "<?>"
+
+        if intent == "request" and source == "sys":
+            value = "dontcare"  # user can "dontcare" system request
+
+        if source == "sys" and intent != "request":
+            return
+
+        if domain not in self.user_goal:
+            self.user_goal[domain] = {}
+            self.user_goal[domain]["none"] = ["none"]
+            self.add_token(domain, "domain")
+            self.add_token("none", "slot")
+            self.add_token("none", "value")
+
+        if slot not in self.user_goal[domain]:
+            self.user_goal[domain][slot] = []
+            self.add_token(domain, "slot")
+
+        if value not in self.user_goal[domain][slot]:
+            value = json.dumps(str(value))[1:-1]
+            self.user_goal[domain][slot].append(value)
+            value = value.replace('"', "'")
+            self.add_token(value, "value")
+
+    def add_token(self, token_name, map_type):
+        if map_type == "value":
+            token_name = token_name.replace('"', "'")
+        if not self.kg_map[map_type].token_name_is_in(token_name):
+            self.kg_map[map_type].add_token(token_name, token_name)
+
+    def _get_max_score(self, outputs, candidate_list, map_type):
+        score = {}
+        if not candidate_list:
+            print(f"ERROR: empty candidate list for {map_type}")
+            score[1] = {"token_id": self._get_token_id(
+                "none"), "token_name": "none"}
+
+        for x in candidate_list:
+            hash_id = self._get_token_id(x)[0]
+            s = outputs[:, hash_id].item()
+            score[s] = {"token_id": self._get_token_id(x),
+                        "token_name": x}
+        return score
+
+    def _select(self, score, mode="max"):
+        probs = [s for s in score]
+        if mode == "max":
+            s = max(probs)
+        elif mode == "sample":
+            s = choices(probs, weights=probs, k=1)
+            s = s[0]
+
+        else:
+            print("unknown select mode")
+
+        return s
+
+    def _get_max_domain_token(self, outputs, candidates, map_type, mode="max"):
+        score = self._get_max_score(outputs, candidates, map_type)
+        s = self._select(score, mode)
+        token_id = score[s]["token_id"]
+        token_name = score[s]["token_name"]
+
+        return {"token_id": token_id, "token_name": token_name}
+
+    def candidate(self, candidate_type, **kwargs):
+        if "intent" in kwargs:
+            intent = kwargs["intent"]
+        if candidate_type == "intent":
+            allow_general_intent = kwargs.get("allow_general_intent", True)
+            if allow_general_intent:
+                return self.domain_intent + self.general_intent
+            else:
+                return self.domain_intent
+        elif candidate_type == "domain":
+            if intent in self.general_intent:
+                return [self.general_domain]
+            else:
+                return [d for d in self.user_goal]
+        elif candidate_type == "slot":
+            if intent in self.general_intent:
+                return ["none"]
+            else:
+                return self._filter_slot(intent, kwargs["domain"], kwargs["is_mentioned"])
+        else:
+            if intent in self.general_intent:
+                return ["none"]
+            elif intent.lower() == "request":
+                return ["<?>"]
+            else:
+                return self._filter_value(intent, kwargs["domain"], kwargs["slot"])
+
+    def get_intent(self, outputs, mode="max", allow_general_intent=True):
+        # return intent, token_id_list
+        # TODO request?
+        canidate_list = self.candidate(
+            "intent", allow_general_intent=allow_general_intent)
+        score = self._get_max_score(outputs, canidate_list, "intent")
+        s = self._select(score, mode)
+
+        return score[s]
+
+    def get_domain(self, outputs, intent, mode="max"):
+        if intent in self.general_intent:
+            token_name = self.general_domain
+            token_id = self.tokenizer(token_name, add_special_tokens=False)
+            token_map = {"token_id": token_id['input_ids'],
+                         "token_name": token_name}
+
+        elif intent in self.domain_intent:
+            # [d for d in self.user_goal]
+            domain_list = self.candidate("domain", intent=intent)
+            token_map = self._get_max_domain_token(
+                outputs=outputs, candidates=domain_list, map_type="domain", mode=mode)
+        else:
+            if self.debug:
+                print("unknown intent", intent)
+
+        return token_map
+
+    def get_slot(self, outputs, intent, domain, mode="max", is_mentioned=False):
+        if intent in self.general_intent:
+            token_name = "none"
+            token_id = self.tokenizer(token_name, add_special_tokens=False)
+            token_map = {"token_id": token_id['input_ids'],
+                         "token_name": token_name}
+
+        elif intent in self.domain_intent:
+            slot_list = self.candidate(
+                candidate_type="slot", intent=intent, domain=domain, is_mentioned=is_mentioned)
+            token_map = self._get_max_domain_token(
+                outputs=outputs, candidates=slot_list, map_type="slot", mode=mode)
+
+        return token_map
+
+    def get_value(self, outputs, intent, domain, slot, mode="max"):
+        if intent in self.general_intent or slot.lower() == "none":
+            token_name = "none"
+            token_id = self.tokenizer(token_name, add_special_tokens=False)
+            token_map = {"token_id": token_id['input_ids'],
+                         "token_name": token_name}
+
+        elif intent.lower() == "request":
+            token_name = "<?>"
+            token_id = self.tokenizer(token_name, add_special_tokens=False)
+            token_map = {"token_id": token_id['input_ids'],
+                         "token_name": token_name}
+
+        elif intent in self.domain_intent:
+            # TODO should not none ?
+            # value_list = [v for v in self.user_goal[domain][slot]]
+            value_list = self.candidate(
+                candidate_type="value", intent=intent, domain=domain, slot=slot)
+
+            token_map = self._get_max_domain_token(
+                outputs=outputs, candidates=value_list, map_type="value", mode=mode)
+
+        return token_map
+
+    def _filter_slot(self, intent, domain, is_mentioned=True):
+        slot_list = []
+        for slot in self.user_goal[domain]:
+            value_list = self._filter_value(intent, domain, slot)
+            if len(value_list) > 0:
+                slot_list.append(slot)
+        if not is_mentioned and intent.lower() != "request":
+            slot_list.append("none")
+        return slot_list
+
+    def _filter_value(self, intent, domain, slot):
+        value_list = [v for v in self.user_goal[domain][slot]]
+        if "none" in value_list:
+            value_list.remove("none")
+        if intent.lower() != "request":
+            if "?" in value_list:
+                value_list.remove("?")
+            if "<?>" in value_list:
+                value_list.remove("<?>")
+        # print(f"{intent}-{domain}-{slot}= {value_list}")
+        return value_list
+
+    def _get_token_id(self, token):
+        return self.tokenizer(token, add_special_tokens=False)["input_ids"]
diff --git a/convlab/policy/genTUS/utils.py b/convlab/policy/genTUS/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..39c822dd35f985790d53a2834e8b6fe437864f24
--- /dev/null
+++ b/convlab/policy/genTUS/utils.py
@@ -0,0 +1,5 @@
+import torch
+
+
+def append_tokens(tokens, new_token, device):
+    return torch.cat((tokens, torch.tensor([new_token]).to(device)), dim=1)
diff --git a/convlab/policy/lava/README.md b/convlab/policy/lava/README.md
index bd06a73b9d4c2954338d448a7f7291652228fc59..9678d76bb7e41b9a8a6035dedfc7f40d1a73395a 100755
--- a/convlab/policy/lava/README.md
+++ b/convlab/policy/lava/README.md
@@ -1,74 +1,17 @@
 ## LAVA: Latent Action Spaces via Variational Auto-encoding for Dialogue Policy Optimization
-Codebase for [LAVA: Latent Action Spaces via Variational Auto-encoding for Dialogue Policy Optimization](https://), published as a long paper in COLING 2020. The code is developed based on the implementations of the [LaRL](https://arxiv.org/abs/1902.08858) paper.
+ConvLab3 interface for [LAVA: Latent Action Spaces via Variational Auto-encoding for Dialogue Policy Optimization](https://aclanthology.org/2020.coling-main.41/), published as a long paper in COLING 2020.
 
-### Requirements
-    python 3
-    pytorch
-    numpy
-            
-### Data
-The pre-processed MultiWoz 2.0 data is included in data.zip. Unzip the compressed file and access the data under **data/norm-multi-woz**.
-            
-### Over structure:
-The implementation of the models, as well as training and evaluation scripts are under **latent_dialog**.
-The scripts for running the experiments are under **experiment_woz**. The trained models and evaluation results are under **experiment_woz/sys_config_log_model**.
+To train a LAVA model, clone and follow instructions from the [original LAVA repository](https://gitlab.cs.uni-duesseldorf.de/general/dsml/lava-public).
 
-There are 3 types of training to achieve the final model.
+With a (pre-)trained LAVA model, it is possible to evaluate or perform online RL with ConvLab3 US by loading the lava module with
 
-### Step 1: Unsupervised training (variational auto-encoding (VAE) task)
-Given a dialogue response, the model is tasked to reproduce it via latent variables. With this task we aim to unsupervisedly capture generative factors of dialogue responses.
+- from convlab.policy.lava.multiwoz import LAVA
 
-    - sl_cat_ae.py: train a VAE model using categorical latent variable
-    - sl_gauss_ae.py: train a VAE model using continuous (Gaussian) latent variable
+and using it as the policy module in the ConvLab pipeline (NLG should be set to None).
 
-### Step 2: Supervised training (response generation task)
-The supervised training step of the variational encoder-decoder model could be done 4 different ways. 
 
-1. from scratch:
+Code example can be found at
+- ConvLab-3/examples/agent_examples/test_LAVA.py
 
+A trained LAVA model can be found at https://huggingface.co/ConvLab/lava-policy-multiwoz21.
 
-    - sl_word: train a standard encoder decoder model using supervised learning (SL)
-    - sl_cat: train a latent action model with categorical latent variables using SL,
-    - sl_gauss: train a latent action model with continuous latent varaibles using SL,
-
-2. using the VAE models as pre-trained model (equivalent to LAVA_pt):
-
-
-    - finetune_cat_ae: use the VAE with categorical latent variables as weight initialization, and then fine-tune the model on response generation task
-    - finetune_gauss_ae: as above but with continuous latent variables 
-    - Note: Fine-tuning can be set to be selective (only fine-tune encoder) or not (fine-tune the entire network) using the "selective_finetune" argument in config
-
-3. using the distribution of the VAE models to obtain informed prior (equivalent to LAVA_kl):
-
-
-    - actz_cat: initialized new encoder is combined with pre-trained VAE decoder and fine-tuned on response generation task. VAE encoder is used to obtain an informed prior of the target response and not trained further.
-    - actz_gauss: as above but with continuous latent variables
-
-4. or simultaneously from scrath with VAE task in a multi-task fashion (equivalent to LAVA_mt):
-
-
-    - mt_cat: train a model to optimize both auto-encoding and response generation in a multi-task fashion, using categorical latent variables
-    - mt_gauss: as above but with continuous latent variables
-
-No.1 and 4 can be directly trained without Step 1. No. 2 and 3 requires a pre-trained VAE model, given via a dictionary 
-
-    pretrained = {"2020-02-26-18-11-37-sl_cat_ae":100}
-
-### Step 3: Reinforcement Learning
-The model can be further optimized with RL to maximize the dialogue success.
-
-Each script is used for:
-
-    - reinforce_word: fine tune a pretrained model with word-level policy gradient (PG)
-    - reinforce_cat: fine tune a pretrained categorical latent action model with latent-level PG.
-    - reinforce_gauss: fine tune a pretrained gaussian latent action model with latent-level PG.
-
-The script takes a file containing list of test results from the SL step.
-
-    f_in = "sys_config_log_model/test_files.lst"
-
-
-### Checking the result
-The evaluation result can be found at the bottom of the test_file.txt. We provide the best model in this repo.
-
-NOTE: when re-running the experiments some variance is to be expected in the numbers due to factors such as random seed and hardware specificiations. Some methods are more sensitive to this than others.
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/__init__.py b/convlab/policy/lava/multiwoz/latent_dialog/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..5cabc5d9c605965db3e69614595ab9bb891814c9
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/__init__.py
@@ -0,0 +1,2 @@
+# @Time    : 10/18/18 1:55 PM
+# @Author  : Tiancheng Zhao
\ No newline at end of file
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/agent_task.py b/convlab/policy/lava/multiwoz/latent_dialog/agent_task.py
new file mode 100644
index 0000000000000000000000000000000000000000..8da7af0e554a09436a80537efccacf5c2b7d0bd5
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/agent_task.py
@@ -0,0 +1,115 @@
+import torch as th
+import torch.nn as nn
+import torch.optim as optim
+import numpy as np
+from convlab.policy.lava.multiwoz.latent_dialog.utils import LONG, FLOAT, Pack, get_detokenize
+from convlab.policy.lava.multiwoz.latent_dialog.main import get_sent
+from convlab.policy.lava.multiwoz.latent_dialog.data_loaders import BeliefDbDataLoaders
+from collections import deque, namedtuple, defaultdict
+import random
+import pdb
+import dill
+
+
+class OfflineRlAgent(object):
+    def __init__(self, model, corpus, args, name, tune_pi_only):
+        self.model = model
+        self.corpus = corpus
+        self.args = args
+        self.name = name
+        self.raw_goal = None
+        self.vec_goals_list = None
+        self.logprobs = None
+        print("Do we only tune the policy: {}".format(tune_pi_only))
+        self.opt = optim.SGD(
+            [p for n, p in self.model.named_parameters() if 'c2z' in n or not tune_pi_only],
+            lr=self.args.rl_lr,
+            momentum=self.args.momentum,
+            nesterov=(self.args.nesterov and self.args.momentum > 0))
+        # self.opt = optim.Adam(self.model.parameters(), lr=0.01)
+        # self.opt = optim.RMSprop(self.model.parameters(), lr=0.0005)
+        self.all_rewards = []
+        self.all_grads = []
+        self.model.train()
+
+    def print_dialog(self, dialog, reward, stats):
+        for t_id, turn in enumerate(dialog):
+            if t_id % 2 == 0:
+                print("Usr: {}".format(' '.join([t for t in turn if t != '<pad>'])))
+            else:
+                print("Sys: {}".format(' '.join(turn)))
+        report = ['{}: {}'.format(k, v) for k, v in stats.items()]
+        print("Reward {}. {}".format(reward, report))
+
+    def run(self, batch, evaluator, max_words=None, temp=0.1):
+        self.logprobs = []
+        self.dlg_history =[]
+        batch_size = len(batch['keys'])
+        logprobs, outs = self.model.forward_rl(batch, max_words, temp)
+        if batch_size == 1:
+            logprobs = [logprobs]
+            outs = [outs]
+
+        key = batch['keys'][0]
+        sys_turns = []
+        # construct the dialog history for printing
+        for turn_id, turn in enumerate(batch['contexts']):
+            user_input = self.corpus.id2sent(turn[-1])
+            self.dlg_history.append(user_input)
+            sys_output = self.corpus.id2sent(outs[turn_id])
+            self.dlg_history.append(sys_output)
+            sys_turns.append(' '.join(sys_output))
+
+        for log_prob in logprobs:
+            self.logprobs.extend(log_prob)
+        # compute reward here
+        generated_dialog = {key: sys_turns}
+        return evaluator.evaluateModel(generated_dialog, mode="offline_rl")
+
+    def update(self, reward, stats):
+        self.all_rewards.append(reward)
+        # standardize the reward
+        r = (reward - np.mean(self.all_rewards)) / max(1e-4, np.std(self.all_rewards))
+        # compute accumulated discounted reward
+        g = self.model.np2var(np.array([r]), FLOAT).view(1, 1)
+        rewards = []
+        for _ in self.logprobs:
+            rewards.insert(0, g)
+            g = g * self.args.gamma
+
+        loss = 0
+        # estimate the loss using one MonteCarlo rollout
+        for lp, r in zip(self.logprobs, rewards):
+            loss -= lp * r
+        self.opt.zero_grad()
+        loss.backward()
+        nn.utils.clip_grad_norm_(self.model.parameters(), self.args.rl_clip)
+        # for name, p in self.model.named_parameters():
+        #    print(name)
+        #    print(p.grad)
+        self.opt.step()
+
+class OfflineLatentRlAgent(OfflineRlAgent):
+    def run(self, batch, evaluator, max_words=None, temp=0.1):
+        self.logprobs = []
+        self.dlg_history =[]
+        batch_size = len(batch['keys'])
+        logprobs, outs, logprob_z, sample_z = self.model.forward_rl(batch, max_words, temp)
+        if batch_size == 1:
+            outs = [outs]
+        key = batch['keys'][0]
+        sys_turns = []
+        # construct the dialog history for printing
+        for turn_id, turn in enumerate(batch['contexts']):
+            user_input = self.corpus.id2sent(turn[-1])
+            self.dlg_history.append(user_input)
+            sys_output = self.corpus.id2sent(outs[turn_id])
+            self.dlg_history.append(sys_output)
+            sys_turns.append(' '.join(sys_output))
+
+        for b_id in range(batch_size):
+            self.logprobs.append(logprob_z[b_id])
+        # compute reward here
+        generated_dialog = {key: sys_turns}
+        return evaluator.evaluateModel(generated_dialog, mode="offline_rl")
+
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/augpt_utils.py b/convlab/policy/lava/multiwoz/latent_dialog/augpt_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..cdf317a08e4a1f9742bfd2f2141d9c9628344184
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/augpt_utils.py
@@ -0,0 +1,558 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+#
+# Copyright © 2021 lubis <lubis@hilbert242>
+#
+# Distributed under terms of the MIT license.
+
+"""
+utils from AuGPT codebase
+"""
+import re
+import os
+import sys
+import types
+import shutil
+import logging
+import requests
+import torch
+import zipfile
+import bisect
+import random
+import copy
+import json
+from collections import OrderedDict, defaultdict
+from typing import Callable, Union, Set, Optional, List, Dict, Any, Tuple, MutableMapping  # noqa: 401
+from dataclasses import dataclass
+import pdb
+
+DATASETS_PATH = os.path.join(os.path.expanduser(os.environ.get('DATASETS_PATH', '~/datasets')), 'augpt')
+pricepat = re.compile("\d{1,3}[.]\d{1,2}")
+
+temp_path = os.path.dirname(os.path.abspath(__file__))
+fin = open(os.path.join("/home/lubis/datasets/augpt/mapping.pair"))
+replacements = []
+for line in fin.readlines():
+    tok_from, tok_to = line.replace('\n', '').split('\t')
+    replacements.append((' ' + tok_from + ' ', ' ' + tok_to + ' '))
+
+
+
+class AutoDatabase:
+    @staticmethod
+    def load(pretrained_model_name_or_path):
+        database_file = os.path.join(pretrained_model_name_or_path, 'database.zip')
+
+        with zipfile.ZipFile(database_file) as zipf:
+            def _build_database():
+                module = types.ModuleType('database')
+                exec(zipf.read('database.py').decode('utf-8'), module.__dict__)
+                return module.Database(zipf)
+
+            database = _build_database()
+
+
+        return database
+
+class BeliefParser:
+    def __init__(self):
+        self.slotval_re = re.compile(r"(\w[\w ]*\w) = ([\w\d: |']+)")
+        self.domain_re = re.compile(r"(\w+) {\s*([\w,= :\d|']*)\s*}", re.IGNORECASE)
+
+    def __call__(self, raw_belief: str):
+        belief = OrderedDict()
+        for match in self.domain_re.finditer(raw_belief):
+            domain, domain_bs = match.group(1), match.group(2)
+            belief[domain] = {}
+            for slot_match in self.slotval_re.finditer(domain_bs):
+                slot, val = slot_match.group(1), slot_match.group(2)
+                belief[domain][slot] = val
+        return belief
+
+class AutoLexicalizer:
+    @staticmethod
+    def load(pretrained_model_name_or_path):
+        lexicalizer_file = os.path.join(pretrained_model_name_or_path, 'lexicalizer.zip')
+        
+        with zipfile.ZipFile(lexicalizer_file) as zipf:
+            def _build_lexicalizer():
+                module = types.ModuleType('lexicalizer')
+                exec(zipf.read('lexicalizer.py').decode('utf-8'), module.__dict__)
+                # return module.Lexicalizer(zipf)
+                return Lexicalizer(zipf)
+
+            lexicalizer = _build_lexicalizer()
+
+
+        return lexicalizer
+
+def build_blacklist(items, domains=None):
+    for i, (dialogue, items) in enumerate(items):
+        if domains is not None and set(dialogue['domains']).difference(domains):
+            yield i
+        elif items[-1]['speaker'] != 'system':
+            yield i
+
+class BlacklistItemsWrapper:
+    def __init__(self, items, blacklist):
+        self.items = items
+        self.key2idx = items.key2idx
+        self._indexmap = []
+        blacklist_pointer = 0
+        for i in range(len(items)):
+            if blacklist_pointer >= len(blacklist):
+                self._indexmap.append(i)
+            elif i < blacklist[blacklist_pointer]:
+                self._indexmap.append(i)
+            elif i == blacklist[blacklist_pointer]:
+                blacklist_pointer += 1
+        assert len(self._indexmap) == len(items) - len(blacklist)
+
+    def __getitem__(self, idx):
+        if isinstance(idx, str):
+            idx = self.key2idx[idx]
+        return self.items[self._indexmap[idx]]
+
+    def __len__(self):
+        return len(self._indexmap)
+def split_name(dataset_name: str):
+    split = dataset_name.rindex('/')
+    return dataset_name[:split], dataset_name[split + 1:]
+
+@dataclass
+class DialogDatasetItem:
+    context: Union[List[str], str]
+    belief: Union[Dict[str, Dict[str, str]], str] = None
+    database: Union[List[Tuple[str, int]], List[Tuple[str, int, Any]], None, str] = None
+    response: str = None
+    positive: bool = True
+    raw_belief: Any = None
+    raw_response: str = None
+    key: str = None
+
+    def __getattribute__(self, name):
+        val = object.__getattribute__(self, name)
+        if name == 'belief' and val is None and self.raw_belief is not None:
+            val = format_belief(self.raw_belief)
+            self.belief = val
+        return val
+
+@dataclass
+class DialogDataset(torch.utils.data.Dataset):
+    items: List[any]
+    database: Any = None
+    domains: List[str] = None
+    lexicalizer: Any = None
+    transform: Callable[[Any], Any] = None
+    normalize_input: Callable[[str], str] = None
+    ontology: Dict[Tuple[str, str], Set[str]] = None
+
+    @staticmethod
+    def build_dataset_without_database(items, *args, **kwargs):
+        return DialogDataset(items, FakeDatabase(), *args, **kwargs)
+
+    def __getitem__(self, index):
+        item = self.items[index]
+        if self.transform is not None:
+            item = self.transform(item)
+        return item
+
+    def __len__(self):
+        return len(self.items)
+
+    def map(self, transformation):
+        def trans(x):
+            x = self.transform(x)
+            x = transformation(x)
+            return x
+        return dataclasses.replace(self, transform=trans)
+
+    def finish(self, progressbar: Union[str, bool] = False):
+        if self.transform is None:
+            return self
+
+        ontology = defaultdict(lambda: set())
+        domains = set(self.domains) if self.domains else set()
+
+        items = []
+        for i in trange(len(self),
+                        desc=progressbar if isinstance(progressbar, str) else 'loading dataset',
+                        disable=not progressbar):
+            item = self[i]
+            for k, bs in item.raw_belief.items():
+                domains.add(k)
+                for k2, val in bs.items():
+                    ontology[(k, k2)].add(val)
+            items.append(item)
+        if self.ontology:
+            ontology = merge_ontologies((self.ontology, ontology))
+        return dataclasses.replace(self, items=items, transform=None, domains=domains, ontology=ontology)
+
+class DialogueItems:
+    @staticmethod
+    def cumsum(sequence):
+        r, s = [], 0
+        for e in sequence:
+            r.append(e + s)
+            s += e
+        return r
+
+    def __init__(self, dialogues):
+        lengths = [len(x['items']) for x in dialogues]
+        self.keys = [x['name'] for x in dialogues]
+        self.key2idx = {k:i for (i, k) in enumerate(self.keys)}
+        self.cumulative_sizes = DialogueItems.cumsum(lengths)
+        self.dialogues = dialogues
+
+    def __getitem__(self, idx):
+        if idx < 0:
+            if -idx > len(self):
+                raise ValueError("absolute value of index should not exceed dataset length")
+            idx = len(self) + idx
+        dialogue_idx = bisect.bisect_right(self.cumulative_sizes, idx)
+        if dialogue_idx == 0:
+            sample_idx = idx
+        else:
+            sample_idx = idx - self.cumulative_sizes[dialogue_idx - 1]
+        return self.dialogues[dialogue_idx], self.dialogues[dialogue_idx]['items'][:sample_idx + 1]
+
+    def __len__(self):
+        if not self.cumulative_sizes:
+            return 0
+        return self.cumulative_sizes[-1]
+
+def insertSpace(token, text):
+    sidx = 0
+    while True:
+        sidx = text.find(token, sidx)
+        if sidx == -1:
+            break
+        if sidx + 1 < len(text) and re.match('[0-9]', text[sidx - 1]) and \
+                re.match('[0-9]', text[sidx + 1]):
+            sidx += 1
+            continue
+        if text[sidx - 1] != ' ':
+            text = text[:sidx] + ' ' + text[sidx:]
+            sidx += 1
+        if sidx + len(token) < len(text) and text[sidx + len(token)] != ' ':
+            text = text[:sidx + 1] + ' ' + text[sidx + 1:]
+        sidx += 1
+    return text
+
+def augpt_normalize(text, delexicalize=True, online=False):
+    # lower case every word
+    text = text.lower()
+
+    text = text.replace(" 1 ", " one ")
+
+    # replace white spaces in front and end
+    text = re.sub(r'^\s*|\s*$', '', text)
+
+    # hotel domain pfb30
+    text = re.sub(r"b&b", "bed and breakfast", text)
+    text = re.sub(r"b and b", "bed and breakfast", text)
+
+    # normalize phone number
+    ms = re.findall('\(?(\d{3})\)?[-.\s]?(\d{3})[-.\s]?(\d{4,5})', text)
+    if ms:
+        sidx = 0
+        for m in ms:
+            sidx = text.find(m[0], sidx)
+            if text[sidx - 1] == '(':
+                sidx -= 1
+            eidx = text.find(m[-1], sidx) + len(m[-1])
+            text = text.replace(text[sidx:eidx], ''.join(m))
+
+    # normalize postcode
+    ms = re.findall('([a-z]{1}[\. ]?[a-z]{1}[\. ]?\d{1,2}[, ]+\d{1}[\. ]?[a-z]{1}[\. ]?[a-z]{1}|[a-z]{2}\d{2}[a-z]{2})',
+                    text)
+    if ms:
+        sidx = 0
+        for m in ms:
+            sidx = text.find(m, sidx)
+            eidx = sidx + len(m)
+            text = text[:sidx] + re.sub('[,\. ]', '', m) + text[eidx:]
+
+    # weird unicode bug
+    text = re.sub(u"(\u2018|\u2019)", "'", text)
+
+    # replace time and and price
+    if delexicalize:
+        text = re.sub(pricepat, ' [price] ', text)
+        #text = re.sub(pricepat2, '[value_price]', text)
+
+    # replace st.
+    text = text.replace(';', ',')
+    text = re.sub('$\/', '', text)
+    text = text.replace('/', ' and ')
+
+    # replace other special characters
+    text = text.replace('-', ' ')
+    if delexicalize and not online:
+        text = re.sub('[\"\:<>@\(\)]', '', text)
+    elif delexicalize and online:
+        text = re.sub('[\"\<>@\(\)]', '', text)
+        text = re.sub("(([^0-9]):([^0-9]|.*))|(([^0-9]|.*):([^0-9]))", "\\2\\3\\5\\6", text) #only replace colons if it's not surrounded by digits. this wasn't a problem in standalone LAVA because time is delexicalized before normalization
+    else:
+        text = re.sub('[\"\<>@\(\)]', '', text)
+
+    # insert white space before and after tokens:
+    for token in ['?', '.', ',', '!']:
+        text = insertSpace(token, text)
+
+    # insert white space for 's
+    text = insertSpace('\'s', text)
+
+    # replace it's, does't, you'd ... etc
+    text = re.sub('^\'', '', text)
+    text = re.sub('\'$', '', text)
+    text = re.sub('\'\s', ' ', text)
+    text = re.sub('\s\'', ' ', text)
+    for fromx, tox in replacements:
+        text = ' ' + text + ' '
+        text = text.replace(fromx, tox)[1:-1]
+
+    # remove multiple spaces
+    text = re.sub(' +', ' ', text)
+
+    # concatenate numbers
+    tmp = text
+    tokens = text.split()
+    i = 1
+    while i < len(tokens):
+        if re.match(u'^\d+$', tokens[i]) and \
+                re.match(u'\d+$', tokens[i - 1]):
+            tokens[i - 1] += tokens[i]
+            del tokens[i]
+        else:
+            i += 1
+    text = ' '.join(tokens)
+
+    return text
+
+def load_dataset(name, use_goal=False, context_window_size=15, domains=None, **kwargs) -> DialogDataset:
+    name, split = split_name(name)
+    path = os.path.join(DATASETS_PATH, name)
+    with open(os.path.join(path, f'{split}.json'), 'r') as f:
+        data = json.load(f, object_pairs_hook=OrderedDict)
+    dialogues = data['dialogues']
+    items = DialogueItems(dialogues)
+    items = BlacklistItemsWrapper(items, list(build_blacklist(items, domains)))
+
+    def transform(x):
+        dialogue, items = x
+        context = [s['text'] for s in items[:-1]]
+        if context_window_size is not None and context_window_size > 0:
+            context = context[-context_window_size:]
+        belief = items[-1]['belief']
+        database = items[-1]['database']
+        item = DialogDatasetItem(context,
+                        raw_belief=belief,
+                        database=database,
+                        response=items[-1]['delexicalised_text'],
+                        raw_response=items[-1]['text'],
+                        key=dialogue['name'])
+        if use_goal:
+            setattr(item, 'goal', dialogue['goal'])
+            # MultiWOZ evaluation uses booked domains property
+            if 'booked_domains' in items[-1]:
+                setattr(item, 'booked_domains', items[-1]['booked_domains'])
+            setattr(item, 'dialogue_act', items[-1]['dialogue_act'])
+        setattr(item, 'active_domain', items[-1]['active_domain'])
+        return item
+
+    dataset = DialogDataset(items, transform=transform, domains=data['domains'])
+    if os.path.exists(os.path.join(path, 'database.zip')):
+        dataset.database = AutoDatabase.load(path)
+
+    if os.path.exists(os.path.join(path, 'lexicalizer.zip')):
+        dataset.lexicalizer = AutoLexicalizer.load(path)
+
+    return dataset
+
+def format_belief(belief: OrderedDict) -> str:
+    assert isinstance(belief, OrderedDict)
+    str_bs = []
+    for domain, domain_bs in belief.items():
+        domain_bs = ', '.join([f'{slot} = {val}' for slot, val in sorted(domain_bs.items(), key=lambda x: x[0])])
+        str_bs.extend([domain, '{' + domain_bs + '}'])
+    return ' '.join(str_bs)
+
+class Lexicalizer:
+    def __init__(self, zipf):
+        self.path = zipf.filename
+
+    placeholder_re = re.compile(r'\[(\s*[\w_\s]+)\s*\]')
+    number_re = re.compile(r'.*(\d+|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)\s$')
+    time_re = re.compile(r'((?:\d{1,2}[:]\d{2,3})|(?:\d{1,2} (?:am|pm)))', re.IGNORECASE)
+
+    @staticmethod
+    def ends_with_number(s):
+        return bool(Lexicalizer.number_re.match(s))
+
+    @staticmethod
+    def extend_database_results(database_results, belief):
+        # Augment database results from the belief state
+        database_results = OrderedDict(database_results)
+        if belief is not None:
+            for i, (domain, (num_results, results)) in enumerate(database_results.items()):
+                if domain not in belief:
+                    continue
+                if num_results == 0:
+                    database_results[domain] = (1, [belief[domain]])
+                else:
+                    new_results = []
+                    for r in results:
+                        r = dict(**r)
+                        for k, val in belief[domain].items():
+                            if k not in r:
+                                r[k] = val
+                        new_results.append(r)
+                    database_results[domain] = (num_results, new_results)
+        return database_results
+
+    @staticmethod
+    def extend_empty_database(database_results, belief):
+        # Augment database results from the belief state
+        database_results = OrderedDict(database_results)
+        if belief is not None:
+            for domain in belief.keys():
+                if domain not in database_results.keys():
+                    if any([len(v) > 0 for v in belief[domain]["semi"].values()] + [len(v) > 0 for v in belief[domain]["book"].values()]):
+                        database_results[domain] = (1, [belief[domain]])
+
+        return database_results
+
+    def __call__(self, text, database_results, belief=None, context=None):
+        database_results = Lexicalizer.extend_database_results(database_results, belief)
+        database_results =  Lexicalizer.extend_empty_database(database_results, belief)
+        result_index = 0
+        last_assignment = defaultdict(set)
+
+        def trans(label, span, force=False, loop=100):
+            nonlocal result_index
+            nonlocal last_assignment
+            result_str = None
+
+            for domain, (count, results) in database_results.items():
+                if domain in ["hotel", "attraction"] and label == "price":
+                    label = "price range"
+
+                # if count == 0:
+                    # pdb.set_trace()
+                    # # continue
+                    # if label in result['semi']:
+                        # result_str = result['semi'][label]
+                    # elif label is result['book']:
+                        # result_str = result['book'][label]
+                # else:
+                if domain == "train" and "arrive by" in results[0]["semi"]:
+                    result = results[-1]
+                else:
+                    result = results[result_index % len(results)]
+                # if domain == "train" and label == "id":
+                #     label = "trainID"
+                if label in result:
+                    result_str = result[label]
+                    if result_str == '?':
+                        result_str = 'unknown'
+                    if label == 'price range' and result_str == 'moderate' and \
+                            not text[span[1]:].startswith(' price range') and \
+                            not text[span[1]:].startswith(' in price'):
+                        result_str = 'moderately priced'
+                elif label in result['book']:
+                    result_str = result['book'][label]
+                elif label in result['semi']:
+                    result_str = result['semi'][label]
+                
+                # if label == 'type':
+                    # pdb.set_trace()
+                    # if text[:span[0]].endswith('no ') or text[:span[0]].endswith('any ') or \
+                            # text[:span[0]].endswith('some ') or Lexicalizer.ends_with_number(text[:span[0]]):
+                        # if not result_str.endswith('s'):
+                            # result_str += 's'
+                if label == 'time' and ('[leave at]' in text or '[arrive by]' in text) and \
+                    belief is not None and 'train' in belief and \
+                        any([k in belief['train'] for k in ('leave at', 'arrive by')]):
+                    # this is a specific case in which additional [time] slot needs to be lexicalised
+                    # directly from the belief state
+                    # "The earliest train after [time] leaves at ... and arrives by ..."
+                    if 'leave at' in belief['train']:
+                        result_str = belief['train']['leave at']
+                    else:
+                        result_str = belief['train']['arrive by']
+                # elif label == 'time' and 'restaurant' in belief and 'book' in belief['restaurant']:
+                        # result_str = belief['restaurant']['book']['time']
+                elif label == 'count':
+                    result_str = str(count)
+                elif label == 'price' and domain == "train" and "total" in text[:span[0]]: 
+                    try:
+                        num_people = int(result['book']['people'])
+                    except:
+                        num_people = 1
+                    try:
+                        result_str = str(float(result[label].split()[0]) * num_people) + " pounds"
+                    except:
+                        result_str = ""
+                elif force:
+                    if label == 'time':
+                        if 'leave at' in result or 'arrive by' in result:
+                            if 'arrive' in text and 'arrive by' in result:
+                                result_str = result['arrive by'].lstrip('0')
+                            elif 'leave at' in result:
+                                result_str = result['leave at'].lstrip('0')
+                        elif context is not None and len(context) > 0:
+                            last_utt = context[-1]
+                            mtch = Lexicalizer.time_re.search(last_utt)
+                            if mtch is not None:
+                                result_str = mtch.group(1).lstrip('0')
+                    elif label == 'name':
+                        result_str = "the " + domain
+                # if result_str == "not mentioned":
+                    # pdb.set_trace()
+                if result_str is not None:
+                    break
+            if force and result_str is None:
+                # for domains with no database or cases with failed database search
+                # if domain == "hospital":
+                #     if label == 'name':
+                        # result_str = "Addenbrookes hospital"
+                    # elif label == "postcode":
+                        # result_str = "cb20qq"
+                    # elif label == "address":
+                        # result_str = "hills rd , cambridge"
+                    # elif label == "phone":
+                        # result_str = "01223216297"
+                # else:
+                if label == 'reference':
+                    result_str = 'YF86GE4J'
+                elif label == 'phone':
+                    result_str = '01223358966'
+                elif label == 'postcode':
+                    result_str = 'cb11jg'
+                elif label == 'agent':
+                    result_str = 'Cambridge Towninfo Centre'
+                elif label == 'stars':
+                    result_str = '4'
+                elif label == 'car':
+                    result_str = 'black honda taxi'
+                elif label == 'address':
+                    result_str = 'Parkside, Cambridge'
+                elif label == 'name':
+                    result_str = "it"
+
+            if result_str is not None and result_str.lower() in last_assignment[label] and loop > 0:
+                result_index += 1
+                return trans(label, force=force, loop=loop - 1, span=span)
+
+            if result_str is not None:
+                last_assignment[label].add(result_str.lower())
+            return result_str or f'[{label}]'
+
+        text = Lexicalizer.placeholder_re.sub(lambda m: trans(m.group(1), span=m.span()), text)
+        text = Lexicalizer.placeholder_re.sub(lambda m: trans(m.group(1), force=True, span=m.span()), text)
+        return text, database_results
+
+    def save(self, path):
+        shutil.copy(self.path, os.path.join(path, os.path.split(self.path)[-1]))
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/base_data_loaders.py b/convlab/policy/lava/multiwoz/latent_dialog/base_data_loaders.py
new file mode 100644
index 0000000000000000000000000000000000000000..e96372acf9fcdbbf17bf393dc58158751212dacc
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/base_data_loaders.py
@@ -0,0 +1,183 @@
+import numpy as np
+import logging
+
+
+class BaseDataLoaders(object):
+    def __init__(self, name):
+        self.data_size = None
+        self.indexes = None
+        self.name = name
+
+    def _shuffle_indexes(self):
+        np.random.shuffle(self.indexes)
+
+    def _shuffle_batch_indexes(self):
+        np.random.shuffle(self.batch_indexes)
+
+    def epoch_init(self, config, shuffle=True, verbose=True, fix_batch=False):
+        self.ptr = 0
+        self.batch_size = config.batch_size
+        self.num_batch = self.data_size // config.batch_size
+
+        if verbose:
+            print('Number of left over sample = %d' % (self.data_size - config.batch_size * self.num_batch))
+
+        if shuffle and not fix_batch:
+            self._shuffle_indexes()
+
+        self.batch_indexes = []
+        for i in range(self.num_batch):
+            self.batch_indexes.append(self.indexes[i*self.batch_size: (i+1)*self.batch_size])
+
+        if shuffle and fix_batch:
+            self._shuffle_batch_indexes()
+
+        if verbose:
+            print('%s begins with %d batches' % (self.name, self.num_batch))
+
+    def next_batch(self):
+        if self.ptr < self.num_batch:
+            selected_ids = self.batch_indexes[self.ptr]
+            self.ptr += 1
+            return self._prepare_batch(selected_index=selected_ids)
+        else:
+            return None
+
+    def _prepare_batch(self, *args, **kwargs):
+        raise NotImplementedError('Have to override _prepare_batch()')
+
+    def pad_to(self, max_len, tokens, do_pad):
+        if len(tokens) >= max_len:
+            return tokens[: max_len-1] + [tokens[-1]]
+        elif do_pad:
+            return tokens + [0] * (max_len - len(tokens))
+        else:
+            return tokens
+
+
+class LongDataLoader(object):
+    """A special efficient data loader for TBPTT. Assume the data contains
+    N long sequences, each sequence has length k_i
+
+    :ivar batch_size: the size of a minibatch
+    :ivar backward_size: how many steps in time to do BP
+    :ivar step_size: how fast we move the window
+    :ivar ptr: the current idx of batch
+    :ivar num_batch: the total number of batch
+    :ivar batch_indexes: a list of list. Each item is the IDs in this batch
+    :ivar grid_indexes: a list of (b_id, s_id, e_id). b_id is the index of
+    batch, s_id is the starting time id in that batch and e_id is the ending
+    time id.
+    :ivar indexes: a list, the ordered of sequences ID it should go through
+    :ivar data_size: the number of sequences, N.
+    :ivar data_lens: a list containing k_i
+    :ivar prev_alive_size:
+    :ivar name: the name of the this data loader
+    """
+    logger = logging.getLogger()
+
+    def __init__(self, name):
+        self.batch_size = 0
+        self.backward_size = 0
+        self.step_size = 0
+        self.ptr = 0
+        self.num_batch = None
+        self.batch_indexes = None  # one batch is a dialog
+        self.grid_indexes = None  # grid is the tokenized versiion
+        self.indexes = None
+        self.data_lens = None
+        self.data_size = None
+        self.name = name
+
+    def _shuffle_batch_indexes(self):
+        np.random.shuffle(self.batch_indexes)
+
+    def _shuffle_grid_indexes(self):
+        np.random.shuffle(self.grid_indexes)
+
+    def _prepare_batch(self, cur_grid, prev_grid):
+        raise NotImplementedError("Have to override prepare batch")
+
+    def epoch_init(self, config, shuffle=True, verbose=True, fix_batch=False):
+
+        assert len(self.indexes) == self.data_size and \
+               len(self.data_lens) == self.data_size
+        # make sure backward_size can be divided by step size
+        assert config.backward_size % config.step_size == 0
+
+        self.ptr = 0
+        self.batch_size = config.batch_size
+        self.backward_size = config.backward_size
+        self.step_size = config.step_size
+
+        # create batch indexes
+        temp_num_batch = self.data_size // config.batch_size
+        self.batch_indexes = []
+        for i in range(temp_num_batch):
+            self.batch_indexes.append(
+                self.indexes[i * self.batch_size:(i + 1) * self.batch_size])
+
+        left_over = self.data_size - temp_num_batch * config.batch_size
+        if shuffle:
+            self._shuffle_batch_indexes()
+
+        # create grid indexes
+        self.grid_indexes = []
+        for idx, b_ids in enumerate(self.batch_indexes):
+            # assume the b_ids are sorted
+            all_lens = [self.data_lens[i] for i in b_ids]
+            max_len = self.data_lens[b_ids[0]]
+            min_len = self.data_lens[b_ids[-1]]
+            assert np.max(all_lens) == max_len
+            assert np.min(all_lens) == min_len
+            num_seg = (max_len - self.backward_size - self.step_size) // self.step_size
+            cut_start, cut_end = [], []
+            if num_seg > 1:
+                cut_start = list(range(config.step_size, num_seg * config.step_size, config.step_size))
+                cut_end = list(range(config.backward_size + config.step_size,
+                                num_seg * config.step_size + config.backward_size,
+                                config.step_size))
+                assert cut_end[-1] < max_len
+
+            actual_size = min(max_len, config.backward_size)
+            temp_end = list(range(2, actual_size, config.step_size))
+            temp_start = [0] * len(temp_end)
+
+            cut_start = temp_start + cut_start
+            cut_end = temp_end + cut_end
+
+            assert len(cut_end) == len(cut_start)
+            new_grids = [(idx, s_id, e_id) for s_id, e_id in
+                         zip(cut_start, cut_end) if s_id < min_len - 1]
+
+            self.grid_indexes.extend(new_grids)
+
+        # shuffle batch indexes
+        if shuffle:
+            self._shuffle_grid_indexes()
+
+        self.num_batch = len(self.grid_indexes)
+        if verbose:
+            self.logger.info("%s init with %d batches with %d left over samples" %
+                             (self.name, self.num_batch, left_over))
+
+    def next_batch(self):
+        if self.ptr < self.num_batch:
+            current_grid = self.grid_indexes[self.ptr]
+            if self.ptr > 0:
+                prev_grid = self.grid_indexes[self.ptr - 1]
+            else:
+                prev_grid = None
+            self.ptr += 1
+            return self._prepare_batch(cur_grid=current_grid,
+                                       prev_grid=prev_grid)
+        else:
+            return None
+
+    def pad_to(self, max_len, tokens, do_pad=True):
+        if len(tokens) >= max_len:
+            return tokens[0:max_len - 1] + [tokens[-1]]
+        elif do_pad:
+            return tokens + [0] * (max_len - len(tokens))
+        else:
+            return tokens
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/base_models.py b/convlab/policy/lava/multiwoz/latent_dialog/base_models.py
new file mode 100644
index 0000000000000000000000000000000000000000..3aff3014678d9529dbcdae846825e02b7c8b2be8
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/base_models.py
@@ -0,0 +1,110 @@
+import torch as th
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+from torch.autograd import Variable
+import numpy as np
+from convlab.policy.lava.multiwoz.latent_dialog.utils import INT, FLOAT, LONG, cast_type
+import pdb
+
+
+class BaseModel(nn.Module):
+    def __init__(self, config):
+        super(BaseModel, self).__init__()
+        self.use_gpu = config.use_gpu
+        self.config = config
+        self.kl_w = 0.0
+
+    def np2var(self, inputs, dtype):
+        if inputs is None:
+            return None
+        return cast_type(Variable(th.from_numpy(inputs)), 
+                         dtype, 
+                         self.use_gpu)
+
+    def forward(self, *inputs):
+        raise NotImplementedError
+
+    def backward(self, loss, batch_cnt):
+        total_loss = self.valid_loss(loss, batch_cnt)
+        total_loss.backward()
+
+    def valid_loss(self, loss, batch_cnt=None):
+        total_loss = 0.0
+        for k, l in loss.items():
+            if l is not None:
+                total_loss += l
+        return total_loss
+
+    def get_optimizer(self, config, verbose=True):
+        if config.op == 'adam':
+            if verbose:
+                print('Use Adam')
+            return optim.Adam(filter(lambda p: p.requires_grad, self.parameters()), lr=config.init_lr, weight_decay=config.l2_norm)
+        elif config.op == 'sgd':
+            print('Use SGD')
+            return optim.SGD(self.parameters(), lr=config.init_lr, momentum=config.momentum)
+        elif config.op == 'rmsprop':
+            print('Use RMSProp')
+            return optim.RMSprop(self.parameters(), lr=config.init_lr, momentum=config.momentum)
+
+    def get_clf_optimizer(self, config):
+        params = []
+        params.extend(self.gru_attn_encoder.parameters())
+        params.extend(self.feat_projecter.parameters())
+        params.extend(self.sel_classifier.parameters())
+
+        if config.fine_tune_op == 'adam':
+            print('Use Adam')
+            return optim.Adam(params, lr=config.fine_tune_lr)
+        elif config.fine_tune_op == 'sgd':
+            print('Use SGD')
+            return optim.SGD(params, lr=config.fine_tune_lr, momentum=config.fine_tune_momentum)
+        elif config.fine_tune_op == 'rmsprop':
+            print('Use RMSProp')
+            return optim.RMSprop(params, lr=config.fine_tune_lr, momentum=config.fine_tune_momentum)
+
+        
+    def model_sel_loss(self, loss, batch_cnt):
+        return self.valid_loss(loss, batch_cnt)
+
+
+    def extract_short_ctx(self, context, context_lens, backward_size=1):
+        utts = []
+        for b_id in range(context.shape[0]):
+            utts.append(context[b_id, context_lens[b_id]-1])
+        return np.array(utts)
+
+    def flatten_context(self, context, context_lens, align_right=False):
+        utts = []
+        temp_lens = []
+        for b_id in range(context.shape[0]):
+            temp = []
+            for t_id in range(context_lens[b_id]):
+                for token in context[b_id, t_id]:
+                    if token != 0:
+                        temp.append(token)
+            temp_lens.append(len(temp))
+            utts.append(temp)
+        max_temp_len = np.max(temp_lens)
+        results = np.zeros((context.shape[0], max_temp_len))
+        for b_id in range(context.shape[0]):
+            if align_right:
+                results[b_id, -temp_lens[b_id]:] = utts[b_id]
+            else:
+                results[b_id, 0:temp_lens[b_id]] = utts[b_id]
+
+        return results
+
+def frange_cycle_linear(n_iter, start=0.0, stop=1.0,  n_cycle=4, ratio=0.5):
+    L = np.ones(n_iter) * stop
+    period = n_iter/n_cycle
+    step = (stop-start)/(period*ratio) # linear schedule
+
+    for c in range(n_cycle):
+        v, i = start, 0
+        while v <= stop and (int(i+c*period) < n_iter):
+            L[int(i+c*period)] = v
+            v += step
+            i += 1
+    return L 
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/corpora.py b/convlab/policy/lava/multiwoz/latent_dialog/corpora.py
new file mode 100644
index 0000000000000000000000000000000000000000..4d406f394a7d43f44a55d9c3f7030cb6fa1cd103
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/corpora.py
@@ -0,0 +1,419 @@
+from __future__ import unicode_literals
+import numpy as np
+from collections import Counter
+from convlab.policy.lava.multiwoz.latent_dialog.utils import Pack, get_tokenize, get_chat_tokenize, missingdict
+import json
+from nltk.tokenize import WordPunctTokenizer
+import logging
+from collections import defaultdict
+import pdb
+
+PAD = '<pad>'
+UNK = '<unk>'
+USR = 'YOU:'
+SYS = 'THEM:'
+BOD = '<d>'
+EOD = '</d>'
+BOS = '<s>'
+EOS = '<eos>'
+SEL = '<selection>'
+SEP = "|"
+REQ = "<requestable>"
+INF = "<informable>"
+WILD = "%s"
+SPECIAL_TOKENS = [PAD, UNK, USR, SYS, BOS, BOD, EOS, EOD]
+STOP_TOKENS = [EOS, SEL]
+DECODING_MASKED_TOKENS = [PAD, UNK, USR, SYS, BOD]
+
+REQ_TOKENS = {}
+DOMAIN_REQ_TOKEN = ['restaurant', 'hospital', 'hotel','attraction', 'train', 'police', 'taxi']
+ACTIVE_BS_IDX = [13, 30, 35, 61, 72, 91, 93] #indexes in the BS indicating if domain is active
+NO_MATCH_DB_IDX = [-1, 0, -1, 6, 12, 18, -1] # indexes in DB pointer indicating 0 match is found for that domain, -1 mean that domain has no DB
+REQ_TOKENS['attraction'] = ["[attraction_address]", "[attraction_name]", "[attraction_phone]", "[attraction_postcode]", "[attraction_reference]", "[attraction_type]"]
+REQ_TOKENS['hospital'] = ["[hospital_address]", "[hospital_department]", "[hospital_name]", "[hospital_phone]", "[hospital_postcode]"] #, "[hospital_reference]"
+REQ_TOKENS['hotel'] = ["[hotel_address]", "[hotel_name]", "[hotel_phone]", "[hotel_postcode]", "[hotel_reference]", "[hotel_type]"]
+REQ_TOKENS['restaurant'] = ["[restaurant_name]", "[restaurant_address]", "[restaurant_phone]", "[restaurant_postcode]", "[restaurant_reference]"]
+REQ_TOKENS['train'] = ["[train_id]", "[train_reference]"]
+REQ_TOKENS['police'] = ["[police_address]", "[police_phone]", "[police_postcode]"] #"[police_name]", 
+REQ_TOKENS['taxi'] = ["[taxi_phone]", "[taxi_type]"]
+
+GENERIC_TOKENS = ["[value_area]", "[value_count]", "[value_day]", "[value_food]", "[value_place]", "[value_price]", "[value_pricerange]", "[value_time]"]
+
+
+
+class NormMultiWozCorpus(object):
+    logger = logging.getLogger()
+
+    def __init__(self, config):
+        self.bs_size = 94
+        self.db_size = 30
+        self.bs_types =['b', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'b', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'b', 'b', 'c', 'c', 'c', 'b', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'b', 'b', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'b', 'b', 'b']
+        self.domains = ['hotel', 'restaurant', 'train', 'attraction', 'hospital', 'police', 'taxi']
+        self.info_types = ['book', 'fail_book', 'fail_info', 'info', 'reqt']
+        self.config = config
+        self.tokenize = lambda x: x.split()
+        self.train_corpus, self.val_corpus, self.test_corpus = self._read_file(self.config)
+        self._extract_vocab()
+        self._extract_goal_vocab()
+        self.logger.info('Loading corpus finished.')
+
+    def _read_file(self, config):
+        train_data = json.load(open(config.train_path))
+        valid_data = json.load(open(config.valid_path))
+        test_data = json.load(open(config.test_path))
+        
+        train_data = self._process_dialogue(train_data)
+        valid_data = self._process_dialogue(valid_data)
+        test_data = self._process_dialogue(test_data)
+
+        return train_data, valid_data, test_data
+
+    def _process_dialogue(self, data):
+        new_dlgs = []
+        all_sent_lens = []
+        all_dlg_lens = []
+
+        for key, raw_dlg in data.items():
+            norm_dlg = [Pack(speaker=USR, utt=[BOS, BOD, EOS], bs=[0.0]*self.bs_size, db=[0.0]*self.db_size)]
+            for t_id in range(len(raw_dlg['db'])):
+                usr_utt = [BOS] + self.tokenize(raw_dlg['usr'][t_id]) + [EOS]
+                sys_utt = [BOS] + self.tokenize(raw_dlg['sys'][t_id]) + [EOS]
+                norm_dlg.append(Pack(speaker=USR, utt=usr_utt, db=raw_dlg['db'][t_id], bs=raw_dlg['bs'][t_id]))
+                norm_dlg.append(Pack(speaker=SYS, utt=sys_utt, db=raw_dlg['db'][t_id], bs=raw_dlg['bs'][t_id]))
+                all_sent_lens.extend([len(usr_utt), len(sys_utt)])
+            # To stop dialog
+            norm_dlg.append(Pack(speaker=USR, utt=[BOS, EOD, EOS], bs=[0.0]*self.bs_size, db=[0.0]*self.db_size))
+            # if self.config.to_learn == 'usr':
+            #     norm_dlg.append(Pack(speaker=USR, utt=[BOS, EOD, EOS], bs=[0.0]*self.bs_size, db=[0.0]*self.db_size))
+            all_dlg_lens.append(len(raw_dlg['db']))
+            processed_goal = self._process_goal(raw_dlg['goal'])
+            new_dlgs.append(Pack(dlg=norm_dlg, goal=processed_goal, key=key))
+
+        self.logger.info('Max utt len = %d, mean utt len = %.2f' % (
+            np.max(all_sent_lens), float(np.mean(all_sent_lens))))
+        self.logger.info('Max dlg len = %d, mean dlg len = %.2f' % (
+            np.max(all_dlg_lens), float(np.mean(all_dlg_lens))))
+        return new_dlgs
+
+    def _extract_vocab(self):
+        all_words = []
+        for dlg in self.train_corpus:
+            for turn in dlg.dlg:
+                all_words.extend(turn.utt)
+        vocab_count = Counter(all_words).most_common()
+        raw_vocab_size = len(vocab_count)
+        keep_vocab_size = min(self.config.max_vocab_size, raw_vocab_size)
+        oov_rate = np.sum([c for t, c in vocab_count[0:keep_vocab_size]]) / float(len(all_words))
+
+        self.logger.info('cut off at word {} with frequency={},\n'.format(vocab_count[keep_vocab_size - 1][0],
+                                                               vocab_count[keep_vocab_size - 1][1]) +
+              'OOV rate = {:.2f}%'.format(100.0 - oov_rate * 100))
+
+        vocab_count = vocab_count[0:keep_vocab_size]
+        self.vocab = SPECIAL_TOKENS + [t for t, cnt in vocab_count if t not in SPECIAL_TOKENS]
+        self.vocab_dict = {t: idx for idx, t in enumerate(self.vocab)}
+        self.unk_id = self.vocab_dict[UNK]
+        self.logger.info("Raw vocab size {} in train set and final vocab size {}".format(raw_vocab_size, len(self.vocab)))
+
+    def _process_goal(self, raw_goal):
+        res = {}
+        for domain in self.domains:
+            all_words = []
+            d_goal = raw_goal[domain]
+            if d_goal:
+                for info_type in self.info_types:
+                    sv_info = d_goal.get(info_type, dict())
+                    if info_type == 'reqt' and isinstance(sv_info, list):
+                        all_words.extend([info_type + '|' + item for item in sv_info])
+                    elif isinstance(sv_info, dict):
+                        all_words.extend([info_type + '|' + k + '|' + str(v) for k, v in sv_info.items()])
+                    else:
+                        print('Fatal Error!')
+                        exit(-1)
+            res[domain] = all_words
+        return res
+
+    def _extract_goal_vocab(self):
+        self.goal_vocab, self.goal_vocab_dict, self.goal_unk_id = {}, {}, {}
+        for domain in self.domains:
+            all_words = []
+            for dlg in self.train_corpus:
+                all_words.extend(dlg.goal[domain])
+            vocab_count = Counter(all_words).most_common()
+            raw_vocab_size = len(vocab_count)
+            discard_wc = np.sum([c for t, c in vocab_count])
+
+            self.logger.info('================= domain = {}, \n'.format(domain) +
+                  'goal vocab size of train set = %d, \n' % (raw_vocab_size,) +
+                  'cut off at word %s with frequency = %d, \n' % (vocab_count[-1][0], vocab_count[-1][1]) +
+                  'OOV rate = %.2f' % (1 - float(discard_wc) / len(all_words),))
+
+            self.goal_vocab[domain] = [UNK] + [g for g, cnt in vocab_count]
+            self.goal_vocab_dict[domain] = {t: idx for idx, t in enumerate(self.goal_vocab[domain])}
+            self.goal_unk_id[domain] = self.goal_vocab_dict[domain][UNK]
+
+    def get_corpus(self):
+        id_train = self._to_id_corpus('Train', self.train_corpus)
+        id_val = self._to_id_corpus('Valid', self.val_corpus)
+        id_test = self._to_id_corpus('Test', self.test_corpus)
+        return id_train, id_val, id_test
+
+    def _to_id_corpus(self, name, data):
+        results = []
+        for dlg in data:
+            if len(dlg.dlg) < 1:
+                continue
+            id_dlg = []
+            for turn in dlg.dlg:
+                id_turn = Pack(utt=self._sent2id(turn.utt),
+                               speaker=turn.speaker,
+                               db=turn.db, bs=turn.bs)
+                id_dlg.append(id_turn)
+            id_goal = self._goal2id(dlg.goal)
+            results.append(Pack(dlg=id_dlg, goal=id_goal, key=dlg.key))
+        return results
+
+    def _sent2id(self, sent):
+        return [self.vocab_dict.get(t, self.unk_id) for t in sent]
+
+    def _goal2id(self, goal):
+        res = {}
+        for domain in self.domains:
+            d_bow = [0.0] * len(self.goal_vocab[domain])
+            for word in goal[domain]:
+                word_id = self.goal_vocab_dict[domain].get(word, self.goal_unk_id[domain])
+                d_bow[word_id] += 1.0
+            res[domain] = d_bow
+        return res
+
+    def id2sent(self, id_list):
+        return [self.vocab[i] for i in id_list]
+
+    def pad_to(self, max_len, tokens, do_pad):
+        if len(tokens) >= max_len:
+            return tokens[: max_len-1] + [tokens[-1]]
+        elif do_pad:
+            return tokens + [0] * (max_len - len(tokens))
+        else:
+            return tokens
+
+class NormMultiWozCorpusAE(object):
+    logger = logging.getLogger()
+
+    def __init__(self, config):
+        self.bs_size = 94
+        self.db_size = 30
+        self.bs_types =['b', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'b', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'b', 'b', 'c', 'c', 'c', 'b', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'b', 'b', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'b', 'b', 'b', 'b', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'b', 'b', 'b']
+        self.domains = ['hotel', 'restaurant', 'train', 'attraction', 'hospital', 'police', 'taxi']
+        self.info_types = ['book', 'fail_book', 'fail_info', 'info', 'reqt']
+        self.act_types = ['bye', 'inform', 'nobook', 'nooffer', 'offerbook', 'offerbooked', 'recommend', 'reqmore', 'request', 'select', 'welcome']
+        self.act2id = {a:i for i, a in enumerate(self.act_types)}
+        self.id2act = {i:a for i, a in enumerate(self.act_types)}
+        self.act_size = len(self.act_types) #domain agnostic act 
+        # self.act_size = len(domain) * len(self.act_types) #domain dependent act 
+        self.config = config
+        self.tokenize = lambda x: x.split()
+        self.train_corpus, self.val_corpus, self.test_corpus = self._read_file(self.config)
+        self._extract_vocab()
+        self._extract_goal_vocab()
+        self.logger.info('Loading corpus finished.')
+
+    def _read_file(self, config):
+        train_data = json.load(open(config.train_path))
+        valid_data = json.load(open(config.valid_path))
+        test_data = json.load(open(config.test_path))
+        dacts = json.load(open(config.dact_path))
+        
+        train_data = self._process_dialogue(train_data, dacts)
+        valid_data = self._process_dialogue(valid_data, dacts)
+        test_data = self._process_dialogue(test_data, dacts)
+
+        return train_data, valid_data, test_data
+
+    def _process_dialogue(self, data, dacts):
+        new_dlgs = []
+        all_sent_lens = []
+        all_dlg_lens = []
+        dact_skip_count = 0
+
+        for key, raw_dlg in data.items():
+            norm_dlg = [Pack(speaker=USR, utt=[BOS, BOD, EOS], bs=[0.0]*self.bs_size, db=[0.0]*self.db_size, act=[0.0]*self.act_size)]
+            if key.split(".")[0].lower() in dacts:
+                for t_id in range(len(raw_dlg['db'])):
+                    usr_utt = [BOS] + self.tokenize(raw_dlg['usr'][t_id]) + [EOS]
+                    sys_utt = [BOS] + self.tokenize(raw_dlg['sys'][t_id]) + [EOS]
+                    # sys_act = self._process_multidomain_summary_acts(dacts[key.split(".")[0].lower()][str(t_id)])
+                    try:
+                        sys_act = self._process_summary_acts(dacts[key.split(".")[0].lower()][str(t_id)])
+                    except:
+                        sys_act = [float(0)] * self.act_size
+                    norm_dlg.append(Pack(speaker=USR, utt=usr_utt, db=raw_dlg['db'][t_id], bs=raw_dlg['bs'][t_id], act=sys_act))
+                    norm_dlg.append(Pack(speaker=SYS, utt=sys_utt, db=raw_dlg['db'][t_id], bs=raw_dlg['bs'][t_id], act=sys_act))
+                    all_sent_lens.extend([len(usr_utt), len(sys_utt)])
+                # To stop dialog
+                norm_dlg.append(Pack(speaker=USR, utt=[BOS, EOD, EOS], bs=[0.0]*self.bs_size, db=[0.0]*self.db_size, act=[0.0]*self.act_size))
+                # if self.config.to_learn == 'usr':
+                #     norm_dlg.append(Pack(speaker=USR, utt=[BOS, EOD, EOS], bs=[0.0]*self.bs_size, db=[0.0]*self.db_size))
+                all_dlg_lens.append(len(raw_dlg['db']))
+                processed_goal = self._process_goal(raw_dlg['goal'])
+                new_dlgs.append(Pack(dlg=norm_dlg, goal=processed_goal, key=key))
+            else:
+                dact_skip_count += 1
+
+        self.logger.info('{} sessions skipped due to missing dialogue act label'.format(dact_skip_count))
+        self.logger.info('Max utt len = %d, mean utt len = %.2f' % (
+            np.max(all_sent_lens), float(np.mean(all_sent_lens))))
+        self.logger.info('Max dlg len = %d, mean dlg len = %.2f' % (
+            np.max(all_dlg_lens), float(np.mean(all_dlg_lens))))
+        return new_dlgs
+
+    def _extract_vocab(self):
+        all_words = []
+        for dlg in self.train_corpus:
+            for turn in dlg.dlg:
+                all_words.extend(turn.utt)
+        vocab_count = Counter(all_words).most_common()
+        raw_vocab_size = len(vocab_count)
+        keep_vocab_size = min(self.config.max_vocab_size, raw_vocab_size)
+        oov_rate = np.sum([c for t, c in vocab_count[0:keep_vocab_size]]) / float(len(all_words))
+
+        self.logger.info('cut off at word {} with frequency={},\n'.format(vocab_count[keep_vocab_size - 1][0],
+                                                               vocab_count[keep_vocab_size - 1][1]) +
+              'OOV rate = {:.2f}%'.format(100.0 - oov_rate * 100))
+
+        vocab_count = vocab_count[0:keep_vocab_size]
+        self.vocab = SPECIAL_TOKENS + [t for t, cnt in vocab_count if t not in SPECIAL_TOKENS]
+        self.vocab_dict = {t: idx for idx, t in enumerate(self.vocab)}
+        self.unk_id = self.vocab_dict[UNK]
+        self.logger.info("Raw vocab size {} in train set and final vocab size {}".format(raw_vocab_size, len(self.vocab)))
+
+    def _process_goal(self, raw_goal):
+        res = {}
+        for domain in self.domains:
+            all_words = []
+            d_goal = raw_goal[domain]
+            if d_goal:
+                for info_type in self.info_types:
+                    sv_info = d_goal.get(info_type, dict())
+                    if info_type == 'reqt' and isinstance(sv_info, list):
+                        all_words.extend([info_type + '|' + item for item in sv_info])
+                    elif isinstance(sv_info, dict):
+                        all_words.extend([info_type + '|' + k + '|' + str(v) for k, v in sv_info.items()])
+                    else:
+                        print('Fatal Error!')
+                        exit(-1)
+            res[domain] = all_words
+        return res
+    
+    def _process_multidomain_summary_acts(self, dact):
+        """
+        process dialogue action dictionary into binary vector representation
+        each domain has its own vector, and final output is the flattened respresentation of each domain's action
+        """
+        res = {}
+        # dact = {domain:{action:[slot]}, domain:{action:[slot]}}
+        for domain in self.domains:
+            res[domain] = np.zeros(len(self.act_types))
+            if domain in dact.keys(): 
+                for i in range(len(self.act_types)):
+                    if self.act_types[i] in dact[domain].keys():
+                        res[domain][i] = 1
+
+
+        # multiwoz dact = {domain-act:[[slot, value], [slot, value]]}
+        # for domain in self.domains:
+            # res[domain] = np.zeros(len(self.act_types))
+        # for k in dact.keys():
+            # d = k.split("-")[0].lower()
+            # a = k.split("-")[1].lower()
+
+            # res[d][self.act2id[a]] = 1
+
+        flat_res = [act for domain in sorted(self.domains) for act in res[domain]]
+        return flat_res
+    
+    def _process_summary_acts(self, dact):
+        """
+        process dialogue action dictionary into binary vector representation, ignoring domain information
+        """
+        res = np.zeros(len(self.act_types))
+        # damd dact = {domain:{action:[slot]}, domain:{action:[slot]}}
+        for domain in self.domains:
+            if domain in dact.keys(): 
+                for i in range(len(self.act_types)):
+                    if self.act_types[i] in dact[domain].keys():
+                        res[i] = 1
+
+        # multiwoz dact = {domain-act:[[slot, value], [slot, value]]}
+        # for k in dact.keys():
+            # # d = k.split("-")[0].lower()
+            # a = k.split("-")[1].lower()
+
+           #  res[self.act2id[a]] = 1
+
+        return list(res)
+
+    def _extract_goal_vocab(self):
+        self.goal_vocab, self.goal_vocab_dict, self.goal_unk_id = {}, {}, {}
+        for domain in self.domains:
+            all_words = []
+            for dlg in self.train_corpus:
+                all_words.extend(dlg.goal[domain])
+            vocab_count = Counter(all_words).most_common()
+            raw_vocab_size = len(vocab_count)
+            discard_wc = np.sum([c for t, c in vocab_count])
+
+            self.logger.info('================= domain = {}, \n'.format(domain) +
+                  'goal vocab size of train set = %d, \n' % (raw_vocab_size,) +
+                  'cut off at word %s with frequency = %d, \n' % (vocab_count[-1][0], vocab_count[-1][1]) +
+                  'OOV rate = %.2f' % (1 - float(discard_wc) / len(all_words),))
+
+            self.goal_vocab[domain] = [UNK] + [g for g, cnt in vocab_count]
+            self.goal_vocab_dict[domain] = {t: idx for idx, t in enumerate(self.goal_vocab[domain])}
+            self.goal_unk_id[domain] = self.goal_vocab_dict[domain][UNK]
+
+    def get_corpus(self):
+        id_train = self._to_id_corpus('Train', self.train_corpus)
+        id_val = self._to_id_corpus('Valid', self.val_corpus)
+        id_test = self._to_id_corpus('Test', self.test_corpus)
+        return id_train, id_val, id_test
+
+    def _to_id_corpus(self, name, data):
+        results = []
+        for dlg in data:
+            if len(dlg.dlg) < 1:
+                continue
+            id_dlg = []
+            for turn in dlg.dlg:
+                id_turn = Pack(utt=self._sent2id(turn.utt),
+                               speaker=turn.speaker,
+                               db=turn.db, bs=turn.bs, act=turn.act)
+                id_dlg.append(id_turn)
+            id_goal = self._goal2id(dlg.goal)
+            results.append(Pack(dlg=id_dlg, goal=id_goal, key=dlg.key))
+        return results
+
+    def _sent2id(self, sent):
+        return [self.vocab_dict.get(t, self.unk_id) for t in sent]
+
+    def _goal2id(self, goal):
+        res = {}
+        for domain in self.domains:
+            d_bow = [0.0] * len(self.goal_vocab[domain])
+            for word in goal[domain]:
+                word_id = self.goal_vocab_dict[domain].get(word, self.goal_unk_id[domain])
+                d_bow[word_id] += 1.0
+            res[domain] = d_bow
+        return res
+
+    def id2sent(self, id_list):
+        return [self.vocab[i] for i in id_list]
+
+    def pad_to(self, max_len, tokens, do_pad):
+        if len(tokens) >= max_len:
+            return tokens[: max_len-1] + [tokens[-1]]
+        elif do_pad:
+            return tokens + [0] * (max_len - len(tokens))
+        else:
+            return tokens
+
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/criterions.py b/convlab/policy/lava/multiwoz/latent_dialog/criterions.py
new file mode 100644
index 0000000000000000000000000000000000000000..0bba8db136175c2ccbefea67c99c2a53771ba6b5
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/criterions.py
@@ -0,0 +1,200 @@
+import torch as th
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.nn.modules.loss import _Loss
+import numpy as np
+from convlab.policy.lava.multiwoz.latent_dialog import domain
+from convlab.policy.lava.multiwoz.latent_dialog.utils import LONG
+
+
+class NLLEntropy(_Loss):
+    def __init__(self, padding_idx, avg_type):
+        super(NLLEntropy, self).__init__()
+        self.padding_idx = padding_idx
+        self.avg_type = avg_type
+
+    def forward(self, net_output, labels):
+        batch_size = net_output.size(0)
+        pred = net_output.view(-1, net_output.size(-1))
+        target = labels.view(-1)
+
+        if self.avg_type is None:
+            loss = F.nll_loss(pred, target, size_average=False, ignore_index=self.padding_idx)
+        elif self.avg_type == 'seq':
+            loss = F.nll_loss(pred, target, size_average=False, ignore_index=self.padding_idx)
+            loss = loss / batch_size
+        elif self.avg_type == 'real_word':
+            loss = F.nll_loss(pred, target, ignore_index=self.padding_idx, reduce=False)
+            loss = loss.view(-1, net_output.size(1))
+            loss = th.sum(loss, dim=1)
+            word_cnt = th.sum(th.sign(labels), dim=1).float()
+            loss = loss / word_cnt
+            loss = th.mean(loss)
+        elif self.avg_type == 'word':
+            loss = F.nll_loss(pred, target, reduction='mean', ignore_index=self.padding_idx)
+        else:
+            raise ValueError('Unknown average type')
+
+        return loss
+
+class WeightedNLLEntropy(_Loss):
+    def __init__(self, padding_idx, avg_type, weight):
+        super(WeightedNLLEntropy, self).__init__()
+        self.padding_idx = padding_idx
+        self.avg_type = avg_type
+        self.weight = weight
+
+    def forward(self, net_output, labels):
+        batch_size = net_output.size(0)
+        pred = net_output.view(-1, net_output.size(-1))
+        target = labels.view(-1)
+        if self.avg_type == 'slot':
+            loss = F.nll_loss(pred, target, weight = self.weight, reduction='mean', ignore_index=self.padding_idx)
+
+        return loss
+ 
+class NLLEntropy4CLF(_Loss):
+    def __init__(self, dictionary, bad_tokens=['<disconnect>', '<disagree>'], reduction='elementwise_mean'):
+        super(NLLEntropy4CLF, self).__init__()
+        w = th.Tensor(len(dictionary)).fill_(1)
+        for token in bad_tokens:
+            w[dictionary[token]] = 0.0
+        self.crit = nn.CrossEntropyLoss(w, reduction=reduction)
+
+    def forward(self, preds, labels):
+        # preds: (batch_size, outcome_len, outcome_vocab_size)
+        # labels: (batch_size, outcome_len)
+        preds = preds.view(-1, preds.size(-1))
+        labels = labels.view(-1)
+        return self.crit(preds, labels)
+
+
+class CombinedNLLEntropy4CLF(_Loss):
+    def __init__(self, dictionary, corpus, np2var, bad_tokens=['<disconnect>', '<disagree>']):
+        super(CombinedNLLEntropy4CLF, self).__init__()
+        self.dictionary = dictionary
+        self.domain = domain.get_domain('object_division')
+        self.corpus = corpus
+        self.np2var = np2var
+        self.bad_tokens = bad_tokens
+
+    def forward(self, preds, goals_id, outcomes_id):
+        # preds: (batch_size, outcome_len, outcome_vocab_size)
+        # goals_id: list of list, id, batch_size*goal_len
+        # outcomes_id: list of list, id, batch_size*outcome_len
+        batch_size = len(goals_id)
+        losses = []
+        for bth in range(batch_size):
+            pred = preds[bth] # (outcome_len, outcome_vocab_size)
+            goal = goals_id[bth] # list, id, len=goal_len
+            goal_str = self.corpus.id2goal(goal) # list, str, len=goal_len
+            outcome = outcomes_id[bth] # list, id, len=outcome_len
+            outcome_str = self.corpus.id2outcome(outcome) # list, str, len=outcome_len
+
+            if outcome_str[0] in self.bad_tokens:
+                continue
+
+            # get all the possible choices
+            choices = self.domain.generate_choices(goal_str)
+            sel_outs = [pred[i] for i in range(pred.size(0))] # outcome_len*(outcome_vocab_size, )
+
+            choices_logits = [] # outcome_len*(option_amount, 1)
+            for i in range(self.domain.selection_length()):
+                idxs = np.array([self.dictionary[c[i]] for c in choices])
+                idxs_var = self.np2var(idxs, LONG) # (option_amount, )
+                choices_logits.append(th.gather(sel_outs[i], 0, idxs_var).unsqueeze(1))
+
+            choice_logit = th.sum(th.cat(choices_logits, 1), 1, keepdim=False) # (option_amount, )
+            choice_logit = choice_logit.sub(choice_logit.max().item()) # (option_amount, )
+            prob = F.softmax(choice_logit, dim=0) # (option_amount, )
+
+            label = choices.index(outcome_str)
+            target_prob = prob[label]
+            losses.append(-th.log(target_prob))
+        return sum(losses) / float(len(losses))
+
+
+class CatKLLoss(_Loss):
+    def __init__(self):
+        super(CatKLLoss, self).__init__()
+
+    def forward(self, log_qy, log_py, batch_size=None, unit_average=False):
+        """
+        qy * log(q(y)/p(y))
+        """
+        qy = th.exp(log_qy)
+        y_kl = th.sum(qy * (log_qy - log_py), dim=1)
+        if unit_average:
+            return th.mean(y_kl)
+        else:
+            return th.sum(y_kl)/batch_size
+
+
+class Entropy(_Loss):
+    def __init__(self):
+        super(Entropy, self).__init__()
+
+    def forward(self, log_qy, batch_size=None, unit_average=False):
+        """
+        -qy log(qy)
+        """
+        if log_qy.dim() > 2:
+            log_qy = log_qy.squeeze()
+        qy = th.exp(log_qy)
+        h_q = th.sum(-1 * log_qy * qy, dim=1)
+        if unit_average:
+            return th.mean(h_q)
+        else:
+            return th.sum(h_q) / batch_size
+
+
+class GaussianEntropy(_Loss):
+    def __init__(self):
+        super(GaussianEntropy, self).__init__()
+
+    def forward(self, mu, logvar):
+        """
+        0.5 (log(mu*var)) + 0.5
+        """
+        std = th.exp(0.5 * logvar)
+        var = th.square(std)
+        h_q = 0.5 * (th.log(2 * math.pi * var)) + 0.5
+
+        return th.mean(h_q)
+
+
+class BinaryNLLEntropy(_Loss):
+
+    def __init__(self, size_average=True):
+        super(BinaryNLLEntropy, self).__init__()
+        self.size_average = size_average
+
+    def forward(self, net_output, label_output):
+        """
+        :param net_output: batch_size x
+        :param labels:
+        :return:
+        """
+        batch_size = net_output.size(0)
+        loss = F.binary_cross_entropy_with_logits(net_output, label_output, size_average=self.size_average)
+        if self.size_average is False:
+            loss /= batch_size
+        return loss
+
+
+class NormKLLoss(_Loss):
+    def __init__(self, unit_average=False):
+        super(NormKLLoss, self).__init__()
+        self.unit_average = unit_average
+
+    def forward(self, recog_mu, recog_logvar, prior_mu, prior_logvar):
+        # find the KL divergence between two Gaussian distribution
+        loss = 1.0 + (recog_logvar - prior_logvar)
+        loss -= th.div(th.pow(prior_mu - recog_mu, 2), th.exp(prior_logvar))
+        loss -= th.div(th.exp(recog_logvar), th.exp(prior_logvar))
+        if self.unit_average:
+            kl_loss = -0.5 * th.mean(loss, dim=1)
+        else:
+            kl_loss = -0.5 * th.sum(loss, dim=1)
+        avg_kl_loss = th.mean(kl_loss)
+        return avg_kl_loss
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/data_loaders.py b/convlab/policy/lava/multiwoz/latent_dialog/data_loaders.py
new file mode 100644
index 0000000000000000000000000000000000000000..3dc252422ee4a14e18266463e3e598f6c9100047
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/data_loaders.py
@@ -0,0 +1,278 @@
+import numpy as np
+import pdb
+from convlab.policy.lava.multiwoz.latent_dialog.utils import Pack
+from convlab.policy.lava.multiwoz.latent_dialog.base_data_loaders import BaseDataLoaders, LongDataLoader
+from convlab.policy.lava.multiwoz.latent_dialog.corpora import USR, SYS
+import json
+
+
+class BeliefDbDataLoaders(BaseDataLoaders):
+    def __init__(self, name, data, config):
+        super(BeliefDbDataLoaders, self).__init__(name)
+        self.max_utt_len = config.max_utt_len
+        self.data, self.indexes, self.batch_indexes = self.flatten_dialog(data, config.backward_size)
+        self.data_size = len(self.data)
+        self.domains = ['hotel', 'restaurant', 'train', 'attraction', 'hospital', 'police', 'taxi']
+
+    def flatten_dialog(self, data, backward_size):
+        results = []
+        indexes = []
+        batch_indexes = []
+        resp_set = set()
+        for dlg in data:
+            goal = dlg.goal
+            key = dlg.key
+            batch_index = []
+            for i in range(1, len(dlg.dlg)):
+                if dlg.dlg[i].speaker == USR:
+                    continue
+                e_idx = i
+                s_idx = max(0, e_idx - backward_size)
+                response = dlg.dlg[i].copy()
+                response['utt'] = self.pad_to(self.max_utt_len, response.utt, do_pad=False)
+                resp_set.add(json.dumps(response.utt))
+                context = []
+                for turn in dlg.dlg[s_idx: e_idx]:
+                    turn['utt'] = self.pad_to(self.max_utt_len, turn.utt, do_pad=False)
+                    context.append(turn)
+                results.append(Pack(context=context, response=response, goal=goal, key=key))
+                indexes.append(len(indexes))
+                batch_index.append(indexes[-1])
+            if len(batch_index) > 0:
+                batch_indexes.append(batch_index)
+        print("Unique resp {}".format(len(resp_set)))
+        return results, indexes, batch_indexes
+
+    def epoch_init(self, config, shuffle=True, verbose=True, fix_batch=False):
+        self.ptr = 0
+        if fix_batch:
+            self.batch_size = None
+            self.num_batch = len(self.batch_indexes)
+        else:
+            self.batch_size = config.batch_size
+            self.num_batch = self.data_size // config.batch_size
+            self.batch_indexes = []
+            for i in range(self.num_batch):
+                self.batch_indexes.append(self.indexes[i * self.batch_size: (i + 1) * self.batch_size])
+            if verbose:
+                print('Number of left over sample = %d' % (self.data_size - config.batch_size * self.num_batch))
+        if shuffle:
+            if fix_batch:
+                self._shuffle_batch_indexes()
+            else:
+                self._shuffle_indexes()
+
+        if verbose:
+            print('%s begins with %d batches' % (self.name, self.num_batch))
+
+    def _prepare_batch(self, selected_index):
+        rows = [self.data[idx] for idx in selected_index]
+
+        ctx_utts, ctx_lens = [], []
+        out_utts, out_lens = [], []
+
+        out_bs, out_db = [] , []
+        goals, goal_lens = [], [[] for _ in range(len(self.domains))]
+        keys = []
+
+        for row in rows:
+            in_row, out_row, goal_row = row.context, row.response, row.goal
+
+            # source context
+            keys.append(row.key)
+            batch_ctx = []
+            for turn in in_row:
+                batch_ctx.append(self.pad_to(self.max_utt_len, turn.utt, do_pad=True))
+            ctx_utts.append(batch_ctx)
+            ctx_lens.append(len(batch_ctx))
+
+            # target response
+            out_utt = [t for idx, t in enumerate(out_row.utt)]
+            out_utts.append(out_utt)
+            out_lens.append(len(out_utt))
+
+            out_bs.append(out_row.bs)
+            out_db.append(out_row.db)
+
+            # goal
+            goals.append(goal_row)
+            for i, d in enumerate(self.domains):
+                goal_lens[i].append(len(goal_row[d]))
+
+        batch_size = len(ctx_lens)
+        vec_ctx_lens = np.array(ctx_lens) # (batch_size, ), number of turns
+        max_ctx_len = np.max(vec_ctx_lens)
+        vec_ctx_utts = np.zeros((batch_size, max_ctx_len, self.max_utt_len), dtype=np.int32)
+        vec_out_bs = np.array(out_bs) # (batch_size, 94)
+        vec_out_db = np.array(out_db) # (batch_size, 30)
+        vec_out_lens = np.array(out_lens)  # (batch_size, ), number of tokens
+        max_out_len = np.max(vec_out_lens)
+        vec_out_utts = np.zeros((batch_size, max_out_len), dtype=np.int32)
+
+        max_goal_lens, min_goal_lens = [max(ls) for ls in goal_lens], [min(ls) for ls in goal_lens]
+        if max_goal_lens != min_goal_lens:
+            print('Fatal Error!')
+            exit(-1)
+        self.goal_lens = max_goal_lens
+        vec_goals_list = [np.zeros((batch_size, l), dtype=np.float32) for l in self.goal_lens]
+
+        for b_id in range(batch_size):
+            vec_ctx_utts[b_id, :vec_ctx_lens[b_id], :] = ctx_utts[b_id]
+            vec_out_utts[b_id, :vec_out_lens[b_id]] = out_utts[b_id]
+            for i, d in enumerate(self.domains):
+                vec_goals_list[i][b_id, :] = goals[b_id][d]
+
+        return Pack(context_lens=vec_ctx_lens, # (batch_size, )
+                    contexts=vec_ctx_utts, # (batch_size, max_ctx_len, max_utt_len)
+                    output_lens=vec_out_lens, # (batch_size, )
+                    outputs=vec_out_utts, # (batch_size, max_out_len)
+                    bs=vec_out_bs, # (batch_size, 94)
+                    db=vec_out_db, # (batch_size, 30)
+                    goals_list=vec_goals_list, # 7*(batch_size, bow_len), bow_len differs w.r.t. domain
+                    keys=keys)
+
+class BeliefDbDataLoadersAE(BaseDataLoaders):
+    def __init__(self, name, data, config):
+        super(BeliefDbDataLoadersAE, self).__init__(name)
+        self.max_utt_len = config.max_utt_len
+        self.data, self.indexes, self.batch_indexes = self.flatten_dialog(data, config.backward_size)
+        self.data_size = len(self.data)
+        self.domains = ['hotel', 'restaurant', 'train', 'attraction', 'hospital', 'police', 'taxi']
+        self.act_types = ['bye', 'inform', 'nobook', 'nooffer', 'offerbook', 'offerbooked', 'recommend', 'reqmore', 'request', 'select', 'welcome']
+        if "ae_zero_pad" in config.keys():
+            self.zero_pad = config.ae_zero_pad
+        else:
+            self.zero_pad = False
+
+    def flatten_dialog(self, data, backward_size):
+        results = []
+        indexes = []
+        batch_indexes = []
+        resp_set = set()
+        for dlg in data:
+            goal = dlg.goal
+            key = dlg.key
+            batch_index = []
+            for i in range(1, len(dlg.dlg)):
+                if dlg.dlg[i].speaker == USR:
+                    continue
+                e_idx = i
+                s_idx = max(0, e_idx - backward_size)
+                response = dlg.dlg[i].copy()
+                response['utt'] = self.pad_to(self.max_utt_len, response.utt, do_pad=False)
+                resp_set.add(json.dumps(response.utt))
+                context = []
+                for turn in dlg.dlg[s_idx: e_idx]:
+                    turn['utt'] = self.pad_to(self.max_utt_len, turn.utt, do_pad=False)
+                    context.append(turn)
+                results.append(Pack(context=context, response=response, goal=goal, key=key))
+                indexes.append(len(indexes))
+                batch_index.append(indexes[-1])
+            if len(batch_index) > 0:
+                batch_indexes.append(batch_index)
+        print("Unique resp {}".format(len(resp_set)))
+        return results, indexes, batch_indexes
+
+    def epoch_init(self, config, shuffle=True, verbose=True, fix_batch=False):
+        self.ptr = 0
+        if fix_batch:
+            self.batch_size = None
+            self.num_batch = len(self.batch_indexes)
+        else:
+            self.batch_size = config.batch_size
+            self.num_batch = self.data_size // config.batch_size
+            self.batch_indexes = []
+            for i in range(self.num_batch):
+                self.batch_indexes.append(self.indexes[i * self.batch_size: (i + 1) * self.batch_size])
+            if verbose:
+                print('Number of left over sample = %d' % (self.data_size - config.batch_size * self.num_batch))
+        if shuffle:
+            if fix_batch:
+                self._shuffle_batch_indexes()
+            else:
+                self._shuffle_indexes()
+
+        if verbose:
+            print('%s begins with %d batches' % (self.name, self.num_batch))
+
+    def _prepare_batch(self, selected_index):
+        rows = [self.data[idx] for idx in selected_index]
+
+        ctx_utts, ctx_lens = [], []
+        out_utts, out_lens = [], []
+        out_act = []
+        out_bs, out_db = [] , []
+        goals, goal_lens = [], [[] for _ in range(len(self.domains))]
+        keys = []
+
+        for row in rows:
+            in_row, out_row, goal_row = row.context, row.response, row.goal
+
+            # source context
+            keys.append(row.key)
+            
+            # batch_ctx = []
+            # for turn in in_row:
+                # batch_ctx.append(self.pad_to(self.max_utt_len, turn.utt, do_pad=True))
+            # ctx_utts.append(batch_ctx)
+            # ctx_lens.append(len(batch_ctx))
+            
+            # for AE, input = output
+            batch_ctx = []
+            batch_ctx = self.pad_to(self.max_utt_len, out_row.utt, do_pad=True)
+            # batch_ctx = [t for idx, t in enumerate(out_row.utt)]
+            ctx_utts.append(batch_ctx)
+            ctx_lens.append(len(batch_ctx))
+
+            # target response
+            out_utt = [t for idx, t in enumerate(out_row.utt)]
+            out_utts.append(out_utt)
+            out_lens.append(len(out_utt))
+
+            if not self.zero_pad:
+                out_bs.append(out_row.bs)
+                out_db.append(out_row.db)
+            else:
+                out_bs.append([0] * 94)
+                out_db.append([0] * 30)
+            out_act.append(out_row.act)
+
+            # goal
+            goals.append(goal_row)
+            for i, d in enumerate(self.domains):
+                goal_lens[i].append(len(goal_row[d]))
+
+        batch_size = len(ctx_lens)
+        vec_ctx_lens = np.array(ctx_lens) # (batch_size, ), number of turns
+        max_ctx_len = np.max(vec_ctx_lens)
+        vec_ctx_utts = np.zeros((batch_size, max_ctx_len, self.max_utt_len), dtype=np.int32)
+        vec_out_bs = np.array(out_bs) # (batch_size, 94)
+        vec_out_db = np.array(out_db) # (batch_size, 30)
+        vec_out_act = np.array(out_act) # (batch_size, 11)
+        vec_out_lens = np.array(out_lens)  # (batch_size, ), number of tokens
+        max_out_len = np.max(vec_out_lens)
+        vec_out_utts = np.zeros((batch_size, max_out_len), dtype=np.int32)
+
+        max_goal_lens, min_goal_lens = [max(ls) for ls in goal_lens], [min(ls) for ls in goal_lens]
+        if max_goal_lens != min_goal_lens:
+            print('Fatal Error!')
+            exit(-1)
+        self.goal_lens = max_goal_lens
+        vec_goals_list = [np.zeros((batch_size, l), dtype=np.float32) for l in self.goal_lens]
+
+        for b_id in range(batch_size):
+            vec_ctx_utts[b_id, :vec_ctx_lens[b_id], :] = ctx_utts[b_id]
+            vec_out_utts[b_id, :vec_out_lens[b_id]] = out_utts[b_id]
+            for i, d in enumerate(self.domains):
+                vec_goals_list[i][b_id, :] = goals[b_id][d]
+
+        return Pack(context_lens=vec_ctx_lens, # (batch_size, )
+                    contexts=vec_ctx_utts, # (batch_size, max_ctx_len, max_utt_len)
+                    output_lens=vec_out_lens, # (batch_size, )
+                    outputs=vec_out_utts, # (batch_size, max_out_len)
+                    bs=vec_out_bs, # (batch_size, 94)
+                    db=vec_out_db, # (batch_size, 30)
+                    act=vec_out_act, #(batch_size, 11)
+                    goals_list=vec_goals_list, # 7*(batch_size, bow_len), bow_len differs w.r.t. domain
+                    keys=keys)
+
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/dialog_task.py b/convlab/policy/lava/multiwoz/latent_dialog/dialog_task.py
new file mode 100644
index 0000000000000000000000000000000000000000..b5b73e86060b2573638529e6dc16866e50445eb2
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/dialog_task.py
@@ -0,0 +1,129 @@
+from convlab.policy.lava.multiwoz.latent_dialog.metric import MetricsContainer
+from convlab.policy.lava.multiwoz.latent_dialog.corpora import EOD, EOS
+from convlab.policy.lava.multiwoz.latent_dialog import evaluators
+
+
+class Dialog(object):
+    """Dialogue runner."""
+    def __init__(self, agents, args):
+        assert len(agents) == 2
+        self.agents = agents
+        self.system, self.user = agents
+        self.args = args
+        self.metrics = MetricsContainer()
+        self.dlg_evaluator = evaluators.MultiWozEvaluator('SYS_WOZ')
+        self._register_metrics()
+
+    def _register_metrics(self):
+        """Registers valuable metrics."""
+        self.metrics.register_average('dialog_len')
+        self.metrics.register_average('sent_len')
+        self.metrics.register_average('reward')
+        self.metrics.register_time('time')
+
+    def _is_eod(self, out):
+        return len(out) == 2 and out[0] == EOD and out[1] == EOS
+
+    def _eval _dialog(self, conv, g_key, goal):
+        generated_dialog = dict()
+        generated_dialog[g_key] = {'goal': goal, 'log': list()}
+        for t_id, (name, utt) in enumerate(conv):
+            # assert utt[-1] == EOS, utt
+            if t_id % 2 == 0:
+                assert name == 'Baozi'
+            utt = ' '.join(utt[:-1])
+            if utt == EOD:
+                continue
+            generated_dialog[g_key]['log'].append({'text': utt})
+        report, success_r, match_r = self.dlg_evaluator.evaluateModel(generated_dialog, mode='rollout')
+        return success_r + match_r
+
+    def show_metrics(self):
+        return ' '.join(['%s=%s' % (k, v) for k, v in self.metrics.dict().items()])
+
+    def run(self, g_key, goal):
+        """Runs one instance of the dialogue."""
+        # initialize agents by feeding in the goal
+        # initialize BOD utterance for each agent
+        for agent in self.agents:
+            agent.feed_goal(goal)
+            agent.bod_init()
+
+        # role assignment
+        reader, writer = self.system, self.user
+        begin_name = writer.name
+        print('begin_name = {}'.format(begin_name))
+
+        conv = []
+        # reset metrics
+        self.metrics.reset()
+        nturn = 0
+        while True:
+            nturn += 1
+            # produce an utterance
+            out_words = writer.write() # out: list of word, str, len = max_words
+            print('\t{} out_words = {}'.format(writer.name, ' '.join(out_words)))
+
+            self.metrics.record('sent_len', len(out_words))
+            # self.metrics.record('%s_unique' % writer.name, out_words)
+
+            # append the utterance to the conversation
+            conv.append((writer.name, out_words))
+            # make the other agent to read it
+            reader.read(out_words)
+            # check if the end of the conversation was generated
+            if self._is_eod(out_words):
+                break
+
+            if self.args.max_nego_turn > 0 and nturn >= self.args.max_nego_turn:
+                # return conv, 0
+                break
+
+            writer, reader = reader, writer
+
+        # evaluate dialog and produce success
+        reward = self._eval_dialog(conv, g_key, goal)
+        print('Reward = {}'.format(reward))
+        # perform update
+        self.system.update(reward)
+        self.metrics.record('time')
+        self.metrics.record('dialog_len', len(conv))
+        self.metrics.record('reward', int(reward))
+
+        print('='*50)
+        print(self.show_metrics())
+        print('='*50)
+        return conv, reward
+
+
+class DialogEval(Dialog):
+    def run(self, g_key, goal):
+        """Runs one instance of the dialogue."""
+        # initialize agents by feeding in the goal
+        # initialize BOD utterance for each agent
+        for agent in self.agents:
+            agent.feed_goal(goal)
+            agent.bod_init()
+
+        # role assignment
+        reader, writer = self.system, self.user
+        conv = []
+        nturn = 0
+        while True:
+            nturn += 1
+            # produce an utterance
+            out_words = writer.write()  # out: list of word, str, len = max_words
+            conv.append((writer.name, out_words))
+            # make the other agent to read it
+            reader.read(out_words)
+            # check if the end of the conversation was generated
+            if self._is_eod(out_words):
+                break
+
+            writer, reader = reader, writer
+            if self.args.max_nego_turn > 0 and nturn >= self.args.max_nego_turn:
+                return conv, 0
+
+        # evaluate dialog and produce success
+        reward = self._eval_dialog(conv, g_key, goal)
+        return conv, reward
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/domain.py b/convlab/policy/lava/multiwoz/latent_dialog/domain.py
new file mode 100644
index 0000000000000000000000000000000000000000..43d3ff99bcf99042bc65ec3f3fc0da96734b391b
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/domain.py
@@ -0,0 +1,124 @@
+import re
+import random
+import json
+
+
+def get_domain(name):
+    if name == 'object_division':
+        return ObjectDivisionDomain()
+    raise()
+
+
+class ObjectDivisionDomain(object):
+    def __init__(self):
+        self.item_pattern = re.compile('^item([0-9])=([0-9\-])+$')
+
+    def input_length(self):
+        return 3
+
+    def selection_length(self):
+        return 6
+
+    def generate_choices(self, inpt):
+        cnts, _ = self.parse_context(inpt)
+
+        def gen(cnts, idx=0, choice=[]):
+            if idx >= len(cnts):
+                left_choice = ['item%d=%d' % (i, c) for i, c in enumerate(choice)]
+                right_choice = ['item%d=%d' % (i, n - c) for i, (n, c) in enumerate(zip(cnts, choice))]
+                return [left_choice + right_choice]
+            choices = []
+            for c in range(cnts[idx] + 1):
+                choice.append(c)
+                choices += gen(cnts, idx + 1, choice)
+                choice.pop()
+            return choices
+        choices = gen(cnts)
+        choices.append(['<no_agreement>'] * self.selection_length())
+        choices.append(['<disconnect>'] * self.selection_length())
+        return choices
+
+    def parse_context(self, ctx):
+        cnts = [int(n) for n in ctx[0::2]]
+        vals = [int(v) for v in ctx[1::2]]
+        return cnts, vals
+
+    def _to_int(self, x):
+        try:
+            return int(x)
+        except:
+            return 0
+
+    def score_choices(self, choices, ctxs):
+        assert len(choices) == len(ctxs)
+        # print('choices = {}'.format(choices))
+        # print('ctxs = {}'.format(ctxs))
+        cnts = [int(x) for x in ctxs[0][0::2]]
+        agree, scores = True, [0 for _ in range(len(ctxs))]
+        for i, n in enumerate(cnts):
+            for agent_id, (choice, ctx) in enumerate(zip(choices, ctxs)):
+                # taken = self._to_int(choice[i+3][-1])
+                taken = self._to_int(choice[i][-1])
+                n -= taken
+                scores[agent_id] += int(ctx[2 * i + 1]) * taken
+            agree = agree and (n == 0)
+        return agree, scores
+
+
+class ContextGenerator(object):
+    """Dialogue context generator. Generates contexes from the file."""
+    def __init__(self, context_file):
+        self.ctxs = []
+        with open(context_file, 'r') as f:
+            ctx_pair = []
+            for line in f:
+                ctx = line.strip().split()
+                ctx_pair.append(ctx)
+                if len(ctx_pair) == 2:
+                    self.ctxs.append(ctx_pair)
+                    ctx_pair = []
+
+    def sample(self):
+        return random.choice(self.ctxs)
+
+    def iter(self, nepoch=1):
+        for e in range(nepoch):
+            random.shuffle(self.ctxs)
+            for ctx in self.ctxs:
+                yield ctx
+
+    def total_size(self, nepoch):
+        return nepoch*len(self.ctxs)
+
+
+class ContextGeneratorEval(object):
+    """Dialogue context generator. Generates contexes from the file."""
+    def __init__(self, context_file):
+        self.ctxs = []
+        with open(context_file, 'r') as f:
+            ctx_pair = []
+            for line in f:
+                ctx = line.strip().split()
+                ctx_pair.append(ctx)
+                if len(ctx_pair) == 2:
+                    self.ctxs.append(ctx_pair)
+                    ctx_pair = []
+
+
+class TaskGoalGenerator(object):
+    def __init__(self, goal_file):
+        self.goals = []
+        data = json.load(open(goal_file))
+        for key, raw_dlg in data.items():
+            self.goals.append((key, raw_dlg['goal']))
+
+    def sample(self):
+        return random.choice(self.goals)
+
+    def iter(self, nepoch=1):
+        for e in range(nepoch):
+            random.shuffle(self.goals)
+            for goal in self.goals:
+                yield goal
+
+
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/enc2dec/__init__.py b/convlab/policy/lava/multiwoz/latent_dialog/enc2dec/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..6caafab6a74e286872551db1a2edcc848eb00fd0
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/enc2dec/__init__.py
@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+# Author: Tiancheng Zhao
+# Date: 9/15/18
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/enc2dec/base_modules.py b/convlab/policy/lava/multiwoz/latent_dialog/enc2dec/base_modules.py
new file mode 100644
index 0000000000000000000000000000000000000000..bc7d98601596f147a25aa47332483f5502eaf9d3
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/enc2dec/base_modules.py
@@ -0,0 +1,68 @@
+import torch as th
+import torch.nn as nn
+import numpy as np
+from torch.nn.modules.module import _addindent
+
+def summary(model, show_weights=True, show_parameters=True):
+    """
+    Summarizes torch model by showing trainable parameters and weights.
+    """
+    tmpstr = model.__class__.__name__ + ' (\n'
+    total_params = 0
+    for key, module in model._modules.items():
+        # if it contains layers let call it recursively to get params
+        # and weights
+        if type(module) in [
+            th.nn.modules.container.Container,
+            th.nn.modules.container.Sequential
+        ]:
+            modstr = summary(module)
+        else:
+            modstr = module.__repr__()
+        modstr = _addindent(modstr, 2)
+
+        params = sum([np.prod(p.size()) for p in module.parameters()])
+        weights = tuple([tuple(p.size()) for p in module.parameters()])
+        total_params += params
+
+        tmpstr += '  (' + key + '): ' + modstr
+        if show_weights:
+            tmpstr += ', weights={}'.format(weights)
+        if show_parameters:
+            tmpstr += ', parameters={}'.format(params)
+        tmpstr += '\n'
+
+    tmpstr = tmpstr + ') Total Parameters={}'.format(total_params)
+    return tmpstr
+
+
+class BaseRNN(nn.Module):
+    KEY_ATTN_SCORE = 'attention_score'
+    KEY_SEQUENCE = 'sequence'
+
+    def __init__(self, input_dropout_p, rnn_cell, 
+                     input_size, hidden_size, num_layers, 
+                     output_dropout_p, bidirectional):
+        super(BaseRNN, self).__init__()
+        self.input_dropout = nn.Dropout(p=input_dropout_p)
+        if rnn_cell.lower() == 'lstm':
+            self.rnn_cell = nn.LSTM
+        elif rnn_cell.lower() == 'gru':
+            self.rnn_cell = nn.GRU
+        else:
+            raise ValueError('Unsupported RNN Cell Type: {0}'.format(rnn_cell))
+        self.rnn = self.rnn_cell(input_size=input_size, 
+                                 hidden_size=hidden_size,
+                                 num_layers=num_layers, 
+                                 batch_first=True, 
+                                 dropout=output_dropout_p, 
+                                 bidirectional=bidirectional)
+
+        # TODO Trick for initializing LSTM gate parameters
+        if rnn_cell.lower() == 'lstm':
+            for names in self.rnn._all_weights:
+                for name in filter(lambda n: 'bias' in n, names):
+                    bias = getattr(self.rnn, name)
+                    n = bias.size(0)
+                    start, end = n // 4, n // 2
+                    bias.data[start:end].fill_(1.)
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/enc2dec/classifier.py b/convlab/policy/lava/multiwoz/latent_dialog/enc2dec/classifier.py
new file mode 100644
index 0000000000000000000000000000000000000000..96d39e2273af130da10903f0cd4588f69146c5da
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/enc2dec/classifier.py
@@ -0,0 +1,103 @@
+import torch as th
+import torch.nn as nn
+import torch.nn.functional as F
+from convlab.policy.lava.multiwoz.latent_dialog.enc2dec.base_modules import BaseRNN
+
+
+class EncoderGRUATTN(BaseRNN):
+    def __init__(self, input_dropout_p, rnn_cell, input_size, hidden_size, num_layers, output_dropout_p, bidirectional, variable_lengths):
+        super(EncoderGRUATTN, self).__init__(input_dropout_p=input_dropout_p, 
+                                             rnn_cell=rnn_cell, 
+                                             input_size=input_size, 
+                                             hidden_size=hidden_size, 
+                                             num_layers=num_layers, 
+                                             output_dropout_p=output_dropout_p, 
+                                             bidirectional=bidirectional)
+        self.variable_lengths = variable_lengths
+        self.nhid_attn = hidden_size
+        self.output_size = hidden_size*2 if bidirectional else hidden_size
+
+        # attention to combine selection hidden states
+        self.attn = nn.Sequential(
+            nn.Linear(2 * hidden_size, hidden_size), 
+            nn.Tanh(), 
+            nn.Linear(hidden_size, 1)
+        )
+
+    def forward(self, residual_var, input_var, turn_feat, mask=None, init_state=None, input_lengths=None):
+        # residual_var: (batch_size, max_dlg_len, 2*utt_cell_size)
+        # input_var: (batch_size, max_dlg_len, dlg_cell_size)
+
+        # TODO switch of mask
+        # mask = None
+        
+        require_embed = True
+        if require_embed:
+            # input_cat = th.cat([input_var, residual_var], 2) # (batch_size, max_dlg_len, dlg_cell_size+2*utt_cell_size)
+            input_cat = th.cat([input_var, residual_var, turn_feat], 2) # (batch_size, max_dlg_len, dlg_cell_size+2*utt_cell_size)
+        else:
+            # input_cat = th.cat([input_var], 2)
+            input_cat = th.cat([input_var, turn_feat], 2)
+        if mask is not None:
+            input_mask = mask.view(input_cat.size(0), input_cat.size(1), 1) # (batch_size, max_dlg_len*max_utt_len, 1)
+            input_cat = th.mul(input_cat, input_mask)
+        embedded = self.input_dropout(input_cat)
+        
+        require_rnn = True
+        if require_rnn:
+            if init_state is not None:
+                h, _ = self.rnn(embedded, init_state)
+            else:
+                h, _ = self.rnn(embedded) # (batch_size, max_dlg_len, 2*nhid_attn)
+    
+            logit = self.attn(h.contiguous().view(-1, 2*self.nhid_attn)).view(h.size(0), h.size(1)) # (batch_size, max_dlg_len)
+            # if mask is not None:
+            #     logit_mask = mask.view(input_cat.size(0), input_cat.size(1))
+            #     logit_mask = -999.0 * logit_mask
+            #     logit = logit_mask + logit
+    
+            prob = F.softmax(logit, dim=1).unsqueeze(2).expand_as(h) # (batch_size, max_dlg_len, 2*nhid_attn)
+            attn = th.sum(th.mul(h, prob), 1) # (batch_size, 2*nhid_attn)
+            
+            return attn
+
+        else:
+            logit = self.attn(embedded.contiguous().view(input_cat.size(0)*input_cat.size(1), -1)).view(input_cat.size(0), input_cat.size(1))
+            if mask is not None:
+                logit_mask = mask.view(input_cat.size(0), input_cat.size(1))
+                logit_mask = -999.0 * logit_mask
+                logit = logit_mask + logit
+
+            prob = F.softmax(logit, dim=1).unsqueeze(2).expand_as(embedded) # (batch_size, max_dlg_len, 2*nhid_attn)
+            attn = th.sum(th.mul(embedded, prob), 1) # (batch_size, 2*nhid_attn)
+            
+            return attn
+
+
+class FeatureProjecter(nn.Module):
+    def __init__(self, input_dropout_p, input_size, output_size):
+        super(FeatureProjecter, self).__init__()
+        self.input_dropout = nn.Dropout(p=input_dropout_p)
+        self.sel_encoder = nn.Sequential(
+            nn.Linear(input_size, output_size), 
+            nn.Tanh()
+        )
+
+    def forward(self, goals_h, attn_outs):
+        h = th.cat([attn_outs, goals_h], 1) # (batch_size, 2*nhid_attn+goal_nhid)
+        h = self.input_dropout(h)
+        h = self.sel_encoder.forward(h) # (batch_size, nhid_sel)
+        return h
+
+
+class SelectionClassifier(nn.Module):
+    def __init__(self, selection_length, input_size, output_size):
+        super(SelectionClassifier, self).__init__()
+        self.sel_decoders = nn.ModuleList()
+        for _ in range(selection_length):
+            self.sel_decoders.append(nn.Linear(input_size, output_size))
+
+    def forward(self, proj_outs):
+        outs = [decoder.forward(proj_outs).unsqueeze(1) for decoder in self.sel_decoders] # outcome_len*(batch_size, 1, outcome_vocab_size)
+        outs = th.cat(outs, 1) # (batch_size, outcome_len, outcome_vocab_size)
+        return outs
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/enc2dec/decoders.py b/convlab/policy/lava/multiwoz/latent_dialog/enc2dec/decoders.py
new file mode 100644
index 0000000000000000000000000000000000000000..cbb1dac5d64619e20c1df95d67fe4c2d96a12f98
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/enc2dec/decoders.py
@@ -0,0 +1,574 @@
+import torch as th
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+from torch.autograd import Variable
+import numpy as np
+from convlab.policy.lava.multiwoz.latent_dialog.enc2dec.base_modules import BaseRNN
+from convlab.policy.lava.multiwoz.latent_dialog.utils import cast_type, LONG, FLOAT
+from convlab.policy.lava.multiwoz.latent_dialog.corpora import DECODING_MASKED_TOKENS, EOS
+
+
+TEACH_FORCE = 'teacher_forcing'
+TEACH_GEN = 'teacher_gen'
+GEN = 'gen'
+GEN_VALID = 'gen_valid'
+
+
+class Attention(nn.Module):
+    def __init__(self, dec_cell_size, ctx_cell_size, attn_mode, project):
+        super(Attention, self).__init__()
+        self.dec_cell_size = dec_cell_size
+        self.ctx_cell_size = ctx_cell_size
+        self.attn_mode = attn_mode
+        if project:
+            self.linear_out = nn.Linear(dec_cell_size+ctx_cell_size, dec_cell_size)
+        else:
+            self.linear_out = None
+
+        if attn_mode == 'general':
+            self.dec_w = nn.Linear(dec_cell_size, ctx_cell_size)
+        elif attn_mode == 'cat':
+            self.dec_w = nn.Linear(dec_cell_size, dec_cell_size)
+            self.attn_w = nn.Linear(ctx_cell_size, dec_cell_size)
+            self.query_w = nn.Linear(dec_cell_size, 1)
+
+    def forward(self, output, context):
+        # output: (batch_size, output_seq_len, dec_cell_size)
+        # context: (batch_size, max_ctx_len, ctx_cell_size)
+        batch_size = output.size(0)
+        max_ctx_len = context.size(1)
+
+        if self.attn_mode == 'dot':
+            attn = th.bmm(output, context.transpose(1, 2)) # (batch_size, output_seq_len, max_ctx_len)
+        elif self.attn_mode == 'general':
+            mapped_output = self.dec_w(output) # (batch_size, output_seq_len, ctx_cell_size)
+            attn = th.bmm(mapped_output, context.transpose(1, 2)) # (batch_size, output_seq_len, max_ctx_len)
+        elif self.attn_mode == 'cat':
+            mapped_output = self.dec_w(output) # (batch_size, output_seq_len, dec_cell_size)
+            mapped_attn = self.attn_w(context) # (batch_size, max_ctx_len, dec_cell_size)
+            tiled_output = mapped_output.unsqueeze(2).repeat(1, 1, max_ctx_len, 1) # (batch_size, output_seq_len, max_ctx_len, dec_cell_size)
+            tiled_attn = mapped_attn.unsqueeze(1) # (batch_size, 1, max_ctx_len, dec_cell_size)
+            fc1 = th.tanh(tiled_output+tiled_attn) # (batch_size, output_seq_len, max_ctx_len, dec_cell_size)
+            attn = self.query_w(fc1).squeeze(-1) # (batch_size, otuput_seq_len, max_ctx_len)
+        else:
+            raise ValueError('Unknown attention mode')
+
+        # TODO mask
+        # if self.mask is not None:
+
+        attn = F.softmax(attn.view(-1, max_ctx_len), dim=1).view(batch_size, -1, max_ctx_len) # (batch_size, output_seq_len, max_ctx_len)
+        mix = th.bmm(attn, context) # (batch_size, output_seq_len, ctx_cell_size)
+        combined = th.cat((mix, output), dim=2) # (batch_size, output_seq_len, dec_cell_size+ctx_cell_size)
+        if self.linear_out is None:
+            return combined, attn
+        else:
+            output = th.tanh(
+                self.linear_out(combined.view(-1, self.dec_cell_size+self.ctx_cell_size))).view(
+                batch_size, -1, self.dec_cell_size) # (batch_size, output_seq_len, dec_cell_size)
+            return output, attn
+
+
+class DecoderRNN(BaseRNN):
+    def __init__(self, input_dropout_p, rnn_cell, input_size, hidden_size, num_layers, output_dropout_p,
+                 bidirectional, vocab_size, use_attn, ctx_cell_size, attn_mode, sys_id, eos_id, use_gpu,
+                 max_dec_len, embedding=None):
+
+        super(DecoderRNN, self).__init__(input_dropout_p=input_dropout_p, 
+                                         rnn_cell=rnn_cell, 
+                                         input_size=input_size, 
+                                         hidden_size=hidden_size, 
+                                         num_layers=num_layers, 
+                                         output_dropout_p=output_dropout_p, 
+                                         bidirectional=bidirectional)
+
+        # TODO embedding is None or not
+        if embedding is None:
+            self.embedding = nn.Embedding(vocab_size, input_size)
+        else:
+            self.embedding = embedding
+
+        # share parameters between encoder and decoder
+        # self.rnn = ctx_encoder.rnn
+        # self.FC = nn.Linear(input_size, utt_encoder.output_size)
+
+        self.use_attn = use_attn
+        if self.use_attn:
+            self.attention = Attention(dec_cell_size=hidden_size, 
+                                       ctx_cell_size=ctx_cell_size, 
+                                       attn_mode=attn_mode, 
+                                       project=True)
+        
+        self.dec_cell_size = hidden_size
+        self.output_size = vocab_size
+        self.project = nn.Linear(self.dec_cell_size, self.output_size)
+        self.log_softmax = F.log_softmax
+
+        self.sys_id = sys_id
+        self.eos_id = eos_id
+        self.use_gpu = use_gpu
+        self.max_dec_len = max_dec_len
+
+    def forward(self, batch_size, dec_inputs, dec_init_state, attn_context, mode, gen_type, beam_size, goal_hid=None):
+        # dec_inputs: (batch_size, response_size-1)
+        # attn_context: (batch_size, max_ctx_len, ctx_cell_size)
+        # goal_hid: (batch_size, goal_nhid)
+
+        ret_dict = dict()
+
+        if self.use_attn:
+            ret_dict[DecoderRNN.KEY_ATTN_SCORE] = list()
+
+        if mode == GEN:
+            dec_inputs = None
+
+        if gen_type != 'beam':
+            beam_size = 1
+
+        if dec_inputs is not None:
+            decoder_input = dec_inputs
+        else:
+            # prepare the BOS inputs
+            with th.no_grad():
+                bos_var = Variable(th.LongTensor([self.sys_id]))
+            bos_var = cast_type(bos_var, LONG, self.use_gpu)
+            decoder_input = bos_var.expand(batch_size*beam_size, 1) # (batch_size, 1)
+
+        if mode == GEN and gen_type == 'beam':
+            # TODO if beam search, repeat the initial states of the RNN
+            pass
+        else:
+            decoder_hidden_state = dec_init_state
+
+        prob_outputs = [] # list of logprob | max_dec_len*(batch_size, 1, vocab_size)
+        symbol_outputs = [] # list of word ids | max_dec_len*(batch_size, 1)
+        # back_pointers = []
+        # lengths = blabla...
+
+        def decode(step, cum_sum, step_output, step_attn):
+            prob_outputs.append(step_output)
+            step_output_slice = step_output.squeeze(1) # (batch_size, vocab_size)
+            if self.use_attn:
+                ret_dict[DecoderRNN.KEY_ATTN_SCORE].append(step_attn)
+
+            if gen_type == 'greedy':
+                _, symbols = step_output_slice.topk(1) # (batch_size, 1)
+            elif gen_type == 'sample':
+                # TODO FIXME
+                # symbols = self.gumbel_max(step_output_slice)
+                pass
+            elif gen_type == 'beam':
+                # TODO
+                pass
+            else:
+                raise ValueError('Unsupported decoding mode')
+
+            symbol_outputs.append(symbols)
+
+            return cum_sum, symbols
+
+        if mode == TEACH_FORCE:
+            prob_outputs, decoder_hidden_state, attn = self.forward_step(input_var=decoder_input, hidden_state=decoder_hidden_state, encoder_outputs=attn_context, goal_hid=goal_hid)
+        else:
+            # do free running here
+            cum_sum = None
+            for step in range(self.max_dec_len):
+                # Input:
+                #   decoder_input: (batch_size, 1)
+                #   decoder_hidden_state: tuple: (h, c)
+                #   attn_context: (batch_size, max_ctx_len, ctx_cell_size)
+                #   goal_hid: (batch_size, goal_nhid)
+                # Output:
+                #   decoder_output: (batch_size, 1, vocab_size)
+                #   decoder_hidden_state: tuple: (h, c)
+                #   step_attn: (batch_size, 1, max_ctx_len)
+                decoder_output, decoder_hidden_state, step_attn = self.forward_step(decoder_input, decoder_hidden_state, attn_context, goal_hid=goal_hid)
+                cum_sum, symbols = decode(step, cum_sum, decoder_output, step_attn)
+                decoder_input = symbols
+
+            prob_outputs = th.cat(prob_outputs, dim=1) # (batch_size, max_dec_len, vocab_size)
+
+            # back tracking to recover the 1-best in beam search
+            # if gen_type == 'beam':
+
+        ret_dict[DecoderRNN.KEY_SEQUENCE] = symbol_outputs
+
+        # prob_outputs: (batch_size, max_dec_len, vocab_size)
+        # decoder_hidden_state: tuple: (h, c)
+        # ret_dict[DecoderRNN.KEY_ATTN_SCORE]: max_dec_len*(batch_size, 1, max_ctx_len)
+        # ret_dict[DecoderRNN.KEY_SEQUENCE]: max_dec_len*(batch_size, 1) 
+        return prob_outputs, decoder_hidden_state, ret_dict
+
+    def forward_step(self, input_var, hidden_state, encoder_outputs, goal_hid):
+        # input_var: (batch_size, response_size-1 i.e. output_seq_len)
+        # hidden_state: tuple: (h, c)
+        # encoder_outputs: (batch_size, max_ctx_len, ctx_cell_size)
+        # goal_hid: (batch_size, goal_nhid)
+        batch_size, output_seq_len = input_var.size()
+        embedded = self.embedding(input_var) # (batch_size, output_seq_len, embedding_dim)
+
+        # add goals
+        if goal_hid is not None:
+            goal_hid = goal_hid.view(goal_hid.size(0), 1, goal_hid.size(1)) # (batch_size, 1, goal_nhid)
+            goal_rep = goal_hid.repeat(1, output_seq_len, 1) # (batch_size, output_seq_len, goal_nhid)
+            embedded = th.cat([embedded, goal_rep], dim=2) # (batch_size, output_seq_len, embedding_dim+goal_nhid)
+
+        embedded = self.input_dropout(embedded)
+
+        # ############
+        # embedded = self.FC(embedded.view(-1, embedded.size(-1))).view(batch_size, output_seq_len, -1)
+
+        # output: (batch_size, output_seq_len, dec_cell_size)
+        # hidden: tuple: (h, c)
+        output, hidden_s = self.rnn(embedded, hidden_state)
+
+        attn = None
+        if self.use_attn:
+            # output: (batch_size, output_seq_len, dec_cell_size)
+            # encoder_outputs: (batch_size, max_ctx_len, ctx_cell_size)
+            # attn: (batch_size, output_seq_len, max_ctx_len)
+            output, attn = self.attention(output, encoder_outputs)
+
+        logits = self.project(output.contiguous().view(-1, self.dec_cell_size)) # (batch_size*output_seq_len, vocab_size)
+        prediction = self.log_softmax(logits, dim=logits.dim()-1).view(batch_size, output_seq_len, -1) # (batch_size, output_seq_len, vocab_size)
+        return prediction, hidden_s, attn
+
+    # special for rl
+    def _step(self, input_var, hidden_state, encoder_outputs, goal_hid):
+        # input_var: (1, 1)
+        # hidden_state: tuple: (h, c)
+        # encoder_outputs: (1, max_dlg_len, dlg_cell_size)
+        # goal_hid: (1, goal_nhid)
+        batch_size, output_seq_len = input_var.size()
+        embedded = self.embedding(input_var) # (1, 1, embedding_dim)
+
+        if goal_hid is not None:
+            goal_hid = goal_hid.view(goal_hid.size(0), 1, goal_hid.size(1)) # (1, 1, goal_nhid)
+            goal_rep = goal_hid.repeat(1, output_seq_len, 1) # (1, 1, goal_nhid)
+            embedded = th.cat([embedded, goal_rep], dim=2) # (1, 1, embedding_dim+goal_nhid)
+
+        embedded = self.input_dropout(embedded)
+
+        # ############
+        # embedded = self.FC(embedded.view(-1, embedded.size(-1))).view(batch_size, output_seq_len, -1)
+
+        # output: (1, 1, dec_cell_size)
+        # hidden: tuple: (h, c)
+        output, hidden_s = self.rnn(embedded, hidden_state)
+
+        attn = None
+        if self.use_attn:
+            # output: (1, 1, dec_cell_size)
+            # encoder_outputs: (1, max_dlg_len, dlg_cell_size)
+            # attn: (1, 1, max_dlg_len)
+            output, attn = self.attention(output, encoder_outputs)
+
+        logits = self.project(output.view(-1, self.dec_cell_size)) # (1*1, vocab_size)
+        prediction = logits.view(batch_size, output_seq_len, -1) # (1, 1, vocab_size)
+        # prediction = self.log_softmax(logits, dim=logits.dim()-1).view(batch_size, output_seq_len, -1) # (batch_size, output_seq_len, vocab_size)
+        return prediction, hidden_s
+
+    # special for rl
+    def write(self, input_var, hidden_state, encoder_outputs, max_words, vocab, stop_tokens, goal_hid=None, mask=True,
+              decoding_masked_tokens=DECODING_MASKED_TOKENS):
+        # input_var: (1, 1)
+        # hidden_state: tuple: (h, c)
+        # encoder_outputs: max_dlg_len*(1, 1, dlg_cell_size)
+        # goal_hid: (1, goal_nhid)
+        logprob_outputs = [] # list of logprob | max_dec_len*(1, )
+        symbol_outputs = [] # list of word ids | max_dec_len*(1, )
+        decoder_input = input_var
+        decoder_hidden_state = hidden_state
+        if type(encoder_outputs) is list:
+            encoder_outputs = th.cat(encoder_outputs, 1) # (1, max_dlg_len, dlg_cell_size)
+        # print('encoder_outputs.size() = {}'.format(encoder_outputs.size()))
+        
+        if mask:
+            special_token_mask = Variable(th.FloatTensor([-999. if token in decoding_masked_tokens else 0. for token in vocab]))
+            special_token_mask = cast_type(special_token_mask, FLOAT, self.use_gpu) # (vocab_size, )
+
+        def _sample(dec_output, num_i):
+            # dec_output: (1, 1, vocab_size), need to softmax and log_softmax
+            dec_output = dec_output.view(-1) # (vocab_size, )
+            # TODO temperature
+            prob = F.softmax(dec_output/0.6, dim=0) # (vocab_size, )
+            logprob = F.log_softmax(dec_output, dim=0) # (vocab_size, )
+            symbol = prob.multinomial(num_samples=1).detach() # (1, )
+            # _, symbol = prob.topk(1) # (1, )
+            _, tmp_symbol = prob.topk(1) # (1, )
+            # print('multinomial symbol = {}, prob = {}'.format(symbol, prob[symbol.item()]))
+            # print('topk symbol = {}, prob = {}'.format(tmp_symbol, prob[tmp_symbol.item()]))
+            logprob = logprob.gather(0, symbol) # (1, )
+            return logprob, symbol
+
+        for i in range(max_words):
+            decoder_output, decoder_hidden_state = self._step(decoder_input, decoder_hidden_state, encoder_outputs, goal_hid)
+            # disable special tokens from being generated in a normal turn
+            if mask:
+                decoder_output += special_token_mask.expand(1, 1, -1)
+            logprob, symbol = _sample(decoder_output, i)
+            logprob_outputs.append(logprob)
+            symbol_outputs.append(symbol)
+            decoder_input = symbol.view(1, -1)
+
+            if vocab[symbol.item()] in stop_tokens:
+                break
+
+        assert len(logprob_outputs) == len(symbol_outputs)
+        # logprob_list = [t.item() for t in logprob_outputs]
+        logprob_list = logprob_outputs
+        symbol_list = [t.item() for t in symbol_outputs]
+        return logprob_list, symbol_list
+
+    # For MultiWoz RL
+    def forward_rl(self, batch_size, dec_init_state, attn_context, vocab, max_words, goal_hid=None, mask=True, temp=0.1):
+        # prepare the BOS inputs
+        with th.no_grad():
+            bos_var = Variable(th.LongTensor([self.sys_id]))
+        bos_var = cast_type(bos_var, LONG, self.use_gpu)
+        decoder_input = bos_var.expand(batch_size, 1) # (1, 1)
+        decoder_hidden_state = dec_init_state # tuple: (h, c)
+        encoder_outputs = attn_context # (1, ctx_len, ctx_cell_size)
+
+        logprob_outputs = [] # list of logprob | max_dec_len*(1, )
+        symbol_outputs = [] # list of word ids | max_dec_len*(1, )
+
+        if mask:
+            special_token_mask = Variable(th.FloatTensor([-999. if token in DECODING_MASKED_TOKENS else 0. for token in vocab]))
+            special_token_mask = cast_type(special_token_mask, FLOAT, self.use_gpu) # (vocab_size, )
+
+        def _sample(dec_output, num_i):
+            # dec_output: (1, 1, vocab_size), need to softmax and log_softmax
+            dec_output = dec_output.view(batch_size, -1) # (batch_size, vocab_size, )
+            prob = F.softmax(dec_output/temp, dim=1) # (batch_size, vocab_size, )
+            logprob = F.log_softmax(dec_output, dim=1) # (batch_size, vocab_size, )
+            symbol = prob.multinomial(num_samples=1).detach() # (batch_size, 1)
+            # _, symbol = prob.topk(1) # (1, )
+            _, tmp_symbol = prob.topk(1) # (1, )
+            # print('multinomial symbol = {}, prob = {}'.format(symbol, prob[symbol.item()]))
+            # print('topk symbol = {}, prob = {}'.format(tmp_symbol, prob[tmp_symbol.item()]))
+            logprob = logprob.gather(1, symbol) # (1, )
+            return logprob, symbol
+
+        stopped_samples = set()
+        for i in range(max_words):
+            decoder_output, decoder_hidden_state = self._step(decoder_input, decoder_hidden_state, encoder_outputs, goal_hid)
+            # disable special tokens from being generated in a normal turn
+            if mask:
+                decoder_output += special_token_mask.expand(1, 1, -1)
+            logprob, symbol = _sample(decoder_output, i)
+            logprob_outputs.append(logprob)
+            symbol_outputs.append(symbol)
+            decoder_input = symbol.view(batch_size, -1)
+            for b_id in range(batch_size):
+                if vocab[symbol[b_id].item()] == EOS:
+                    stopped_samples.add(b_id)
+
+            if len(stopped_samples) == batch_size:
+                break
+
+        assert len(logprob_outputs) == len(symbol_outputs)
+        symbol_outputs = th.cat(symbol_outputs, dim=1).cpu().data.numpy().tolist()
+        logprob_outputs = th.cat(logprob_outputs, dim=1)
+        logprob_list = []
+        symbol_list = []
+        for b_id in range(batch_size):
+            b_logprob = []
+            b_symbol = []
+            for t_id in range(logprob_outputs.shape[1]):
+                symbol = symbol_outputs[b_id][t_id]
+                if vocab[symbol] == EOS and t_id != 0:
+                    break
+
+                b_symbol.append(symbol_outputs[b_id][t_id])
+                b_logprob.append(logprob_outputs[b_id][t_id])
+
+            logprob_list.append(b_logprob)
+            symbol_list.append(b_symbol)
+
+        # TODO backward compatible, if batch_size == 1, we remove the nested structure
+        if batch_size == 1:
+            logprob_list = logprob_list[0]
+            symbol_list = symbol_list[0]
+
+        return logprob_list, symbol_list
+
+class DecoderPointerGen(BaseRNN):
+
+    def __init__(self, vocab_size, max_len, input_size, hidden_size, sos_id,
+                 eos_id, n_layers=1, rnn_cell='lstm', input_dropout_p=0,
+                 dropout_p=0, attn_mode='cat', attn_size=None, use_gpu=True,
+                 embedding=None):
+
+        super(DecoderPointerGen, self).__init__(vocab_size, input_size,
+                                                hidden_size, input_dropout_p,
+                                                dropout_p, n_layers, rnn_cell, False)
+
+        self.output_size = vocab_size
+        self.max_length = max_len
+        self.eos_id = eos_id
+        self.sos_id = sos_id
+        self.use_gpu = use_gpu
+        self.attn_size = attn_size
+
+        if embedding is None:
+            self.embedding = nn.Embedding(self.output_size, self.input_size)
+        else:
+            self.embedding = embedding
+
+        self.attention = Attention(self.hidden_size, attn_size, attn_mode,
+                                   project=True)
+
+        self.project = nn.Linear(self.hidden_size, self.output_size)
+        self.sentinel = nn.Parameter(torch.randn((1, 1, attn_size)), requires_grad=True)
+        self.register_parameter('sentinel', self.sentinel)
+
+    def forward_step(self, input_var, hidden, attn_ctxs, attn_words, ctx_embed=None):
+        """
+        attn_size: number of context to attend
+        :param input_var: 
+        :param hidden: 
+        :param attn_ctxs: batch_size x attn_size+1 x ctx_size. If None, then leave it empty
+        :param attn_words: batch_size x attn_size
+        :return: 
+        """
+        # we enable empty attention context
+        batch_size = input_var.size(0)
+        seq_len = input_var.size(1)
+        embedded = self.embedding(input_var)
+        if ctx_embed is not None:
+            embedded += ctx_embed
+
+        embedded = self.input_dropout(embedded)
+        output, hidden = self.rnn(embedded, hidden)
+
+        if attn_ctxs is None:
+            # pointer network here
+            logits = self.project(output.contiguous().view(-1, self.hidden_size))
+            predicted_softmax = F.log_softmax(logits, dim=1)
+            return predicted_softmax, None, hidden, None, None
+        else:
+            attn_size = attn_words.size(1)
+            combined_output, attn = self.attention(output, attn_ctxs)
+
+            # output: batch_size x seq_len x hidden_size
+            # attn: batch_size x seq_len x (attn_size+1)
+
+            # pointer network here
+            rnn_softmax = F.softmax(self.project(output.view(-1, self.hidden_size)), dim=1)
+            g = attn[:, :, 0].contiguous()
+            ptr_attn = attn[:, :, 1:].contiguous()
+            ptr_softmax = Variable(torch.zeros((batch_size * seq_len * attn_size, self.vocab_size)))
+            ptr_softmax = cast_type(ptr_softmax, FLOAT, self.use_gpu)
+
+            # convert words and ids into 1D
+            flat_attn_words = attn_words.unsqueeze(1).repeat(1, seq_len, 1).view(-1, 1)
+            flat_attn = ptr_attn.view(-1, 1)
+
+            # fill in the attention into ptr_softmax
+            ptr_softmax = ptr_softmax.scatter_(1, flat_attn_words, flat_attn)
+            ptr_softmax = ptr_softmax.view(batch_size * seq_len, attn_size, self.vocab_size)
+            ptr_softmax = torch.sum(ptr_softmax, dim=1)
+
+            # mix the softmax from rnn and pointer
+            mixture_softmax = rnn_softmax * g.view(-1, 1) + ptr_softmax
+
+            # take the log to get logsoftmax
+            logits = torch.log(mixture_softmax.clamp(min=1e-8))
+            predicted_softmax = logits.view(batch_size, seq_len, -1)
+            ptr_softmax = ptr_softmax.view(batch_size, seq_len, -1)
+
+            return predicted_softmax, ptr_softmax, hidden, ptr_attn, g
+
+    def forward(self, batch_size, attn_context, attn_words,
+                inputs=None, init_state=None, mode=TEACH_FORCE,
+                gen_type='greedy', ctx_embed=None):
+
+        # sanity checks
+        ret_dict = dict()
+
+        if mode == GEN:
+            inputs = None
+
+        if inputs is not None:
+            decoder_input = inputs
+        else:
+            # prepare the BOS inputs
+            bos_var = Variable(torch.LongTensor([self.sos_id]), volatile=True)
+            bos_var = cast_type(bos_var, LONG, self.use_gpu)
+            decoder_input = bos_var.expand(batch_size, 1)
+
+        # append sentinel to the attention
+        if attn_context is not None:
+            attn_context = torch.cat([self.sentinel.expand(batch_size, 1, self.attn_size),
+                                      attn_context], dim=1)
+
+        decoder_hidden = init_state
+        decoder_outputs = [] # a list of logprob
+        sequence_symbols = [] # a list word ids
+        attentions = []
+        pointer_gs = []
+        pointer_outputs = []
+        lengths = np.array([self.max_length] * batch_size)
+
+        def decode(step, step_output):
+            decoder_outputs.append(step_output)
+            step_output_slice = step_output.squeeze(1)
+
+            if gen_type == 'greedy':
+                symbols = step_output_slice.topk(1)[1]
+            elif gen_type == 'sample':
+                symbols = self.gumbel_max(step_output_slice)
+            else:
+                raise ValueError("Unsupported decoding mode")
+
+            sequence_symbols.append(symbols)
+
+            eos_batches = symbols.data.eq(self.eos_id)
+            if eos_batches.dim() > 0:
+                eos_batches = eos_batches.cpu().view(-1).numpy()
+                update_idx = ((lengths > di) & eos_batches) != 0
+                lengths[update_idx] = len(sequence_symbols)
+            return symbols
+
+        # Manual unrolling is used to support random teacher forcing.
+        # If teacher_forcing_ratio is True or False instead of a probability,
+        # the unrolling can be done in graph
+        if mode == TEACH_FORCE:
+            pred_softmax, ptr_softmax, decoder_hidden, attn, step_g = self.forward_step(
+                decoder_input, decoder_hidden, attn_context, attn_words, ctx_embed)
+
+            # in teach forcing mode, we don't need symbols.
+            attentions = attn
+            decoder_outputs = pred_softmax
+            pointer_gs = step_g
+            pointer_outputs = ptr_softmax
+
+        else:
+            # do free running here
+            for di in range(self.max_length):
+                pred_softmax, ptr_softmax, decoder_hidden, step_attn, step_g = self.forward_step(
+                    decoder_input, decoder_hidden, attn_context, attn_words, ctx_embed)
+
+                symbols = decode(di, pred_softmax)
+
+                # append the results into ctx dictionary
+                attentions.append(step_attn)
+                pointer_gs.append(step_g)
+                pointer_outputs.append(ptr_softmax)
+                decoder_input = symbols
+
+            # make list be a tensor
+            decoder_outputs = torch.cat(decoder_outputs, dim=1)
+            pointer_outputs = torch.cat(pointer_outputs, dim=1)
+            pointer_gs = torch.cat(pointer_gs, dim=1)
+
+        # save the decoded sequence symbols and sequence length
+        ret_dict[self.KEY_ATTN_SCORE] = attentions
+        ret_dict[self.KEY_SEQUENCE] = sequence_symbols
+        ret_dict[self.KEY_LENGTH] = lengths
+        ret_dict[self.KEY_G] = pointer_gs
+        ret_dict[self.KEY_PTR_SOFTMAX] = pointer_outputs
+        ret_dict[self.KEY_PTR_CTX] = attn_words
+
+        return decoder_outputs, decoder_hidden, ret_dict
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/enc2dec/encoders.py b/convlab/policy/lava/multiwoz/latent_dialog/enc2dec/encoders.py
new file mode 100644
index 0000000000000000000000000000000000000000..33c754e55c4e60b668357c0f4790b4595d3b4d30
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/enc2dec/encoders.py
@@ -0,0 +1,215 @@
+import torch as th
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Variable
+import numpy as np
+from convlab.policy.lava.multiwoz.latent_dialog.enc2dec.base_modules import BaseRNN
+
+
+class EncoderRNN(BaseRNN):
+    def __init__(self, input_dropout_p, rnn_cell, input_size, hidden_size, num_layers, output_dropout_p, bidirectional, variable_lengths):
+        super(EncoderRNN, self).__init__(input_dropout_p=input_dropout_p, 
+                                         rnn_cell=rnn_cell, 
+                                         input_size=input_size, 
+                                         hidden_size=hidden_size, 
+                                         num_layers=num_layers, 
+                                         output_dropout_p=output_dropout_p, 
+                                         bidirectional=bidirectional)
+        self.variable_lengths = variable_lengths
+        self.output_size = hidden_size*2 if bidirectional else hidden_size
+
+    def forward(self, input_var, init_state=None, input_lengths=None, goals=None):
+        # add goals
+        if goals is not None:
+            batch_size, max_ctx_len, ctx_nhid = input_var.size()
+            goals = goals.view(goals.size(0), 1, goals.size(1))
+            goals_rep = goals.repeat(1, max_ctx_len, 1).view(batch_size, max_ctx_len, -1) # (batch_size, max_ctx_len, goal_nhid)
+            input_var = th.cat([input_var, goals_rep], dim=2)
+
+        embedded = self.input_dropout(input_var)
+
+        if self.variable_lengths:
+            embedded = nn.utils.rnn.pack_padded_sequence(embedded, input_lengths,
+                                                         batch_first=True)
+        if init_state is not None:
+            output, hidden = self.rnn(embedded, init_state)
+        else:
+            output, hidden = self.rnn(embedded)
+        if self.variable_lengths:
+            output, _ = nn.utils.rnn.pad_packed_sequence(output, batch_first=True)
+
+        return output, hidden
+
+
+class RnnUttEncoder(nn.Module):
+    def __init__(self, vocab_size, embedding_dim, feat_size, goal_nhid, rnn_cell,
+                 utt_cell_size, num_layers, input_dropout_p, output_dropout_p,
+                 bidirectional, variable_lengths, use_attn, embedding=None):
+        super(RnnUttEncoder, self).__init__()
+        if embedding is None:
+            self.embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=embedding_dim)
+        else:
+            self.embedding = embedding
+
+        self.rnn = EncoderRNN(input_dropout_p=input_dropout_p,
+                              rnn_cell=rnn_cell, 
+                              input_size=embedding_dim+feat_size+goal_nhid, 
+                              hidden_size=utt_cell_size, 
+                              num_layers=num_layers, 
+                              output_dropout_p=output_dropout_p, 
+                              bidirectional=bidirectional, 
+                              variable_lengths=variable_lengths)
+
+        self.utt_cell_size = utt_cell_size
+        self.multiplier = 2 if bidirectional else 1
+        self.output_size = self.multiplier * self.utt_cell_size
+        self.use_attn = use_attn
+        if self.use_attn:
+            self.key_w = nn.Linear(self.output_size, self.utt_cell_size)
+            self.query = nn.Linear(self.utt_cell_size, 1)
+
+    def forward(self, utterances, feats=None, init_state=None, goals=None):
+        batch_size, max_ctx_len, max_utt_len = utterances.size()
+        # get word embeddings
+        flat_words = utterances.view(-1, max_utt_len) # (batch_size*max_ctx_len, max_utt_len)
+        word_embeddings = self.embedding(flat_words) # (batch_size*max_ctx_len, max_utt_len, embedding_dim)
+        flat_mask = th.sign(flat_words).float()
+        # add features
+        if feats is not None:
+            flat_feats = feats.view(-1, 1) # (batch_size*max_ctx_len, 1)
+            flat_feats = flat_feats.unsqueeze(1).repeat(1, max_utt_len, 1) # (batch_size*max_ctx_len, max_utt_len, 1)
+            word_embeddings = th.cat([word_embeddings, flat_feats], dim=2) # (batch_size*max_ctx_len, max_utt_len, embedding_dim+1)
+
+        # add goals
+        if goals is not None:
+            goals = goals.view(goals.size(0), 1, 1, goals.size(1))
+            goals_rep = goals.repeat(1, max_ctx_len, max_utt_len, 1).view(batch_size*max_ctx_len, max_utt_len, -1) # (batch_size*max_ctx_len, max_utt_len, goal_nhid)
+            word_embeddings = th.cat([word_embeddings, goals_rep], dim=2)
+
+        # enc_outs: (batch_size*max_ctx_len, max_utt_len, num_directions*utt_cell_size)
+        # enc_last: (num_layers*num_directions, batch_size*max_ctx_len, utt_cell_size)
+        enc_outs, enc_last = self.rnn(word_embeddings, init_state=init_state)
+
+        if self.use_attn:
+            fc1 = th.tanh(self.key_w(enc_outs)) # (batch_size*max_ctx_len, max_utt_len, utt_cell_size)
+            attn = self.query(fc1).squeeze(2)
+            # (batch_size*max_ctx_len, max_utt_len)
+            attn = F.softmax(attn, attn.dim()-1) # (batch_size*max_ctx_len, max_utt_len, 1)
+            attn = attn * flat_mask
+            attn = (attn / (th.sum(attn, dim=1, keepdim=True)+1e-10)).unsqueeze(2)
+            utt_embedded = attn * enc_outs # (batch_size*max_ctx_len, max_utt_len, num_directions*utt_cell_size)
+            utt_embedded = th.sum(utt_embedded, dim=1) # (batch_size*max_ctx_len, num_directions*utt_cell_size)
+        else:
+            # FIXME bug for multi-layer
+            attn = None
+            utt_embedded = enc_last.transpose(0, 1).contiguous() # (batch_size*max_ctx_lens, num_layers*num_directions, utt_cell_size)
+            utt_embedded = utt_embedded.view(-1, self.output_size) # (batch_size*max_ctx_len*num_layers, num_directions*utt_cell_size)
+
+        utt_embedded = utt_embedded.view(batch_size, max_ctx_len, self.output_size)
+        return utt_embedded, word_embeddings.contiguous().view(batch_size, max_ctx_len*max_utt_len, -1), \
+               enc_outs.contiguous().view(batch_size, max_ctx_len*max_utt_len, -1)
+
+
+class MlpGoalEncoder(nn.Module):
+    def __init__(self, goal_vocab_size, k, nembed, nhid, init_range):
+        super(MlpGoalEncoder, self).__init__()
+
+        # create separate embedding for counts and values
+        self.cnt_enc = nn.Embedding(goal_vocab_size, nembed)
+        self.val_enc = nn.Embedding(goal_vocab_size, nembed)
+
+        self.encoder = nn.Sequential(
+            nn.Tanh(),
+            nn.Linear(k*nembed, nhid) 
+        )
+
+        self.cnt_enc.weight.data.uniform_(-init_range, init_range)
+        self.val_enc.weight.data.uniform_(-init_range, init_range)
+        self._init_cont(self.encoder, init_range)
+
+    def _init_cont(self, cont, init_range):
+        """initializes a container uniformly."""
+        for m in cont:
+            if hasattr(m, 'weight'):
+                m.weight.data.uniform_(-init_range, init_range)
+            if hasattr(m, 'bias'):
+                m.bias.data.fill_(0)
+
+    def forward(self, goal):
+        # goal: (batch_size, goal_len)
+        goal = goal.transpose(0, 1).contiguous() # (goal_len, batch_size)
+        idx = np.arange(goal.size(0) // 2)
+        
+        # extract counts and values
+        cnt_idx = Variable(th.from_numpy(2 * idx + 0))
+        val_idx = Variable(th.from_numpy(2 * idx + 1))
+
+        if goal.is_cuda:
+            cnt_idx = cnt_idx.type(th.cuda.LongTensor)
+            val_idx = val_idx.type(th.cuda.LongTensor)
+        else:
+            cnt_idx = cnt_idx.type(th.LongTensor)
+            val_idx = val_idx.type(th.LongTensor)
+
+        cnt = goal.index_select(0, cnt_idx) # (3, batch_size)
+        val = goal.index_select(0, val_idx) # (3, batch_size)
+
+        # embed counts and values
+        cnt_emb = self.cnt_enc(cnt) # (3, batch_size, nembed)
+        val_emb = self.val_enc(val) # (3, batch_size, nembed)
+
+        # element wise multiplication to get a hidden state
+        h = th.mul(cnt_emb, val_emb) # (3, batch_size, nembed)
+        # run the hidden state through the MLP
+        h = h.transpose(0, 1).contiguous().view(goal.size(1), -1) # (batch_size, 3*nembed)
+        goal_h = self.encoder(h) # (batch_size, nhid)
+
+        return goal_h
+
+
+class TaskMlpGoalEncoder(nn.Module):
+    def __init__(self, goal_vocab_sizes, nhid, init_range):
+        super(TaskMlpGoalEncoder, self).__init__()
+        
+        self.encoder = nn.ModuleList()
+        for v_size in goal_vocab_sizes:
+            domain_encoder = nn.Sequential(
+                nn.Linear(v_size, nhid), 
+                nn.Tanh()
+            )
+            self._init_cont(domain_encoder, init_range)
+            self.encoder.append(domain_encoder)
+
+    def _init_cont(self, cont, init_range):
+        """initializes a container uniformly."""
+        for m in cont:
+            if hasattr(m, 'weight'):
+                m.weight.data.uniform_(-init_range, init_range)
+            if hasattr(m, 'bias'):
+                m.bias.data.fill_(0)
+
+    def forward(self, goals_list):
+        # goals_list: list of tensor, 7*(batch_size, goal_len), goal_len varies among differnet domains
+        outs = [encoder.forward(goal) for goal, encoder in zip(goals_list, self.encoder)] # 7*(batch_size, goal_nhid)
+        outs = th.sum(th.stack(outs), dim=0) # (batch_size, goal_nhid)
+        return outs
+
+
+class SelfAttn(nn.Module):
+    def __init__(self, hidden_size):
+        super(SelfAttn, self).__init__()
+        self.query = nn.Linear(hidden_size, 1)
+
+    def forward(self, keys, values, attn_mask=None):
+        """
+        :param attn_inputs: batch_size x time_len x hidden_size
+        :param attn_mask: batch_size x time_len
+        :return: summary state
+        """
+        alpha = F.softmax(self.query(keys), dim=1)
+        if attn_mask is not None:
+            alpha = alpha * attn_mask.unsqueeze(2)
+            alpha = alpha / th.sum(alpha, dim=1, keepdim=True)
+
+        summary = th.sum(values * alpha, dim=1)
+        return summary
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/evaluators.py b/convlab/policy/lava/multiwoz/latent_dialog/evaluators.py
new file mode 100644
index 0000000000000000000000000000000000000000..f2b03076fa90648a42a5cd669ef6bfe073272353
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/evaluators.py
@@ -0,0 +1,1325 @@
+from nltk.translate.bleu_score import corpus_bleu, SmoothingFunction
+import math
+import numpy as np
+import convlab.policy.lava.multiwoz.latent_dialog.normalizer.delexicalize as delex
+from convlab.policy.lava.multiwoz.latent_dialog.utils import get_tokenize, get_detokenize
+from collections import Counter, defaultdict
+from nltk.util import ngrams
+from convlab.policy.lava.multiwoz.latent_dialog.corpora import SYS, USR, BOS, EOS
+from sklearn.feature_extraction.text import CountVectorizer
+import json
+from convlab.policy.lava.multiwoz.latent_dialog.normalizer.delexicalize import normalize
+import sqlite3
+import os
+import random
+import logging
+import pdb
+from sklearn.multiclass import OneVsRestClassifier
+from sklearn.linear_model import SGDClassifier
+from sklearn import metrics
+from nltk.translate import bleu_score
+from nltk.translate.bleu_score import SmoothingFunction
+from scipy.stats import gmean
+
+
+
+class BaseEvaluator(object):
+    def initialize(self):
+        raise NotImplementedError
+
+    def add_example(self, ref, hyp):
+        raise NotImplementedError
+
+    def get_report(self, *args, **kwargs):
+        raise NotImplementedError
+
+    @staticmethod
+    def _get_prec_recall(tp, fp, fn):
+        precision = tp / (tp + fp + 10e-20)
+        recall = tp / (tp + fn + 10e-20)
+        f1 = 2 * precision * recall / (precision + recall + 1e-20)
+        return precision, recall, f1
+
+    @staticmethod
+    def _get_tp_fp_fn(label_list, pred_list):
+        tp = len([t for t in pred_list if t in label_list])
+        fp = max(0, len(pred_list) - tp)
+        fn = max(0, len(label_list) - tp)
+        return tp, fp, fn
+
+
+class BLEUScorer(object):
+    ## BLEU score calculator via GentScorer interface
+    ## it calculates the BLEU-4 by taking the entire corpus in
+    ## Calulate based multiple candidates against multiple references
+    def score(self, hypothesis, corpus, n=1):
+        # containers
+        count = [0, 0, 0, 0]
+        clip_count = [0, 0, 0, 0]
+        r = 0
+        c = 0
+        weights = [0.25, 0.25, 0.25, 0.25]
+
+        # accumulate ngram statistics
+        for hyps, refs in zip(hypothesis, corpus):
+            # if type(hyps[0]) is list:
+            #    hyps = [hyp.split() for hyp in hyps[0]]
+            # else:
+            #    hyps = [hyp.split() for hyp in hyps]
+
+            # refs = [ref.split() for ref in refs]
+            hyps = [hyps]
+            # Shawn's evaluation
+            # refs[0] = [u'GO_'] + refs[0] + [u'EOS_']
+            # hyps[0] = [u'GO_'] + hyps[0] + [u'EOS_']
+
+            for idx, hyp in enumerate(hyps):
+                for i in range(4):
+                    # accumulate ngram counts
+                    hypcnts = Counter(ngrams(hyp, i + 1))
+                    cnt = sum(hypcnts.values())
+                    count[i] += cnt
+
+                    # compute clipped counts
+                    max_counts = {}
+                    for ref in refs:
+                        refcnts = Counter(ngrams(ref, i + 1))
+                        for ng in hypcnts:
+                            max_counts[ng] = max(max_counts.get(ng, 0), refcnts[ng])
+                    clipcnt = dict((ng, min(count, max_counts[ng])) \
+                                   for ng, count in hypcnts.items())
+                    clip_count[i] += sum(clipcnt.values())
+
+                # accumulate r & c
+                bestmatch = [1000, 1000]
+                for ref in refs:
+                    if bestmatch[0] == 0: break
+                    diff = abs(len(ref) - len(hyp))
+                    if diff < bestmatch[0]:
+                        bestmatch[0] = diff
+                        bestmatch[1] = len(ref)
+                r += bestmatch[1]
+                c += len(hyp)
+                if n == 1:
+                    break
+        # computing bleu score
+        p0 = 1e-7
+        bp = 1 if c > r else math.exp(1 - float(r) / float(c))
+        p_ns = [float(clip_count[i]) / float(count[i] + p0) + p0 \
+                for i in range(4)]
+        s = math.fsum(w * math.log(p_n) \
+                      for w, p_n in zip(weights, p_ns) if p_n)
+        bleu = bp * math.exp(s)
+        return bleu
+
+
+class BleuEvaluator(BaseEvaluator):
+    def __init__(self, data_name):
+        self.data_name = data_name
+        self.labels = list()
+        self.hyps = list()
+
+    def initialize(self):
+        self.labels = list()
+        self.hyps = list()
+
+    def add_example(self, ref, hyp):
+        self.labels.append(ref)
+        self.hyps.append(hyp)
+
+    def get_report(self):
+        tokenize = get_tokenize()
+        print('Generate report for {} samples'.format(len(self.hyps)))
+        refs, hyps = [], []
+        for label, hyp in zip(self.labels, self.hyps):
+            # label = label.replace(EOS, '')
+            # hyp = hyp.replace(EOS, '')
+            # ref_tokens = tokenize(label)[1:]
+            # hyp_tokens = tokenize(hyp)[1:]
+            ref_tokens = tokenize(label)
+            hyp_tokens = tokenize(hyp)
+            refs.append([ref_tokens])
+            hyps.append(hyp_tokens)
+        bleu = corpus_bleu(refs, hyps, smoothing_function=SmoothingFunction().method1)
+        report = '\n===== BLEU = %f =====\n' % (bleu,)
+        return '\n===== REPORT FOR DATASET {} ====={}'.format(self.data_name, report)
+
+
+class MultiWozDB(object):
+    # loading databases
+    domains = ['restaurant', 'hotel', 'attraction', 'train', 'taxi', 'hospital']  # , 'police']
+    dbs = {}
+    CUR_DIR = os.path.dirname(__file__).replace('latent_dialog', '')
+
+    for domain in domains:
+        db = os.path.join(CUR_DIR, 'data/norm-multi-woz/db/{}-dbase.db'.format(domain))
+        conn = sqlite3.connect(db)
+        c = conn.cursor()
+        dbs[domain] = c
+
+    def queryResultVenues(self, domain, turn, real_belief=False):
+        # query the db
+        sql_query = "select * from {}".format(domain)
+
+        if real_belief == True:
+            items = turn.items()
+        else:
+            items = turn['metadata'][domain]['semi'].items()
+
+        flag = True
+        for key, val in items:
+            if val == "" or val == "dontcare" or val == 'not mentioned' or val == "don't care" or val == "dont care" or val == "do n't care":
+                pass
+            else:
+                if flag:
+                    sql_query += " where "
+                    val2 = val.replace("'", "''")
+                    val2 = normalize(val2)
+                    if key == 'leaveAt':
+                        sql_query += r" " + key + " > " + r"'" + val2 + r"'"
+                    elif key == 'arriveBy':
+                        sql_query += r" " + key + " < " + r"'" + val2 + r"'"
+                    else:
+                        sql_query += r" " + key + "=" + r"'" + val2 + r"'"
+                    flag = False
+                else:
+                    val2 = val.replace("'", "''")
+                    val2 = normalize(val2)
+                    if key == 'leaveAt':
+                        sql_query += r" and " + key + " > " + r"'" + val2 + r"'"
+                    elif key == 'arriveBy':
+                        sql_query += r" and " + key + " < " + r"'" + val2 + r"'"
+                    else:
+                        sql_query += r" and " + key + "=" + r"'" + val2 + r"'"
+
+        try:  # "select * from attraction  where name = 'queens college'"
+            return self.dbs[domain].execute(sql_query).fetchall()
+        except:
+            return []  # TODO test it
+
+
+class MultiWozEvaluator(BaseEvaluator):
+    CUR_DIR = os.path.dirname(__file__).replace('latent_dialog', '')
+    logger = logging.getLogger()
+    def __init__(self, data_name):
+        self.data_name = data_name
+        self.slot_dict = delex.prepareSlotValuesIndependent()
+        self.delex_dialogues = json.load(open(os.path.join(self.CUR_DIR, 'data/norm-multi-woz/delex.json')))
+        self.db = MultiWozDB()
+        self.labels = list()
+        self.hyps = list()
+
+    def initialize(self):
+        self.labels = list()
+        self.hyps = list()
+
+    def add_example(self, ref, hyp):
+        self.labels.append(ref)
+        self.hyps.append(hyp)
+
+    def _parseGoal(self, goal, d, domain):
+        """Parses user goal into dictionary format."""
+        goal[domain] = {}
+        goal[domain] = {'informable': [], 'requestable': [], 'booking': []}
+        if 'info' in d['goal'][domain]:
+        # if d['goal'][domain].has_key('info'):
+            if domain == 'train':
+                # we consider dialogues only where train had to be booked!
+                if 'book' in d['goal'][domain]:
+                # if d['goal'][domain].has_key('book'):
+                    goal[domain]['requestable'].append('reference')
+                if 'reqt' in d['goal'][domain]:
+                # if d['goal'][domain].has_key('reqt'):
+                    if 'trainID' in d['goal'][domain]['reqt']:
+                        goal[domain]['requestable'].append('id')
+            else:
+                if 'reqt' in d['goal'][domain]:
+                # if d['goal'][domain].has_key('reqt'):
+                    for s in d['goal'][domain]['reqt']:  # addtional requests:
+                        if s in ['phone', 'address', 'postcode', 'reference', 'id']:
+                            # ones that can be easily delexicalized
+                            goal[domain]['requestable'].append(s)
+                if 'book' in d['goal'][domain]:
+                # if d['goal'][domain].has_key('book'):
+                    goal[domain]['requestable'].append("reference")
+
+            goal[domain]["informable"] = d['goal'][domain]['info']
+            if 'book' in d['goal'][domain]:
+            # if d['goal'][domain].has_key('book'):
+                goal[domain]["booking"] = d['goal'][domain]['book']
+
+        return goal
+
+    def _evaluateGeneratedDialogue(self, dialog, goal, realDialogue, real_requestables, soft_acc=False):
+        """Evaluates the dialogue created by the model.
+        First we load the user goal of the dialogue, then for each turn
+        generated by the system we look for key-words.
+        For the Inform rate we look whether the entity was proposed.
+        For the Success rate we look for requestables slots"""
+        # for computing corpus success
+        requestables = ['phone', 'address', 'postcode', 'reference', 'id']
+
+        # CHECK IF MATCH HAPPENED
+        provided_requestables = {}
+        venue_offered = {}
+        domains_in_goal = []
+
+        for domain in goal.keys():
+            venue_offered[domain] = []
+            provided_requestables[domain] = []
+            domains_in_goal.append(domain)
+
+        for t, sent_t in enumerate(dialog):
+            for domain in goal.keys():
+                # for computing success
+                if '[' + domain + '_name]' in sent_t or '_id' in sent_t: # undo delexicalization if system generates [domain_name] or [domain_id]
+                    if domain in ['restaurant', 'hotel', 'attraction', 'train']: 
+                        # HERE YOU CAN PUT YOUR BELIEF STATE ESTIMATION
+                        # in this case, look for the actual offered venues based on true belief state
+                        venues = self.db.queryResultVenues(domain, realDialogue['log'][t * 2 + 1])
+
+                        # if venue has changed
+                        if len(venue_offered[domain]) == 0 and venues:
+                            venue_offered[domain] = random.sample(venues, 1)
+                        else:
+                            flag = False
+                            for ven in venues:
+                                if venue_offered[domain][0] == ven:
+                                    flag = True
+                                    break
+                            if not flag and venues:  # sometimes there are no results so sample won't work
+                                # print venues
+                                venue_offered[domain] = random.sample(venues, 1)
+                    else:  # not limited so we can provide one
+                        venue_offered[domain] = '[' + domain + '_name]'
+
+                # ATTENTION: assumption here - we didn't provide phone or address twice! etc
+                for requestable in requestables:
+                    if requestable == 'reference':
+                        if domain + '_reference' in sent_t:
+                            if 'restaurant_reference' in sent_t:
+                                if realDialogue['log'][t * 2]['db_pointer'][
+                                    -5] == 1:  # if pointer was allowing for that?
+                                    provided_requestables[domain].append('reference')
+
+                            elif 'hotel_reference' in sent_t:
+                                if realDialogue['log'][t * 2]['db_pointer'][
+                                    -3] == 1:  # if pointer was allowing for that?
+                                    provided_requestables[domain].append('reference')
+
+                            elif 'train_reference' in sent_t:
+                                if realDialogue['log'][t * 2]['db_pointer'][
+                                    -1] == 1:  # if pointer was allowing for that?
+                                    provided_requestables[domain].append('reference')
+
+                            else:
+                                provided_requestables[domain].append('reference')
+                    else:
+                        if '[' + domain + '_' + requestable + ']' in sent_t:
+                            provided_requestables[domain].append(requestable)
+
+        # if name was given in the task
+        for domain in goal.keys():
+            # if name was provided for the user, the match is being done automatically
+            # assumption doesn't always hold, maybe it's better if name is provided by user that it is ignored?
+            if 'info' in realDialogue['goal'][domain]:
+                if 'name' in realDialogue['goal'][domain]['info']:
+                    venue_offered[domain] = '[' + domain + '_name]'
+
+            # special domains - entity does not need to be provided
+            if domain in ['taxi', 'police', 'hospital']:
+                venue_offered[domain] = '[' + domain + '_name]'
+
+            if domain == 'train':
+                if not venue_offered[domain]:
+                    # if realDialogue['goal'][domain].has_key('reqt') and 'id' not in realDialogue['goal'][domain]['reqt']:
+                    if 'reqt' in realDialogue['goal'][domain] and 'id' not in realDialogue['goal'][domain]['reqt']:
+                        venue_offered[domain] = '[' + domain + '_name]'
+
+        """
+        Given all inform and requestable slots
+        we go through each domain from the user goal
+        and check whether right entity was provided and
+        all requestable slots were given to the user.
+        The dialogue is successful if that's the case for all domains.
+        """
+        # HARD EVAL
+        stats = {'restaurant': [0, 0, 0], 'hotel': [0, 0, 0], 'attraction': [0, 0, 0], 'train': [0, 0, 0],
+                 'taxi': [0, 0, 0],
+                 'hospital': [0, 0, 0], 'police': [0, 0, 0]}
+
+        match = 0
+        success = 0
+        # MATCH (between offered venue by generated dialogue and venue actually fitting to the criteria)
+        for domain in goal.keys():
+            match_stat = 0
+            if domain in ['restaurant', 'hotel', 'attraction', 'train']:
+                goal_venues = self.db.queryResultVenues(domain, goal[domain]['informable'], real_belief=True)
+                # if venue offered is not dict
+                if type(venue_offered[domain]) is str and '_name' in venue_offered[domain]: # yields false positive, does not match what is offered with real dialogue?
+                    match += 1
+                    match_stat = 1
+                # if venue offered is dict
+                elif len(venue_offered[domain]) > 0 and venue_offered[domain][0] in goal_venues: # actually checks the offered venue
+                    match += 1
+                    match_stat = 1
+            # other domains
+            else:
+                if domain + '_name]' in venue_offered[domain]: # yields false positive, in terms of occurence and correctness
+                    match += 1
+                    match_stat = 1
+
+            stats[domain][0] = match_stat
+            stats[domain][2] = 1
+
+        if soft_acc:
+            match = float(match)/len(goal.keys())
+        else:
+            # only count success if all domain has matches
+            if match == len(goal.keys()):
+                match = 1.0
+            else:
+                match = 0.0
+
+        # SUCCESS (whether the requestable info in realDialogue is generated by the system)
+        # if no match, then success is assumed to be 0
+        if match == 1.0:
+            for domain in domains_in_goal:
+                success_stat = 0
+                domain_success = 0
+                if len(real_requestables[domain]) == 0: # if there is no requestable, assume to be succesful. incorrect, cause does not count false positives. 
+                    success += 1
+                    success_stat = 1
+                    stats[domain][1] = success_stat
+                    continue
+                # if values in sentences are super set of requestables
+                for request in set(provided_requestables[domain]):
+                    if request in real_requestables[domain]:
+                        domain_success += 1
+
+                if domain_success >= len(real_requestables[domain]):
+                    success += 1
+                    success_stat = 1
+
+                stats[domain][1] = success_stat
+
+            # final eval
+            if soft_acc:
+                success = float(success)/len(real_requestables)
+            else:
+                if success >= len(real_requestables):
+                    success = 1
+                else:
+                    success = 0
+
+        # rint requests, 'DIFF', requests_real, 'SUCC', success
+        return success, match, stats
+
+    def _evaluateGeneratedDialogue_new(self, dialog, goal, realDialogue, real_requestables, soft_acc=False):
+        """Evaluates the dialogue created by the model.
+        First we load the user goal of the dialogue, then for each turn
+        generated by the system we look for key-words.
+        For the Inform rate we look whether the entity was proposed.
+        For the Success rate we look for requestables slots"""
+        # for computing corpus success
+        requestables = ['phone', 'address', 'postcode', 'reference', 'id']
+
+        # CHECK IF MATCH HAPPENED
+        provided_requestables = {}
+        venue_offered = {}
+        domains_in_goal = []
+
+        for domain in goal.keys():
+            venue_offered[domain] = []
+            provided_requestables[domain] = []
+            domains_in_goal.append(domain)
+
+        for t, sent_t in enumerate(dialog):
+            for domain in goal.keys():
+                # for computing success
+                if '[' + domain + '_name]' in sent_t or '_id' in sent_t:
+                    if domain in ['restaurant', 'hotel', 'attraction', 'train']:
+                        # HERE YOU CAN PUT YOUR BELIEF STATE ESTIMATION
+                        venues = self.db.queryResultVenues(domain, realDialogue['log'][t * 2 + 1])
+
+                        # if venue has changed
+                        if len(venue_offered[domain]) == 0 and venues:
+                            venue_offered[domain] = random.sample(venues, 1)
+                        else:
+                            flag = False
+                            for ven in venues:
+                                if venue_offered[domain][0] == ven:
+                                    flag = True
+                                    break
+                            if not flag and venues:  # sometimes there are no results so sample won't work
+                                # print venues
+                                venue_offered[domain] = random.sample(venues, 1)
+                    else:  # not limited so we can provide one
+                        venue_offered[domain] = '[' + domain + '_name]'
+
+                # ATTENTION: assumption here - we didn't provide phone or address twice! etc
+                for requestable in requestables:
+                    if requestable == 'reference':
+                        if domain + '_reference' in sent_t:
+                            if 'restaurant_reference' in sent_t:
+                                if realDialogue['log'][t * 2]['db_pointer'][
+                                    -5] == 1:  # if pointer was allowing for that?
+                                    provided_requestables[domain].append('reference')
+
+                            elif 'hotel_reference' in sent_t:
+                                if realDialogue['log'][t * 2]['db_pointer'][
+                                    -3] == 1:  # if pointer was allowing for that?
+                                    provided_requestables[domain].append('reference')
+
+                            elif 'train_reference' in sent_t:
+                                if realDialogue['log'][t * 2]['db_pointer'][
+                                    -1] == 1:  # if pointer was allowing for that?
+                                    provided_requestables[domain].append('reference')
+
+                            else:
+                                provided_requestables[domain].append('reference')
+                    else:
+                        if domain + '_' + requestable + ']' in sent_t:
+                            provided_requestables[domain].append(requestable)
+
+        # if name was given in the task
+        for domain in goal.keys():
+            # if name was provided for the user, the match is being done automatically
+            # if realDialogue['goal'][domain].has_key('info'):
+            if 'info' in realDialogue['goal'][domain]:
+                # if realDialogue['goal'][domain]['info'].has_key('name'):
+                if 'name' in realDialogue['goal'][domain]['info']:
+                    venue_offered[domain] = '[' + domain + '_name]'
+
+            # special domains - entity does not need to be provided
+            if domain in ['taxi', 'police', 'hospital']:
+                venue_offered[domain] = '[' + domain + '_name]'
+
+            # the original method
+            # if domain == 'train':
+            #     if not venue_offered[domain]:
+            #         # if realDialogue['goal'][domain].has_key('reqt') and 'id' not in realDialogue['goal'][domain]['reqt']:
+            #         if 'reqt' in realDialogue['goal'][domain] and 'id' not in realDialogue['goal'][domain]['reqt']:
+            #             venue_offered[domain] = '[' + domain + '_name]'
+
+            # Wrong one in HDSA
+            # if domain == 'train':
+            #     if not venue_offered[domain]:
+            #         if goal[domain]['requestable'] and 'id' not in goal[domain]['requestable']:
+            #             venue_offered[domain] = '[' + domain + '_name]'
+
+            # if id was not requested but train was found we dont want to override it to check if we booked the right train
+            if domain == 'train' and (not venue_offered[domain] and 'id' not in goal['train']['requestable']):
+                venue_offered[domain] = '[' + domain + '_name]'
+
+        """
+        Given all inform and requestable slots
+        we go through each domain from the user goal
+        and check whether right entity was provided and
+        all requestable slots were given to the user.
+        The dialogue is successful if that's the case for all domains.
+        """
+        # HARD EVAL
+        stats = {'restaurant': [0, 0, 0], 'hotel': [0, 0, 0], 'attraction': [0, 0, 0], 'train': [0, 0, 0],
+                 'taxi': [0, 0, 0],
+                 'hospital': [0, 0, 0], 'police': [0, 0, 0]}
+
+        match = 0
+        success = 0
+        # MATCH
+        for domain in goal.keys():
+            match_stat = 0
+            if domain in ['restaurant', 'hotel', 'attraction', 'train']:
+                goal_venues = self.db.queryResultVenues(domain, goal[domain]['informable'], real_belief=True)
+                if type(venue_offered[domain]) is str and '_name' in venue_offered[domain]:
+                    match += 1
+                    match_stat = 1
+                elif len(venue_offered[domain]) > 0 and venue_offered[domain][0] in goal_venues:
+                    match += 1
+                    match_stat = 1
+            else:
+                if domain + '_name]' in venue_offered[domain]:
+                    match += 1
+                    match_stat = 1
+
+            stats[domain][0] = match_stat
+            stats[domain][2] = 1
+
+        if soft_acc:
+            match = float(match)/len(goal.keys())
+        else:
+            if match == len(goal.keys()):
+                match = 1.0
+            else:
+                match = 0.0
+
+        # SUCCESS
+        if match == 1.0:
+            for domain in domains_in_goal:
+                success_stat = 0
+                domain_success = 0
+                if len(real_requestables[domain]) == 0:
+                    success += 1
+                    success_stat = 1
+                    stats[domain][1] = success_stat
+                    continue
+                # if values in sentences are super set of requestables
+                for request in set(provided_requestables[domain]):
+                    if request in real_requestables[domain]:
+                        domain_success += 1
+
+                if domain_success >= len(real_requestables[domain]):
+                    success += 1
+                    success_stat = 1
+
+                stats[domain][1] = success_stat
+
+            # final eval
+            if soft_acc:
+                success = float(success)/len(real_requestables)
+            else:
+                if success >= len(real_requestables):
+                    success = 1
+                else:
+                    success = 0
+
+        # rint requests, 'DIFF', requests_real, 'SUCC', success
+        return success, match, stats
+
+    def _evaluateRealDialogue(self, dialog, filename):
+        """Evaluation of the real dialogue from corpus.
+        First we loads the user goal and then go through the dialogue history.
+        Similar to evaluateGeneratedDialogue above."""
+        domains = ['restaurant', 'hotel', 'attraction', 'train', 'taxi', 'hospital', 'police']
+        requestables = ['phone', 'address', 'postcode', 'reference', 'id']
+
+        # get the list of domains in the goal
+        domains_in_goal = []
+        goal = {}
+        for domain in domains:
+            if dialog['goal'][domain]:
+                goal = self._parseGoal(goal, dialog, domain)
+                domains_in_goal.append(domain)
+
+        # compute corpus success
+        real_requestables = {}
+        provided_requestables = {}
+        venue_offered = {}
+        for domain in goal.keys():
+            provided_requestables[domain] = []
+            venue_offered[domain] = []
+            real_requestables[domain] = goal[domain]['requestable']
+
+        # iterate each turn
+        m_targetutt = [turn['text'] for idx, turn in enumerate(dialog['log']) if idx % 2 == 1]
+        for t in range(len(m_targetutt)):
+            for domain in domains_in_goal:
+                sent_t = m_targetutt[t]
+                # for computing match - where there are limited entities
+                if domain + '_name' in sent_t or '_id' in sent_t:
+                    if domain in ['restaurant', 'hotel', 'attraction', 'train']:
+                        # HERE YOU CAN PUT YOUR BELIEF STATE ESTIMATION
+                        venues = self.db.queryResultVenues(domain, dialog['log'][t * 2 + 1])
+
+                        # if venue has changed
+                        if len(venue_offered[domain]) == 0 and venues:
+                            venue_offered[domain] = random.sample(venues, 1)
+                        else:
+                            flag = False
+                            for ven in venues:
+                                if venue_offered[domain][0] == ven:
+                                    flag = True
+                                    break
+                            if not flag and venues:  # sometimes there are no results so sample won't work
+                                # print venues
+                                venue_offered[domain] = random.sample(venues, 1)
+                    else:  # not limited so we can provide one
+                        venue_offered[domain] = '[' + domain + '_name]'
+
+                for requestable in requestables:
+                    # check if reference could be issued
+                    if requestable == 'reference':
+                        if domain + '_reference' in sent_t:
+                            if 'restaurant_reference' in sent_t:
+                                if dialog['log'][t * 2]['db_pointer'][-5] == 1:  # if pointer was allowing for that?
+                                    provided_requestables[domain].append('reference')
+
+                            elif 'hotel_reference' in sent_t:
+                                if dialog['log'][t * 2]['db_pointer'][-3] == 1:  # if pointer was allowing for that?
+                                    provided_requestables[domain].append('reference')
+
+                                    # return goal, 0, match, real_requestables
+                            elif 'train_reference' in sent_t:
+                                if dialog['log'][t * 2]['db_pointer'][-1] == 1:  # if pointer was allowing for that?
+                                    provided_requestables[domain].append('reference')
+
+                            else:
+                                provided_requestables[domain].append('reference')
+                    else:
+                        if domain + '_' + requestable in sent_t:
+                            provided_requestables[domain].append(requestable)
+
+        # offer was made?
+        for domain in domains_in_goal:
+            # if name was provided for the user, the match is being done automatically
+            # if dialog['goal'][domain].has_key('info'):
+            if 'info' in dialog['goal'][domain]:
+                # if dialog['goal'][domain]['info'].has_key('name'):
+                if 'name' in dialog['goal'][domain]['info']:
+                    venue_offered[domain] = '[' + domain + '_name]'
+
+            # special domains - entity does not need to be provided
+            if domain in ['taxi', 'police', 'hospital']:
+                venue_offered[domain] = '[' + domain + '_name]'
+
+            # if id was not requested but train was found we dont want to override it to check if we booked the right train
+            if domain == 'train' and (not venue_offered[domain] and 'id' not in goal['train']['requestable']):
+                venue_offered[domain] = '[' + domain + '_name]'
+
+        # HARD (0-1) EVAL
+        stats = {'restaurant': [0, 0, 0], 'hotel': [0, 0, 0], 'attraction': [0, 0, 0], 'train': [0, 0, 0],
+                 'taxi': [0, 0, 0],
+                 'hospital': [0, 0, 0], 'police': [0, 0, 0]}
+
+        match, success = 0, 0
+        # MATCH
+        for domain in goal.keys():
+            match_stat = 0
+            if domain in ['restaurant', 'hotel', 'attraction', 'train']:
+                goal_venues = self.db.queryResultVenues(domain, dialog['goal'][domain]['info'], real_belief=True)
+                # print(goal_venues)
+                if type(venue_offered[domain]) is str and '_name' in venue_offered[domain]:
+                    match += 1
+                    match_stat = 1
+                elif len(venue_offered[domain]) > 0 and venue_offered[domain][0] in goal_venues:
+                    match += 1
+                    match_stat = 1
+
+            else:
+                if domain + '_name' in venue_offered[domain]:
+                    match += 1
+                    match_stat = 1
+
+            stats[domain][0] = match_stat
+            stats[domain][2] = 1
+
+        if match == len(goal.keys()):
+            match = 1
+        else:
+            match = 0
+
+        # SUCCESS
+        if match:
+            for domain in domains_in_goal:
+                domain_success = 0
+                success_stat = 0
+                if len(real_requestables[domain]) == 0:
+                    # check that
+                    success += 1
+                    success_stat = 1
+                    stats[domain][1] = success_stat
+                    continue
+                # if values in sentences are super set of requestables
+                for request in set(provided_requestables[domain]):
+                    if request in real_requestables[domain]:
+                        domain_success += 1
+
+                if domain_success >= len(real_requestables[domain]):
+                    success += 1
+                    success_stat = 1
+
+                stats[domain][1] = success_stat
+
+            # final eval
+            if success >= len(real_requestables):
+                success = 1
+            else:
+                success = 0
+
+        return goal, success, match, real_requestables, stats
+
+    def _evaluateRolloutDialogue(self, dialog):
+        domains = ['restaurant', 'hotel', 'attraction', 'train', 'taxi', 'hospital', 'police']
+        requestables = ['phone', 'address', 'postcode', 'reference', 'id']
+
+        # get the list of domains in the goal
+        domains_in_goal = []
+        goal = {}
+        for domain in domains:
+            if dialog['goal'][domain]:
+                goal = self._parseGoal(goal, dialog, domain)
+                domains_in_goal.append(domain)
+
+        # compute corpus success
+        real_requestables = {}
+        provided_requestables = {}
+        venue_offered = {}
+        for domain in goal.keys():
+            provided_requestables[domain] = []
+            venue_offered[domain] = []
+            real_requestables[domain] = goal[domain]['requestable']
+
+        # iterate each turn
+        m_targetutt = [turn['text'] for idx, turn in enumerate(dialog['log']) if idx % 2 == 1]
+        for t in range(len(m_targetutt)):
+            for domain in domains_in_goal:
+                sent_t = m_targetutt[t]
+                # for computing match - where there are limited entities
+                if domain + '_name' in sent_t or domain+'_id' in sent_t:
+                    if domain in ['restaurant', 'hotel', 'attraction', 'train']:
+                        venue_offered[domain] = '[' + domain + '_name]'
+                        """
+                        venues = self.db.queryResultVenues(domain, dialog['log'][t * 2 + 1])
+                        if len(venue_offered[domain]) == 0 and venues:
+                            venue_offered[domain] = random.sample(venues, 1)
+                        else:
+                            flag = False
+                            for ven in venues:
+                                if venue_offered[domain][0] == ven:
+                                    flag = True
+                                    break
+                            if not flag and venues:  # sometimes there are no results so sample won't work
+                                # print venues
+                                venue_offered[domain] = random.sample(venues, 1)
+                        """
+                    else:  # not limited so we can provide one
+                        venue_offered[domain] = '[' + domain + '_name]'
+
+                for requestable in requestables:
+                    # check if reference could be issued
+                    if requestable == 'reference':
+                        if domain + '_reference' in sent_t:
+                            if 'restaurant_reference' in sent_t:
+                                if True or dialog['log'][t * 2]['db_pointer'][-5] == 1:  # if pointer was allowing for that?
+                                    provided_requestables[domain].append('reference')
+
+                            elif 'hotel_reference' in sent_t:
+                                if True or dialog['log'][t * 2]['db_pointer'][-3] == 1:  # if pointer was allowing for that?
+                                    provided_requestables[domain].append('reference')
+                                    # return goal, 0, match, real_requestables
+                            elif 'train_reference' in sent_t:
+                                if True or dialog['log'][t * 2]['db_pointer'][-1] == 1:  # if pointer was allowing for that?
+                                    provided_requestables[domain].append('reference')
+
+                            else:
+                                provided_requestables[domain].append('reference')
+                    else:
+                        if domain + '_' + requestable in sent_t:
+                            provided_requestables[domain].append(requestable)
+
+        # offer was made?
+        for domain in domains_in_goal:
+            # if name was provided for the user, the match is being done automatically
+            # if dialog['goal'][domain].has_key('info'):
+            if 'info' in dialog['goal'][domain]:
+                # if dialog['goal'][domain]['info'].has_key('name'):
+                if 'name' in dialog['goal'][domain]['info']:
+                    venue_offered[domain] = '[' + domain + '_name]'
+
+            # special domains - entity does not need to be provided
+            if domain in ['taxi', 'police', 'hospital']:
+                venue_offered[domain] = '[' + domain + '_name]'
+
+            # if id was not requested but train was found we dont want to override it to check if we booked the right train
+            if domain == 'train' and (not venue_offered[domain] and 'id' not in goal['train']['requestable']):
+                venue_offered[domain] = '[' + domain + '_name]'
+
+        # REWARD CALCULATION
+        stats = {'restaurant': [0, 0, 0], 'hotel': [0, 0, 0], 'attraction': [0, 0, 0], 'train': [0, 0, 0],
+                 'taxi': [0, 0, 0], 'hospital': [0, 0, 0], 'police': [0, 0, 0]}
+        match, success = 0.0, 0.0
+        # MATCH
+        for domain in goal.keys():
+            match_stat = 0
+            if domain in ['restaurant', 'hotel', 'attraction', 'train']:
+                goal_venues = self.db.queryResultVenues(domain, dialog['goal'][domain]['info'], real_belief=True)
+                if type(venue_offered[domain]) is str and '_name' in venue_offered[domain]:
+                    match += 1
+                    match_stat = 1
+                elif len(venue_offered[domain]) > 0 and venue_offered[domain][0] in goal_venues:
+                    match += 1
+                    match_stat = 1
+            else:
+                if domain + '_name' in venue_offered[domain]:
+                    match += 1
+                    match_stat = 1
+
+            stats[domain][0] = match_stat
+            stats[domain][2] = 1
+
+        match = min(1.0, float(match) / len(goal.keys()))
+
+        # SUCCESS
+        if match:
+            for domain in domains_in_goal:
+                domain_success = 0
+                success_stat = 0
+                if len(real_requestables[domain]) == 0:
+                    # check that
+                    success += 1
+                    success_stat = 1
+                    stats[domain][1] = success_stat
+                    continue
+                # if values in sentences are super set of requestables
+                for request in set(provided_requestables[domain]):
+                    if request in real_requestables[domain]:
+                        domain_success += 1
+
+                if domain_success >= len(real_requestables[domain]):
+                    success += 1
+                    success_stat = 1
+
+                stats[domain][1] = success_stat
+
+            # final eval
+            success = min(1.0, float(success) / len(real_requestables))
+
+        return success, match, stats
+
+    def _parse_entities(self, tokens):
+        entities = []
+        for t in tokens:
+            if '[' in t and ']' in t:
+                entities.append(t)
+        return entities
+
+    def evaluateModel(self, dialogues, mode='valid', new_version=False):
+        """Gathers statistics for the whole sets."""
+        delex_dialogues = self.delex_dialogues
+        # pdb.set_trace()
+        successes, matches = 0, 0
+        corpus_successes, corpus_matches = 0, 0
+        total = 0
+
+        gen_stats = {'restaurant': [0, 0, 0], 'hotel': [0, 0, 0], 'attraction': [0, 0, 0], 'train': [0, 0, 0],
+                     'taxi': [0, 0, 0],
+                     'hospital': [0, 0, 0], 'police': [0, 0, 0]}
+        sng_gen_stats = {'restaurant': [0, 0, 0], 'hotel': [0, 0, 0], 'attraction': [0, 0, 0], 'train': [0, 0, 0],
+                         'taxi': [0, 0, 0], 'hospital': [0, 0, 0], 'police': [0, 0, 0]}
+
+        for filename, dial in dialogues.items():
+            if mode == 'rollout':
+                success, match, stats = self._evaluateRolloutDialogue(dial)
+            else:
+                # data is ground truth, dial is generated
+                data = delex_dialogues[filename]
+                goal, success, match, requestables, _ = self._evaluateRealDialogue(data, filename) # only goal and requestables are kept
+                corpus_successes += success
+                corpus_matches += match
+                if new_version:
+                    success, match, stats = self._evaluateGeneratedDialogue_new(dial, goal, data, requestables,
+                                                                            soft_acc=mode =='offline_rl')
+                else:
+                    success, match, stats = self._evaluateGeneratedDialogue(dial, goal, data, requestables,
+                                                                            soft_acc=mode =='offline_rl')
+
+            successes += success
+            matches += match
+            total += 1
+
+            for domain in gen_stats.keys():
+                gen_stats[domain][0] += stats[domain][0]
+                gen_stats[domain][1] += stats[domain][1]
+                gen_stats[domain][2] += stats[domain][2]
+
+            if 'SNG' in filename:
+                for domain in gen_stats.keys():
+                    sng_gen_stats[domain][0] += stats[domain][0]
+                    sng_gen_stats[domain][1] += stats[domain][1]
+                    sng_gen_stats[domain][2] += stats[domain][2]
+
+        report = ""
+        report += '{} Corpus Matches : {:2.2f}%, Groundtruth {} Matches : {:2.2f}%'.format(mode, (matches / float(total) * 100), mode, (corpus_matches / float(total) * 100)) + "\n"
+        report += '{} Corpus Success : {:2.2f}%, Groundtruth {} Success : {:2.2f}%'.format(mode, (successes / float(total) * 100), mode, (corpus_successes / float(total) * 100)) + "\n"
+        report += 'Total number of dialogues: %s, new version=%s ' % (total, new_version)
+
+        self.logger.info(report)
+        return report, successes/float(total), matches/float(total)
+    
+    def get_report(self):
+        tokenize = lambda x: x.split()
+        print('Generate report for {} samples'.format(len(self.hyps)))
+        refs, hyps = [], []
+        tp, fp, fn = 0, 0, 0
+        for label, hyp in zip(self.labels, self.hyps):
+            ref_tokens = [BOS] + tokenize(label.replace(SYS, '').replace(USR, '').strip()) + [EOS]
+            hyp_tokens = [BOS] + tokenize(hyp.replace(SYS, '').replace(USR, '').strip()) + [EOS]
+            refs.append([ref_tokens])
+            hyps.append(hyp_tokens)
+
+            ref_entities = self._parse_entities(ref_tokens)
+            hyp_entities = self._parse_entities(hyp_tokens)
+            tpp, fpp, fnn = self._get_tp_fp_fn(ref_entities, hyp_entities)
+            tp += tpp
+            fp += fpp
+            fn += fnn
+
+        # bleu = corpus_bleu(refs, hyps, smoothing_function=SmoothingFunction().method1)
+        bleu = BLEUScorer().score(hyps, refs) 
+        prec, rec, f1 = self._get_prec_recall(tp, fp, fn)
+        report = "\nBLEU score {}\nEntity precision {:.4f} recall {:.4f} and f1 {:.4f}\n".format(bleu, prec, rec, f1)
+        return report, bleu, prec, rec, f1
+
+    def get_groundtruth_report(self):
+        tokenize = lambda x: x.split()
+        print('Generate report for {} samples'.format(len(self.hyps)))
+        refs, hyps = [], []
+        tp, fp, fn = 0, 0, 0
+        for label, hyp in zip(self.labels, self.hyps):
+            ref_tokens = [BOS] + tokenize(label.replace(SYS, '').replace(USR, '').strip()) + [EOS]
+            refs.append([ref_tokens])
+
+            ref_entities = self._parse_entities(ref_tokens)
+            tpp, fpp, fnn = self._get_tp_fp_fn(ref_entities, ref_entities)
+            tp += tpp
+            fp += fpp
+            fn += fnn
+
+        # bleu = corpus_bleu(refs, hyps, smoothing_function=SmoothingFunction().method1)
+        # bleu = BLEUScorer().score(refs, refs) 
+        prec, rec, f1 = self._get_prec_recall(tp, fp, fn)
+        # report = "\nGroundtruth BLEU score {}\nEntity precision {:.4f} recall {:.4f} and f1 {:.4f}\n".format(bleu, prec, rec, f1)
+        report = "\nGroundtruth\nEntity precision {:.4f} recall {:.4f} and f1 {:.4f}\n".format(prec, rec, f1)
+        return report, 0, prec, rec, f1
+
+class SimDialEvaluator(BaseEvaluator):
+    CUR_DIR = os.path.dirname(__file__).replace('latent_dialog', '')
+    logger = logging.getLogger()
+    def __init__(self, data_name):
+        self.data_name = data_name
+        self.slot_dict = delex.prepareSlotValuesIndependent()
+        # self.delex_dialogues = json.load(open(os.path.join(self.CUR_DIR, 'data/norm-multi-woz/delex.json')))
+        # self.db = MultiWozDB()
+        self.labels = list()
+        self.hyps = list()
+
+    def initialize(self):
+        self.labels = list()
+        self.hyps = list()
+
+    def add_example(self, ref, hyp):
+        self.labels.append(ref)
+        self.hyps.append(hyp)
+
+    def _parse_entities(self, tokens):
+        entities = []
+        for t in tokens:
+            if '[' in t and ']' in t:
+                entities.append(t)
+        return entities
+
+    def get_report(self):
+        tokenize = lambda x: x.split()
+        print('Generate report for {} samples'.format(len(self.hyps)))
+        refs, hyps = [], []
+        tp, fp, fn = 0, 0, 0
+        for label, hyp in zip(self.labels, self.hyps):
+            ref_tokens = [BOS] + tokenize(label.replace(SYS, '').replace(USR, '').strip()) + [EOS]
+            hyp_tokens = [BOS] + tokenize(hyp.replace(SYS, '').replace(USR, '').strip()) + [EOS]
+            refs.append([ref_tokens])
+            hyps.append(hyp_tokens)
+
+            ref_entities = self._parse_entities(ref_tokens)
+            hyp_entities = self._parse_entities(hyp_tokens)
+            tpp, fpp, fnn = self._get_tp_fp_fn(ref_entities, hyp_entities)
+            tp += tpp
+            fp += fpp
+            fn += fnn
+
+        # bleu = corpus_bleu(refs, hyps, smoothing_function=SmoothingFunction().method1)
+        bleu = BLEUScorer().score(hyps, refs) 
+        prec, rec, f1 = self._get_prec_recall(tp, fp, fn)
+        report = "\nBLEU score {}\nEntity precision {:.4f} recall {:.4f} and f1 {:.4f}\n".format(bleu, prec, rec, f1)
+        return report, bleu, prec, rec, f1
+
+    def get_groundtruth_report(self):
+        tokenize = lambda x: x.split()
+        print('Generate report for {} samples'.format(len(self.hyps)))
+        refs, hyps = [], []
+        tp, fp, fn = 0, 0, 0
+        for label, hyp in zip(self.labels, self.hyps):
+            ref_tokens = [BOS] + tokenize(label.replace(SYS, '').replace(USR, '').strip()) + [EOS]
+            refs.append([ref_tokens])
+
+            ref_entities = self._parse_entities(ref_tokens)
+            tpp, fpp, fnn = self._get_tp_fp_fn(ref_entities, ref_entities)
+            tp += tpp
+            fp += fpp
+            fn += fnn
+
+        # bleu = corpus_bleu(refs, hyps, smoothing_function=SmoothingFunction().method1)
+        # bleu = BLEUScorer().score(refs, refs) 
+        prec, rec, f1 = self._get_prec_recall(tp, fp, fn)
+        # report = "\nGroundtruth BLEU score {}\nEntity precision {:.4f} recall {:.4f} and f1 {:.4f}\n".format(bleu, prec, rec, f1)
+        report = "\nGroundtruth\nEntity precision {:.4f} recall {:.4f} and f1 {:.4f}\n".format(prec, rec, f1)
+        return report, 0, prec, rec, f1
+
+class TurnEvaluator(BaseEvaluator):
+    """
+    Use string matching to find the F-1 score of slots
+    Use logistic regression to find F-1 score of acts
+    Use string matching to find F-1 score of KB_SEARCH
+    """
+    CLF = "clf"
+    REPRESENTATION = "rep"
+    ID2TAG = "id2tag"
+    TAG2ID = "tag2id"
+    logger = logging.getLogger()
+
+    def __init__(self, data_name, turn_corpus, domain_meta):
+        self.data_name = data_name
+        # train a dialog act classifier
+        domain2ids = defaultdict(list)
+        for d_id, d in enumerate(turn_corpus):
+            domain2ids[d.domain].append(d_id)
+        selected_ids = [v[0:1000] for v in domain2ids.values()]
+        corpus = [turn_corpus[idx] for idxs in selected_ids for idx in idxs]
+
+        self.model = self.get_intent_tagger(corpus)
+
+        # get entity value vocabulary
+        self.domain_id2ent = self.get_entity_dict_from_meta(domain_meta)
+
+        # Initialize containers
+        self.domain_labels = defaultdict(list)
+        self.domain_hyps = defaultdict(list)
+
+    def get_entity_dict_from_meta(self, domain_meta):
+        # get entity value vocabulary
+        domain_id2ent = defaultdict(set)
+        for domain, meta in domain_meta.items():
+            domain_id2ent[domain].add("QUERY")
+            domain_id2ent[domain].add("GOALS")
+            for slot, vocab in meta.sys_slots.items():
+                domain_id2ent[domain].add(slot)
+                for v in vocab:
+                    domain_id2ent[domain].add(v)
+
+            for slot, vocab in meta.usr_slots.items():
+                domain_id2ent[domain].add(slot)
+                for v in vocab:
+                    domain_id2ent[domain].add(v)
+
+        domain_id2ent = {k: list(v) for k, v in domain_id2ent.items()}
+        return domain_id2ent
+
+    def get_entity_dict(self, turn_corpus):
+        utt2act = {}
+        for msg in turn_corpus:
+            utt2act[" ".join(msg.utt[1:-1])] = msg
+
+        detokenize = get_detokenize()
+        utt2act = {detokenize(k.split()): v for k, v in utt2act.items()}
+        self.logger.info("Compress utt2act from {}->{}".format(len(turn_corpus), len(utt2act)))
+
+        # get entity value vocabulary
+        domain_id2ent = defaultdict(set)
+        for utt, msg in utt2act.items():
+            for act in msg.actions:
+                paras = act['parameters']
+                intent = act['act']
+                if intent == 'inform':
+                    for v in paras[0].values():
+                        domain_id2ent[msg.domain].add(str(v))
+                elif intent == 'query':
+                    for v in paras[0].values():
+                        domain_id2ent[msg.domain].add(v)
+                else:
+                    for k, v in paras:
+                        if v:
+                            domain_id2ent[msg.domain].add(v)
+        domain_id2ent = {k: list(v) for k, v in domain_id2ent.items()}
+        return domain_id2ent
+
+    def get_intent_tagger(self, corpus):
+        """
+        :return: train a dialog act tagger for system utterances 
+        """
+        self.logger.info("Train a new intent tagger")
+        all_tags, utts, tags = [], [], []
+        de_tknize = get_detokenize()
+        for msg in corpus:
+            utts.append(de_tknize(msg.utt[1:-1]))
+            tags.append([a['act'] for a in msg.actions])
+            all_tags.extend([a['act'] for a in msg.actions])
+
+        most_common = Counter(all_tags).most_common()
+        self.logger.info(most_common)
+        tag_set = [t for t, c, in most_common]
+        rev_tag_set = {t: i for i, t in enumerate(tag_set)}
+
+        # create train and test set:
+        data_size = len(corpus)
+        train_size = int(data_size * 0.7)
+        train_utts = utts[0:train_size]
+        test_utts = utts[train_size:]
+
+        # create y:
+        sparse_y = np.zeros([data_size, len(tag_set)])
+        for idx, utt_tags in enumerate(tags):
+            for tag in utt_tags:
+                sparse_y[idx, rev_tag_set[tag]] = 1
+        train_y = sparse_y[0:train_size, :]
+        test_y = sparse_y[train_size:, :]
+
+        # train classifier
+        representation = CountVectorizer(ngram_range=[1, 2]).fit(train_utts)
+        train_x = representation.transform(train_utts)
+        test_x = representation.transform(test_utts)
+
+        clf = OneVsRestClassifier(SGDClassifier(loss='hinge', max_iter=10)).fit(train_x, train_y)
+        pred_test_y = clf.predict(test_x)
+
+        def print_report(score_name, scores, names):
+            for s, n in zip(scores, names):
+                self.logger.info("%s: %s -> %f" % (score_name, n, s))
+
+        print_report('F1', metrics.f1_score(test_y, pred_test_y, average=None),
+                     tag_set)
+
+        x = representation.transform(utts)
+        clf = OneVsRestClassifier(SGDClassifier(loss='hinge', max_iter=20)) \
+            .fit(x, sparse_y)
+
+        model_dump = {self.CLF: clf, self.REPRESENTATION: representation,
+                      self.ID2TAG: tag_set,
+                      self.TAG2ID: rev_tag_set}
+        # pkl.dump(model_dump, open("{}.pkl".format(self.data_name), "wb"))
+        return model_dump
+
+    def pred_ents(self, sentence, tokenize, domain):
+        pred_ents = []
+        padded_hyp = "/{}/".format("/".join(tokenize(sentence)))
+        for e in self.domain_id2ent[domain]:
+            count = padded_hyp.count("/{}/".format(e))
+            if domain =='movie' and e == 'I':
+                continue
+            pred_ents.extend([e] * count)
+        return pred_ents
+
+    def pred_acts(self, utts):
+        test_x = self.model[self.REPRESENTATION].transform(utts)
+        pred_test_y = self.model[self.CLF].predict(test_x)
+        pred_tags = []
+        for ys in pred_test_y:
+            temp = []
+            for i in range(len(ys)):
+                if ys[i] == 1:
+                    temp.append(self.model[self.ID2TAG][i])
+            pred_tags.append(temp)
+        return pred_tags
+
+    """
+    Public Functions
+    """
+    def initialize(self):
+        self.domain_labels = defaultdict(list)
+        self.domain_hyps = defaultdict(list)
+
+    def add_example(self, ref, hyp, domain='default'):
+        self.domain_labels[domain].append(ref)
+        self.domain_hyps[domain].append(hyp)
+
+    def get_report(self, include_error=False):
+        reports = []
+
+        errors = []
+
+        for domain, labels in self.domain_labels.items():
+            intent2refs = defaultdict(list)
+            intent2hyps = defaultdict(list)
+
+            predictions = self.domain_hyps[domain]
+            self.logger.info("Generate report for {} for {} samples".format(domain, len(predictions)))
+
+            # find entity precision, recall and f1
+            tp, fp, fn = 0.0, 0.0, 0.0
+
+            # find intent precision recall f1
+            itp, ifp, ifn = 0.0, 0.0, 0.0
+
+            # backend accuracy
+            btp, bfp, bfn = 0.0, 0.0, 0.0
+
+            # BLEU score
+            refs, hyps = [], []
+
+            pred_intents = self.pred_acts(predictions)
+            label_intents = self.pred_acts(labels)
+
+            tokenize = get_tokenize()
+            bad_predictions = []
+
+            for label, hyp, label_ints, pred_ints in zip(labels, predictions, label_intents, pred_intents):
+                refs.append([label.split()])
+                hyps.append(hyp.split())
+
+                # pdb.set_trace()
+
+                label_ents = self.pred_ents(label, tokenize, domain)
+                pred_ents = self.pred_ents(hyp, tokenize, domain)
+
+                for intent in label_ints:
+                    intent2refs[intent].append([label.split()])
+                    intent2hyps[intent].append(hyp.split())
+
+                # update the intent
+                ttpp, ffpp, ffnn = self._get_tp_fp_fn(label_ints, pred_ints)
+                itp += ttpp
+                ifp += ffpp
+                ifn += ffnn
+
+                # entity or KB search
+                ttpp, ffpp, ffnn = self._get_tp_fp_fn(label_ents, pred_ents)
+                if ffpp > 0 or ffnn > 0:
+                    bad_predictions.append((label, hyp))
+
+                if "query" in label_ints:
+                    btp += ttpp
+                    bfp += ffpp
+                    bfn += ffnn
+                else:
+                    tp += ttpp
+                    fp += ffpp
+                    fn += ffnn
+
+            # compute corpus level scores
+            bleu = bleu_score.corpus_bleu(refs, hyps, smoothing_function=SmoothingFunction().method1)
+            ent_precision, ent_recall, ent_f1 = self._get_prec_recall(tp, fp, fn)
+            int_precision, int_recall, int_f1 = self._get_prec_recall(itp, ifp, ifn)
+            back_precision, back_recall, back_f1 = self._get_prec_recall(btp, bfp, bfn)
+
+            # compute BLEU w.r.t intents
+            intent_report = []
+            for intent in intent2refs.keys():
+                i_bleu = bleu_score.corpus_bleu(intent2refs[intent], intent2hyps[intent],
+                                                smoothing_function=SmoothingFunction().method1)
+                intent_report.append("{}: {}".format(intent, i_bleu))
+
+            intent_report = "\n".join(intent_report)
+
+            # create bad cases
+            error = ''
+            if include_error:
+                error = '\nDomain {} errors\n'.format(domain)
+                error += "\n".join(['True: {} ||| Pred: {}'.format(r, h)
+                                    for r, h in bad_predictions])
+            report = "\nDomain: %s\n" \
+                     "Entity precision %f recall %f and f1 %f\n" \
+                     "Intent precision %f recall %f and f1 %f\n" \
+                     "KB precision %f recall %f and f1 %f\n" \
+                     "BLEU %f BEAK %f\n\n%s\n" \
+                     % (domain,
+                        ent_precision, ent_recall, ent_f1,
+                        int_precision, int_recall, int_f1,
+                        back_precision, back_recall, back_f1,
+                        bleu, gmean([ent_f1, int_f1, back_f1, bleu]),
+                        intent_report)
+            reports.append(report)
+            errors.append(error)
+
+        if include_error:
+            return "\n==== REPORT===={error}\n========\n {report}".format(error="========".join(errors),
+                                                                          report="========".join(reports))
+        else:
+            return "\n==== REPORT===={report}".format(report="========".join(reports))
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/main.py b/convlab/policy/lava/multiwoz/latent_dialog/main.py
new file mode 100644
index 0000000000000000000000000000000000000000..44c1b053c3728098dc161da6af96135b40b6b5ad
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/main.py
@@ -0,0 +1,708 @@
+import os
+import sys
+import numpy as np
+import torch as th
+from torch import nn
+from collections import defaultdict, Counter
+from convlab.policy.lava.multiwoz.latent_dialog.enc2dec.base_modules import summary
+from convlab.policy.lava.multiwoz.latent_dialog.enc2dec.decoders import TEACH_FORCE, GEN, DecoderRNN
+from datetime import datetime
+from convlab.policy.lava.multiwoz.latent_dialog.utils import get_detokenize, LONG, FLOAT
+from convlab.policy.lava.multiwoz.latent_dialog.corpora import EOS, PAD
+from convlab.policy.lava.multiwoz.latent_dialog.data_loaders import BeliefDbDataLoaders
+from convlab.policy.lava.multiwoz.latent_dialog import evaluators
+from convlab.policy.lava.multiwoz.latent_dialog.record import record, record_task, UniquenessSentMetric, UniquenessWordMetric
+import logging
+import pdb
+
+logger = logging.getLogger()
+
+class LossManager(object):
+    def __init__(self):
+        self.losses = defaultdict(list)
+        self.backward_losses = []
+
+    def add_loss(self, loss):
+        for key, val in loss.items():
+            # print('key = %s\nval = %s' % (key, val))
+            if val is not None and type(val) is not bool:
+                self.losses[key].append(val.item())
+
+    def pprint(self, name, window=None, prefix=None):
+        str_losses = []
+        for key, loss in self.losses.items():
+            if loss is None:
+                continue
+            aver_loss = np.average(loss) if window is None else np.average(loss[-window:])
+            if 'nll' in key:
+                str_losses.append('{} PPL {:.3f}'.format(key, np.exp(aver_loss)))
+            else:
+                str_losses.append('{} {:.3f}'.format(key, aver_loss))
+
+
+        if prefix:
+            return '{}: {} {}'.format(prefix, name, ' '.join(str_losses))
+        else:
+            return '{} {}'.format(name, ' '.join(str_losses))
+
+    def clear(self):
+        self.losses = defaultdict(list)
+        self.backward_losses = []
+
+    def add_backward_loss(self, loss):
+        self.backward_losses.append(loss.item())
+
+    def avg_loss(self):
+        return np.mean(self.backward_losses)
+
+class OfflineTaskReinforce(object):
+    def __init__(self, agent, corpus, sv_config, sys_model, rl_config, generate_func):
+        self.agent = agent
+        self.corpus = corpus
+        self.sv_config = sv_config
+        self.sys_model = sys_model
+        self.rl_config = rl_config
+        # training func for supervised learning
+        self.train_func = task_train_single_batch
+        self.record_func = record_task
+        self.validate_func = validate
+
+        # prepare data loader
+        train_dial, val_dial, test_dial = self.corpus.get_corpus()
+        self.train_data = BeliefDbDataLoaders('Train', train_dial, self.sv_config)
+        self.sl_train_data = BeliefDbDataLoaders('Train', train_dial, self.sv_config)
+        self.val_data = BeliefDbDataLoaders('Val', val_dial, self.sv_config)
+        self.test_data = BeliefDbDataLoaders('Test', test_dial, self.sv_config)
+
+        # create log files
+        if self.rl_config.record_freq > 0:
+            self.learning_exp_file = open(os.path.join(self.rl_config.record_path, 'offline-learning.tsv'), 'w')
+            self.ppl_val_file = open(os.path.join(self.rl_config.record_path, 'val-ppl.tsv'), 'w')
+            self.rl_val_file = open(os.path.join(self.rl_config.record_path, 'val-rl.tsv'), 'w')
+            self.ppl_test_file = open(os.path.join(self.rl_config.record_path, 'test-ppl.tsv'), 'w')
+            self.rl_test_file = open(os.path.join(self.rl_config.record_path, 'test-rl.tsv'), 'w')
+        # evaluation
+        self.evaluator = evaluators.MultiWozEvaluator('SYS_WOZ')
+        self.generate_func = generate_func
+
+    def run(self):
+        n = 0
+        best_valid_loss = np.inf
+        best_rewards = -1 * np.inf
+
+        # BEFORE RUN, RECORD INITIAL PERFORMANCE
+        test_loss = self.validate_func(self.sys_model, self.test_data, self.sv_config, use_py=True)
+        t_success, t_match, t_bleu, t_f1 = self.generate_func(self.sys_model, self.test_data, self.sv_config,
+                                                              self.evaluator, None, verbose=False)
+
+        self.ppl_test_file.write('{}\t{}\t{}\t{}\n'.format(n, np.exp(test_loss), t_bleu, t_f1))
+        self.ppl_test_file.flush()
+        self.rl_test_file.write('{}\t{}\t{}\t{}\n'.format(n, (t_success + t_match), t_success, t_match))
+        self.rl_test_file.flush()
+
+        self.sys_model.train()
+        try:
+            for epoch_id in range(self.rl_config.nepoch):
+                self.train_data.epoch_init(self.sv_config, shuffle=True, verbose=epoch_id == 0, fix_batch=True)
+                while True:
+                    if n % self.rl_config.episode_repeat == 0:
+                        batch = self.train_data.next_batch()
+
+                    if batch is None:
+                        break
+
+                    n += 1
+                    if n % 50 == 0:
+                        print("Reinforcement Learning {}/{} episode".format(n, self.train_data.num_batch*self.rl_config.nepoch))
+                        self.learning_exp_file.write(
+                            '{}\t{}\n'.format(n, np.mean(self.agent.all_rewards[-50:])))
+                        self.learning_exp_file.flush()
+
+                    # reinforcement learning
+                    # make sure it's the same dialo
+                    assert len(set(batch['keys'])) == 1
+                    task_report, success, match = self.agent.run(batch, self.evaluator, max_words=self.rl_config.max_words, temp=self.rl_config.temperature)
+                    reward = float(success) # + float(match)
+                    stats = {'Match': match, 'Success': success}
+                    self.agent.update(reward, stats)
+
+                    # supervised learning
+                    if self.rl_config.sv_train_freq > 0 and n % self.rl_config.sv_train_freq == 0:
+                        self.train_func(self.sys_model, self.sl_train_data, self.sv_config)
+
+                    # record model performance in terms of several evaluation metrics
+                    if self.rl_config.record_freq > 0 and n % self.rl_config.record_freq == 0:
+                         self.agent.print_dialog(self.agent.dlg_history, reward, stats)
+                         print('-'*15, 'Recording start', '-'*15)
+                         # save train reward
+                         self.learning_exp_file.write('{}\t{}\n'.format(n, np.mean(self.agent.all_rewards[-self.rl_config.record_freq:])))
+                         self.learning_exp_file.flush()
+
+                         # PPL & reward on validation
+                         valid_loss = self.validate_func(self.sys_model, self.val_data, self.sv_config, use_py=True)
+                         v_success, v_match, v_bleu, v_f1 = self.generate_func(self.sys_model, self.val_data, self.sv_config, self.evaluator, None, verbose=False)
+                         self.ppl_val_file.write('{}\t{}\t{}\t{}\n'.format(n, np.exp(valid_loss), v_bleu, v_f1))
+                         self.ppl_val_file.flush()
+                         self.rl_val_file.write('{}\t{}\t{}\t{}\n'.format(n, (v_success + v_match), v_success, v_match))
+                         self.rl_val_file.flush()
+
+                         test_loss = self.validate_func(self.sys_model, self.test_data, self.sv_config, use_py=True)
+                         t_success, t_match, t_bleu, t_f1 = self.generate_func(self.sys_model, self.test_data, self.sv_config, self.evaluator, None, verbose=False)
+                         self.ppl_test_file.write('{}\t{}\t{}\t{}\n'.format(n, np.exp(test_loss), t_bleu, t_f1))
+                         self.ppl_test_file.flush()
+                         self.rl_test_file.write('{}\t{}\t{}\t{}\n'.format(n, (t_success + t_match), t_success, t_match))
+                         self.rl_test_file.flush()
+
+                         # save model is needed
+                         if v_success+v_match > best_rewards:
+                             print("Model saved with success {} match {}".format(v_success, v_match))
+                             th.save(self.sys_model.state_dict(), self.rl_config.reward_best_model_path)
+                             best_rewards = v_success+v_match
+
+
+                         self.sys_model.train()
+                         print('-'*15, 'Recording end', '-'*15)
+        except KeyboardInterrupt:
+            print("RL training stopped from keyboard")
+
+        print("$$$ Load {}-model".format(self.rl_config.reward_best_model_path))
+        self.sv_config.batch_size = 32
+        self.sys_model.load_state_dict(th.load(self.rl_config.reward_best_model_path))
+
+        validate(self.sys_model, self.val_data, self.sv_config, use_py=True)
+        validate(self.sys_model, self.test_data, self.sv_config, use_py=True)
+
+        with open(os.path.join(self.rl_config.record_path, 'valid_file.txt'), 'w') as f:
+            self.generate_func(self.sys_model, self.val_data, self.sv_config, self.evaluator, num_batch=None, dest_f=f)
+
+        with open(os.path.join(self.rl_config.record_path, 'test_file.txt'), 'w') as f:
+            self.generate_func(self.sys_model, self.test_data, self.sv_config, self.evaluator, num_batch=None, dest_f=f)
+
+def validate_rl(dialog_eval, ctx_gen, num_episode=200):
+    print("Validate on training goals for {} episode".format(num_episode))
+    reward_list = []
+    agree_list = []
+    sent_metric = UniquenessSentMetric()
+    word_metric = UniquenessWordMetric()
+    for _ in range(num_episode):
+        ctxs = ctx_gen.sample()
+        conv, agree, rewards = dialog_eval.run(ctxs)
+        true_reward = rewards[0] if agree else 0
+        reward_list.append(true_reward)
+        agree_list.append(float(agree if agree is not None else 0.0))
+        for turn in conv:
+            if turn[0] == 'Elder':
+                sent_metric.record(turn[1])
+                word_metric.record(turn[1])
+    results = {'sys_rew': np.average(reward_list),
+               'avg_agree': np.average(agree_list),
+               'sys_sent_unique': sent_metric.value(),
+               'sys_unique': word_metric.value()}
+    return results
+
+def train_single_batch(model, train_data, config):
+    batch_cnt = 0
+    optimizer = model.get_optimizer(config, verbose=False)
+    model.train()
+    
+    # decoding CE
+    train_data.epoch_init(config, shuffle=True, verbose=False)
+    for i in range(16):
+        batch = train_data.next_batch()
+        if batch is None:
+            train_data.epoch_init(config, shuffle=True, verbose=False)
+            batch = train_data.next_batch()
+        optimizer.zero_grad()
+        loss = model(batch, mode=TEACH_FORCE)
+        model.backward(loss, batch_cnt)
+        nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip)
+        optimizer.step()
+
+def task_train_single_batch(model, train_data, config):
+    batch_cnt = 0
+    optimizer = model.get_optimizer(config, verbose=False)
+    model.train()
+
+    # decoding CE
+    train_data.epoch_init(config, shuffle=True, verbose=False)
+    for i in range(16):
+        batch = train_data.next_batch()
+        if batch is None:
+            train_data.epoch_init(config, shuffle=True, verbose=False)
+            batch = train_data.next_batch()
+        optimizer.zero_grad()
+        loss = model(batch, mode=TEACH_FORCE)
+        model.backward(loss, batch_cnt)
+        nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip)
+        optimizer.step()
+
+def train(model, train_data, val_data, test_data, config, evaluator, gen=None):
+    patience = 10
+    valid_loss_threshold = np.inf
+    best_valid_loss = np.inf
+    batch_cnt = 0
+    optimizer = model.get_optimizer(config)
+    done_epoch = 0
+    best_epoch = 0
+    train_loss = LossManager()
+    model.train()
+    logger.info(summary(model, show_weights=False))
+    saved_models = []
+    last_n_model = config.last_n_model if hasattr(config, 'last_n_model') else 5
+
+    logger.info('***** Training Begins at {} *****'.format(datetime.now().strftime("%Y-%m-%d %H-%M-%S")))
+    logger.info('***** Epoch 0/{} *****'.format(config.max_epoch))
+    while True:
+        train_data.epoch_init(config, shuffle=True, verbose=done_epoch==0, fix_batch=config.fix_train_batch)
+        while True:
+            batch = train_data.next_batch()
+            if batch is None:
+                break
+    
+            optimizer.zero_grad()
+            loss = model(batch, mode=TEACH_FORCE)
+            model.backward(loss, batch_cnt)
+            nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip)
+            optimizer.step()
+            batch_cnt += 1
+            train_loss.add_loss(loss)
+    
+            if batch_cnt % config.print_step == 0:
+                # print('Print step at {}'.format(datetime.now().strftime("%Y-%m-%d %H-%M-%S")))
+                logger.info(train_loss.pprint('Train',
+                                        window=config.print_step, 
+                                        prefix='{}/{}-({:.3f})'.format(batch_cnt%config.ckpt_step, config.ckpt_step, model.kl_w)))
+                sys.stdout.flush()
+    
+            if batch_cnt % config.ckpt_step == 0:
+                logger.info('Checkpoint step at {}'.format(datetime.now().strftime("%Y-%m-%d %H-%M-%S")))
+                logger.info('==== Evaluating Model ====')
+                logger.info(train_loss.pprint('Train'))
+                done_epoch += 1
+                logger.info('done epoch {} -> {}'.format(done_epoch-1, done_epoch))
+
+                # generation
+                if gen is not None:
+                    gen(model, val_data, config, evaluator, num_batch=config.preview_batch_num)
+
+                # validation
+                valid_loss = validate(model, val_data, config, batch_cnt)
+                _ = validate(model, test_data, config, batch_cnt)
+
+                # update early stopping stats
+                if valid_loss < best_valid_loss:
+                    if valid_loss <= valid_loss_threshold * config.improve_threshold:
+                        patience = max(patience, done_epoch*config.patient_increase)
+                        valid_loss_threshold = valid_loss
+                        logger.info('Update patience to {}'.format(patience))
+    
+                    if config.save_model:
+                        cur_time = datetime.now().strftime("%Y-%m-%d %H-%M-%S")
+                        logger.info('!!Model Saved with loss = {},at {}.'.format(valid_loss, cur_time))
+                        th.save(model.state_dict(), os.path.join(config.saved_path, '{}-model'.format(done_epoch)))
+                        best_epoch = done_epoch
+                        saved_models.append(done_epoch)
+                        if len(saved_models) > last_n_model:
+                            remove_model = saved_models[0]
+                            saved_models = saved_models[-last_n_model:]
+                            os.remove(os.path.join(config.saved_path, "{}-model".format(remove_model)))
+    
+                    best_valid_loss = valid_loss
+    
+                if done_epoch >= config.max_epoch \
+                        or config.early_stop and patience <= done_epoch:
+                    if done_epoch < config.max_epoch:
+                        logger.info('!!!!! Early stop due to run out of patience !!!!!')
+                    print('Best validation loss = %f' % (best_valid_loss, ))
+                    return best_epoch
+    
+                # exit eval model
+                model.train()
+                train_loss.clear()
+                logger.info('\n***** Epoch {}/{} *****'.format(done_epoch, config.max_epoch))
+                sys.stdout.flush()
+
+def mt_train(model, train_data, val_data, test_data, aux_train_data, aux_val_data, aux_test_data, config, evaluator, gen=None):
+    patience = 10
+    valid_loss_threshold = np.inf
+    best_valid_loss = np.inf
+    batch_cnt = 0
+    optimizer = model.get_optimizer(config)
+    done_epoch = 0
+    best_epoch = 0
+    train_loss = LossManager()
+    model.train()
+    logger.info(summary(model, show_weights=False))
+    saved_models = []
+    last_n_model = config.last_n_model if hasattr(config, 'last_n_model') else 5
+
+    logger.info('***** Training Begins at {} *****'.format(datetime.now().strftime("%Y-%m-%d %H-%M-%S")))
+    logger.info('***** Epoch 0/{} *****'.format(config.max_epoch))
+    while True:
+        train_data.epoch_init(config, shuffle=True, verbose=done_epoch==0, fix_batch=config.fix_train_batch)
+        while True:
+            batch = train_data.next_batch()
+            if batch is None:
+                break
+    
+            optimizer.zero_grad()
+            loss = model(batch, mode=TEACH_FORCE)
+            model.backward(loss, batch_cnt)
+            nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip)
+            optimizer.step()
+            batch_cnt += 1
+            train_loss.add_loss(loss)
+    
+            if batch_cnt % config.print_step == 0:
+                # print('Print step at {}'.format(datetime.now().strftime("%Y-%m-%d %H-%M-%S")))
+                logger.info(train_loss.pprint('Train',
+                                        window=config.print_step, 
+                                        prefix='{}/{}-({:.3f})'.format(batch_cnt%config.ckpt_step, config.ckpt_step, model.kl_w)))
+                sys.stdout.flush()
+    
+            if batch_cnt % config.ckpt_step == 0:
+                logger.info('Checkpoint step at {}'.format(datetime.now().strftime("%Y-%m-%d %H-%M-%S")))
+                logger.info('==== Evaluating Model ====')
+                logger.info(train_loss.pprint('Train'))
+                done_epoch += 1
+                logger.info('done epoch {} -> {}'.format(done_epoch-1, done_epoch))
+
+                # generation
+                if gen is not None:
+                    gen(model, val_data, config, evaluator, num_batch=config.preview_batch_num)
+
+                # validation
+                valid_loss = validate(model, val_data, config, batch_cnt)
+                _ = validate(model, test_data, config, batch_cnt)
+
+                # update early stopping stats
+                if valid_loss < best_valid_loss:
+                    if valid_loss <= valid_loss_threshold * config.improve_threshold:
+                        patience = max(patience, done_epoch*config.patient_increase)
+                        valid_loss_threshold = valid_loss
+                        logger.info('Update patience to {}'.format(patience))
+    
+                    if config.save_model:
+                        cur_time = datetime.now().strftime("%Y-%m-%d %H-%M-%S")
+                        logger.info('!!Model Saved with loss = {},at {}.'.format(valid_loss, cur_time))
+                        th.save(model.state_dict(), os.path.join(config.saved_path, '{}-model'.format(done_epoch)))
+                        best_epoch = done_epoch
+                        saved_models.append(done_epoch)
+                        if len(saved_models) > last_n_model:
+                            remove_model = saved_models[0]
+                            saved_models = saved_models[-last_n_model:]
+                            os.remove(os.path.join(config.saved_path, "{}-model".format(remove_model)))
+    
+                    best_valid_loss = valid_loss
+    
+                if done_epoch >= config.max_epoch \
+                        or config.early_stop and patience <= done_epoch:
+                    if done_epoch < config.max_epoch:
+                        logger.info('!!!!! Early stop due to run out of patience !!!!!')
+                    print('Best validation loss = %f' % (best_valid_loss, ))
+                    return best_epoch
+    
+                
+                if done_epoch % config.aux_train_freq == 0:
+                    model.train()
+                    train_aux(model, aux_train_data, aux_val_data, aux_test_data, config, evaluator)
+                
+                # exit eval model
+                model.train()
+                train_loss.clear()
+
+                logger.info('\n***** Epoch {}/{} *****'.format(done_epoch, config.max_epoch))
+                sys.stdout.flush()
+                
+def train_aux(model, train_data, val_data, test_data, config, evaluator, gen=None):
+    patience = 10
+    valid_loss_threshold = np.inf
+    best_valid_loss = np.inf
+    batch_cnt = 0
+    optimizer = model.get_optimizer(config)
+    done_epoch = 0
+    best_epoch = 0
+    train_loss = LossManager()
+    model.train()
+    logger.info(summary(model, show_weights=False))
+    saved_models = []
+    last_n_model = config.last_n_model if hasattr(config, 'last_n_model') else 5
+
+    logger.info('+++++ Aux Training Begins at {} +++++'.format(datetime.now().strftime("%Y-%m-%d %H-%M-%S")))
+    logger.info('+++++ Epoch 0/{} +++++'.format(config.aux_max_epoch))
+    while True:
+        train_data.epoch_init(config, shuffle=True, verbose=done_epoch==0, fix_batch=config.fix_train_batch)
+        while True:
+            batch = train_data.next_batch()
+            if batch is None:
+                break
+    
+            optimizer.zero_grad()
+            loss = model.forward_aux(batch, mode=TEACH_FORCE)
+            model.backward(loss, batch_cnt)
+            nn.utils.clip_grad_norm_(model.parameters(), config.grad_clip)
+            optimizer.step()
+            batch_cnt += 1
+            train_loss.add_loss(loss)
+    
+            if batch_cnt % config.print_step == 0:
+                # print('Print step at {}'.format(datetime.now().strftime("%Y-%m-%d %H-%M-%S")))
+                logger.info(train_loss.pprint('Train',
+                                        window=config.print_step, 
+                                        prefix='{}/{}-({:.3f})'.format(batch_cnt%config.ckpt_step, config.ckpt_step, model.kl_w)))
+                sys.stdout.flush()
+    
+            if batch_cnt % config.ckpt_step == 0:
+                logger.info('Checkpoint step at {}'.format(datetime.now().strftime("%Y-%m-%d %H-%M-%S")))
+                logger.info('++++ Evaluating Model ++++')
+                logger.info(train_loss.pprint('Aux train'))
+                done_epoch += 1
+                logger.info('done epoch {} -> {}'.format(done_epoch-1, done_epoch))
+
+                # generation
+                if gen is not None:
+                    gen(model, val_data, config, evaluator, num_batch=config.preview_batch_num)
+
+                # validation
+                valid_loss = aux_validate(model, val_data, config, batch_cnt)
+                _ = aux_validate(model, test_data, config, batch_cnt)
+
+                # update early stopping stats
+                if valid_loss < best_valid_loss:
+                    if valid_loss <= valid_loss_threshold * config.improve_threshold:
+                        patience = max(patience, done_epoch*config.patient_increase)
+                        valid_loss_threshold = valid_loss
+                        logger.info('Update patience to {}'.format(patience))
+    
+                    if config.save_model:
+                        cur_time = datetime.now().strftime("%Y-%m-%d %H-%M-%S")
+                        logger.info('!!New best model with loss = {},at {}.'.format(valid_loss, cur_time))
+                        th.save(model.state_dict(), os.path.join(config.saved_path, 'aux-{}-model'.format(done_epoch)))
+                        best_epoch = done_epoch
+                        saved_models.append(done_epoch)
+                        if len(saved_models) > last_n_model:
+                            remove_model = saved_models[0]
+                            saved_models = saved_models[-last_n_model:]
+                            os.remove(os.path.join(config.saved_path, "{}-model".format(remove_model)))
+    
+                    best_valid_loss = valid_loss
+    
+                if done_epoch >= config.aux_max_epoch \
+                        or config.early_stop and patience <= done_epoch:
+                    if done_epoch < config.aux_max_epoch:
+                        logger.info('!!!!! Early stop due to run out of patience !!!!!')
+                    print('Best validation loss = %f' % (best_valid_loss, ))
+                    return best_epoch
+    
+                # exit eval model
+                model.train()
+                train_loss.clear()
+                logger.info('\n***** Epoch {}/{} *****'.format(done_epoch, config.aux_max_epoch))
+                sys.stdout.flush()
+
+def validate(model, val_data, config, batch_cnt=None, use_py=None):
+    model.eval()
+    val_data.epoch_init(config, shuffle=False, verbose=False)
+    losses = LossManager()
+    while True:
+        batch = val_data.next_batch()
+        if batch is None:
+            break
+        if use_py is not None:
+            # loss = model(batch, mode=TEACH_FORCE, use_py=use_py)
+            loss = model(batch, mode=TEACH_FORCE)
+        else:
+            loss = model(batch, mode=TEACH_FORCE)
+
+        losses.add_loss(loss)
+        losses.add_backward_loss(model.model_sel_loss(loss, batch_cnt))
+
+    valid_loss = losses.avg_loss()
+    # print('Validation finished at {}'.format(datetime.now().strftime("%Y-%m-%d %H-%M-%S")))
+    logger.info(losses.pprint(val_data.name))
+    logger.info('Total valid loss = {}'.format(valid_loss))
+    sys.stdout.flush()
+    return valid_loss
+
+def validate_actz(model, val_data, config, enc="utt", batch_cnt=None, use_py=None):
+    model.eval()
+    val_data.epoch_init(config, shuffle=False, verbose=False)
+    losses = LossManager()
+    while True:
+        batch = val_data.next_batch()
+        if batch is None:
+            break
+        if use_py is not None:
+            # loss = model(batch, mode=TEACH_FORCE, use_py=use_py)
+            if enc=="aux":
+                loss = model.forward_aez(batch, mode=TEACH_FORCE)
+            else:
+                loss = model(batch, mode=TEACH_FORCE)
+        else:
+            if enc=="aux":
+                loss = model.forward_aez(batch, mode=TEACH_FORCE)
+            else:
+                loss = model(batch, mode=TEACH_FORCE)
+
+        losses.add_loss(loss)
+        losses.add_backward_loss(model.model_sel_loss(loss, batch_cnt))
+
+    valid_loss = losses.avg_loss()
+    # print('Validation finished at {}'.format(datetime.now().strftime("%Y-%m-%d %H-%M-%S")))
+    logger.info(losses.pprint(val_data.name))
+    logger.info('Total valid loss = {}'.format(valid_loss))
+    sys.stdout.flush()
+    return valid_loss
+
+def validate_mt(model, val_data, config, batch_cnt=None, use_py=None):
+    model.eval()
+    val_data.epoch_init(config, shuffle=False, verbose=False)
+    losses = LossManager()
+    while True:
+        batch = val_data.next_batch()
+        if batch is None:
+            break
+        loss = model(batch, mode=TEACH_FORCE)
+        losses.add_loss(loss)
+        losses.add_backward_loss(model.model_sel_loss(loss, batch_cnt))
+
+    valid_loss = losses.avg_loss()
+    # print('Validation finished at {}'.format(datetime.now().strftime("%Y-%m-%d %H-%M-%S")))
+    logger.info(losses.pprint(val_data.name))
+    logger.info('Total valid loss = {}'.format(valid_loss))
+    sys.stdout.flush()
+    return valid_loss
+
+def aux_validate(model, val_data, config, batch_cnt=None, use_py=None):
+    model.eval()
+    val_data.epoch_init(config, shuffle=False, verbose=False)
+    losses = LossManager()
+    while True:
+        batch = val_data.next_batch()
+        if batch is None:
+            break
+        loss = model.forward_aux(batch, mode=TEACH_FORCE)
+
+        losses.add_loss(loss)
+        losses.add_backward_loss(model.model_sel_loss(loss, batch_cnt))
+
+    valid_loss = losses.avg_loss()
+    # print('Validation finished at {}'.format(datetime.now().strftime("%Y-%m-%d %H-%M-%S")))
+    logger.info(losses.pprint(val_data.name))
+    logger.info('Total aux valid loss = {}'.format(valid_loss))
+    sys.stdout.flush()
+    return valid_loss
+
+def generate(model, data, config, evaluator, num_batch, dest_f=None):
+    
+    def write(msg):
+        if msg is None or msg == '':
+            return
+        if dest_f is None:
+            print(msg)
+        else:
+            dest_f.write(msg + '\n')
+
+    model.eval()
+    de_tknize = get_detokenize()
+    data.epoch_init(config, shuffle=num_batch is not None, verbose=False)
+    evaluator.initialize()
+    logger.info('Generation: {} batches'.format(data.num_batch
+                                          if num_batch is None
+                                          else num_batch))
+    batch_cnt = 0
+    print_cnt = 0
+    while True:
+        batch_cnt += 1
+        batch = data.next_batch()
+        if batch is None or (num_batch is not None and data.ptr > num_batch):
+            break
+        outputs, labels = model(batch, mode=GEN, gen_type=config.gen_type)
+
+        # move from GPU to CPU
+        labels = labels.cpu()
+        pred_labels = [t.cpu().data.numpy() for t in outputs[DecoderRNN.KEY_SEQUENCE]]
+        pred_labels = np.array(pred_labels, dtype=int).squeeze(-1).swapaxes(0, 1) # (batch_size, max_dec_len)
+        true_labels = labels.data.numpy() # (batch_size, output_seq_len)
+
+        # get attention if possible
+        if config.dec_use_attn:
+            pred_attns = [t.cpu().data.numpy() for t in outputs[DecoderRNN.KEY_ATTN_SCORE]]
+            pred_attns = np.array(pred_attns, dtype=float).squeeze(2).swapaxes(0, 1) # (batch_size, max_dec_len, max_ctx_len)
+        else:
+            pred_attns = None
+        # get context
+        ctx = batch.get('contexts') # (batch_size, max_ctx_len, max_utt_len)
+        ctx_len = batch.get('context_lens') # (batch_size, )
+
+        for b_id in range(pred_labels.shape[0]):
+            # TODO attn
+            pred_str = get_sent(model.vocab, de_tknize, pred_labels, b_id) 
+            true_str = get_sent(model.vocab, de_tknize, true_labels, b_id)
+            prev_ctx = ''
+            if ctx is not None:
+                ctx_str = []
+                for t_id in range(ctx_len[b_id]):
+                    temp_str = get_sent(model.vocab, de_tknize, ctx[:, t_id, :], b_id, stop_eos=False)
+                    # print('temp_str = %s' % (temp_str, ))
+                    # print('ctx[:, t_id, :] = %s' % (ctx[:, t_id, :], ))
+                    ctx_str.append(temp_str)
+                ctx_str = '|'.join(ctx_str)[-200::]
+                prev_ctx = 'Source context: {}'.format(ctx_str)
+
+            evaluator.add_example(true_str, pred_str)
+
+            if num_batch is None or batch_cnt < 2:
+                print_cnt += 1
+                write('prev_ctx = %s' % (prev_ctx, ))
+                write('True: {}'.format(true_str, ))
+                write('Pred: {}'.format(pred_str, ))
+                write('='*30)
+                if num_batch is not None and print_cnt > 10:
+                    break
+
+    write(evaluator.get_report())
+    # write(evaluator.get_groundtruth_report())
+    write('Generation Done')
+
+def get_sent(vocab, de_tknize, data, b_id, stop_eos=True, stop_pad=True):
+    ws = []
+    for t_id in range(data.shape[1]):
+        w = vocab[data[b_id, t_id]]
+        # TODO EOT
+        if (stop_eos and w == EOS) or (stop_pad and w == PAD):
+            break
+        if w != PAD:
+            ws.append(w)
+
+    return de_tknize(ws)
+
+def most_frequent(List):
+    occ_count = Counter(List)
+    return occ_count.most_common(1)[0][0]
+
+def generate_with_name(model, data, config):
+    model.eval()
+    de_tknize = get_detokenize()
+    data.epoch_init(config, shuffle=False, verbose=False)
+    logger.info('Generation With Name: {} batches.'.format(data.num_batch))
+
+    from collections import defaultdict
+    res = defaultdict(dict)
+    while True:
+        batch = data.next_batch()
+        if batch is None:
+            break
+        keys, outputs, labels = model(batch, mode=GEN, gen_type=config.gen_type)
+        
+        pred_labels = [t.cpu().data.numpy() for t in outputs[DecoderRNN.KEY_SEQUENCE]]
+        pred_labels = np.array(pred_labels, dtype=int).squeeze(-1).swapaxes(0, 1) # (batch_size, max_dec_len)
+        true_labels = labels.cpu().data.numpy() # (batch_size, output_seq_len)
+
+        for b_id in range(pred_labels.shape[0]):
+            pred_str = get_sent(model.vocab, de_tknize, pred_labels, b_id) 
+            true_str = get_sent(model.vocab, de_tknize, true_labels, b_id)
+            dlg_name, dlg_turn = keys[b_id]
+            res[dlg_name][dlg_turn] = {'pred': pred_str, 'true': true_str}
+
+    return res
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/metric.py b/convlab/policy/lava/multiwoz/latent_dialog/metric.py
new file mode 100644
index 0000000000000000000000000000000000000000..433c19e369fd7c8274cd1629a053137278f7079b
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/metric.py
@@ -0,0 +1,151 @@
+import time
+from collections import OrderedDict
+
+
+class NumericMetric(object):
+    """Base class for a numeric metric."""
+    def __init__(self):
+        self.k = 0
+        self.n = 0
+
+    def reset(self):
+        pass
+
+    def record(self, k, n=1):
+        self.k += k
+        self.n += n
+
+    def value(self):
+        self.n = max(1, self.n)
+        return 1.0 * self.k / self.n
+
+
+class AverageMetric(NumericMetric):
+    """Average."""
+    def show(self):
+        return '%.2f' % (1. * self.value())
+
+
+class PercentageMetric(NumericMetric):
+    """Percentage."""
+    def show(self):
+        return '%2.2f%%' % (100. * self.value())
+
+
+class TimeMetric(object):
+    """Time based metric."""
+    def __init__(self):
+        self.t = 0
+        self.n = 0
+
+    def reset(self):
+        self.last_t = time.time()
+
+    def record(self, n=1):
+        self.t += time.time() - self.last_t
+        self.n += 1
+
+    def value(self):
+        self.n = max(1, self.n)
+        return 1.0 * self.t / self.n
+
+    def show(self):
+        return '%.3fs' % (1. * self.value())
+
+
+class UniquenessMetric(object):
+    """Metric that evaluates the number of unique sentences."""
+    def __init__(self):
+        self.seen = set()
+
+    def reset(self):
+        pass
+
+    def record(self, sen):
+        self.seen.add(' '.join(sen))
+
+    def value(self):
+        return len(self.seen)
+
+    def show(self):
+        return str(self.value())
+
+
+class TextMetric(object):
+    """Text based metric."""
+    def __init__(self, text):
+        self.text = text
+        self.k = 0
+        self.n = 0
+
+    def reset(self):
+        pass
+
+    def value(self):
+        self.n = max(1, self.n)
+        return 1. * self.k / self.n
+
+    def show(self):
+        return '%.2f' % (1. * self.value())
+
+
+class NGramMetric(TextMetric):
+    """Metric that evaluates n grams."""
+    def __init__(self, text, ngram=-1):
+        super(NGramMetric, self).__init__(text)
+        self.ngram = ngram
+
+    def record(self, sen):
+        n = len(sen) if self.ngram == -1 else self.ngram
+        for i in range(len(sen) - n + 1):
+            self.n += 1
+            target = ' '.join(sen[i:i + n])
+            if self.text.find(target) != -1:
+                self.k += 1
+
+
+class MetricsContainer(object):
+    """A container that stores and updates several metrics."""
+    def __init__(self):
+        self.metrics = OrderedDict()
+
+    def _register(self, name, ty, *args, **kwargs):
+        name = name.lower()
+        assert name not in self.metrics
+        self.metrics[name] = ty(*args, **kwargs)
+
+    def register_average(self, name, *args, **kwargs):
+        self._register(name, AverageMetric, *args, **kwargs)
+
+    def register_time(self, name, *args, **kwargs):
+        self._register(name, TimeMetric, *args, **kwargs)
+
+    def register_percentage(self, name, *args, **kwargs):
+        self._register(name, PercentageMetric, *args, **kwargs)
+
+    def register_ngram(self, name, *args, **kwargs):
+        self._register(name, NGramMetric, *args, **kwargs)
+
+    def register_uniqueness(self, name, *args, **kwargs):
+        self._register(name, UniquenessMetric, *args, **kwargs)
+
+    def record(self, name, *args, **kwargs):
+        name = name.lower()
+        assert name in self.metrics
+        self.metrics[name].record(*args, **kwargs)
+
+    def reset(self):
+        for m in self.metrics.values():
+            m.reset()
+
+    def value(self, name):
+        return self.metrics[name].value()
+
+    def show(self):
+        return ' '.join(['%s=%s' % (k, v.show()) for k, v in self.metrics.iteritems()])
+
+    def dict(self):
+        d = OrderedDict()
+        for k, v in self.metrics.items():
+            d[k] = v.show()
+        return d
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/models_task.py b/convlab/policy/lava/multiwoz/latent_dialog/models_task.py
new file mode 100644
index 0000000000000000000000000000000000000000..84a1eb986f91301d6c463ada7c180ba0b6227927
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/models_task.py
@@ -0,0 +1,3958 @@
+import torch as th
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Variable
+from convlab.policy.lava.multiwoz.latent_dialog.base_models import BaseModel
+from convlab.policy.lava.multiwoz.latent_dialog.corpora import SYS, EOS, PAD, BOS
+from convlab.policy.lava.multiwoz.latent_dialog.utils import INT, FLOAT, LONG, Pack, cast_type
+from convlab.policy.lava.multiwoz.latent_dialog.enc2dec.encoders import RnnUttEncoder
+from convlab.policy.lava.multiwoz.latent_dialog.enc2dec.decoders import DecoderRNN, GEN, TEACH_FORCE
+from convlab.policy.lava.multiwoz.latent_dialog.criterions import NLLEntropy, CatKLLoss, Entropy, NormKLLoss, WeightedNLLEntropy
+from convlab.policy.lava.multiwoz.latent_dialog import nn_lib
+import numpy as np
+import pdb
+
+
+class SysPerfectBD2Word(BaseModel):
+    def __init__(self, corpus, config):
+        super(SysPerfectBD2Word, self).__init__(config)
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+
+        self.embedding = None
+        self.utt_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+        self.policy = nn.Sequential(nn.Linear(self.utt_encoder.output_size + self.db_size + self.bs_size,
+                                              config.dec_cell_size), nn.Tanh(), nn.Dropout(config.dropout))
+
+        self.decoder = DecoderRNN(input_dropout_p=config.dropout,
+                                  rnn_cell=config.dec_rnn_cell,
+                                  input_size=config.embed_size,
+                                  hidden_size=config.dec_cell_size,
+                                  num_layers=config.num_layers,
+                                  output_dropout_p=config.dropout,
+                                  bidirectional=False,
+                                  vocab_size=self.vocab_size,
+                                  use_attn=config.dec_use_attn,
+                                  ctx_cell_size=self.utt_encoder.output_size,
+                                  attn_mode=config.dec_attn_mode,
+                                  sys_id=self.bos_id,
+                                  eos_id=self.eos_id,
+                                  use_gpu=config.use_gpu,
+                                  max_dec_len=config.max_dec_len,
+                                  embedding=self.embedding)
+
+        self.nll = NLLEntropy(self.pad_id, config.avg_type)
+
+    def forward(self, data_feed, mode, clf=False, gen_type='greedy', return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            attn_context = enc_outs
+        else:
+            attn_context = None
+
+        # create decoder initial states
+        dec_init_state = self.policy(th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)).unsqueeze(0)
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            # h_dec_init_state = utt_summary.squeeze(1).unsqueeze(0)
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               # (batch_size, response_size-1)
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               # (batch_size, max_ctx_len, ctx_cell_size)
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            return ret_dict, labels
+        if return_latent:
+            return Pack(nll=self.nll(dec_outputs, labels),
+                        latent_action=dec_init_state)
+        else:
+            return Pack(nll=self.nll(dec_outputs, labels))
+
+    def forward_rl(self, data_feed, max_words, temp=0.1):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            attn_context = enc_outs
+        else:
+            attn_context = None
+
+        # create decoder initial states
+        dec_init_state = self.policy(th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)).unsqueeze(0)
+
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        # decode
+        logprobs, outs = self.decoder.forward_rl(batch_size=batch_size,
+                                                 dec_init_state=dec_init_state,
+                                                 attn_context=attn_context,
+                                                 vocab=self.vocab,
+                                                 max_words=max_words,
+                                                 temp=temp)
+        return logprobs, outs
+
+class SysPerfectBD2Cat(BaseModel):
+    def __init__(self, corpus, config):
+        super(SysPerfectBD2Cat, self).__init__(config)
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        self.k_size = config.k_size
+        self.y_size = config.y_size
+        self.simple_posterior = config.simple_posterior
+        self.contextual_posterior = config.contextual_posterior
+
+        self.embedding = None
+        self.utt_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+        self.c2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size + self.db_size + self.bs_size,
+                                          config.y_size, config.k_size, is_lstm=False)
+        self.z_embedding = nn.Linear(self.y_size * self.k_size, config.dec_cell_size, bias=False)
+        self.gumbel_connector = nn_lib.GumbelConnector(config.use_gpu)
+        if not self.simple_posterior:
+            if self.contextual_posterior:
+                self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size * 2 + self.db_size + self.bs_size,
+                                                   config.y_size, config.k_size, is_lstm=False)
+            else:
+                self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size, config.y_size, config.k_size, is_lstm=False)
+
+        self.decoder = DecoderRNN(input_dropout_p=config.dropout,
+                                  rnn_cell=config.dec_rnn_cell,
+                                  input_size=config.embed_size,
+                                  hidden_size=config.dec_cell_size,
+                                  num_layers=config.num_layers,
+                                  output_dropout_p=config.dropout,
+                                  bidirectional=False,
+                                  vocab_size=self.vocab_size,
+                                  use_attn=config.dec_use_attn,
+                                  ctx_cell_size=config.dec_cell_size,
+                                  attn_mode=config.dec_attn_mode,
+                                  sys_id=self.bos_id,
+                                  eos_id=self.eos_id,
+                                  use_gpu=config.use_gpu,
+                                  max_dec_len=config.max_dec_len,
+                                  embedding=self.embedding)
+
+        if config.avg_type == "slot":
+            # give slot_weight:1 ratio between slot tokens and other words
+            self.loss_weight = th.tensor([(config.slot_weight - 1) * int(k[0] == '[' and k[-1] == ']' ) + 1 for k in self.vocab_dict.keys()]).type(th.FloatTensor)
+            if self.use_gpu:
+                self.loss_weight = self.loss_weight.cuda()
+            self.nll = WeightedNLLEntropy(self.pad_id, config.avg_type, self.loss_weight)
+        else:
+            self.nll = NLLEntropy(self.pad_id, config.avg_type)
+
+        self.cat_kl_loss = CatKLLoss()
+        self.entropy_loss = Entropy()
+        self.log_uniform_y = Variable(th.log(th.ones(1) / config.k_size))
+        self.eye = Variable(th.eye(self.config.y_size).unsqueeze(0))
+        self.beta = self.config.beta if hasattr(self.config, 'beta') else 0.0
+        if self.use_gpu:
+            self.log_uniform_y = self.log_uniform_y.cuda()
+            self.eye = self.eye.cuda()
+
+    def valid_loss(self, loss, batch_cnt=None):
+        if self.simple_posterior:
+            total_loss = loss.nll
+            if self.config.use_pr > 0.0:
+                total_loss += self.beta * loss.pi_kl
+        else:
+            total_loss = loss.nll + loss.pi_kl
+
+        if self.config.use_mi:
+            total_loss += (loss.b_pr * self.beta)
+
+        if self.config.use_diversity:
+            total_loss += loss.diversity
+
+        return total_loss
+
+    def forward(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        # create decoder initial states
+        if self.simple_posterior:
+            logits_qy, log_qy = self.c2z(enc_last)
+            sample_y = self.gumbel_connector(logits_qy, hard=mode==GEN)
+            log_py = self.log_uniform_y
+        else:
+            logits_py, log_py = self.c2z(enc_last)
+            # encode response and use posterior to find q(z|x, c)
+            x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            if self.contextual_posterior:
+                logits_qy, log_qy = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            else:
+                logits_qy, log_qy = self.xc2z(x_h.squeeze(1))
+
+            # use prior at inference time, otherwise use posterior
+            if mode == GEN or (use_py is not None and use_py is True):
+                sample_y = self.gumbel_connector(logits_py, hard=False)
+            else:
+                sample_y = self.gumbel_connector(logits_qy, hard=True)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               # (batch_size, response_size-1)
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               # (batch_size, max_ctx_len, ctx_cell_size)
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_y
+            ret_dict['log_qy'] = log_qy
+            return ret_dict, labels
+
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            # regularization qy to be uniform
+            avg_log_qy = th.exp(log_qy.view(-1, self.config.y_size, self.config.k_size))
+            avg_log_qy = th.log(th.mean(avg_log_qy, dim=0) + 1e-15)
+            b_pr = self.cat_kl_loss(avg_log_qy, self.log_uniform_y, batch_size, unit_average=True)
+            mi = self.entropy_loss(avg_log_qy, unit_average=True) - self.entropy_loss(log_qy, unit_average=True)
+            pi_kl = self.cat_kl_loss(log_qy, log_py, batch_size, unit_average=True)
+            q_y = th.exp(log_qy).view(-1, self.config.y_size, self.config.k_size)  # b
+            p = th.pow(th.bmm(q_y, th.transpose(q_y, 1, 2)) - self.eye, 2)
+
+            result['pi_kl'] = pi_kl
+
+            result['diversity'] = th.mean(p)
+            result['nll'] = self.nll(dec_outputs, labels)
+            result['b_pr'] = b_pr
+            result['mi'] = mi
+            return result
+
+    def forward_rl(self, data_feed, max_words, temp=0.1):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        # create decoder initial states
+        if self.simple_posterior:
+            logits_py, log_qy = self.c2z(enc_last)
+        else:
+            logits_py, log_qy = self.c2z(enc_last)
+
+        qy = F.softmax(logits_py / temp, dim=1)  # (batch_size, vocab_size, )
+        log_qy = F.log_softmax(logits_py, dim=1)  # (batch_size, vocab_size, )
+        idx = th.multinomial(qy, 1).detach()
+        logprob_sample_z = log_qy.gather(1, idx).view(-1, self.y_size)
+        joint_logpz = th.sum(logprob_sample_z, dim=1)
+        sample_y = cast_type(Variable(th.zeros(log_qy.size())), FLOAT, self.use_gpu)
+        sample_y.scatter_(1, idx, 1.0)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        # decode
+        logprobs, outs = self.decoder.forward_rl(batch_size=batch_size,
+                                                 dec_init_state=dec_init_state,
+                                                 attn_context=attn_context,
+                                                 vocab=self.vocab,
+                                                 max_words=max_words,
+                                                 temp=0.1)
+        return logprobs, outs, joint_logpz, sample_y
+
+class SysAECat(BaseModel):
+    def __init__(self, corpus, config):
+        super(SysAECat, self).__init__(config)
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        # self.bs_size = corpus.bs_size
+        # self.db_size = corpus.db_size
+        # self.act_size = corpus.act_size
+        self.k_size = config.k_size
+        self.y_size = config.y_size
+        self.simple_posterior = True # minimize kl to uninformed prior instead of dist conditioned by context
+        self.contextual_posterior = False # does not use context cause AE task
+
+        self.embedding = None
+        self.utt_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+        self.c2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size,
+                                          config.y_size, config.k_size, is_lstm=False)
+        self.z_embedding = nn.Linear(self.y_size * self.k_size, config.dec_cell_size, bias=False)
+        self.gumbel_connector = nn_lib.GumbelConnector(config.use_gpu)
+        # if not self.simple_posterior: #q(z|x,c)
+            # if self.contextual_posterior:
+                # # x, c, BS, and DB
+                # self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size,
+                                                   # config.y_size, config.k_size, is_lstm=False)
+            # else:
+                # self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size, config.y_size, config.k_size, is_lstm=False)
+
+        self.decoder = DecoderRNN(input_dropout_p=config.dropout,
+                                  rnn_cell=config.dec_rnn_cell,
+                                  input_size=config.embed_size,
+                                  hidden_size=config.dec_cell_size,
+                                  num_layers=config.num_layers,
+                                  output_dropout_p=config.dropout,
+                                  bidirectional=False,
+                                  vocab_size=self.vocab_size,
+                                  use_attn=config.dec_use_attn,
+                                  ctx_cell_size=config.dec_cell_size,
+                                  attn_mode=config.dec_attn_mode,
+                                  sys_id=self.bos_id,
+                                  eos_id=self.eos_id,
+                                  use_gpu=config.use_gpu,
+                                  max_dec_len=config.max_dec_len,
+                                  embedding=self.embedding)
+        if config.avg_type == "slot":
+            # give slot_weight:1 ratio between slot tokens and other words
+            self.loss_weight = th.tensor([(config.slot_weight - 1) * int(k[0] == '[' and k[-1] == ']' ) + 1 for k in self.vocab_dict.keys()]).type(th.FloatTensor)
+            if self.use_gpu:
+                self.loss_weight = self.loss_weight.cuda()
+            self.nll = WeightedNLLEntropy(self.pad_id, config.avg_type, self.loss_weight)
+        else:
+            self.nll = NLLEntropy(self.pad_id, config.avg_type)
+
+        self.cat_kl_loss = CatKLLoss()
+        self.entropy_loss = Entropy()
+        self.log_uniform_y = Variable(th.log(th.ones(1) / config.k_size))
+        self.eye = Variable(th.eye(self.config.y_size).unsqueeze(0))
+        self.beta = self.config.beta if hasattr(self.config, 'beta') else 0.0
+        if self.use_gpu:
+            self.log_uniform_y = self.log_uniform_y.cuda()
+            self.eye = self.eye.cuda()
+
+    def valid_loss(self, loss, batch_cnt=None):
+        if self.simple_posterior:
+            total_loss = loss.nll
+            if self.config.use_pr > 0.0:
+                total_loss += self.beta * loss.pi_kl
+        else:
+            total_loss = loss.nll + loss.pi_kl
+
+        if self.config.use_mi:
+            total_loss += (loss.b_pr * self.beta)
+
+        if self.config.use_diversity:
+            total_loss += loss.diversity
+
+        return total_loss
+
+    def forward(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        # bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        # db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        # act_label = self.np2var(data_feed['act'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        enc_last = utt_summary.squeeze(1)
+        # create decoder initial states
+        if self.simple_posterior:
+            logits_qy, log_qy = self.c2z(enc_last)
+            sample_y = self.gumbel_connector(logits_qy, hard=mode==GEN)
+            log_py = self.log_uniform_y
+        # else:
+            # logits_py, log_py = self.c2z(enc_last)
+            # # encode response and use posterior to find q(z|x, c)
+            # x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            # if self.contextual_posterior:
+                # logits_qy, log_qy = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            # else:
+                # logits_qy, log_qy = self.xc2z(x_h.squeeze(1))
+
+            # # use prior at inference time, otherwise use posterior
+            # if mode == GEN or (use_py is not None and use_py is True):
+                # sample_y = self.gumbel_connector(logits_py, hard=False)
+            # else:
+                # sample_y = self.gumbel_connector(logits_qy, hard=True)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               # (batch_size, response_size-1)
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               # (batch_size, max_ctx_len, ctx_cell_size)
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_y
+            ret_dict['log_qy'] = log_qy
+            return ret_dict, labels
+
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            # regularization qy to be uniform
+            avg_log_qy = th.exp(log_qy.view(-1, self.config.y_size, self.config.k_size))
+            avg_log_qy = th.log(th.mean(avg_log_qy, dim=0) + 1e-15)
+            b_pr = self.cat_kl_loss(avg_log_qy, self.log_uniform_y, batch_size, unit_average=True)
+            mi = self.entropy_loss(avg_log_qy, unit_average=True) - self.entropy_loss(log_qy, unit_average=True)
+            pi_kl = self.cat_kl_loss(log_qy, log_py, batch_size, unit_average=True)
+            q_y = th.exp(log_qy).view(-1, self.config.y_size, self.config.k_size)  # b
+            p = th.pow(th.bmm(q_y, th.transpose(q_y, 1, 2)) - self.eye, 2)
+
+            result['pi_kl'] = pi_kl
+
+            result['diversity'] = th.mean(p)
+            result['nll'] = self.nll(dec_outputs, labels)
+            result['b_pr'] = b_pr
+            result['mi'] = mi
+            return result
+
+    def forward_rl(self, data_feed, max_words, temp=0.1):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        # create decoder initial states
+        if self.simple_posterior:
+            logits_py, log_qy = self.c2z(enc_last)
+        else:
+            logits_py, log_qy = self.c2z(enc_last)
+
+        qy = F.softmax(logits_py / temp, dim=1)  # (batch_size, vocab_size, )
+        log_qy = F.log_softmax(logits_py, dim=1)  # (batch_size, vocab_size, )
+        idx = th.multinomial(qy, 1).detach()
+        logprob_sample_z = log_qy.gather(1, idx).view(-1, self.y_size)
+        joint_logpz = th.sum(logprob_sample_z, dim=1)
+        sample_y = cast_type(Variable(th.zeros(log_qy.size())), FLOAT, self.use_gpu)
+        sample_y.scatter_(1, idx, 1.0)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        # decode
+        logprobs, outs = self.decoder.forward_rl(batch_size=batch_size,
+                                                 dec_init_state=dec_init_state,
+                                                 attn_context=attn_context,
+                                                 vocab=self.vocab,
+                                                 max_words=max_words,
+                                                 temp=0.1)
+        return logprobs, outs, joint_logpz, sample_y
+
+class SysMTCat(BaseModel):
+    def __init__(self, corpus, config): 
+        super(SysMTCat, self).__init__(config)
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        # self.act_size = corpus.act_size
+        self.k_size = config.k_size
+        self.y_size = config.y_size
+        self.simple_posterior = config.simple_posterior # minimize kl to uninformed prior instead of dist conditioned by context
+        self.contextual_posterior = config.contextual_posterior # does not use context cause AE task
+
+        if "use_aux_kl" in config:
+            self.use_aux_kl = config.use_aux_kl
+        else:
+            self.use_aux_kl = False
+
+        self.embedding = None
+        self.aux_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+        self.utt_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+
+        self.c2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size + self.db_size + self.bs_size,
+                                          config.y_size, config.k_size, is_lstm=False)
+        self.z_embedding = nn.Linear(self.y_size * self.k_size, config.dec_cell_size, bias=False)
+        self.gumbel_connector = nn_lib.GumbelConnector(config.use_gpu)
+        
+        if not self.simple_posterior: #q(z|x,c)
+            if self.contextual_posterior:
+                # x, c, BS, and DB
+                self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size,
+                                                   config.y_size, config.k_size, is_lstm=False)
+            else:
+                self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size, config.y_size, config.k_size, is_lstm=False)
+
+        self.decoder = DecoderRNN(input_dropout_p=config.dropout,
+                                  rnn_cell=config.dec_rnn_cell,
+                                  input_size=config.embed_size,
+                                  hidden_size=config.dec_cell_size,
+                                  num_layers=config.num_layers,
+                                  output_dropout_p=config.dropout,
+                                  bidirectional=False,
+                                  vocab_size=self.vocab_size,
+                                  use_attn=config.dec_use_attn,
+                                  ctx_cell_size=config.dec_cell_size,
+                                  attn_mode=config.dec_attn_mode,
+                                  sys_id=self.bos_id,
+                                  eos_id=self.eos_id,
+                                  use_gpu=config.use_gpu,
+                                  max_dec_len=config.max_dec_len,
+                                  embedding=self.embedding)
+
+        if config.avg_type == "slot":
+            # give slot_weight:1 ratio between slot tokens and other words
+            self.loss_weight = th.tensor([(config.slot_weight - 1) * int(k[0] == '[' and k[-1] == ']' ) + 1 for k in self.vocab_dict.keys()]).type(th.FloatTensor)
+            if self.use_gpu:
+                self.loss_weight = self.loss_weight.cuda()
+            self.nll = WeightedNLLEntropy(self.pad_id, config.avg_type, self.loss_weight)
+        else:
+            self.nll = NLLEntropy(self.pad_id, config.avg_type)
+        self.cat_kl_loss = CatKLLoss()
+        self.entropy_loss = Entropy()
+        self.log_uniform_y = Variable(th.log(th.ones(1) / config.k_size))
+        self.eye = Variable(th.eye(self.config.y_size).unsqueeze(0))
+        self.beta = self.config.beta if hasattr(self.config, 'beta') else 0.0
+        if self.use_gpu:
+            self.log_uniform_y = self.log_uniform_y.cuda()
+            self.eye = self.eye.cuda()
+
+    def valid_loss(self, loss, batch_cnt=None):
+        if self.simple_posterior:
+            total_loss = loss.nll
+            if self.config.use_pr > 0.0:
+                total_loss += self.beta * loss.pi_kl
+        else:
+            total_loss = loss.nll + loss.pi_kl
+
+        if self.config.use_mi:
+            total_loss += (loss.b_pr * self.beta)
+
+        if self.config.use_diversity:
+            total_loss += loss.diversity
+
+        if self.use_aux_kl:
+            try:
+                total_loss += loss.aux_pi_kl
+            except KeyError:
+                total_loss += 0
+
+        return total_loss
+
+    def forward_aux(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        act_label = self.np2var(data_feed['act'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.aux_encoder(short_ctx_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        
+        # how to use z, alone or in combination with bs and db
+        if self.simple_posterior:
+            logits_qy, log_qy = self.c2z(enc_last)
+            sample_y = self.gumbel_connector(logits_qy, hard=mode==GEN)
+            log_py = self.log_uniform_y
+        # else:
+            # logits_py, log_py = self.c2z(enc_last)
+            # # encode response and use posterior to find q(z|x, c)
+            # x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            # if self.contextual_posterior:
+                # logits_qy, log_qy = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            # else:
+                # logits_qy, log_qy = self.xc2z(x_h.squeeze(1))
+
+            # # use prior at inference time, otherwise use posterior
+            # if mode == GEN or (use_py is not None and use_py is True):
+                # sample_y = self.gumbel_connector(logits_py, hard=False)
+            # else:
+                # sample_y = self.gumbel_connector(logits_qy, hard=True)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               # (batch_size, response_size-1)
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               # (batch_size, max_ctx_len, ctx_cell_size)
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_y
+            ret_dict['log_qy'] = log_qy
+            return ret_dict, labels
+
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            # regularization qy to be uniform
+            avg_log_qy = th.exp(log_qy.view(-1, self.config.y_size, self.config.k_size))
+            avg_log_qy = th.log(th.mean(avg_log_qy, dim=0) + 1e-15)
+            b_pr = self.cat_kl_loss(avg_log_qy, self.log_uniform_y, batch_size, unit_average=True)
+            mi = self.entropy_loss(avg_log_qy, unit_average=True) - self.entropy_loss(log_qy, unit_average=True)
+            pi_kl = self.cat_kl_loss(log_qy, log_py, batch_size, unit_average=True)
+            q_y = th.exp(log_qy).view(-1, self.config.y_size, self.config.k_size)  # b
+            p = th.pow(th.bmm(q_y, th.transpose(q_y, 1, 2)) - self.eye, 2)
+
+            result['pi_kl'] = pi_kl
+            result['diversity'] = th.mean(p)
+            result['nll'] = self.nll(dec_outputs, labels)
+            result['b_pr'] = b_pr
+            result['mi'] = mi
+            return result
+
+    def forward(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        # create decoder initial states
+        if self.simple_posterior:
+            logits_qy, log_qy = self.c2z(enc_last)
+            sample_y = self.gumbel_connector(logits_qy, hard=mode==GEN)
+            log_py = self.log_uniform_y
+        else:
+            logits_py, log_py = self.c2z(enc_last)
+            # encode response and use posterior to find q(z|x, c)
+            x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            if self.contextual_posterior:
+                logits_qy, log_qy = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            else:
+                logits_qy, log_qy = self.xc2z(x_h.squeeze(1))
+
+            # use prior at inference time, otherwise use posterior
+            if mode == GEN or (use_py is not None and use_py is True):
+                sample_y = self.gumbel_connector(logits_py, hard=False)
+            else:
+                sample_y = self.gumbel_connector(logits_qy, hard=True)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               # (batch_size, response_size-1)
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               # (batch_size, max_ctx_len, ctx_cell_size)
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_y
+            ret_dict['log_qy'] = log_qy
+            return ret_dict, labels
+
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            # regularization qy to be uniform
+            avg_log_qy = th.exp(log_qy.view(-1, self.config.y_size, self.config.k_size))
+            avg_log_qy = th.log(th.mean(avg_log_qy, dim=0) + 1e-15)
+            b_pr = self.cat_kl_loss(avg_log_qy, self.log_uniform_y, batch_size, unit_average=True)
+            mi = self.entropy_loss(avg_log_qy, unit_average=True) - self.entropy_loss(log_qy, unit_average=True)
+            pi_kl = self.cat_kl_loss(log_qy, log_py, batch_size, unit_average=True)
+            q_y = th.exp(log_qy).view(-1, self.config.y_size, self.config.k_size)  # b
+            p = th.pow(th.bmm(q_y, th.transpose(q_y, 1, 2)) - self.eye, 2)
+
+            result['pi_kl'] = pi_kl
+            result['diversity'] = th.mean(p)
+            result['nll'] = self.nll(dec_outputs, labels)
+            result['b_pr'] = b_pr
+            result['mi'] = mi
+            return result
+    
+    def shared_forward(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        short_target_utts = self.np2var(data_feed['outputs'], LONG)
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+        aux_utt_summary, _, aux_enc_outs = self.aux_encoder(short_target_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        aux_enc_last = th.cat([bs_label, db_label, aux_utt_summary.squeeze(1)], dim=1)
+        # create decoder initial states
+        if self.simple_posterior:
+            logits_qy, log_qy = self.c2z(enc_last)
+            aux_logits_qy, aux_log_qy = self.c2z(aux_enc_last)
+            sample_y = self.gumbel_connector(logits_qy, hard=mode==GEN)
+            log_py = self.log_uniform_y
+        else:
+            logits_py, log_py = self.c2z(enc_last)
+            aux_logits_qy, aux_log_qy = self.c2z(aux_enc_last)
+            # encode response and use posterior to find q(z|x, c)
+            x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            if self.contextual_posterior:
+                logits_qy, log_qy = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            else:
+                logits_qy, log_qy = self.xc2z(x_h.squeeze(1))
+
+            # use prior at inference time, otherwise use posterior
+            if mode == GEN or (use_py is not None and use_py is True):
+                sample_y = self.gumbel_connector(logits_py, hard=False)
+            else:
+                sample_y = self.gumbel_connector(logits_qy, hard=True)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               # (batch_size, response_size-1)
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               # (batch_size, max_ctx_len, ctx_cell_size)
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_y
+            ret_dict['log_qy'] = log_qy
+            return ret_dict, labels
+
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            # regularization qy to be uniform
+            avg_log_qy = th.exp(log_qy.view(-1, self.config.y_size, self.config.k_size))
+            avg_log_qy = th.log(th.mean(avg_log_qy, dim=0) + 1e-15)
+            b_pr = self.cat_kl_loss(avg_log_qy, self.log_uniform_y, batch_size, unit_average=True)
+            mi = self.entropy_loss(avg_log_qy, unit_average=True) - self.entropy_loss(log_qy, unit_average=True)
+            pi_kl = self.cat_kl_loss(log_qy, log_py, batch_size, unit_average=True)
+            aux_pi_kl = self.cat_kl_loss(log_qy, aux_log_qy, batch_size, unit_average=True)
+            q_y = th.exp(log_qy).view(-1, self.config.y_size, self.config.k_size)  # b
+            p = th.pow(th.bmm(q_y, th.transpose(q_y, 1, 2)) - self.eye, 2)
+
+            result['pi_kl'] = pi_kl
+            result['aux_pi_kl'] = aux_pi_kl
+            result['diversity'] = th.mean(p)
+            result['nll'] = self.nll(dec_outputs, labels)
+            result['b_pr'] = b_pr
+            result['mi'] = mi
+            return result
+
+class SysActZCat(BaseModel):
+    def __init__(self, corpus, config): 
+        super(SysActZCat, self).__init__(config)
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        # self.act_size = corpus.act_size
+        self.k_size = config.k_size
+        self.y_size = config.y_size
+        self.simple_posterior = config.simple_posterior # minimize kl to uninformed prior instead of dist conditioned by context
+        self.contextual_posterior = config.contextual_posterior # does not use context cause AE task
+
+        if "use_aux_kl" in config:
+            self.use_aux_kl = config.use_aux_kl
+        else:
+            self.use_aux_kl = False
+
+        self.embedding = None
+        self.aux_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+        self.utt_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+
+        self.c2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size + self.db_size + self.bs_size,
+                                          config.y_size, config.k_size, is_lstm=False)
+        self.z_embedding = nn.Linear(self.y_size * self.k_size, config.dec_cell_size, bias=False)
+        self.gumbel_connector = nn_lib.GumbelConnector(config.use_gpu)
+        
+        if not self.simple_posterior: #q(z|x,c)
+            if self.contextual_posterior:
+                # x, c, BS, and DB
+                self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size,
+                                                   config.y_size, config.k_size, is_lstm=False)
+            else:
+                self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size, config.y_size, config.k_size, is_lstm=False)
+
+        self.decoder = DecoderRNN(input_dropout_p=config.dropout,
+                                  rnn_cell=config.dec_rnn_cell,
+                                  input_size=config.embed_size,
+                                  hidden_size=config.dec_cell_size,
+                                  num_layers=config.num_layers,
+                                  output_dropout_p=config.dropout,
+                                  bidirectional=False,
+                                  vocab_size=self.vocab_size,
+                                  use_attn=config.dec_use_attn,
+                                  ctx_cell_size=config.dec_cell_size,
+                                  attn_mode=config.dec_attn_mode,
+                                  sys_id=self.bos_id,
+                                  eos_id=self.eos_id,
+                                  use_gpu=config.use_gpu,
+                                  max_dec_len=config.max_dec_len,
+                                  embedding=self.embedding)
+
+        if config.avg_type == "slot":
+            # give slot_weight:1 ratio between slot tokens and other words
+            self.loss_weight = th.tensor([(config.slot_weight - 1) * int(k[0] == '[' and k[-1] == ']' ) + 1 for k in self.vocab_dict.keys()]).type(th.FloatTensor)
+            if self.use_gpu:
+                self.loss_weight = self.loss_weight.cuda()
+            self.nll = WeightedNLLEntropy(self.pad_id, config.avg_type, self.loss_weight)
+        else:
+            self.nll = NLLEntropy(self.pad_id, config.avg_type)
+        self.cat_kl_loss = CatKLLoss()
+        self.entropy_loss = Entropy()
+        self.log_uniform_y = Variable(th.log(th.ones(1) / config.k_size))
+        self.eye = Variable(th.eye(self.config.y_size).unsqueeze(0))
+        self.beta = self.config.beta if hasattr(self.config, 'beta') else 0.0
+        if self.use_gpu:
+            self.log_uniform_y = self.log_uniform_y.cuda()
+            self.eye = self.eye.cuda()
+
+    def valid_loss(self, loss, batch_cnt=None):
+        if self.simple_posterior:
+            total_loss = loss.nll
+            if self.config.use_pr > 0.0:
+                total_loss += self.beta * loss.pi_kl
+        else:
+            total_loss = loss.nll + loss.pi_kl
+
+        if self.config.use_mi:
+            total_loss += (loss.b_pr * self.beta)
+
+        if self.config.use_diversity:
+            total_loss += loss.diversity
+
+        return total_loss
+    
+    def forward(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        short_target_utts = self.np2var(data_feed['outputs'], LONG)
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+        aux_utt_summary, _, aux_enc_outs = self.aux_encoder(short_target_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        aux_enc_last = th.cat([bs_label, db_label, aux_utt_summary.squeeze(1)], dim=1)
+        # create decoder initial states
+        if self.simple_posterior:
+            logits_qy, log_qy = self.c2z(enc_last)
+            aux_logits_qy, aux_log_qy = self.c2z(aux_enc_last)
+            sample_y = self.gumbel_connector(logits_qy, hard=mode==GEN)
+            log_py = aux_log_qy
+        else: 
+            logits_py, log_py = self.c2z(enc_last)
+            aux_logits_qy, aux_log_qy = self.c2z(aux_enc_last)
+            # encode response and use posterior to find q(z|x, c)
+            x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            if self.contextual_posterior:
+                logits_qy, log_qy = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            else:
+                logits_qy, log_qy = self.xc2z(x_h.squeeze(1))
+
+            # use prior at inference time, otherwise use posterior
+            if mode == GEN or (use_py is not None and use_py is True):
+                sample_y = self.gumbel_connector(logits_py, hard=False)
+            else:
+                sample_y = self.gumbel_connector(logits_qy, hard=True)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               # (batch_size, response_size-1)
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               # (batch_size, max_ctx_len, ctx_cell_size)
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_y
+            ret_dict['log_qy'] = log_qy
+            return ret_dict, labels
+
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            # regularization qy to be uniform
+            avg_log_qy = th.exp(log_qy.view(-1, self.config.y_size, self.config.k_size))
+            avg_log_qy = th.log(th.mean(avg_log_qy, dim=0) + 1e-15)
+            b_pr = self.cat_kl_loss(avg_log_qy, self.log_uniform_y, batch_size, unit_average=True)
+            mi = self.entropy_loss(avg_log_qy, unit_average=True) - self.entropy_loss(log_qy, unit_average=True)
+            pi_kl = self.cat_kl_loss(log_qy, log_py, batch_size, unit_average=True)
+            q_y = th.exp(log_qy).view(-1, self.config.y_size, self.config.k_size)  # b
+            p = th.pow(th.bmm(q_y, th.transpose(q_y, 1, 2)) - self.eye, 2)
+
+            result['pi_kl'] = pi_kl
+            result['diversity'] = th.mean(p)
+            result['nll'] = self.nll(dec_outputs, labels)
+            result['b_pr'] = b_pr
+            result['mi'] = mi
+            return result
+
+    def forward_rl(self, data_feed, max_words, temp=0.1):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        # create decoder initial states
+        if self.simple_posterior:
+            logits_py, log_qy = self.c2z(enc_last)
+        else:
+            logits_py, log_qy = self.c2z(enc_last)
+
+        qy = F.softmax(logits_py / temp, dim=1)  # (batch_size, vocab_size, )
+        log_qy = F.log_softmax(logits_py, dim=1)  # (batch_size, vocab_size, )
+        idx = th.multinomial(qy, 1).detach()
+        logprob_sample_z = log_qy.gather(1, idx).view(-1, self.y_size)
+        joint_logpz = th.sum(logprob_sample_z, dim=1)
+        sample_y = cast_type(Variable(th.zeros(log_qy.size())), FLOAT, self.use_gpu)
+        sample_y.scatter_(1, idx, 1.0)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        # decode
+        logprobs, outs = self.decoder.forward_rl(batch_size=batch_size,
+                                                 dec_init_state=dec_init_state,
+                                                 attn_context=attn_context,
+                                                 vocab=self.vocab,
+                                                 max_words=max_words,
+                                                 temp=0.1)
+        return logprobs, outs, joint_logpz, sample_y
+
+class SysE2ECat(BaseModel):
+    def __init__(self, corpus, config):
+        super(SysE2ECat, self).__init__(config)
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        self.k_size = config.k_size
+        self.y_size = config.y_size
+        self.simple_posterior = config.simple_posterior
+        self.contextual_posterior = config.contextual_posterior
+
+        self.embedding = None
+        self.utt_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+        self.c2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size,
+                                          config.y_size, config.k_size, is_lstm=False)
+        self.z_embedding = nn.Linear(self.y_size * self.k_size, config.dec_cell_size, bias=False)
+        self.gumbel_connector = nn_lib.GumbelConnector(config.use_gpu)
+        if not self.simple_posterior:
+            if self.contextual_posterior:
+                self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size * 2 + self.db_size + self.bs_size,
+                                                   config.y_size, config.k_size, is_lstm=False)
+            else:
+                self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size, config.y_size, config.k_size, is_lstm=False)
+
+        self.decoder = DecoderRNN(input_dropout_p=config.dropout,
+                                  rnn_cell=config.dec_rnn_cell,
+                                  input_size=config.embed_size,
+                                  hidden_size=config.dec_cell_size,
+                                  num_layers=config.num_layers,
+                                  output_dropout_p=config.dropout,
+                                  bidirectional=False,
+                                  vocab_size=self.vocab_size,
+                                  use_attn=config.dec_use_attn,
+                                  ctx_cell_size=config.dec_cell_size,
+                                  attn_mode=config.dec_attn_mode,
+                                  sys_id=self.bos_id,
+                                  eos_id=self.eos_id,
+                                  use_gpu=config.use_gpu,
+                                  max_dec_len=config.max_dec_len,
+                                  embedding=self.embedding)
+
+        if config.avg_type == "slot":
+            # give slot_weight:1 ratio between slot tokens and other words
+            self.loss_weight = th.tensor([(config.slot_weight - 1) * int(k[0] == '[' and k[-1] == ']' ) + 1 for k in self.vocab_dict.keys()]).type(th.FloatTensor)
+            if self.use_gpu:
+                self.loss_weight = self.loss_weight.cuda()
+            self.nll = WeightedNLLEntropy(self.pad_id, config.avg_type, self.loss_weight)
+        else:
+            self.nll = NLLEntropy(self.pad_id, config.avg_type)
+
+        self.cat_kl_loss = CatKLLoss()
+        self.entropy_loss = Entropy()
+        self.log_uniform_y = Variable(th.log(th.ones(1) / config.k_size))
+        self.eye = Variable(th.eye(self.config.y_size).unsqueeze(0))
+        self.beta = self.config.beta if hasattr(self.config, 'beta') else 0.0
+        if self.use_gpu:
+            self.log_uniform_y = self.log_uniform_y.cuda()
+            self.eye = self.eye.cuda()
+
+    def valid_loss(self, loss, batch_cnt=None):
+        if self.simple_posterior:
+            total_loss = loss.nll
+            if self.config.use_pr > 0.0:
+                total_loss += self.beta * loss.pi_kl
+        else:
+            total_loss = loss.nll + loss.pi_kl
+
+        if self.config.use_mi:
+            total_loss += (loss.b_pr * self.beta)
+
+        if self.config.use_diversity:
+            total_loss += loss.diversity
+
+        return total_loss
+
+    def forward(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        # enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        enc_last = utt_summary.squeeze(1)
+        # create decoder initial states
+        if self.simple_posterior:
+            logits_qy, log_qy = self.c2z(enc_last)
+            sample_y = self.gumbel_connector(logits_qy, hard=mode==GEN)
+            log_py = self.log_uniform_y
+        else:
+            logits_py, log_py = self.c2z(enc_last)
+            # encode response and use posterior to find q(z|x, c)
+            x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            if self.contextual_posterior:
+                logits_qy, log_qy = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            else:
+                logits_qy, log_qy = self.xc2z(x_h.squeeze(1))
+
+            # use prior at inference time, otherwise use posterior
+            if mode == GEN or (use_py is not None and use_py is True):
+                sample_y = self.gumbel_connector(logits_py, hard=False)
+            else:
+                sample_y = self.gumbel_connector(logits_qy, hard=True)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               # (batch_size, response_size-1)
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               # (batch_size, max_ctx_len, ctx_cell_size)
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_y
+            ret_dict['log_qy'] = log_qy
+            return ret_dict, labels
+
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            # regularization qy to be uniform
+            avg_log_qy = th.exp(log_qy.view(-1, self.config.y_size, self.config.k_size))
+            avg_log_qy = th.log(th.mean(avg_log_qy, dim=0) + 1e-15)
+            b_pr = self.cat_kl_loss(avg_log_qy, self.log_uniform_y, batch_size, unit_average=True)
+            mi = self.entropy_loss(avg_log_qy, unit_average=True) - self.entropy_loss(log_qy, unit_average=True)
+            pi_kl = self.cat_kl_loss(log_qy, log_py, batch_size, unit_average=True)
+            q_y = th.exp(log_qy).view(-1, self.config.y_size, self.config.k_size)  # b
+            p = th.pow(th.bmm(q_y, th.transpose(q_y, 1, 2)) - self.eye, 2)
+
+            result['pi_kl'] = pi_kl
+
+            result['diversity'] = th.mean(p)
+            result['nll'] = self.nll(dec_outputs, labels)
+            result['b_pr'] = b_pr
+            result['mi'] = mi
+            return result
+
+    def forward_rl(self, data_feed, max_words, temp=0.1):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # create decoder initial states
+        # enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        enc_last = utt_summary.squeeze(1)
+        # create decoder initial states
+        if self.simple_posterior:
+            logits_py, log_qy = self.c2z(enc_last)
+        else:
+            logits_py, log_qy = self.c2z(enc_last)
+
+        qy = F.softmax(logits_py / temp, dim=1)  # (batch_size, vocab_size, )
+        log_qy = F.log_softmax(logits_py, dim=1)  # (batch_size, vocab_size, )
+        idx = th.multinomial(qy, 1).detach()
+        logprob_sample_z = log_qy.gather(1, idx).view(-1, self.y_size)
+        joint_logpz = th.sum(logprob_sample_z, dim=1)
+        sample_y = cast_type(Variable(th.zeros(log_qy.size())), FLOAT, self.use_gpu)
+        sample_y.scatter_(1, idx, 1.0)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        # decode
+        logprobs, outs = self.decoder.forward_rl(batch_size=batch_size,
+                                                 dec_init_state=dec_init_state,
+                                                 attn_context=attn_context,
+                                                 vocab=self.vocab,
+                                                 max_words=max_words,
+                                                 temp=0.1)
+        return logprobs, outs, joint_logpz, sample_y
+
+class SysE2EActZCat(BaseModel):
+    def __init__(self, corpus, config): 
+        super(SysE2EActZCat, self).__init__(config)
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        self.k_size = config.k_size
+        self.y_size = config.y_size
+        self.simple_posterior = config.simple_posterior # minimize kl to uninformed prior instead of dist conditioned by context
+        self.contextual_posterior = config.contextual_posterior # does not use context cause AE task
+        if "use_act_label" in config:
+            self.use_act_label = config.use_act_label
+        else:
+            self.use_act_label = False
+
+        if "use_aux_c2z" in config:
+            self.use_aux_c2z = config.use_aux_c2z
+        else:
+            self.use_aux_c2z = False
+
+        self.embedding = None
+        self.aux_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+        self.utt_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+
+        if self.use_act_label:
+            self.c2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size + self.act_size, config.y_size, config.k_size, is_lstm=False)
+        else:
+            self.c2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size,
+                                          config.y_size, config.k_size, is_lstm=False)
+            if self.use_aux_c2z:
+                self.aux_c2z = nn_lib.Hidden2Discrete(self.aux_encoder.output_size, config.y_size, config.k_size, is_lstm=False)
+        self.z_embedding = nn.Linear(self.y_size * self.k_size, config.dec_cell_size, bias=False)
+        self.gumbel_connector = nn_lib.GumbelConnector(config.use_gpu)
+        
+        if not self.simple_posterior: #q(z|x,c)
+            if self.contextual_posterior:
+                # x, c, BS, and DB
+                self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size,
+                                                   config.y_size, config.k_size, is_lstm=False)
+            else:
+                self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size, config.y_size, config.k_size, is_lstm=False)
+
+        self.decoder = DecoderRNN(input_dropout_p=config.dropout,
+                                  rnn_cell=config.dec_rnn_cell,
+                                  input_size=config.embed_size,
+                                  hidden_size=config.dec_cell_size,
+                                  num_layers=config.num_layers,
+                                  output_dropout_p=config.dropout,
+                                  bidirectional=False,
+                                  vocab_size=self.vocab_size,
+                                  use_attn=config.dec_use_attn,
+                                  ctx_cell_size=config.dec_cell_size,
+                                  attn_mode=config.dec_attn_mode,
+                                  sys_id=self.bos_id,
+                                  eos_id=self.eos_id,
+                                  use_gpu=config.use_gpu,
+                                  max_dec_len=config.max_dec_len,
+                                  embedding=self.embedding)
+
+        if config.avg_type == "slot":
+            # give slot_weight:1 ratio between slot tokens and other words
+            self.loss_weight = th.tensor([(config.slot_weight - 1) * int(k[0] == '[' and k[-1] == ']' ) + 1 for k in self.vocab_dict.keys()]).type(th.FloatTensor)
+            if self.use_gpu:
+                self.loss_weight = self.loss_weight.cuda()
+            self.nll = WeightedNLLEntropy(self.pad_id, config.avg_type, self.loss_weight)
+        else:
+            self.nll = NLLEntropy(self.pad_id, config.avg_type)
+        self.cat_kl_loss = CatKLLoss()
+        self.entropy_loss = Entropy()
+        self.log_uniform_y = Variable(th.log(th.ones(1) / config.k_size))
+        self.eye = Variable(th.eye(self.config.y_size).unsqueeze(0))
+        self.beta = self.config.beta if hasattr(self.config, 'beta') else 0.0
+        if self.use_gpu:
+            self.log_uniform_y = self.log_uniform_y.cuda()
+            self.eye = self.eye.cuda()
+
+    def valid_loss(self, loss, batch_cnt=None):
+        if self.simple_posterior:
+            total_loss = loss.nll
+            if self.config.use_pr > 0.0:
+                total_loss += self.beta * loss.pi_kl
+        else:
+            total_loss = loss.nll + loss.pi_kl
+
+        if self.config.use_mi:
+            total_loss += (loss.b_pr * self.beta)
+
+        if self.config.use_diversity:
+            total_loss += loss.diversity
+
+        return total_loss
+    
+    def forward(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        short_target_utts = self.np2var(data_feed['outputs'], LONG)
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+        aux_utt_summary, _, aux_enc_outs = self.aux_encoder(short_target_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        enc_last = utt_summary.squeeze(1)
+        aux_enc_last = aux_utt_summary.squeeze(1)
+        # enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        # aux_enc_last = th.cat([bs_label, db_label, aux_utt_summary.squeeze(1)], dim=1)
+        # create decoder initial states
+        if self.simple_posterior:
+            logits_qy, log_qy = self.c2z(enc_last)
+            if self.use_aux_c2z:
+                aux_logits_qy, aux_log_qy = self.aux_c2z(aux_utt_summary.squeeze(1))
+            else:
+                aux_logits_qy, aux_log_qy = self.c2z(aux_enc_last)
+            sample_y = self.gumbel_connector(logits_qy, hard=mode==GEN)
+            log_py = aux_log_qy
+        else: 
+            logits_py, log_py = self.c2z(enc_last)
+            aux_logits_qy, aux_log_qy = self.c2z(aux_enc_last)
+            # encode response and use posterior to find q(z|x, c)
+            x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            if self.contextual_posterior:
+                logits_qy, log_qy = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            else:
+                logits_qy, log_qy = self.xc2z(x_h.squeeze(1))
+
+            # use prior at inference time, otherwise use posterior
+            if mode == GEN or (use_py is not None and use_py is True):
+                sample_y = self.gumbel_connector(logits_py, hard=False)
+            else:
+                sample_y = self.gumbel_connector(logits_qy, hard=True)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               # (batch_size, response_size-1)
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               # (batch_size, max_ctx_len, ctx_cell_size)
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_y
+            ret_dict['log_qy'] = log_qy
+            return ret_dict, labels
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            # regularization qy to be uniform
+            avg_log_qy = th.exp(log_qy.view(-1, self.config.y_size, self.config.k_size))
+            avg_log_qy = th.log(th.mean(avg_log_qy, dim=0) + 1e-15)
+            b_pr = self.cat_kl_loss(avg_log_qy, self.log_uniform_y, batch_size, unit_average=True)
+            mi = self.entropy_loss(avg_log_qy, unit_average=True) - self.entropy_loss(log_qy, unit_average=True)
+            pi_kl = self.cat_kl_loss(log_qy, log_py, batch_size, unit_average=True)
+            q_y = th.exp(log_qy).view(-1, self.config.y_size, self.config.k_size)  # b
+            p = th.pow(th.bmm(q_y, th.transpose(q_y, 1, 2)) - self.eye, 2)
+
+            result['pi_kl'] = pi_kl
+            result['diversity'] = th.mean(p)
+            result['nll'] = self.nll(dec_outputs, labels)
+            result['b_pr'] = b_pr
+            result['mi'] = mi
+            return result
+    
+    def forward_rl(self, data_feed, max_words, temp=0.1, enc="utt"):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        batch_size = len(ctx_lens)
+        if enc == "utt":
+            short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+            bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+            db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+
+            utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+            # create decoder initial states
+            enc_last = utt_summary.squeeze(1)
+            # create decoder initial states
+            if self.simple_posterior:
+                logits_py, log_qy = self.c2z(enc_last)
+            else:
+                logits_py, log_qy = self.c2z(enc_last)
+
+        elif enc == "aux":
+            short_target_utts = self.np2var(data_feed['outputs'], LONG)
+            # short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['outputs'], ctx_lens), LONG)
+            bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+            db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+
+            aux_utt_summary, _, aux_enc_outs = self.aux_encoder(short_target_utts.unsqueeze(1))
+            if self.simple_posterior:
+                if self.use_aux_c2z:
+                    aux_logits_qy, aux_log_qy = self.aux_c2z(aux_utt_summary.squeeze(1))
+                else:
+                    aux_enc_last = aux_utt_summary.squeeze(1)
+                    aux_logits_qy, aux_log_qy = self.c2z(aux_enc_last)
+            logits_py = aux_logits_qy
+            log_qy = aux_log_qy
+
+        
+        qy = F.softmax(logits_py / temp, dim=1)  # (batch_size, vocab_size, )
+        log_qy = F.log_softmax(logits_py, dim=1)  # (batch_size, vocab_size, )
+        idx = th.multinomial(qy, 1).detach()
+        logprob_sample_z = log_qy.gather(1, idx).view(-1, self.y_size)
+        joint_logpz = th.sum(logprob_sample_z, dim=1)
+        sample_y = cast_type(Variable(th.zeros(log_qy.size())), FLOAT, self.use_gpu)
+        sample_y.scatter_(1, idx, 1.0)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        # decode
+        logprobs, outs = self.decoder.forward_rl(batch_size=batch_size,
+                                                 dec_init_state=dec_init_state,
+                                                 attn_context=attn_context,
+                                                 vocab=self.vocab,
+                                                 max_words=max_words,
+                                                 temp=0.1)
+        return logprobs, outs, joint_logpz, sample_y
+
+class SysPerfectBD2Gauss(BaseModel):
+    def __init__(self, corpus, config):
+        super(SysPerfectBD2Gauss, self).__init__(config)
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        self.y_size = config.y_size
+        self.simple_posterior = config.simple_posterior
+        if "contextual posterior" in config: 
+            self.contextual_posterior = config.contextual_posterior
+        else:
+            self.contextual_posterior = True # default value is true, i.e. q(z|x,c)
+
+        self.embedding = None
+        self.utt_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+        self.c2z = nn_lib.Hidden2Gaussian(self.utt_encoder.output_size + self.db_size + self.bs_size,
+                                          config.y_size, is_lstm=False)
+        self.gauss_connector = nn_lib.GaussianConnector(self.use_gpu)
+        self.z_embedding = nn.Linear(self.y_size, config.dec_cell_size)
+        if not self.simple_posterior:
+            # self.xc2z = nn_lib.Hidden2Gaussian(self.utt_encoder.output_size * 2 + self.db_size + self.bs_size,
+                                               # config.y_size, is_lstm=False)
+            if self.contextual_posterior:
+                self.xc2z = nn_lib.Hidden2Gaussian(self.utt_encoder.output_size * 2 + self.db_size + self.bs_size,
+                                                   config.y_size, is_lstm=False)
+            else:
+                self.xc2z = nn_lib.Hidden2Gaussian(self.utt_encoder.output_size, config.y_size, is_lstm=False)
+
+
+        self.decoder = DecoderRNN(input_dropout_p=config.dropout,
+                                  rnn_cell=config.dec_rnn_cell,
+                                  input_size=config.embed_size,
+                                  hidden_size=config.dec_cell_size,
+                                  num_layers=config.num_layers,
+                                  output_dropout_p=config.dropout,
+                                  bidirectional=False,
+                                  vocab_size=self.vocab_size,
+                                  use_attn=config.dec_use_attn,
+                                  ctx_cell_size=config.dec_cell_size,
+                                  attn_mode=config.dec_attn_mode,
+                                  sys_id=self.bos_id,
+                                  eos_id=self.eos_id,
+                                  use_gpu=config.use_gpu,
+                                  max_dec_len=config.max_dec_len,
+                                  embedding=self.embedding)
+
+        if config.avg_type == "slot":
+            # give slot_weight:1 ratio between slot tokens and other words
+            self.loss_weight = th.tensor([(config.slot_weight - 1) * int(k[0] == '[' and k[-1] == ']' ) + 1 for k in self.vocab_dict.keys()]).type(th.FloatTensor)
+            if self.use_gpu:
+                self.loss_weight = self.loss_weight.cuda()
+            self.nll = WeightedNLLEntropy(self.pad_id, config.avg_type, self.loss_weight)
+        else:
+            self.nll = NLLEntropy(self.pad_id, config.avg_type)
+
+        self.gauss_kl = NormKLLoss(unit_average=True)
+        self.zero = cast_type(th.zeros(1), FLOAT, self.use_gpu)
+
+    def valid_loss(self, loss, batch_cnt=None):
+        if self.simple_posterior:
+            total_loss = loss.nll
+            if self.config.use_pr > 0.0:
+                total_loss += self.config.beta * loss.pi_kl
+        else:
+            total_loss = loss.nll + loss.pi_kl
+
+        return total_loss
+
+    def forward(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+
+        # create decoder initial states
+        if self.simple_posterior:
+            q_mu, q_logvar = self.c2z(enc_last)
+            sample_z = self.gauss_connector(q_mu, q_logvar)
+            p_mu, p_logvar = self.zero, self.zero
+        else:
+            p_mu, p_logvar = self.c2z(enc_last)
+            # encode response and use posterior to find q(z|x, c)
+            x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            if self.contextual_posterior:
+                q_mu, q_logvar = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            else:
+                q_mu, q_logvar = self.xc2z(x_h.squeeze(1))
+
+            # use prior at inference time, otherwise use posterior
+            if mode == GEN or use_py:
+                sample_z = self.gauss_connector(p_mu, p_logvar)
+            else:
+                sample_z = self.gauss_connector(q_mu, q_logvar)
+
+        # pack attention context
+        dec_init_state = self.z_embedding(sample_z.unsqueeze(0))
+        attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_z
+            ret_dict['q_mu'] = q_mu
+            ret_dict['q_logvar'] = q_logvar
+            return ret_dict, labels
+
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            pi_kl = self.gauss_kl(q_mu, q_logvar, p_mu, p_logvar)
+            result['pi_kl'] = pi_kl
+            result['nll'] = self.nll(dec_outputs, labels)
+            return result
+
+    def gaussian_logprob(self, mu, logvar, sample_z):
+        var = th.exp(logvar)
+        constant = float(-0.5 * np.log(2*np.pi))
+        logprob = constant - 0.5 * logvar - th.pow((mu-sample_z), 2) / (2.0*var)
+        return logprob
+
+    def forward_rl(self, data_feed, max_words, temp=0.1):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        # create decoder initial states
+        p_mu, p_logvar = self.c2z(enc_last)
+
+        sample_z = th.normal(p_mu, th.sqrt(th.exp(p_logvar))).detach()
+        logprob_sample_z = self.gaussian_logprob(p_mu, self.zero, sample_z)
+        joint_logpz = th.sum(logprob_sample_z, dim=1)
+
+        # pack attention context
+        dec_init_state = self.z_embedding(sample_z.unsqueeze(0))
+        attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        # decode
+        logprobs, outs = self.decoder.forward_rl(batch_size=batch_size,
+                                                 dec_init_state=dec_init_state,
+                                                 attn_context=attn_context,
+                                                 vocab=self.vocab,
+                                                 max_words=max_words,
+                                                 temp=0.1)
+        return logprobs, outs, joint_logpz, sample_z
+
+class SysAEGauss(BaseModel):
+    def __init__(self, corpus, config):
+        super(SysAEGauss, self).__init__(config)
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        # self.bs_size = corpus.bs_size
+        # self.db_size = corpus.db_size
+        # self.act_size = corpus.act_size
+        self.y_size = config.y_size
+        self.simple_posterior = True
+        self.contextual_posterior = False
+
+        self.embedding = None
+        self.utt_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+        
+        self.c2z = nn_lib.Hidden2Gaussian(self.utt_encoder.output_size,
+                                          config.y_size, is_lstm=False)
+        self.gauss_connector = nn_lib.GaussianConnector(self.use_gpu)
+       
+        self.z_embedding = nn.Linear(self.y_size, config.dec_cell_size)
+        if not self.simple_posterior:
+            # self.xc2z = nn_lib.Hidden2Gaussian(self.utt_encoder.output_size * 2 + self.db_size + self.bs_size,
+                                               # config.y_size, is_lstm=False)
+            if self.contextual_posterior:
+                self.xc2z = nn_lib.Hidden2Gaussian(self.utt_encoder.output_size * 2 + self.db_size + self.bs_size,
+                                                   config.y_size, is_lstm=False)
+            else:
+                self.xc2z = nn_lib.Hidden2Gaussian(self.utt_encoder.output_size, config.y_size, is_lstm=False)
+
+
+        self.decoder = DecoderRNN(input_dropout_p=config.dropout,
+                                  rnn_cell=config.dec_rnn_cell,
+                                  input_size=config.embed_size,
+                                  hidden_size=config.dec_cell_size,
+                                  num_layers=config.num_layers,
+                                  output_dropout_p=config.dropout,
+                                  bidirectional=False,
+                                  vocab_size=self.vocab_size,
+                                  use_attn=config.dec_use_attn,
+                                  ctx_cell_size=config.dec_cell_size,
+                                  attn_mode=config.dec_attn_mode,
+                                  sys_id=self.bos_id,
+                                  eos_id=self.eos_id,
+                                  use_gpu=config.use_gpu,
+                                  max_dec_len=config.max_dec_len,
+                                  embedding=self.embedding)
+
+        if config.avg_type == "slot":
+            # give slot_weight:1 ratio between slot tokens and other words
+            self.loss_weight = th.tensor([(config.slot_weight - 1) * int(k[0] == '[' and k[-1] == ']' ) + 1 for k in self.vocab_dict.keys()]).type(th.FloatTensor)
+            if self.use_gpu:
+                self.loss_weight = self.loss_weight.cuda()
+            self.nll = WeightedNLLEntropy(self.pad_id, config.avg_type, self.loss_weight)
+        else:
+            self.nll = NLLEntropy(self.pad_id, config.avg_type)
+
+        self.gauss_kl = NormKLLoss(unit_average=True)
+        self.zero = cast_type(th.zeros(1), FLOAT, self.use_gpu)
+
+    def valid_loss(self, loss, batch_cnt=None):
+        if self.simple_posterior:
+            total_loss = loss.nll
+            if self.config.use_pr > 0.0:
+                total_loss += self.config.beta * loss.pi_kl
+        else:
+            total_loss = loss.nll + loss.pi_kl
+
+        return total_loss
+
+    def forward(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        # bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        # db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        # act_label = self.np2var(data_feed['act'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        # enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        enc_last = utt_summary.squeeze(1)
+
+
+        # create decoder initial states
+        if self.simple_posterior:
+            q_mu, q_logvar = self.c2z(enc_last)
+            sample_z = self.gauss_connector(q_mu, q_logvar)
+            p_mu, p_logvar = self.zero, self.zero
+        # else:
+            # p_mu, p_logvar = self.c2z(enc_last)
+            # # encode response and use posterior to find q(z|x, c)
+            # x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            # if self.contextual_posterior:
+                # q_mu, q_logvar = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            # else:
+                # q_mu, q_logvar = self.xc2z(x_h.squeeze(1))
+
+            # # use prior at inference time, otherwise use posterior
+            # if mode == GEN or use_py:
+                # sample_z = self.gauss_connector(p_mu, p_logvar)
+            # else:
+                # sample_z = self.gauss_connector(q_mu, q_logvar)
+
+        # pack attention context
+        dec_init_state = self.z_embedding(sample_z.unsqueeze(0))
+        attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_z
+            ret_dict['q_mu'] = q_mu
+            ret_dict['q_logvar'] = q_logvar
+            return ret_dict, labels
+
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            pi_kl = self.gauss_kl(q_mu, q_logvar, p_mu, p_logvar)
+            result['pi_kl'] = pi_kl
+            result['nll'] = self.nll(dec_outputs, labels)
+            return result
+
+    def gaussian_logprob(self, mu, logvar, sample_z):
+        var = th.exp(logvar)
+        constant = float(-0.5 * np.log(2*np.pi))
+        logprob = constant - 0.5 * logvar - th.pow((mu-sample_z), 2) / (2.0*var)
+        return logprob
+
+    def forward_rl(self, data_feed, max_words, temp=0.1):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        # create decoder initial states
+        p_mu, p_logvar = self.c2z(enc_last)
+
+        sample_z = th.normal(p_mu, th.sqrt(th.exp(p_logvar))).detach()
+        logprob_sample_z = self.gaussian_logprob(p_mu, self.zero, sample_z)
+        joint_logpz = th.sum(logprob_sample_z, dim=1)
+
+        # pack attention context
+        dec_init_state = self.z_embedding(sample_z.unsqueeze(0))
+        attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        # decode
+        logprobs, outs = self.decoder.forward_rl(batch_size=batch_size,
+                                                 dec_init_state=dec_init_state,
+                                                 attn_context=attn_context,
+                                                 vocab=self.vocab,
+                                                 max_words=max_words,
+                                                 temp=0.1)
+        return logprobs, outs, joint_logpz, sample_z
+
+class SysMTGauss(BaseModel):
+    def __init__(self, corpus, config):
+        super(SysMTGauss, self).__init__(config)
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        self.y_size = config.y_size
+        self.simple_posterior = config.simple_posterior
+        self.contextual_posterior = config.contextual_posterior
+
+        if "use_aux_kl" in config:
+            self.use_aux_kl = config.use_aux_kl
+        else:
+            self.use_aux_kl = False
+
+
+        self.embedding = None
+        self.utt_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+        
+        self.aux_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+        self.c2z = nn_lib.Hidden2Gaussian(self.utt_encoder.output_size + self.db_size + self.bs_size,
+                                          config.y_size, is_lstm=False)
+        self.gauss_connector = nn_lib.GaussianConnector(self.use_gpu)
+        self.z_embedding = nn.Linear(self.y_size, config.dec_cell_size)
+        if not self.simple_posterior:
+            # self.xc2z = nn_lib.Hidden2Gaussian(self.utt_encoder.output_size * 2 + self.db_size + self.bs_size,
+                                               # config.y_size, is_lstm=False)
+            if self.contextual_posterior:
+                self.xc2z = nn_lib.Hidden2Gaussian(self.utt_encoder.output_size * 2 + self.db_size + self.bs_size,
+                                                   config.y_size, is_lstm=False)
+            else:
+                self.xc2z = nn_lib.Hidden2Gaussian(self.utt_encoder.output_size, config.y_size, is_lstm=False)
+
+
+        self.decoder = DecoderRNN(input_dropout_p=config.dropout,
+                                  rnn_cell=config.dec_rnn_cell,
+                                  input_size=config.embed_size,
+                                  hidden_size=config.dec_cell_size,
+                                  num_layers=config.num_layers,
+                                  output_dropout_p=config.dropout,
+                                  bidirectional=False,
+                                  vocab_size=self.vocab_size,
+                                  use_attn=config.dec_use_attn,
+                                  ctx_cell_size=config.dec_cell_size,
+                                  attn_mode=config.dec_attn_mode,
+                                  sys_id=self.bos_id,
+                                  eos_id=self.eos_id,
+                                  use_gpu=config.use_gpu,
+                                  max_dec_len=config.max_dec_len,
+                                  embedding=self.embedding)
+
+        if config.avg_type == "slot":
+            # give slot_weight:1 ratio between slot tokens and other words
+            self.loss_weight = th.tensor([(config.slot_weight - 1) * int(k[0] == '[' and k[-1] == ']' ) + 1 for k in self.vocab_dict.keys()]).type(th.FloatTensor)
+            if self.use_gpu:
+                self.loss_weight = self.loss_weight.cuda()
+            self.nll = WeightedNLLEntropy(self.pad_id, config.avg_type, self.loss_weight)
+        else:
+            self.nll = NLLEntropy(self.pad_id, config.avg_type)
+
+        self.gauss_kl = NormKLLoss(unit_average=True)
+        self.zero = cast_type(th.zeros(1), FLOAT, self.use_gpu)
+
+    def valid_loss(self, loss, batch_cnt=None):
+        if self.simple_posterior:
+            total_loss = loss.nll
+            if self.config.use_pr > 0.0:
+                total_loss += self.config.beta * loss.pi_kl
+        else:
+            total_loss = loss.nll + loss.pi_kl
+
+        if self.use_aux_kl:
+            try:
+                total_loss += loss.aux_pi_kl
+            except KeyError:
+                total_loss += 0
+
+        return total_loss
+
+    def forward(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+
+
+        # create decoder initial states
+        if self.simple_posterior:
+            q_mu, q_logvar = self.c2z(enc_last)
+            sample_z = self.gauss_connector(q_mu, q_logvar)
+            p_mu, p_logvar = self.zero, self.zero
+        else:
+            p_mu, p_logvar = self.c2z(enc_last)
+            # encode response and use posterior to find q(z|x, c)
+            x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            if self.contextual_posterior:
+                q_mu, q_logvar = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            else:
+                q_mu, q_logvar = self.xc2z(x_h.squeeze(1))
+
+            # use prior at inference time, otherwise use posterior
+            if mode == GEN or use_py:
+                sample_z = self.gauss_connector(p_mu, p_logvar)
+            else:
+                sample_z = self.gauss_connector(q_mu, q_logvar)
+
+        # pack attention context
+        dec_init_state = self.z_embedding(sample_z.unsqueeze(0))
+        attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_z
+            ret_dict['q_mu'] = q_mu
+            ret_dict['q_logvar'] = q_logvar
+            return ret_dict, labels
+
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            pi_kl = self.gauss_kl(q_mu, q_logvar, p_mu, p_logvar)
+            result['pi_kl'] = pi_kl
+            result['nll'] = self.nll(dec_outputs, labels)
+            return result
+    
+    def shared_forward(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        short_target_utts = self.np2var(data_feed['outputs'], LONG)
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+        aux_utt_summary, _, aux_enc_outs = self.aux_encoder(short_target_utts.unsqueeze(1))
+        
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        aux_enc_last = th.cat([bs_label, db_label, aux_utt_summary.squeeze(1)], dim=1)
+
+        # create decoder initial states
+        if self.simple_posterior:
+            q_mu, q_logvar = self.c2z(enc_last)
+            aux_q_mu, aux_q_logvar = self.c2z(aux_enc_last)
+            sample_z = self.gauss_connector(q_mu, q_logvar)
+            p_mu, p_logvar = self.zero, self.zero
+        else:
+            p_mu, p_logvar = self.c2z(enc_last)
+            # encode response and use posterior to find q(z|x, c)
+            x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            if self.contextual_posterior:
+                q_mu, q_logvar = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            else:
+                q_mu, q_logvar = self.xc2z(x_h.squeeze(1))
+
+            aux_q_mu, aux_q_logvar = self.c2z(aux_enc_last)
+            
+            # use prior at inference time, otherwise use posterior
+            if mode == GEN or use_py:
+                sample_z = self.gauss_connector(p_mu, p_logvar)
+            else:
+                sample_z = self.gauss_connector(q_mu, q_logvar)
+
+        # pack attention context
+        dec_init_state = self.z_embedding(sample_z.unsqueeze(0))
+        attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_z
+            ret_dict['q_mu'] = q_mu
+            ret_dict['q_logvar'] = q_logvar
+            return ret_dict, labels
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            pi_kl = self.gauss_kl(q_mu, q_logvar, p_mu, p_logvar)
+            aux_pi_kl = self.gauss_kl(q_mu, q_logvar, aux_q_mu, aux_q_logvar)
+            result['pi_kl'] = pi_kl
+            result['aux_pi_kl'] = aux_pi_kl
+            result['nll'] = self.nll(dec_outputs, labels)
+            return result
+
+    def gaussian_logprob(self, mu, logvar, sample_z):
+        var = th.exp(logvar)
+        constant = float(-0.5 * np.log(2*np.pi))
+        logprob = constant - 0.5 * logvar - th.pow((mu-sample_z), 2) / (2.0*var)
+        return logprob
+
+        return logprobs, outs, joint_logpz, sample_z
+
+    def forward_aux(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        act_label = self.np2var(data_feed['act'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.aux_encoder(short_ctx_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+
+        # create decoder initial states
+        if self.simple_posterior:
+            q_mu, q_logvar = self.c2z(enc_last)
+            sample_z = self.gauss_connector(q_mu, q_logvar)
+            p_mu, p_logvar = self.zero, self.zero
+        # else:
+            # p_mu, p_logvar = self.c2z(enc_last)
+            # # encode response and use posterior to find q(z|x, c)
+            # x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            # if self.contextual_posterior:
+                # q_mu, q_logvar = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            # else:
+                # q_mu, q_logvar = self.xc2z(x_h.squeeze(1))
+
+            # # use prior at inference time, otherwise use posterior
+            # if mode == GEN or use_py:
+                # sample_z = self.gauss_connector(p_mu, p_logvar)
+            # else:
+                # sample_z = self.gauss_connector(q_mu, q_logvar)
+
+        # pack attention context
+        dec_init_state = self.z_embedding(sample_z.unsqueeze(0))
+        attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_z
+            ret_dict['q_mu'] = q_mu
+            ret_dict['q_logvar'] = q_logvar
+            return ret_dict, labels
+
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            pi_kl = self.gauss_kl(q_mu, q_logvar, p_mu, p_logvar)
+            result['pi_kl'] = pi_kl
+            result['nll'] = self.nll(dec_outputs, labels)
+            return result
+
+class SysActZGauss(BaseModel):
+    def __init__(self, corpus, config):
+        super(SysActZGauss, self).__init__(config)
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        self.y_size = config.y_size
+        self.simple_posterior = config.simple_posterior
+        self.contextual_posterior = config.contextual_posterior
+
+        self.embedding = None
+        self.utt_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+        
+        self.aux_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+        self.c2z = nn_lib.Hidden2Gaussian(self.utt_encoder.output_size + self.db_size + self.bs_size,
+                                          config.y_size, is_lstm=False)
+        self.gauss_connector = nn_lib.GaussianConnector(self.use_gpu)
+        self.z_embedding = nn.Linear(self.y_size, config.dec_cell_size)
+        if not self.simple_posterior:
+            # self.xc2z = nn_lib.Hidden2Gaussian(self.utt_encoder.output_size * 2 + self.db_size + self.bs_size,
+                                               # config.y_size, is_lstm=False)
+            if self.contextual_posterior:
+                self.xc2z = nn_lib.Hidden2Gaussian(self.utt_encoder.output_size * 2 + self.db_size + self.bs_size,
+                                                   config.y_size, is_lstm=False)
+            else:
+                self.xc2z = nn_lib.Hidden2Gaussian(self.utt_encoder.output_size, config.y_size, is_lstm=False)
+
+
+        self.decoder = DecoderRNN(input_dropout_p=config.dropout,
+                                  rnn_cell=config.dec_rnn_cell,
+                                  input_size=config.embed_size,
+                                  hidden_size=config.dec_cell_size,
+                                  num_layers=config.num_layers,
+                                  output_dropout_p=config.dropout,
+                                  bidirectional=False,
+                                  vocab_size=self.vocab_size,
+                                  use_attn=config.dec_use_attn,
+                                  ctx_cell_size=config.dec_cell_size,
+                                  attn_mode=config.dec_attn_mode,
+                                  sys_id=self.bos_id,
+                                  eos_id=self.eos_id,
+                                  use_gpu=config.use_gpu,
+                                  max_dec_len=config.max_dec_len,
+                                  embedding=self.embedding)
+
+        if config.avg_type == "slot":
+            # give slot_weight:1 ratio between slot tokens and other words
+            self.loss_weight = th.tensor([(config.slot_weight - 1) * int(k[0] == '[' and k[-1] == ']' ) + 1 for k in self.vocab_dict.keys()]).type(th.FloatTensor)
+            if self.use_gpu:
+                self.loss_weight = self.loss_weight.cuda()
+            self.nll = WeightedNLLEntropy(self.pad_id, config.avg_type, self.loss_weight)
+        else:
+            self.nll = NLLEntropy(self.pad_id, config.avg_type)
+
+        self.gauss_kl = NormKLLoss(unit_average=True)
+        self.zero = cast_type(th.zeros(1), FLOAT, self.use_gpu)
+    
+    def valid_loss(self, loss, batch_cnt=None):
+        if self.simple_posterior:
+            total_loss = loss.nll
+            if self.config.use_pr > 0.0:
+                total_loss += self.config.beta * loss.pi_kl
+        else:
+            total_loss = loss.nll + loss.pi_kl
+
+        if self.config.use_mi:
+            total_loss += (loss.b_pr * self.beta)
+
+        if self.config.use_diversity:
+            total_loss += loss.diversity
+
+        return total_loss
+    
+    def forward(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        short_target_utts = self.np2var(data_feed['outputs'], LONG)
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+        aux_utt_summary, _, aux_enc_outs = self.aux_encoder(short_target_utts.unsqueeze(1))
+        
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        aux_enc_last = th.cat([bs_label, db_label, aux_utt_summary.squeeze(1)], dim=1)
+
+        # create decoder initial states
+        if self.simple_posterior:
+            q_mu, q_logvar = self.c2z(enc_last)
+            p_mu, p_logvar = self.c2z(aux_enc_last)
+            sample_z = self.gauss_connector(q_mu, q_logvar)
+        else:
+            p_mu, p_logvar = self.c2z(enc_last)
+            # encode response and use posterior to find q(z|x, c)
+            x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            if self.contextual_posterior:
+                q_mu, q_logvar = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            else:
+                q_mu, q_logvar = self.xc2z(x_h.squeeze(1))
+
+            aux_q_mu, aux_q_logvar = self.c2z(aux_enc_last)
+            
+            # use prior at inference time, otherwise use posterior
+            if mode == GEN or use_py:
+                sample_z = self.gauss_connector(p_mu, p_logvar)
+            else:
+                sample_z = self.gauss_connector(q_mu, q_logvar)
+
+        # pack attention context
+        dec_init_state = self.z_embedding(sample_z.unsqueeze(0))
+        attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_z
+            ret_dict['q_mu'] = q_mu
+            ret_dict['q_logvar'] = q_logvar
+            return ret_dict, labels
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            pi_kl = self.gauss_kl(q_mu, q_logvar, p_mu, p_logvar)
+            result['pi_kl'] = pi_kl
+            result['nll'] = self.nll(dec_outputs, labels)
+            return result
+
+    def forward_rl(self, data_feed, max_words, temp=0.1):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed['contexts'], ctx_lens), LONG)
+        bs_label = self.np2var(data_feed['bs'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = self.np2var(data_feed['db'], FLOAT)  # (batch_size, max_ctx_len, max_utt_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        # create decoder initial states
+        if self.simple_posterior:
+            logits_py, log_qy = self.c2z(enc_last)
+        else:
+            logits_py, log_qy = self.c2z(enc_last)
+
+        qy = F.softmax(logits_py / temp, dim=1)  # (batch_size, vocab_size, )
+        log_qy = F.log_softmax(logits_py, dim=1)  # (batch_size, vocab_size, )
+        idx = th.multinomial(qy, 1).detach()
+        logprob_sample_z = log_qy.gather(1, idx).view(-1, self.y_size)
+        joint_logpz = th.sum(logprob_sample_z, dim=1)
+        sample_y = cast_type(Variable(th.zeros(log_qy.size())), FLOAT, self.use_gpu)
+        sample_y.scatter_(1, idx, 1.0)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        # decode
+        logprobs, outs = self.decoder.forward_rl(batch_size=batch_size,
+                                                 dec_init_state=dec_init_state,
+                                                 attn_context=attn_context,
+                                                 vocab=self.vocab,
+                                                 max_words=max_words,
+                                                 temp=0.1)
+        return logprobs, outs, joint_logpz, sample_y
+
+
+# Grounded Models
+
+class SysEncodedBD2Cat(BaseModel):
+    def __init__(self, corpus, config):
+        super(SysEncodedBD2Cat, self).__init__(config)
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        self.k_size = config.k_size
+        self.y_size = config.y_size
+        self.simple_posterior = config.simple_posterior
+        self.contextual_posterior = config.contextual_posterior
+
+        self.embedding = None
+        self.utt_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+        if config.use_metadata_for_decoding:
+            self.metadata_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                             embedding_dim=int(config.embed_size / 2),
+                                             feat_size=0,
+                                             goal_nhid=0,
+                                             rnn_cell=config.utt_rnn_cell,
+                                             utt_cell_size=int(config.utt_cell_size / 2),
+                                             num_layers=config.num_layers,
+                                             input_dropout_p=config.dropout,
+                                             output_dropout_p=config.dropout,
+                                             bidirectional=config.bi_utt_cell,
+                                             variable_lengths=False,
+                                             use_attn=config.enc_use_attn,
+                                             embedding=self.embedding)
+
+        if "policy_dropout" in config and config.policy_dropout:
+            if "policy_dropout_rate" in config:
+                self.c2z = nn_lib.Hidden2DiscretewDropout(self.utt_encoder.output_size,
+                                  config.y_size, config.k_size, is_lstm=False, p_dropout=config.policy_dropout_rate, dropout_on_eval=config.dropout_on_eval)
+            else:
+                self.c2z = nn_lib.Hidden2DiscretewDropout(self.utt_encoder.output_size,
+                                  config.y_size, config.k_size, is_lstm=False)
+
+        else:
+            self.c2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size,
+                                              config.y_size, config.k_size, is_lstm=False)
+        self.z_embedding = nn.Linear(self.y_size * self.k_size, config.dec_cell_size, bias=False)
+        self.gumbel_connector = nn_lib.GumbelConnector(config.use_gpu)
+        if not self.simple_posterior:
+            if self.contextual_posterior:
+                self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size * 2 + self.db_size + self.bs_size,
+                                                   config.y_size, config.k_size, is_lstm=False)
+            else:
+                self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size, config.y_size, config.k_size, is_lstm=False)
+
+        if config.use_metadata_for_decoding:
+            dec_hidden_size = config.dec_cell_size + self.metadata_encoder.output_size
+        else:
+            dec_hidden_size = config.dec_cell_size
+
+        self.decoder = DecoderRNN(input_dropout_p=config.dropout,
+                                  rnn_cell=config.dec_rnn_cell,
+                                  input_size=config.embed_size,
+                                  hidden_size=dec_hidden_size,
+                                  num_layers=config.num_layers,
+                                  output_dropout_p=config.dropout,
+                                  bidirectional=False,
+                                  vocab_size=self.vocab_size,
+                                  use_attn=config.dec_use_attn,
+                                  ctx_cell_size=config.dec_cell_size,
+                                  attn_mode=config.dec_attn_mode,
+                                  sys_id=self.bos_id,
+                                  eos_id=self.eos_id,
+                                  use_gpu=config.use_gpu,
+                                  max_dec_len=config.max_dec_len,
+                                  embedding=self.embedding)
+
+        self.nll = NLLEntropy(self.pad_id, config.avg_type)
+        if config.avg_type == "weighted" and config.nll_weight=="no_match_penalty":
+            req_tokens = []
+            for d in REQ_TOKENS.keys():
+                req_tokens.extend(REQ_TOKENS[d])
+            nll_weight = Variable(th.FloatTensor([10. if token in req_tokens  else 1. for token in self.vocab]))
+            print("req tokens assigned with special weights")
+            if config.use_gpu:
+                nll_weight = nll_weight.cuda()
+            self.nll.set_weight(nll_weight)
+
+        self.cat_kl_loss = CatKLLoss()
+        self.entropy_loss = Entropy()
+        self.log_uniform_y = Variable(th.log(th.ones(1) / config.k_size))
+        self.eye = Variable(th.eye(self.config.y_size).unsqueeze(0))
+        self.beta = self.config.beta if hasattr(self.config, 'beta') else 0.0
+        if self.use_gpu:
+            self.log_uniform_y = self.log_uniform_y.cuda()
+            self.eye = self.eye.cuda()
+
+    def valid_loss(self, loss, batch_cnt=None):
+        if self.simple_posterior:
+            total_loss = loss.nll
+            if self.config.use_pr > 0.0:
+                total_loss += self.beta * loss.pi_kl
+        else:
+            total_loss = loss.nll + loss.pi_kl
+
+        if self.config.use_mi:
+            total_loss += (loss.b_pr * self.beta)
+
+        if self.config.use_diversity:
+            total_loss += loss.diversity
+
+        return total_loss
+
+    def extract_short_ctx(self, data_feed):
+        utts = []
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        context = data_feed['contexts']
+        bs = data_feed['bs']
+        db = data_feed['db']
+        if not isinstance(bs, list):
+            bs = data_feed['bs'].tolist()
+            db = data_feed['db'].tolist()
+
+        for b_id in range(len(context)):
+            utt = []
+            for t_id in range(ctx_lens[b_id]):
+                utt.extend(context[b_id][t_id])
+            try:
+                utt.extend(bs[b_id] + db[b_id])
+            except:
+                pdb.set_trace()
+            utts.append(self.pad_to(self.config.max_utt_len, utt, do_pad=True))
+        return np.array(utts)
+    
+    def extract_metadata(self, data_feed):
+        utts = []
+        bs = data_feed['bs']
+        db = data_feed['db']
+        if not isinstance(bs, list):
+            bs = data_feed['bs'].tolist()
+            db = data_feed['db'].tolist()
+
+        for b_id in range(len(bs)):
+            utt = []
+            utt.extend(bs[b_id] + db[b_id])
+            utts.append(self.pad_to(self.config.max_metadata_len, utt, do_pad=True))
+        return np.array(utts)
+
+    def pad_to(self, max_len, tokens, do_pad):
+        if len(tokens) >= max_len:
+            # print("cutting off, ", tokens)
+            return tokens[: max_len-1] + [tokens[-1]]
+        elif do_pad:
+            return tokens + [0] * (max_len - len(tokens))
+        else:
+            return tokens
+
+    def forward(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed), LONG) # contains bs and db
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        # enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        enc_last = utt_summary.unsqueeze(1)
+        # create decoder initial states
+        if self.simple_posterior:
+            logits_qy, log_qy = self.c2z(enc_last)
+            sample_y = self.gumbel_connector(logits_qy, hard=mode==GEN)
+            log_py = self.log_uniform_y
+        else:
+            logits_py, log_py = self.c2z(enc_last)
+            # encode response and use posterior to find q(z|x, c)
+            x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            if self.contextual_posterior:
+                logits_qy, log_qy = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            else:
+                logits_qy, log_qy = self.xc2z(x_h.squeeze(1))
+
+            # use prior at inference time, otherwise use posterior
+            if mode == GEN or (use_py is not None and use_py is True):
+                sample_y = self.gumbel_connector(logits_py, hard=False)
+            else:
+                sample_y = self.gumbel_connector(logits_qy, hard=True)
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+        
+        if self.config.use_metadata_for_decoding:
+            metadata = self.np2var(self.extract_metadata(data_feed), LONG) 
+            metadata_summary, _, metadata_enc_outs = self.metadata_encoder(metadata.unsqueeze(1))
+            dec_init_state = th.cat((dec_init_state, metadata_summary.view(1, batch_size, -1)), dim=2)
+        
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               # (batch_size, response_size-1)
+                                                               dec_init_state=dec_init_state,   # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               # (batch_size, max_ctx_len, ctx_cell_size)
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_y
+            ret_dict['log_qy'] = log_qy
+            return ret_dict, labels
+
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            # regularization qy to be uniform
+            avg_log_qy = th.exp(log_qy.view(-1, self.config.y_size, self.config.k_size))
+            avg_log_qy = th.log(th.mean(avg_log_qy, dim=0) + 1e-15) # averaged over all samples
+            b_pr = self.cat_kl_loss(avg_log_qy, self.log_uniform_y, batch_size, unit_average=True)
+            mi = self.entropy_loss(avg_log_qy, unit_average=True) - self.entropy_loss(log_qy, unit_average=True)
+            pi_kl = self.cat_kl_loss(log_qy, log_py, batch_size, unit_average=True)
+            q_y = th.exp(log_qy).view(-1, self.config.y_size, self.config.k_size)  # b
+            p = th.pow(th.bmm(q_y, th.transpose(q_y, 1, 2)) - self.eye, 2)
+
+            result['pi_kl'] = pi_kl
+
+            result['diversity'] = th.mean(p)
+            result['b_pr'] = b_pr
+            result['mi'] = mi
+            result['pi_entropy'] = self.entropy_loss(log_qy, unit_average=True)
+            return result
+
+    def forward_rl(self, data_feed, max_words, temp=0.1):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        # pdb.set_trace()
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed), LONG) # contains bs and db
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # create decoder initial states
+        # enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        enc_last = utt_summary.unsqueeze(1)
+        # create decoder initial states
+        logits_py, log_qy = self.c2z(enc_last)
+        qy = F.softmax(logits_py / temp, dim=1)  # (batch_size, vocab_size, )
+        log_qy = F.log_softmax(logits_py, dim=1)  # (batch_size, vocab_size, )
+        idx = th.multinomial(qy, 1).detach()
+        
+        logprob_sample_z = log_qy.gather(1, idx).view(-1, self.y_size)
+        joint_logpz = th.sum(logprob_sample_z, dim=1)
+        sample_y = cast_type(Variable(th.zeros(log_qy.size())), FLOAT, self.use_gpu)
+        sample_y.scatter_(1, idx, 1.0)
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+        
+        if self.config.use_metadata_for_decoding:
+            metadata = self.np2var(self.extract_metadata(data_feed), LONG) 
+            metadata_summary, _, metadata_enc_outs = self.metadata_encoder(metadata.unsqueeze(1))
+            dec_init_state = th.cat((dec_init_state, metadata_summary.view(1, batch_size, -1)), dim=2)
+        
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        logprobs, outs = self.decoder.forward_rl(batch_size=batch_size,
+                                                 dec_init_state=dec_init_state,
+                                                 attn_context=attn_context,
+                                                 vocab=self.vocab,
+                                                 max_words=max_words,
+                                                 temp=0.1)
+        return logprobs, outs, joint_logpz, sample_y
+    
+    def sample_z(self, data_feed, n_z=1, temp=0.1):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed), LONG) # contains bs and db
+        # metadata = self.np2var(self.extract_metadata(data_feed), LONG) 
+        # out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+        # metadata_summary, _, metadata_enc_outs = self.utt_encoder(metadata.unsqueeze(1))
+
+
+        # create decoder initial states
+        enc_last = utt_summary.unsqueeze(1)
+        if self.simple_posterior:
+            logits_py, log_qy = self.c2z(enc_last)
+        else:
+            logits_py, log_qy = self.c2z(enc_last)
+
+        qy = F.softmax(logits_py / temp, dim=1)  # (batch_size, vocab_size, )
+        log_qy = F.log_softmax(logits_py, dim=1)  # (batch_size, vocab_size, )
+
+        zs = []
+        logpzs = []
+        for i in range(n_z):
+            idx = th.multinomial(qy, 1).detach()
+            logprob_sample_z = log_qy.gather(1, idx).view(-1, self.y_size)
+            joint_logpz = th.sum(logprob_sample_z, dim=1)
+            sample_y = cast_type(Variable(th.zeros(log_qy.size())), FLOAT, self.use_gpu)
+            sample_y.scatter_(1, idx, 1.0)
+
+            zs.append(sample_y)
+            logpzs.append(joint_logpz)
+
+        
+        return th.stack(zs), th.stack(logpzs)
+
+    def decode_z(self, sample_y, batch_size, max_words=None, temp=0.1, gen_type='greedy'):
+        """
+        generate response from latent var
+        """
+        # pack attention context
+        metadata = self.np2var(self.extract_metadata(data_feed), LONG) 
+        metadata_summary, _, metadata_enc_outs = self.utt_encoder(metadata.unsqueeze(1))
+
+        if isinstance(sample_y, np.ndarray):
+            sample_y = self.np2var(sample_y, FLOAT)
+
+        if self.config.dec_use_attn:
+           z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+           attn_context = []
+           temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+           for z_id in range(self.y_size):
+               attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+           attn_context = th.cat(attn_context, dim=1)
+           dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+           dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+           attn_context = None
+
+        dec_init_state = th.cat((dec_init_state, metadata_summary.view(1, batch_size, -1)), dim=2)
+
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        # has to be forward_rl because we don't have the golden target
+        logprobs, outs = self.decoder.forward_rl(batch_size=batch_size,
+                                                 dec_init_state=dec_init_state,
+                                                 attn_context=attn_context,
+                                                 vocab=self.vocab,
+                                                 max_words=max_words,
+                                                temp=temp)
+        return logprobs, outs
+
+class SysGroundedActZCat(BaseModel):
+    def __init__(self, corpus, config): 
+        super(SysGroundedActZCat, self).__init__(config)
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        # self.act_size = corpus.act_size
+        self.k_size = config.k_size
+        self.y_size = config.y_size
+        self.simple_posterior = config.simple_posterior # minimize kl to uninformed prior instead of dist conditioned by context
+        self.contextual_posterior = config.contextual_posterior # does not use context cause AE task
+
+        if "use_aux_kl" in config:
+            self.use_aux_kl = config.use_aux_kl
+        else:
+            self.use_aux_kl = False
+
+        self.embedding = None
+        self.aux_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+        self.utt_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+        if config.use_metadata_for_decoding:
+            self.metadata_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                             embedding_dim=int(config.embed_size / 2),
+                                             feat_size=0,
+                                             goal_nhid=0,
+                                             rnn_cell=config.utt_rnn_cell,
+                                             utt_cell_size=int(config.utt_cell_size / 2),
+                                             num_layers=config.num_layers,
+                                             input_dropout_p=config.dropout,
+                                             output_dropout_p=config.dropout,
+                                             bidirectional=config.bi_utt_cell,
+                                             variable_lengths=False,
+                                             use_attn=config.enc_use_attn,
+                                             embedding=self.embedding)
+
+
+
+        if "policy_dropout" in config and config.policy_dropout:
+            self.c2z = nn_lib.Hidden2DiscretewDropout(self.utt_encoder.output_size,
+                                              config.y_size, config.k_size, is_lstm=False, p_dropout=config.policy_dropout_rate, dropout_on_eval=config.dropout_on_eval)
+        else:
+            self.c2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size,
+                                              config.y_size, config.k_size, is_lstm=False)
+
+        self.z_embedding = nn.Linear(self.y_size * self.k_size, config.dec_cell_size, bias=False)
+        self.gumbel_connector = nn_lib.GumbelConnector(config.use_gpu)
+        
+        if not self.simple_posterior: #q(z|x,c)
+            if self.contextual_posterior:
+                # x, c, BS, and DB
+                self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size,
+                                                   config.y_size, config.k_size, is_lstm=False)
+            else:
+                self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size, config.y_size, config.k_size, is_lstm=False)
+        if config.use_metadata_for_decoding:
+            dec_hidden_size = config.dec_cell_size + self.metadata_encoder.output_size
+        else:
+            dec_hidden_size = config.dec_cell_size
+
+
+        self.decoder = DecoderRNN(input_dropout_p=config.dropout,
+                                  rnn_cell=config.dec_rnn_cell,
+                                  input_size=config.embed_size,
+                                  hidden_size=dec_hidden_size,
+                                  num_layers=config.num_layers,
+                                  output_dropout_p=config.dropout,
+                                  bidirectional=False,
+                                  vocab_size=self.vocab_size,
+                                  use_attn=config.dec_use_attn,
+                                  ctx_cell_size=config.dec_cell_size,
+                                  attn_mode=config.dec_attn_mode,
+                                  sys_id=self.bos_id,
+                                  eos_id=self.eos_id,
+                                  use_gpu=config.use_gpu,
+                                  max_dec_len=config.max_dec_len,
+                                  embedding=self.embedding)
+
+
+        self.nll = NLLEntropy(self.pad_id, config.avg_type)
+        if config.avg_type == "weighted" and config.nll_weight=="no_match_penalty":
+            req_tokens = []
+            for d in REQ_TOKENS.keys():
+                req_tokens.extend(REQ_TOKENS[d])
+            nll_weight = Variable(th.FloatTensor([10. if token in req_tokens  else 1. for token in self.vocab]))
+            print("req tokens assigned with special weights")
+            if config.use_gpu:
+                nll_weight = nll_weight.cuda()
+            self.nll.set_weight(nll_weight)
+
+        self.cat_kl_loss = CatKLLoss()
+        self.entropy_loss = Entropy()
+        self.log_uniform_y = Variable(th.log(th.ones(1) / config.k_size))
+        self.eye = Variable(th.eye(self.config.y_size).unsqueeze(0))
+        self.beta = self.config.beta if hasattr(self.config, 'beta') else 0.0
+        if self.use_gpu:
+            self.log_uniform_y = self.log_uniform_y.cuda()
+            self.eye = self.eye.cuda()
+
+    def extract_short_ctx(self, data_feed):
+        utts = []
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        context = data_feed['contexts']
+        bs = data_feed['bs']
+        db = data_feed['db']
+        if not isinstance(bs, list):
+            bs = data_feed['bs'].tolist()
+            db = data_feed['db'].tolist()
+
+        for b_id in range(len(context)):
+            utt = []
+            for t_id in range(ctx_lens[b_id]):
+                utt.extend(context[b_id][t_id])
+            try:
+                utt.extend(bs[b_id] + db[b_id])
+            except:
+                pdb.set_trace()
+            utts.append(self.pad_to(self.config.max_utt_len, utt, do_pad=True))
+        return np.array(utts)
+
+    def extract_metadata(self, data_feed):
+        utts = []
+        bs = data_feed['bs']
+        db = data_feed['db']
+        if not isinstance(bs, list):
+            bs = data_feed['bs'].tolist()
+            db = data_feed['db'].tolist()
+
+        for b_id in range(len(bs)):
+            utt = []
+            if "metadata_db_only" in config and self.config.metadata_db_only:
+                utt.extend(db[b_id])
+            else:
+                utt.extend(bs[b_id] + db[b_id])
+            utts.append(self.pad_to(self.config.max_metadata_len, utt, do_pad=True))
+        return np.array(utts)
+
+    def extract_AE_ctx(self, data_feed):
+        utts = []
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        context = data_feed['outputs']
+        bs = data_feed['bs']
+        db = data_feed['db']
+        if not isinstance(bs, list):
+            bs = data_feed['bs'].tolist()
+            db = data_feed['db'].tolist()
+
+        for b_id in range(len(context)):
+            utt = []
+            utt.extend(context[b_id])
+            try:
+                utt.extend(bs[b_id] + db[b_id])
+            except:
+                pdb.set_trace()
+            utts.append(self.pad_to(self.config.max_utt_len, utt, do_pad=True))
+        return np.array(utts)
+
+    def pad_to(self, max_len, tokens, do_pad):
+        if len(tokens) >= max_len:
+            return tokens[: max_len-1] + [tokens[-1]]
+        elif do_pad:
+            return tokens + [0] * (max_len - len(tokens))
+        else:
+            return tokens
+
+    def valid_loss(self, loss, batch_cnt=None):
+        if self.simple_posterior:
+            total_loss = loss.nll
+            if self.config.use_pr > 0.0:
+                total_loss += self.beta * loss.pi_kl
+        else:
+            total_loss = loss.nll + loss.pi_kl
+
+        if self.config.use_mi:
+            total_loss += (loss.b_pr * self.beta)
+
+        if self.config.use_diversity:
+            total_loss += loss.diversity
+
+        return total_loss
+    
+    def forward(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed), LONG) # contains bs and db
+        metadata = self.np2var(self.extract_metadata(data_feed), LONG) 
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+        if self.config.use_metadata_for_aux_encoder:
+            ctx_outs = self.np2var(self.extract_AE_ctx(data_feed), LONG) # contains bs and db
+            aux_utt_summary, _, aux_enc_outs = self.aux_encoder(ctx_outs.unsqueeze(1))
+        else:
+            short_target_utts = self.np2var(data_feed['outputs'], LONG)
+            aux_utt_summary, _, aux_enc_outs = self.aux_encoder(short_target_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        enc_last = utt_summary.unsqueeze(1)
+        aux_enc_last = aux_utt_summary.unsqueeze(1)
+        # enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        # aux_enc_last = th.cat([bs_label, db_label, aux_utt_summary.squeeze(1)], dim=1)
+        # create decoder initial states
+        if self.simple_posterior:
+            logits_qy, log_qy = self.c2z(enc_last)
+            aux_logits_qy, aux_log_qy = self.c2z(aux_enc_last)
+            sample_y = self.gumbel_connector(logits_qy, hard=mode==GEN)
+            log_py = aux_log_qy
+        else: 
+            logits_py, log_py = self.c2z(enc_last)
+            aux_logits_qy, aux_log_qy = self.c2z(aux_enc_last)
+            # encode response and use posterior to find q(z|x, c)
+            x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            if self.contextual_posterior:
+                logits_qy, log_qy = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            else:
+                logits_qy, log_qy = self.xc2z(x_h.squeeze(1))
+
+            # use prior at inference time, otherwise use posterior
+            if mode == GEN or (use_py is not None and use_py is True):
+                sample_y = self.gumbel_connector(logits_py, hard=False)
+            else:
+                sample_y = self.gumbel_connector(logits_qy, hard=True)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        if self.config.use_metadata_for_decoding:
+            metadata = self.np2var(self.extract_metadata(data_feed), LONG) 
+            metadata_summary, _, metadata_enc_outs = self.metadata_encoder(metadata.unsqueeze(1))
+            dec_init_state = th.cat((dec_init_state, metadata_summary.view(1, batch_size, -1)), dim=2)
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               # (batch_size, response_size-1)
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               # (batch_size, max_ctx_len, ctx_cell_size)
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_y
+            ret_dict['log_qy'] = log_qy
+            return ret_dict, labels
+
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            # regularization qy to be uniform
+            avg_log_qy = th.exp(log_qy.view(-1, self.config.y_size, self.config.k_size))
+            avg_log_qy = th.log(th.mean(avg_log_qy, dim=0) + 1e-15)
+            b_pr = self.cat_kl_loss(avg_log_qy, self.log_uniform_y, batch_size, unit_average=True)
+            mi = self.entropy_loss(avg_log_qy, unit_average=True) - self.entropy_loss(log_qy, unit_average=True)
+            pi_kl = self.cat_kl_loss(log_qy, log_py, batch_size, unit_average=True)
+            q_y = th.exp(log_qy).view(-1, self.config.y_size, self.config.k_size)  # b
+            p = th.pow(th.bmm(q_y, th.transpose(q_y, 1, 2)) - self.eye, 2)
+
+            result['pi_kl'] = pi_kl
+            result['diversity'] = th.mean(p)
+            result['nll'] = self.nll(dec_outputs, labels)
+            result['b_pr'] = b_pr
+            result['mi'] = mi
+            result['pi_entropy'] = self.entropy_loss(log_qy, unit_average=True)
+            return result
+    
+    def forward_rl(self, data_feed, max_words, temp=0.1):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        # pdb.set_trace()
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed), LONG) # contains bs and db
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # create decoder initial states
+        # enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        enc_last = utt_summary.unsqueeze(1)
+        # create decoder initial states
+        logits_py, log_qy = self.c2z(enc_last)
+        qy = F.softmax(logits_py / temp, dim=1)  # (batch_size, vocab_size, )
+        log_qy = F.log_softmax(logits_py, dim=1)  # (batch_size, vocab_size, )
+        idx = th.multinomial(qy, 1).detach()
+        
+        logprob_sample_z = log_qy.gather(1, idx).view(-1, self.y_size)
+        joint_logpz = th.sum(logprob_sample_z, dim=1)
+        sample_y = cast_type(Variable(th.zeros(log_qy.size())), FLOAT, self.use_gpu)
+        sample_y.scatter_(1, idx, 1.0)
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+        
+        if self.config.use_metadata_for_decoding:
+            metadata = self.np2var(self.extract_metadata(data_feed), LONG) 
+            metadata_summary, _, metadata_enc_outs = self.metadata_encoder(metadata.unsqueeze(1))
+            dec_init_state = th.cat((dec_init_state, metadata_summary.view(1, batch_size, -1)), dim=2)
+        
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        logprobs, outs = self.decoder.forward_rl(batch_size=batch_size,
+                                                 dec_init_state=dec_init_state,
+                                                 attn_context=attn_context,
+                                                 vocab=self.vocab,
+                                                 max_words=max_words,
+                                                 temp=0.1)
+        return logprobs, outs, joint_logpz, sample_y
+
+class SysGroundedMTCat(BaseModel):
+    def __init__(self, corpus, config): 
+        super(SysGroundedMTCat, self).__init__(config)
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        # self.act_size = corpus.act_size
+        self.k_size = config.k_size
+        self.y_size = config.y_size
+        self.simple_posterior = config.simple_posterior # minimize kl to uninformed prior instead of dist conditioned by context
+        self.contextual_posterior = config.contextual_posterior # does not use context cause AE task
+
+        if "use_aux_kl" in config:
+            self.use_aux_kl = config.use_aux_kl
+        else:
+            self.use_aux_kl = False
+
+        self.embedding = None
+        self.aux_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                         embedding=self.embedding)
+
+        self.utt_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                         embedding_dim=config.embed_size,
+                                         feat_size=0,
+                                         goal_nhid=0,
+                                         rnn_cell=config.utt_rnn_cell,
+                                         utt_cell_size=config.utt_cell_size,
+                                         num_layers=config.num_layers,
+                                         input_dropout_p=config.dropout,
+                                         output_dropout_p=config.dropout,
+                                         bidirectional=config.bi_utt_cell,
+                                         variable_lengths=False,
+                                         use_attn=config.enc_use_attn,
+                                        embedding=self.embedding)
+
+        if config.use_metadata_for_decoding:
+            self.metadata_encoder = RnnUttEncoder(vocab_size=self.vocab_size,
+                                             embedding_dim=int(config.embed_size / 2),
+                                             feat_size=0,
+                                             goal_nhid=0,
+                                             rnn_cell=config.utt_rnn_cell,
+                                             utt_cell_size=int(config.utt_cell_size / 2),
+                                             num_layers=config.num_layers,
+                                             input_dropout_p=config.dropout,
+                                             output_dropout_p=config.dropout,
+                                             bidirectional=config.bi_utt_cell,
+                                             variable_lengths=False,
+                                             use_attn=config.enc_use_attn,
+                                             embedding=self.embedding)
+
+
+
+        if "policy_dropout" in config and config.policy_dropout:
+            self.c2z = nn_lib.Hidden2DiscretewDropout(self.utt_encoder.output_size,
+                                              config.y_size, config.k_size, is_lstm=False, p_dropout=config.policy_dropout_rate, dropout_on_eval=config.dropout_on_eval)
+        else:
+            self.c2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size,
+                                              config.y_size, config.k_size, is_lstm=False)
+
+
+        self.z_embedding = nn.Linear(self.y_size * self.k_size, config.dec_cell_size, bias=False)
+        self.gumbel_connector = nn_lib.GumbelConnector(config.use_gpu)
+        
+        if not self.simple_posterior: #q(z|x,c)
+            if self.contextual_posterior:
+                # x, c, BS, and DB
+                self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size,
+                                                   config.y_size, config.k_size, is_lstm=False)
+            else:
+                self.xc2z = nn_lib.Hidden2Discrete(self.utt_encoder.output_size, config.y_size, config.k_size, is_lstm=False)
+
+        self.decoder = DecoderRNN(input_dropout_p=config.dropout,
+                                  rnn_cell=config.dec_rnn_cell,
+                                  input_size=config.embed_size,
+                                  hidden_size=config.dec_cell_size,
+                                  num_layers=config.num_layers,
+                                  output_dropout_p=config.dropout,
+                                  bidirectional=False,
+                                  vocab_size=self.vocab_size,
+                                  use_attn=config.dec_use_attn,
+                                  ctx_cell_size=config.dec_cell_size,
+                                  attn_mode=config.dec_attn_mode,
+                                  sys_id=self.bos_id,
+                                  eos_id=self.eos_id,
+                                  use_gpu=config.use_gpu,
+                                  max_dec_len=config.max_dec_len,
+                                  embedding=self.embedding)
+
+        self.nll = NLLEntropy(self.pad_id, config.avg_type)
+        self.cat_kl_loss = CatKLLoss()
+        self.entropy_loss = Entropy()
+        self.log_uniform_y = Variable(th.log(th.ones(1) / config.k_size))
+        self.eye = Variable(th.eye(self.config.y_size).unsqueeze(0))
+        self.beta = self.config.beta if hasattr(self.config, 'beta') else 0.0
+        if self.use_gpu:
+            self.log_uniform_y = self.log_uniform_y.cuda()
+            self.eye = self.eye.cuda()
+
+    def valid_loss(self, loss, batch_cnt=None):
+        if self.simple_posterior:
+            total_loss = loss.nll
+            if self.config.use_pr > 0.0:
+                total_loss += self.beta * loss.pi_kl
+        else:
+            total_loss = loss.nll + loss.pi_kl
+
+        if self.config.use_mi:
+            total_loss += (loss.b_pr * self.beta)
+
+        if self.config.use_diversity:
+            total_loss += loss.diversity
+
+        if self.use_aux_kl:
+            try:
+                total_loss += loss.aux_pi_kl
+            except KeyError:
+                total_loss += 0
+
+        return total_loss
+
+    def forward_aux(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        batch_size = len(ctx_lens)
+
+        if self.config.use_metadata_for_aux_encoder:
+            ctx_outs = self.np2var(self.extract_AE_ctx(data_feed), LONG) # contains bs and db
+            utt_summary, _, _ = self.aux_encoder(ctx_outs.unsqueeze(1))
+        else:
+            short_target_utts = self.np2var(data_feed['outputs'], LONG)
+            utt_summary, _, _ = self.aux_encoder(short_target_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        enc_last = utt_summary.unsqueeze(1)
+        
+        # how to use z, alone or in combination with bs and db
+        if self.simple_posterior:
+            logits_qy, log_qy = self.c2z(enc_last)
+            sample_y = self.gumbel_connector(logits_qy, hard=mode==GEN)
+            log_py = self.log_uniform_y
+        # else:
+            # logits_py, log_py = self.c2z(enc_last)
+            # # encode response and use posterior to find q(z|x, c)
+            # x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            # if self.contextual_posterior:
+                # logits_qy, log_qy = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            # else:
+                # logits_qy, log_qy = self.xc2z(x_h.squeeze(1))
+
+            # # use prior at inference time, otherwise use posterior
+            # if mode == GEN or (use_py is not None and use_py is True):
+                # sample_y = self.gumbel_connector(logits_py, hard=False)
+            # else:
+                # sample_y = self.gumbel_connector(logits_qy, hard=True)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               # (batch_size, response_size-1)
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               # (batch_size, max_ctx_len, ctx_cell_size)
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_y
+            ret_dict['log_qy'] = log_qy
+            return ret_dict, labels
+
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            # regularization qy to be uniform
+            avg_log_qy = th.exp(log_qy.view(-1, self.config.y_size, self.config.k_size))
+            avg_log_qy = th.log(th.mean(avg_log_qy, dim=0) + 1e-15)
+            b_pr = self.cat_kl_loss(avg_log_qy, self.log_uniform_y, batch_size, unit_average=True)
+            mi = self.entropy_loss(avg_log_qy, unit_average=True) - self.entropy_loss(log_qy, unit_average=True)
+            pi_kl = self.cat_kl_loss(log_qy, log_py, batch_size, unit_average=True)
+            q_y = th.exp(log_qy).view(-1, self.config.y_size, self.config.k_size)  # b
+            p = th.pow(th.bmm(q_y, th.transpose(q_y, 1, 2)) - self.eye, 2)
+
+            result['pi_kl'] = pi_kl
+            result['diversity'] = th.mean(p)
+            result['nll'] = self.nll(dec_outputs, labels)
+            result['b_pr'] = b_pr
+            result['mi'] = mi
+            return result
+
+    def forward(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed), LONG) # contains bs and db
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        enc_last = utt_summary.unsqueeze(1)
+        # create decoder initial states
+        if self.simple_posterior:
+            logits_qy, log_qy = self.c2z(enc_last)
+            sample_y = self.gumbel_connector(logits_qy, hard=mode==GEN)
+            log_py = self.log_uniform_y
+        else:
+            logits_py, log_py = self.c2z(enc_last)
+            # encode response and use posterior to find q(z|x, c)
+            x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            if self.contextual_posterior:
+                logits_qy, log_qy = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            else:
+                logits_qy, log_qy = self.xc2z(x_h.squeeze(1))
+
+            # use prior at inference time, otherwise use posterior
+            if mode == GEN or (use_py is not None and use_py is True):
+                sample_y = self.gumbel_connector(logits_py, hard=False)
+            else:
+                sample_y = self.gumbel_connector(logits_qy, hard=True)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               # (batch_size, response_size-1)
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               # (batch_size, max_ctx_len, ctx_cell_size)
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_y
+            ret_dict['log_qy'] = log_qy
+            return ret_dict, labels
+
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            # regularization qy to be uniform
+            avg_log_qy = th.exp(log_qy.view(-1, self.config.y_size, self.config.k_size))
+            avg_log_qy = th.log(th.mean(avg_log_qy, dim=0) + 1e-15)
+            b_pr = self.cat_kl_loss(avg_log_qy, self.log_uniform_y, batch_size, unit_average=True)
+            mi = self.entropy_loss(avg_log_qy, unit_average=True) - self.entropy_loss(log_qy, unit_average=True)
+            pi_kl = self.cat_kl_loss(log_qy, log_py, batch_size, unit_average=True)
+            q_y = th.exp(log_qy).view(-1, self.config.y_size, self.config.k_size)  # b
+            p = th.pow(th.bmm(q_y, th.transpose(q_y, 1, 2)) - self.eye, 2)
+
+            result['pi_kl'] = pi_kl
+            result['diversity'] = th.mean(p)
+            result['nll'] = self.nll(dec_outputs, labels)
+            result['b_pr'] = b_pr
+            result['mi'] = mi
+            return result
+    
+    def shared_forward(self, data_feed, mode, clf=False, gen_type='greedy', use_py=None, return_latent=False):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        out_utts = self.np2var(data_feed['outputs'], LONG)  # (batch_size, max_out_len)
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed), LONG) # contains bs and db
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        if self.config.use_metadata_for_aux_encoder:
+            ctx_outs = self.np2var(self.extract_AE_ctx(data_feed), LONG) # contains bs and db
+            aux_utt_summary, _, aux_enc_outs = self.aux_encoder(ctx_outs.unsqueeze(1))
+        else:
+            short_target_utts = self.np2var(data_feed['outputs'], LONG)
+            aux_utt_summary, _, aux_enc_outs = self.aux_encoder(short_target_utts.unsqueeze(1))
+
+        # get decoder inputs
+        dec_inputs = out_utts[:, :-1]
+        labels = out_utts[:, 1:].contiguous()
+
+        # create decoder initial states
+        enc_last = utt_summary.unsqueeze(1)
+        aux_enc_last = aux_utt_summary.unsqueeze(1)
+
+        # create decoder initial states
+        if self.simple_posterior:
+            logits_qy, log_qy = self.c2z(enc_last)
+            aux_logits_qy, aux_log_qy = self.c2z(aux_enc_last)
+            sample_y = self.gumbel_connector(logits_qy, hard=mode==GEN)
+            log_py = self.log_uniform_y
+        else:
+            logits_py, log_py = self.c2z(enc_last)
+            aux_logits_qy, aux_log_qy = self.c2z(aux_enc_last)
+            # encode response and use posterior to find q(z|x, c)
+            x_h, _, _ = self.utt_encoder(out_utts.unsqueeze(1))
+            if self.contextual_posterior:
+                logits_qy, log_qy = self.xc2z(th.cat([enc_last, x_h.squeeze(1)], dim=1))
+            else:
+                logits_qy, log_qy = self.xc2z(x_h.squeeze(1))
+
+            # use prior at inference time, otherwise use posterior
+            if mode == GEN or (use_py is not None and use_py is True):
+                sample_y = self.gumbel_connector(logits_py, hard=False)
+            else:
+                sample_y = self.gumbel_connector(logits_qy, hard=True)
+
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        dec_outputs, dec_hidden_state, ret_dict = self.decoder(batch_size=batch_size,
+                                                               dec_inputs=dec_inputs,
+                                                               # (batch_size, response_size-1)
+                                                               dec_init_state=dec_init_state,  # tuple: (h, c)
+                                                               attn_context=attn_context,
+                                                               # (batch_size, max_ctx_len, ctx_cell_size)
+                                                               mode=mode,
+                                                               gen_type=gen_type,
+                                                               beam_size=self.config.beam_size)  # (batch_size, goal_nhid)
+        if mode == GEN:
+            ret_dict['sample_z'] = sample_y
+            ret_dict['log_qy'] = log_qy
+            return ret_dict, labels
+
+        else:
+            result = Pack(nll=self.nll(dec_outputs, labels))
+            # regularization qy to be uniform
+            avg_log_qy = th.exp(log_qy.view(-1, self.config.y_size, self.config.k_size))
+            avg_log_qy = th.log(th.mean(avg_log_qy, dim=0) + 1e-15)
+            b_pr = self.cat_kl_loss(avg_log_qy, self.log_uniform_y, batch_size, unit_average=True)
+            mi = self.entropy_loss(avg_log_qy, unit_average=True) - self.entropy_loss(log_qy, unit_average=True)
+            pi_kl = self.cat_kl_loss(log_qy, log_py, batch_size, unit_average=True)
+            aux_pi_kl = self.cat_kl_loss(log_qy, aux_log_qy, batch_size, unit_average=True)
+            q_y = th.exp(log_qy).view(-1, self.config.y_size, self.config.k_size)  # b
+            p = th.pow(th.bmm(q_y, th.transpose(q_y, 1, 2)) - self.eye, 2)
+
+            result['pi_kl'] = pi_kl
+            result['aux_pi_kl'] = aux_pi_kl
+            result['diversity'] = th.mean(p)
+            result['nll'] = self.nll(dec_outputs, labels)
+            result['b_pr'] = b_pr
+            result['mi'] = mi
+            return result
+
+    def extract_metadata(self, data_feed):
+        utts = []
+        bs = data_feed['bs']
+        db = data_feed['db']
+        if not isinstance(bs, list):
+            bs = data_feed['bs'].tolist()
+            db = data_feed['db'].tolist()
+
+        for b_id in range(len(bs)):
+            utt = []
+            if "metadata_db_only" in self.config and self.config.metadata_db_only:
+                utt.extend(db[b_id])
+            else:
+                utt.extend(bs[b_id] + db[b_id])
+            utts.append(self.pad_to(self.config.max_metadata_len, utt, do_pad=True))
+        return np.array(utts)
+
+    def extract_AE_ctx(self, data_feed):
+        utts = []
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        context = data_feed['outputs']
+        bs = data_feed['bs']
+        db = data_feed['db']
+        if not isinstance(bs, list):
+            bs = data_feed['bs'].tolist()
+            db = data_feed['db'].tolist()
+
+        for b_id in range(len(context)):
+            utt = []
+            utt.extend(context[b_id])
+            try:
+                utt.extend(bs[b_id] + db[b_id])
+            except:
+                pdb.set_trace()
+            utts.append(self.pad_to(self.config.max_utt_len, utt, do_pad=True))
+        return np.array(utts)
+
+    def extract_short_ctx(self, data_feed):
+        utts = []
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        context = data_feed['contexts']
+        bs = data_feed['bs']
+        db = data_feed['db']
+        if not isinstance(bs, list):
+            bs = data_feed['bs'].tolist()
+            db = data_feed['db'].tolist()
+
+        for b_id in range(len(context)):
+            utt = []
+            for t_id in range(ctx_lens[b_id]):
+                utt.extend(context[b_id][t_id])
+            try:
+                utt.extend(bs[b_id] + db[b_id])
+            except:
+                pdb.set_trace()
+            utts.append(self.pad_to(self.config.max_utt_len, utt, do_pad=True))
+        return np.array(utts)
+
+    def pad_to(self, max_len, tokens, do_pad):
+        if len(tokens) >= max_len:
+            return tokens[: max_len-1] + [tokens[-1]]
+        elif do_pad:
+            return tokens + [0] * (max_len - len(tokens))
+        else:
+            return tokens
+
+    def forward_rl(self, data_feed, max_words, temp=0.1):
+        ctx_lens = data_feed['context_lens']  # (batch_size, )
+        # pdb.set_trace()
+        short_ctx_utts = self.np2var(self.extract_short_ctx(data_feed), LONG) # contains bs and db
+        batch_size = len(ctx_lens)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+
+        # create decoder initial states
+        # enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+        enc_last = utt_summary.unsqueeze(1)
+        # create decoder initial states
+        logits_py, log_qy = self.c2z(enc_last)
+        qy = F.softmax(logits_py / temp, dim=1)  # (batch_size, vocab_size, )
+        log_qy = F.log_softmax(logits_py, dim=1)  # (batch_size, vocab_size, )
+        idx = th.multinomial(qy, 1).detach()
+        
+        logprob_sample_z = log_qy.gather(1, idx).view(-1, self.y_size)
+        joint_logpz = th.sum(logprob_sample_z, dim=1)
+        sample_y = cast_type(Variable(th.zeros(log_qy.size())), FLOAT, self.use_gpu)
+        sample_y.scatter_(1, idx, 1.0)
+        # pack attention context
+        if self.config.dec_use_attn:
+            z_embeddings = th.t(self.z_embedding.weight).split(self.k_size, dim=0)
+            attn_context = []
+            temp_sample_y = sample_y.view(-1, self.config.y_size, self.config.k_size)
+            for z_id in range(self.y_size):
+                attn_context.append(th.mm(temp_sample_y[:, z_id], z_embeddings[z_id]).unsqueeze(1))
+            attn_context = th.cat(attn_context, dim=1)
+            dec_init_state = th.sum(attn_context, dim=1).unsqueeze(0)
+        else:
+            dec_init_state = self.z_embedding(sample_y.view(1, -1, self.config.y_size * self.config.k_size))
+            attn_context = None
+        
+        if self.config.use_metadata_for_decoding:
+            metadata = self.np2var(self.extract_metadata(data_feed), LONG) 
+            metadata_summary, _, metadata_enc_outs = self.metadata_encoder(metadata.unsqueeze(1))
+            dec_init_state = th.cat((dec_init_state, metadata_summary.view(1, batch_size, -1)), dim=2)
+        
+        # decode
+        if self.config.dec_rnn_cell == 'lstm':
+            dec_init_state = tuple([dec_init_state, dec_init_state])
+
+        logprobs, outs = self.decoder.forward_rl(batch_size=batch_size,
+                                                 dec_init_state=dec_init_state,
+                                                 attn_context=attn_context,
+                                                 vocab=self.vocab,
+                                                 max_words=max_words,
+                                                 temp=0.1)
+        return logprobs, outs, joint_logpz, sample_y
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/nn_lib.py b/convlab/policy/lava/multiwoz/latent_dialog/nn_lib.py
new file mode 100644
index 0000000000000000000000000000000000000000..e843684c9b6ac3303425074378c5a368076b9b82
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/nn_lib.py
@@ -0,0 +1,182 @@
+import torch as th
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+from torch.autograd import Variable
+from convlab.policy.lava.multiwoz.latent_dialog.utils import cast_type, FLOAT
+
+
+class IdentityConnector(nn.Module):
+    def __init(self):
+        super(IdentityConnector, self).__init__()
+
+    def forward(self, hidden_state):
+        return hidden_state
+
+
+class Bi2UniConnector(nn.Module):
+    def __init__(self, rnn_cell, num_layer, hidden_size, output_size):
+        super(Bi2UniConnector, self).__init__()
+        if rnn_cell == 'lstm':
+            self.fch = nn.Linear(hidden_size*2*num_layer, output_size)
+            self.fcc = nn.Linear(hidden_size*2*num_layer, output_size)
+        else:
+            self.fc = nn.Linear(hidden_size*2*num_layer, output_size)
+
+        self.rnn_cell = rnn_cell
+        self.hidden_size = hidden_size
+        self.output_size = output_size
+
+    def forward(self, hidden_state):
+        """
+        :param hidden_state: [num_layer, batch_size, feat_size]
+        :param inputs: [batch_size, feat_size]
+        :return: 
+        """
+        if self.rnn_cell == 'lstm':
+            h, c = hidden_state
+            num_layer = h.size()[0]
+            flat_h = h.transpose(0, 1).contiguous()
+            flat_c = c.transpose(0, 1).contiguous()
+            new_h = self.fch(flat_h.view(-1, self.hidden_size*num_layer))
+            new_c = self.fch(flat_c.view(-1, self.hidden_size*num_layer))
+            return (new_h.view(1, -1, self.output_size),
+                    new_c.view(1, -1, self.output_size))
+        else:
+            # FIXME fatal error here!
+            num_layer = hidden_state.size()[0]
+            new_s = self.fc(hidden_state.view(-1, self.hidden_size*num_layer))
+            new_s = new_s.view(1, -1, self.output_size)
+            return new_s
+
+
+class Hidden2Gaussian(nn.Module):
+    def __init__(self, input_size, output_size, is_lstm=False, has_bias=True):
+        super(Hidden2Gaussian, self).__init__()
+        if is_lstm:
+            self.mu_h = nn.Linear(input_size, output_size, bias=has_bias)
+            self.logvar_h = nn.Linear(input_size, output_size, bias=has_bias)
+
+            self.mu_c = nn.Linear(input_size, output_size, bias=has_bias)
+            self.logvar_c = nn.Linear(input_size, output_size, bias=has_bias)
+        else:
+            self.mu = nn.Linear(input_size, output_size, bias=has_bias)
+            self.logvar = nn.Linear(input_size, output_size, bias=has_bias)
+
+        self.is_lstm = is_lstm
+
+    def forward(self, inputs):
+        """
+        :param inputs: batch_size x input_size
+        :return:
+        """
+        if self.is_lstm:
+            h, c= inputs
+            if h.dim() == 3:
+                h = h.squeeze(0)
+                c = c.squeeze(0)
+
+            mu_h, mu_c = self.mu_h(h), self.mu_c(c)
+            logvar_h, logvar_c = self.logvar_h(h), self.logvar_c(c)
+            return mu_h+mu_c, logvar_h+logvar_c
+        else:
+            # if inputs.dim() == 3:
+            #    inputs = inputs.squeeze(0)
+            mu = self.mu(inputs)
+            logvar = self.logvar(inputs)
+            return mu, logvar
+
+
+class Hidden2Discrete(nn.Module):
+    def __init__(self, input_size, y_size, k_size, is_lstm=False, has_bias=True):
+        super(Hidden2Discrete, self).__init__()
+        self.y_size = y_size
+        self.k_size = k_size
+        latent_size = self.k_size*self.y_size
+        if is_lstm:
+            self.p_h = nn.Linear(input_size, latent_size, bias=has_bias)
+
+            self.p_c = nn.Linear(input_size, latent_size, bias=has_bias)
+        else:
+            self.p_h = nn.Linear(input_size, latent_size, bias=has_bias)
+
+        self.is_lstm = is_lstm
+
+    def forward(self, inputs):
+        """
+        :param inputs: batch_size x input_size
+        :return:
+        """
+        if self.is_lstm:
+            h, c= inputs
+            if h.dim() == 3:
+                h = h.squeeze(0)
+                c = c.squeeze(0)
+            logits = self.p_h(h) + self.p_c(c)
+        else:
+            logits = self.p_h(inputs)
+        logits = logits.view(-1, self.k_size)
+        log_qy = F.log_softmax(logits, dim=1)
+        return logits, log_qy
+
+
+class GaussianConnector(nn.Module):
+    def __init__(self, use_gpu):
+        super(GaussianConnector, self).__init__()
+        self.use_gpu = use_gpu
+
+    def forward(self, mu, logvar):
+        """
+        Sample a sample from a multivariate Gaussian distribution with a diagonal covariance matrix using the
+        reparametrization trick.
+        TODO: this should be better be a instance method in a Gaussian class.
+        :param mu: a tensor of size [batch_size, variable_dim]. Batch_size can be None to support dynamic batching
+        :param logvar: a tensor of size [batch_size, variable_dim]. Batch_size can be None.
+        :return:
+        """
+        epsilon = th.randn(logvar.size())
+        epsilon = cast_type(Variable(epsilon), FLOAT, self.use_gpu)
+        std = th.exp(0.5 * logvar)
+        z = mu + std * epsilon
+        return z
+
+
+class GumbelConnector(nn.Module):
+    def __init__(self, use_gpu):
+        super(GumbelConnector, self).__init__()
+        self.use_gpu = use_gpu
+
+    def sample_gumbel(self, logits, use_gpu, eps=1e-20):
+        u = th.rand(logits.size())
+        sample = Variable(-th.log(-th.log(u + eps) + eps))
+        sample = cast_type(sample, FLOAT, use_gpu)
+        return sample
+
+    def gumbel_softmax_sample(self, logits, temperature, use_gpu):
+        """ Draw a sample from the Gumbel-Softmax distribution"""
+        eps = self.sample_gumbel(logits, use_gpu)
+        y = logits + eps
+        return F.softmax(y / temperature, dim=y.dim()-1)
+
+    def forward(self, logits, temperature=1.0, hard=False,
+                return_max_id=False):
+        """
+        :param logits: [batch_size, n_class] unnormalized log-prob
+        :param temperature: non-negative scalar
+        :param hard: if True take argmax
+        :param return_max_id
+        :return: [batch_size, n_class] sample from gumbel softmax
+        """
+        y = self.gumbel_softmax_sample(logits, temperature, self.use_gpu)
+        _, y_hard = th.max(y, dim=1, keepdim=True)
+        if hard:
+            y_onehot = cast_type(Variable(th.zeros(y.size())), FLOAT, self.use_gpu)
+            y_onehot.scatter_(1, y_hard, 1.0)
+            y = y_onehot
+        if return_max_id:
+            return y, y_hard
+        else:
+            return y
+
+
+
diff --git a/convlab/base_models/gpt/__init__.py b/convlab/policy/lava/multiwoz/latent_dialog/normalizer/__init__.py
similarity index 100%
rename from convlab/base_models/gpt/__init__.py
rename to convlab/policy/lava/multiwoz/latent_dialog/normalizer/__init__.py
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/normalizer/delexicalize.py b/convlab/policy/lava/multiwoz/latent_dialog/normalizer/delexicalize.py
new file mode 100644
index 0000000000000000000000000000000000000000..df4636b24d0083bffb9b6d1303dc05d447720970
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/normalizer/delexicalize.py
@@ -0,0 +1,283 @@
+import re
+import os
+# import simplejson as json
+import json
+
+
+digitpat = re.compile('\d+')
+timepat = re.compile("\d{1,2}[:]\d{1,2}")
+pricepat = re.compile("\d{1,3}[.]\d{1,2}")
+
+CUR_PATH = os.path.join(os.path.dirname(__file__))
+fin = open(os.path.join(CUR_PATH, 'mapping.pair'), 'r')
+replacements = []
+for line in fin.readlines():
+    tok_from, tok_to = line.replace('\n', '').split('\t')
+    replacements.append((' ' + tok_from + ' ', ' ' + tok_to + ' '))
+
+# FORMAT
+# domain_value
+# restaurant_postcode
+# restaurant_address
+# taxi_car8
+# taxi_number
+# train_id etc..
+
+def insertSpace(token, text):
+    sidx = 0
+    while True:
+        sidx = text.find(token, sidx)
+        if sidx == -1:
+            break
+        if sidx + 1 < len(text) and re.match('[0-9]', text[sidx - 1]) and \
+                re.match('[0-9]', text[sidx + 1]):
+            sidx += 1
+            continue
+        if text[sidx - 1] != ' ':
+            text = text[:sidx] + ' ' + text[sidx:]
+            sidx += 1
+        if sidx + len(token) < len(text) and text[sidx + len(token)] != ' ':
+            text = text[:sidx + 1] + ' ' + text[sidx + 1:]
+        sidx += 1
+    return text
+
+
+def normalize(text):
+    # lower case every word
+    text = text.lower()
+
+    # replace white spaces in front and end
+    text = re.sub(r'^\s*|\s*$', '', text)
+
+    # hotel domain pfb30
+    text = re.sub(r"b&b", "bed and breakfast", text)
+    text = re.sub(r"b and b", "bed and breakfast", text)
+
+    # normalize phone number
+    ms = re.findall('\(?(\d{3})\)?[-.\s]?(\d{3})[-.\s]?(\d{4,5})', text)
+    if ms:
+        sidx = 0
+        for m in ms:
+            sidx = text.find(m[0], sidx)
+            if text[sidx - 1] == '(':
+                sidx -= 1
+            eidx = text.find(m[-1], sidx) + len(m[-1])
+            text = text.replace(text[sidx:eidx], ''.join(m))
+
+    # normalize postcode
+    ms = re.findall('([a-z]{1}[\. ]?[a-z]{1}[\. ]?\d{1,2}[, ]+\d{1}[\. ]?[a-z]{1}[\. ]?[a-z]{1}|[a-z]{2}\d{2}[a-z]{2})',
+                    text)
+    if ms:
+        sidx = 0
+        for m in ms:
+            sidx = text.find(m, sidx)
+            eidx = sidx + len(m)
+            text = text[:sidx] + re.sub('[,\. ]', '', m) + text[eidx:]
+
+    # weird unicode bug
+    text = re.sub(u"(\u2018|\u2019)", "'", text)
+
+    # replace time and and price
+    text = re.sub(timepat, ' [value_time] ', text)
+    text = re.sub(pricepat, ' [value_price] ', text)
+
+    # replace st.
+    text = text.replace(';', ',')
+    text = re.sub('$\/', '', text)
+    text = text.replace('/', ' and ')
+
+    # replace other special characters
+    text = text.replace('-', ' ')
+    text = re.sub('[\":\<>@\(\)]', '', text)
+
+    # insert white space before and after tokens:
+    for token in ['?', '.', ',', '!']:
+        text = insertSpace(token, text)
+
+    # insert white space for 's
+    text = insertSpace('\'s', text)
+
+    # replace it's, does't, you'd ... etc
+    text = re.sub('^\'', '', text)
+    text = re.sub('\'$', '', text)
+    text = re.sub('\'\s', ' ', text)
+    text = re.sub('\s\'', ' ', text)
+    for fromx, tox in replacements:
+        text = ' ' + text + ' '
+        text = text.replace(fromx, tox)[1:-1]
+
+    # remove multiple spaces
+    text = re.sub(' +', ' ', text)
+
+    # concatenate numbers
+    tmp = text
+    tokens = text.split()
+    i = 1
+    while i < len(tokens):
+        if re.match(u'^\d+$', tokens[i]) and \
+                re.match(u'\d+$', tokens[i - 1]):
+            tokens[i - 1] += tokens[i]
+            del tokens[i]
+        else:
+            i += 1
+    text = ' '.join(tokens)
+
+    return text
+
+
+def prepareSlotValuesIndependent():
+    domains = ['restaurant', 'hotel', 'attraction', 'train', 'taxi', 'hospital', 'police']
+    requestables = ['phone', 'address', 'postcode', 'reference', 'id']
+    dic = []
+    dic_area = []
+    dic_food = []
+    dic_price = []
+
+    # read databases
+    for domain in domains:
+        try:
+            fin = file(os.path.join(CUR_PATH.replace('latent_dialog/normalizer', ''), 'data/norm-multi-woz/' + domain + '_db.json'))
+            db_json = json.load(fin)
+            fin.close()
+
+            for ent in db_json:
+                for key, val in ent.items():
+                    if val == '?' or val == 'free':
+                        pass
+                    elif key == 'address':
+                        dic.append((normalize(val), '[' + domain + '_' + 'address' + ']'))
+                        if "road" in val:
+                            val = val.replace("road", "rd")
+                            dic.append((normalize(val), '[' + domain + '_' + 'address' + ']'))
+                        elif "rd" in val:
+                            val = val.replace("rd", "road")
+                            dic.append((normalize(val), '[' + domain + '_' + 'address' + ']'))
+                        elif "st" in val:
+                            val = val.replace("st", "street")
+                            dic.append((normalize(val), '[' + domain + '_' + 'address' + ']'))
+                        elif "street" in val:
+                            val = val.replace("street", "st")
+                            dic.append((normalize(val), '[' + domain + '_' + 'address' + ']'))
+                    elif key == 'name':
+                        dic.append((normalize(val), '[' + domain + '_' + 'name' + ']'))
+                        if "b & b" in val:
+                            val = val.replace("b & b", "bed and breakfast")
+                            dic.append((normalize(val), '[' + domain + '_' + 'name' + ']'))
+                        elif "bed and breakfast" in val:
+                            val = val.replace("bed and breakfast", "b & b")
+                            dic.append((normalize(val), '[' + domain + '_' + 'name' + ']'))
+                        elif "hotel" in val and 'gonville' not in val:
+                            val = val.replace("hotel", "")
+                            dic.append((normalize(val), '[' + domain + '_' + 'name' + ']'))
+                        elif "restaurant" in val:
+                            val = val.replace("restaurant", "")
+                            dic.append((normalize(val), '[' + domain + '_' + 'name' + ']'))
+                    elif key == 'postcode':
+                        dic.append((normalize(val), '[' + domain + '_' + 'postcode' + ']'))
+                    elif key == 'phone':
+                        dic.append((val, '[' + domain + '_' + 'phone' + ']'))
+                    elif key == 'trainID':
+                        dic.append((normalize(val), '[' + domain + '_' + 'id' + ']'))
+                    elif key == 'department':
+                        dic.append((normalize(val), '[' + domain + '_' + 'department' + ']'))
+
+                    # NORMAL DELEX
+                    elif key == 'area':
+                        dic_area.append((normalize(val), '[' + 'value' + '_' + 'area' + ']'))
+                    elif key == 'food':
+                        dic_food.append((normalize(val), '[' + 'value' + '_' + 'food' + ']'))
+                    elif key == 'pricerange':
+                        dic_price.append((normalize(val), '[' + 'value' + '_' + 'pricerange' + ']'))
+                    else:
+                        pass
+                    # TODO car type?
+        except:
+            pass
+
+        if domain == 'hospital':
+            dic.append((normalize('Hills Rd'), '[' + domain + '_' + 'address' + ']'))
+            dic.append((normalize('Hills Road'), '[' + domain + '_' + 'address' + ']'))
+            dic.append((normalize('CB20QQ'), '[' + domain + '_' + 'postcode' + ']'))
+            dic.append(('01223245151', '[' + domain + '_' + 'phone' + ']'))
+            dic.append(('1223245151', '[' + domain + '_' + 'phone' + ']'))
+            dic.append(('0122324515', '[' + domain + '_' + 'phone' + ']'))
+            dic.append((normalize('Addenbrookes Hospital'), '[' + domain + '_' + 'name' + ']'))
+
+        elif domain == 'police':
+            dic.append((normalize('Parkside'), '[' + domain + '_' + 'address' + ']'))
+            dic.append((normalize('CB11JG'), '[' + domain + '_' + 'postcode' + ']'))
+            dic.append(('01223358966', '[' + domain + '_' + 'phone' + ']'))
+            dic.append(('1223358966', '[' + domain + '_' + 'phone' + ']'))
+            dic.append((normalize('Parkside Police Station'), '[' + domain + '_' + 'name' + ']'))
+
+    # add at the end places from trains
+    fin = open(os.path.join(CUR_PATH.replace('latent_dialog/normalizer', ''), 'data/norm-multi-woz/' + 'train' + '_db.json'))
+    db_json = json.load(fin)
+    fin.close()
+
+    for ent in db_json:
+        for key, val in ent.items():
+            if key == 'departure' or key == 'destination':
+                dic.append((normalize(val), '[' + 'value' + '_' + 'place' + ']'))
+
+    # add specific values:
+    for key in ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']:
+        dic.append((normalize(key), '[' + 'value' + '_' + 'day' + ']'))
+
+    # more general values add at the end
+    dic.extend(dic_area)
+    dic.extend(dic_food)
+    dic.extend(dic_price)
+
+    return dic
+
+
+def delexicalise(utt, dictionary):
+    for key, val in dictionary:
+        utt = (' ' + utt + ' ').replace(' ' + key + ' ', ' ' + val + ' ')
+        utt = utt[1:-1]  # why this?
+
+    return utt
+
+
+def delexicaliseReferenceNumber(sent, metadata):
+    """Based on the belief state, we can find reference number that
+    during data gathering was created randomly."""
+    domains = ['restaurant', 'hotel', 'attraction', 'train', 'taxi', 'hospital']  # , 'police']
+    if metadata:
+        for domain in domains:
+            if metadata[domain]['book']['booked']:
+                for slot in metadata[domain]['book']['booked'][0]:
+                    if slot == 'reference':
+                        val = '[' + domain + '_' + slot + ']'
+                    else:
+                        val = '[' + domain + '_' + slot + ']'
+                    key = normalize(metadata[domain]['book']['booked'][0][slot])
+                    sent = (' ' + sent + ' ').replace(' ' + key + ' ', ' ' + val + ' ')
+
+                    # try reference with hashtag
+                    key = normalize("#" + metadata[domain]['book']['booked'][0][slot])
+                    sent = (' ' + sent + ' ').replace(' ' + key + ' ', ' ' + val + ' ')
+
+                    # try reference with ref#
+                    key = normalize("ref#" + metadata[domain]['book']['booked'][0][slot])
+                    sent = (' ' + sent + ' ').replace(' ' + key + ' ', ' ' + val + ' ')
+    return sent
+
+
+def delexicalse_num(sent):
+    # changes to numbers only here
+    digitpat = re.compile('\d+')
+    sent = re.sub(digitpat, '[value_count]', sent)
+    return sent
+
+
+def e2e_delecalise(utt, dictionary, metadata):
+    utt = normalize(utt)
+    utt = delexicalise(utt, dictionary)
+    utt = delexicaliseReferenceNumber(utt, metadata)
+    return delexicalse_num(utt)
+
+
+if __name__ == '__main__':
+    prepareSlotValuesIndependent()
diff --git a/convlab/dst/setsumbt/multiwoz/dataset/mapping.pair b/convlab/policy/lava/multiwoz/latent_dialog/normalizer/mapping.pair
similarity index 100%
rename from convlab/dst/setsumbt/multiwoz/dataset/mapping.pair
rename to convlab/policy/lava/multiwoz/latent_dialog/normalizer/mapping.pair
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/offlinerl_utils.py b/convlab/policy/lava/multiwoz/latent_dialog/offlinerl_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..85f76eb767b518063751db6d9d1d8ac8e2bbb367
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/offlinerl_utils.py
@@ -0,0 +1,845 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim:fenc=utf-8
+#
+# Copyright © 2021 lubis <lubis@hilbert50>
+#
+# Distributed under terms of the MIT license.
+
+"""
+
+"""
+
+import torch as th
+import torch.nn as nn
+import torch.optim as optim
+import torch.nn.functional as F
+import numpy as np
+import pdb
+import random
+import copy
+from collections import namedtuple, deque
+from torch.autograd import Variable
+from convlab.policy.lava.multiwoz.latent_dialog.enc2dec.encoders import RnnUttEncoder
+from convlab.policy.lava.multiwoz.latent_dialog.utils import get_detokenize, cast_type, extract_short_ctx, np2var, LONG, FLOAT
+from convlab.policy.lava.multiwoz.latent_dialog.corpora import SYS, EOS, PAD, BOS, DOMAIN_REQ_TOKEN, ACTIVE_BS_IDX, NO_MATCH_DB_IDX, REQ_TOKENS
+import dill
+
+class Actor(nn.Module):
+    def __init__(self, model, corpus, config):
+        super(Actor, self).__init__()
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        self.config = config
+
+        self.use_gpu = config.use_gpu
+
+        self.embedding = None
+        self.is_stochastic = config.is_stochastic
+        self.y_size = config.y_size
+        if 'k_size' in config:
+            self.k_size = config.k_size
+            self.is_gauss = False
+        else:
+            self.max_action = config.max_action if "max_action" in config else None
+            self.is_gauss = True
+
+        self.utt_encoder = copy.deepcopy(model.utt_encoder)
+        self.c2z = copy.deepcopy(model.c2z)
+        if not self.is_gauss:
+            self.gumbel_connector = copy.deepcopy(model.gumbel_connector)
+        else:
+            self.gauss_connector = copy.deepcopy(model.gauss_connector)
+            self.gaussian_logprob = model.gaussian_logprob
+            self.zero = cast_type(th.zeros(1), FLOAT, self.use_gpu)
+
+        # self.l1 = nn.Linear(self.utt_encoder.output_size, 400)
+        # self.l2 = nn.Linear(400, 300)
+        # self.l3 = nn.Linear(300, config.y_size * config.k_size)
+
+    def forward(self, data_feed, hard=False):
+        short_ctx_utts = np2var(extract_short_ctx(data_feed['contexts'], data_feed['context_lens']), LONG, self.use_gpu)
+        bs_label = np2var(data_feed['bs'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = np2var(data_feed['db'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+
+        if self.is_gauss:
+            q_mu, q_logvar = self.c2z(enc_last)
+            # sample_z = q_mu
+            if self.is_stochastic:
+                sample_z = self.gauss_connector(q_mu, q_logvar)
+            else:
+                sample_z = q_mu
+            logprob_sample_z = self.gaussian_logprob(q_mu, q_logvar, sample_z)
+            # joint_logpz = th.sum(logprob_sample_z, dim=1)
+            # return self.max_action * torch.tanh(z)
+        else:
+            logits_qy, log_qy = self.c2z(enc_last)
+            qy = F.softmax(logits_qy / 1.0, dim=1)  # (batch_size, vocab_size, )
+            log_qy = F.log_softmax(logits_qy, dim=1)  # (batch_size, vocab_size, )
+
+            if self.is_stochastic:
+                idx = th.multinomial(qy, 1).detach()
+                soft_z = self.gumbel_connector(logits_qy, hard=False)
+            else:
+                idx = th.argmax(th.exp(log_qy), dim=1, keepdim=True)
+                soft_z = th.exp(log_qy)
+            sample_z = cast_type(Variable(th.zeros(log_qy.size())), FLOAT, self.use_gpu)
+            sample_z.scatter_(1, idx, 1.0)
+            logprob_sample_z = log_qy.gather(1, idx).view(-1, self.y_size)
+
+        joint_logpz = th.sum(logprob_sample_z, dim=1)
+        # for i in range(logprob_sample_z.shape[0]):
+            # print(logprob_sample_z[i])
+            # print(joint_logpz[i])
+        return joint_logpz, sample_z
+
+class DeterministicGaussianActor(nn.Module):
+    def __init__(self, model, corpus, config):
+        super(DeterministicGaussianActor, self).__init__()
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        self.config = config
+
+        self.use_gpu = config.use_gpu
+
+        self.embedding = None
+        self.y_size = config.y_size
+        self.max_action = config.max_action if "max_action" in config else None
+        self.is_gauss = True
+
+        self.utt_encoder = copy.deepcopy(model.utt_encoder)
+
+        self.policy = copy.deepcopy(model.c2z)
+        # self.gauss_connector = copy.deepcopy(model.gauss_connector)
+
+    def forward(self, data_feed):
+        short_ctx_utts = np2var(extract_short_ctx(data_feed['contexts'], data_feed['context_lens']), LONG, self.use_gpu)
+        bs_label = np2var(data_feed['bs'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = np2var(data_feed['db'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+
+        mu, logvar = self.policy(enc_last)
+        z = mu
+        if self.max_action is not None:
+            z =  self.max_action * th.tanh(z)
+
+        return z, mu, logvar
+
+class StochasticGaussianActor(nn.Module):
+    def __init__(self, model, corpus, config):
+        super(StochasticGaussianActor, self).__init__()
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        self.config = config
+
+        self.use_gpu = config.use_gpu
+
+        self.embedding = None
+        self.y_size = config.y_size
+        self.max_action = config.max_action if "max_action" in config else None
+        self.is_gauss = True
+
+        self.utt_encoder = copy.deepcopy(model.utt_encoder)
+        self.policy = copy.deepcopy(model.c2z)
+        self.gauss_connector = copy.deepcopy(model.gauss_connector)
+
+    def forward(self, data_feed, n_z=1):
+        short_ctx_utts = np2var(extract_short_ctx(data_feed['contexts'], data_feed['context_lens']), LONG, self.use_gpu)
+        bs_label = np2var(data_feed['bs'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = np2var(data_feed['db'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+
+        q_mu, q_logvar = self.policy(enc_last)
+        if n_z > 1:
+            z = [self.gauss_connector(q_mu, q_logvar) for _ in range(n_z)]
+        else:
+            z = self.gauss_connector(q_mu, q_logvar)
+
+        return z, q_mu, q_logvar
+
+class RecurrentCritic(nn.Module):
+    def __init__(self,cvae, corpus, config, args):
+        super(RecurrentLatentCritic, self).__init__()
+
+        # self.vocab = corpus.vocab
+        # self.vocab_dict = corpus.vocab_dict
+        # self.vocab_size = len(self.vocab)
+        # self.bos_id = self.vocab_dict[BOS]
+        # self.eos_id = self.vocab_dict[EOS]
+        # self.pad_id = self.vocab_dict[PAD]
+
+        self.embedding = None
+        self.word_plas = args.word_plas
+        self.state_dim = cvae.utt_encoder.output_size
+        if self.word_plas:
+            self.action_dim = cvae.aux_encoder.output_size
+        else:
+            self.action_dim = config.y_size #TODO adjust for categorical
+        # if "k_size"  in config:
+            # if args.embed_z_for_critic:
+                # self.action_dim = config.dec_cell_size # for categorical, the action can be embedded
+            # else:
+        #         self.action_dim *= config.k_size
+
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        self.input_dim = self.state_dim + self.bs_size + self.db_size + self.action_dim
+        # self.input_dim = self.state_dim + 50 + self.action_dim
+        self.goal_to_critic = args.goal_to_critic
+        if self.goal_to_critic:
+            raise NotImplementedError
+
+        self.use_gpu = config.use_gpu
+
+        self.state_encoder = copy.deepcopy(cvae.utt_encoder)
+        if self.word_plas:
+            self.action_encoder = copy.deepcopy(cvae.aux_encoder)
+        else:
+            self.action_encoder = None
+
+        # self.q11 = nn.Linear(self.state_dim + self.action_dim + 50, 500)
+        self.q11 = nn.Linear(self.input_dim, 500)
+        self.q12 = nn.Linear(500, 300)
+        self.q13 = nn.Linear(300, 100)
+        self.q14 = nn.Linear(100, 20)
+        self.q15 = nn.Linear(20, 1)
+
+        self.q21 = nn.Linear(self.input_dim, 500)
+        self.q22 = nn.Linear(500, 300)
+        self.q23 = nn.Linear(300, 100)
+        self.q24 = nn.Linear(100, 20)
+        self.q25 = nn.Linear(20, 1)
+
+    def forward(self, data_feed, act):
+        ctx = np2var(extract_short_ctx(data_feed['contexts'], data_feed['context_lens']), LONG, self.use_gpu)
+        bs_label = np2var(data_feed['bs'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = np2var(data_feed['db'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+
+        ctx_summary, _, _ = self.state_encoder(ctx.unsqueeze(1))
+        if self.word_plas:
+            resp_summary, _, _ = self.action_encoder(act.unsqueeze(1))
+            sa = th.cat([ctx_summary.squeeze(1), bs_label, db_label, resp_summary.squeeze(1)], dim=1)
+        else:
+            sa = th.cat([ctx_summary.squeeze(1), bs_label, db_label, act], dim=1)
+
+        q1 = self.q11(sa)
+        #-
+        # q1 = F.relu(self.q12(th.cat([q1, metadata_summary], dim=1)))
+        q1 = F.relu(self.q12(q1))
+        # q1 = self.q12(q1)
+        #-
+        # q1 = th.sigmoid(self.q13(q1))
+        q1 = F.relu(self.q13(q1))
+        # q1 = F.softmax(self.q13(q1))
+        #-
+        # q1 = th.sigmoid(self.q14(q1))
+        q1 = F.relu(self.q14(q1))
+        # q1 = self.q14(q1)
+        #-
+        q1 = self.q15(q1)
+
+
+        q2 = self.q21(sa)
+        #-
+        # q2 = F.relu(self.lq22(th.cat([q2, metadata_summary], dim=1)))
+        q2 = F.relu(self.q22(q2))
+        # q2 = self.q22(q2)
+        #-
+        # q2 = th.sigmoid(self.q23(q2))
+        q2 = F.relu(self.q23(q2))
+        # q2 = F.softmax(self.q23(q2))
+        #-
+        # q2 = th.sigmoid(self.q24(q2))
+        q2 = F.relu(self.q24(q2))
+        # q2 = self.q24(q2)
+        #-
+        q2 = self.q25(q2)
+
+        return q1, q2
+
+    def q1(self, data_feed, act):
+        ctx = np2var(extract_short_ctx(data_feed['contexts'], data_feed['context_lens']), LONG, self.use_gpu)
+        bs_label = np2var(data_feed['bs'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = np2var(data_feed['db'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+
+        ctx_summary, _, _ = self.state_encoder(ctx.unsqueeze(1))
+        if self.word_plas:
+            resp_summary, _, _ = self.action_encoder(act.unsqueeze(1))
+            sa = th.cat([ctx_summary.squeeze(1), bs_label, db_label, resp_summary.squeeze(0)], dim=1)
+        else:
+            sa = th.cat([ctx_summary.squeeze(1), bs_label, db_label, act], dim=1)
+        
+        q1 = self.q11(sa)
+        #-
+        # q1 = F.relu(self.q12(th.cat([q1, metadata_summary], dim=1)))
+        q1 = F.relu(self.q12(q1))
+        # q1 = self.q12(q1)
+        #-
+        # q1 = th.sigmoid(self.q13(q1))
+        q1 = F.relu(self.q13(q1))
+        # q1 = F.softmax(self.q13(q1))
+        #-
+        # q1 = th.sigmoid(self.q14(q1))
+        q1 = F.relu(self.q14(q1))
+        # q1 = self.q14(q1)
+        #-
+        q1 = self.q15(q1)
+
+        return q1
+
+class SingleRecurrentCritic(nn.Module):
+    def __init__(self, cvae, corpus, config, args):
+        super(SingleRecurrentCritic, self).__init__()
+
+        # self.vocab = corpus.vocab
+        # self.vocab_dict = corpus.vocab_dict
+        # self.vocab_size = len(self.vocab)
+        # self.bos_id = self.vocab_dict[BOS]
+        # self.eos_id = self.vocab_dict[EOS]
+        # self.pad_id = self.vocab_dict[PAD]
+
+        if "gauss" in args.sv_config_path:
+            self.is_gauss = True
+        else:
+            self.is_gauss = False
+        self.embedding = None
+        self.word_plas = args.word_plas
+        self.state_dim = cvae.utt_encoder.output_size
+        if self.word_plas:
+            self.action_dim = cvae.aux_encoder.output_size
+        else:
+            if self.is_gauss:
+                self.action_dim = config.y_size 
+            else:
+                if args.embed_z_for_critic:
+                    self.action_dim = config.dec_cell_size
+                else:
+                    self.action_dim = config.y_size * config.k_size
+
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        self.input_dim = self.state_dim + self.bs_size + self.db_size + self.action_dim
+
+        self.goal_to_critic = args.goal_to_critic
+        if self.goal_to_critic:
+            self.goal_size = corpus.goal_size
+            self.input_dim += self.goal_size
+
+        # self.input_dim = self.state_dim + 50 + self.action_dim
+
+        self.use_gpu = config.use_gpu
+
+        self.state_encoder = copy.deepcopy(cvae.utt_encoder)
+        if self.word_plas:
+            self.action_encoder = copy.deepcopy(cvae.aux_encoder)
+        else:
+            self.action_encoder = None
+
+        # if self.goal_to_critic:
+            # self.q11 = nn.Linear(self.input_dim, 500)
+            # self.q12 = nn.Linear(500, 1)
+        # else:
+        self.q11 = nn.Linear(self.input_dim, 1)
+        self.activation_function = args.critic_actf if "critic_actf" in args else "none"
+
+        self.critic_dropout = args.critic_dropout
+        if self.critic_dropout:
+            self.d = th.nn.Dropout(p=args.critic_dropout_rate, inplace=False)
+        else:
+            self.d = th.nn.Dropout(p=0.0, inplace=False)
+
+    def forward(self, data_feed, act):
+        ctx = np2var(extract_short_ctx(data_feed['contexts'], data_feed['context_lens']), LONG, self.use_gpu)
+        bs_label = np2var(data_feed['bs'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = np2var(data_feed['db'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+
+        ctx_summary, _, _ = self.state_encoder(ctx.unsqueeze(1))
+        if self.word_plas:
+            resp_summary, _, _ = self.action_encoder(act.unsqueeze(1))
+            sa = th.cat([ctx_summary.squeeze(1), bs_label, db_label, resp_summary.squeeze(1)], dim=1)
+        else:
+            sa = th.cat([ctx_summary.squeeze(1), bs_label, db_label, act], dim=1)
+
+        if self.goal_to_critic:
+            try:
+                goals = np2var(data_feed['goals'], FLOAT, self.use_gpu)
+            except KeyError:
+                goals = []
+                for turn_id in range(len(ctx_summary)):
+                    goals.append(np.concatenate([data_feed['goals_list'][d][turn_id] for d in range(7)]))
+                goals = np2var(np.asarray(goals), FLOAT, self.use_gpu)
+            sa = th.cat([sa, goals], dim = 1)
+
+        # metadata_summary = self.metadata_encoder(th.cat([bs_label, db_label], dim=1))
+        # sa = th.cat([ctx_summary.squeeze(1), metadata_summary, act], dim=1)
+        # if self.is_gauss:
+            # q1 = F.relu(self.q11(self.d(sa)))
+        # else:
+        # q1 = F.sigmoid(self.q11(self.d(sa)))
+        q1 = self.q11(self.d(sa))
+        # if self.goal_to_critic:
+            # q1 = self.q12(q1)
+
+        if self.activation_function == "relu":
+            q1 = F.relu(q1)
+        elif self.activation_function == "sigmoid":
+            q1 = F.sigmoid(q1)
+
+        return q1
+
+class SingleHierarchicalRecurrentCritic(nn.Module):
+    def __init__(self, cvae, corpus, config, args):
+        super(SingleHierarchicalRecurrentCritic, self).__init__()
+
+        # self.vocab = corpus.vocab
+        # self.vocab_dict = corpus.vocab_dict
+        # self.vocab_size = len(self.vocab)
+        # self.bos_id = self.vocab_dict[BOS]
+        # self.eos_id = self.vocab_dict[EOS]
+        # self.pad_id = self.vocab_dict[PAD]
+
+        self.hidden_size = 500
+
+        if "gauss" in args.sv_config_path:
+            self.is_gauss = True
+        else:
+            self.is_gauss = False
+        self.embedding = None
+        self.word_plas = args.word_plas
+        self.state_dim = cvae.utt_encoder.output_size
+        if self.word_plas:
+            self.action_dim = cvae.aux_encoder.output_size
+        else:
+            if self.is_gauss:
+                self.action_dim = config.y_size 
+            else:
+                if args.embed_z_for_critic:
+                    self.action_dim = config.dec_cell_size
+                else:
+                    self.action_dim = config.y_size * config.k_size
+
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        self.input_dim = self.state_dim + self.bs_size + self.db_size + self.action_dim
+        # self.input_dim = self.state_dim + 50 + self.action_dim
+
+        self.goal_to_critic = args.goal_to_critic
+        self.add_goal = args.add_goal
+        if self.goal_to_critic:
+            self.goal_size = corpus.goal_size
+            if self.add_goal == "early":
+                self.input_dim += self.goal_size
+
+
+        self.use_gpu = config.use_gpu
+
+        self.state_encoder = copy.deepcopy(cvae.utt_encoder)
+        if self.word_plas:
+            self.action_encoder = copy.deepcopy(cvae.aux_encoder)
+        else:
+            self.action_encoder = None
+
+        self.dialogue_encoder = nn.LSTM(
+                input_size = self.input_dim,
+                hidden_size = self.hidden_size,
+                dropout=0.1
+                )
+
+        if self.add_goal=="late":
+            self.q11 = nn.Linear(self.hidden_size + self.goal_size, 1)
+        else:
+            self.q11 = nn.Linear(self.hidden_size, 1)
+        self.activation_function = args.critic_actf if "critic_actf" in args else "none"
+
+        self.critic_dropout = args.critic_dropout
+        if self.critic_dropout:
+            self.d = th.nn.Dropout(p=args.critic_dropout_rate, inplace=False)
+        else:
+            self.d = th.nn.Dropout(p=0.0, inplace=False)
+
+        if args.critic_actf == "tanh" or args.critic_actf == "sigmoid":
+            self.maxq = args.critic_maxq
+        else:
+            self.maxq = None
+
+    def forward(self, data_feed, act):
+        ctx = np2var(extract_short_ctx(data_feed['contexts'], data_feed['context_lens']), LONG, self.use_gpu)
+        bs_label = np2var(data_feed['bs'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = np2var(data_feed['db'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+
+        ctx_summary, _, _ = self.state_encoder(ctx.unsqueeze(1))
+        if self.word_plas:
+            resp_summary, _, _ = self.action_encoder(act.unsqueeze(1))
+            sa = th.cat([ctx_summary.squeeze(1), bs_label, db_label, resp_summary.squeeze(1)], dim=1)
+        else:
+            sa = th.cat([ctx_summary.squeeze(1), bs_label, db_label, act], dim=1)
+
+        if self.goal_to_critic:
+            try:
+                goals = np2var(data_feed['goals'], FLOAT, self.use_gpu)
+            except KeyError:
+                goals = []
+                for turn_id in range(len(ctx_summary)):
+                    goals.append(np.concatenate([data_feed['goals_list'][d][turn_id] for d in range(7)]))
+                goals = np2var(np.asarray(goals), FLOAT, self.use_gpu)
+
+        #OPTION 1 add goal to encoder for each time step
+        if self.goal_to_critic and self.add_goal=="early":
+            sa = th.cat([sa, goals], dim = 1)
+
+        output, (hn, cn) = self.dialogue_encoder(self.d(sa.unsqueeze(1)))
+
+        #OPTION 2 add goal combined with hidden state to predict final score
+        if self.goal_to_critic and self.add_goal=="late":
+            output = th.cat([output, goals.unsqueeze(1)], dim = 2)
+
+        q1 = self.q11(output.squeeze(1))
+
+        if self.activation_function == "relu":
+            q1 = F.relu(q1)
+        elif self.activation_function == "sigmoid":
+            q1 = th.sigmoid(q1)
+        elif self.activation_function == "tanh":
+            q1 = F.tanh(q1)
+
+        if self.maxq is not None:
+            q1 *= self.maxq
+
+        return q1
+    
+    def forward_target(self, data_feed, act, corpus_act):
+        ctx = np2var(extract_short_ctx(data_feed['contexts'], data_feed['context_lens']), LONG, self.use_gpu)
+        bs_label = np2var(data_feed['bs'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = np2var(data_feed['db'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+
+        ctx_summary, _, _ = self.state_encoder(ctx.unsqueeze(1))
+        q1s =[]
+        for i in range(bs_label.shape[0]):
+            if self.word_plas:
+                corpus_resp_summary, _, _ = self.action_encoder(corpus_act[:-i].unsqueeze(1))
+                actor_resp_summary, _, _ = self.action_encoder(act[i].unsqueeze(1))
+                sa = th.cat([ctx_summary[:i+1].squeeze(1), bs_label[:i+1], db_label[:i+1], th.cat([corpus_resp_summary[:i], actor_resp_summary[i]], dim=0).squeeze(1)], dim=1)
+            else:
+                sa = th.cat([ctx_summary[:i+1].squeeze(1), bs_label[:i+1], db_label[:i+1], th.cat([corpus_act[:i], act[i].unsqueeze(0)], dim=0)], dim=1)
+
+            if self.goal_to_critic:
+                try:
+                    goals = np2var(data_feed['goals'][:i+1], FLOAT, self.use_gpu)
+                except KeyError:
+                    goals = []
+                    for turn_id in range(i+1):
+                        goals.append(np.concatenate([data_feed['goals_list'][d][turn_id] for d in range(7)]))
+                    goals = np2var(np.asarray(goals), FLOAT, self.use_gpu)
+
+            #OPTION 1 add goal to encoder for each time step
+            if self.goal_to_critic and self.add_goal=="early":
+                sa = th.cat([sa, goals], dim = 1)
+
+            output, (hn, cn) = self.dialogue_encoder(self.d(sa.unsqueeze(1)))
+
+            #OPTION 2 add goal combined with hidden state to predict final score
+            if self.goal_to_critic and self.add_goal=="late":
+                output = th.cat([output, goals.unsqueeze(1)], dim = 2)
+
+            q1 = self.q11(output.squeeze(1))
+
+            if self.activation_function == "relu":
+                q1 = F.relu(q1)
+            elif self.activation_function == "sigmoid":
+                q1 = F.sigmoid(q1)
+            elif self.activation_function == "tanh":
+                q1 = F.tanh(q1) * self.maxq
+
+            q1s.append(q1[-1])
+
+        return th.cat(q1s, dim=0).unsqueeze(1)
+
+class SingleTransformersCritic(nn.Module):
+    def __init__(self, cvae, corpus, config, args):
+        super(SingleTransformersCritic, self).__init__()
+
+        # self.vocab = corpus.vocab
+        # self.vocab_dict = corpus.vocab_dict
+        # self.vocab_size = len(self.vocab)
+        # self.bos_id = self.vocab_dict[BOS]
+        # self.eos_id = self.vocab_dict[EOS]
+        # self.pad_id = self.vocab_dict[PAD]
+
+        self.hidden_size = 128
+
+        if "gauss" in args.sv_config_path:
+            self.is_gauss = True
+        else:
+            self.is_gauss = False
+        self.embedding = None
+        self.word_plas = args.word_plas
+        self.state_dim = cvae.utt_encoder.output_size
+        if self.word_plas:
+            self.action_dim = cvae.aux_encoder.output_size
+        else:
+            if self.is_gauss:
+                self.action_dim = config.y_size 
+            else:
+                if args.embed_z_for_critic:
+                    self.action_dim = config.dec_cell_size
+                else:
+                    self.action_dim = config.y_size * config.k_size
+
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        self.input_dim = self.state_dim + self.bs_size + self.db_size + self.action_dim
+        self.db_embedding = nn.Linear(self.db_size, config.embed_size)
+        self.bs_embedding = nn.Linear(self.bs_size, config.embed_size)
+
+        self.goal_to_critic = args.goal_to_critic
+        if self.goal_to_critic:
+            raise NotImplementedError
+
+
+        self.use_gpu = config.use_gpu
+
+        self.state_encoder = copy.deepcopy(cvae.utt_encoder)
+        if self.word_plas:
+            self.action_encoder = copy.deepcopy(cvae.aux_encoder)
+        else:
+            self.action_encoder = None
+
+        self.trans_encoder_layer = nn.TransformerEncoderLayer(nhead=8, d_model=config.embed_size)
+        self.trans_encoder = nn.TransformerEncoder(self.trans_encoder_layer, num_layers=4)
+
+        self.dialogue_encoder = nn.LSTM(
+                input_size = config.embed_size,
+                hidden_size = self.hidden_size,
+                dropout=0.1
+                )
+        if not self.word_plas:
+            self.act_embedding = nn.Linear(self.action_dim, config.embed_size)
+        self.bs_encoder = nn.Linear(self.db_size, config.embed_size)
+        self.db_encoder = nn.Linear(self.db_size, config.embed_size)
+
+        self.q11 = nn.Linear(self.hidden_size, 1)
+
+        self.critic_dropout = args.critic_dropout
+        if self.critic_dropout:
+            self.d = th.nn.Dropout(p=args.critic_dropout_rate, inplace=False)
+        else:
+            self.d = th.nn.Dropout(p=0.0, inplace=False)
+
+    def forward(self, data_feed, act):
+        ctx = np2var(extract_short_ctx(data_feed['contexts'], data_feed['context_lens']), LONG, self.use_gpu)
+        bs_label = np2var(data_feed['bs'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = np2var(data_feed['db'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+
+        ctx_summary, word_emb, enc_outs = self.state_encoder(ctx.unsqueeze(1))
+        # word_emb : (batch_size, max_len, 256)
+        # enc_outs : (batch_size, max_len, 600)
+        metadata_embedding = th.cat([self.bs_embedding(bs_label).unsqueeze(1), self.db_embedding(db_label).unsqueeze(1)], dim=1)
+
+        if self.word_plas:
+            resp_summary, resp_word_emb, resp_enc_outs = self.action_encoder(act.unsqueeze(1))
+            act_embedding = resp_word_emb
+        else:
+            act_embedding = self.act_embedding(act).unsqueeze(1)
+
+        sa = th.cat([word_emb, metadata_embedding, act_embedding], dim=1)
+        sa = self.trans_encoder(self.d(sa))
+        output, (hn, cn) = self.dialogue_encoder(self.d(sa))
+        q1 = F.sigmoid(self.q11(output[:, -1].squeeze(1)))
+        # q1 = self.q11(q1[:, 0])
+
+
+        return q1
+
+
+class CatActor(nn.Module):
+    def __init__(self, model, corpus, config):
+        super(CatActor, self).__init__()
+        self.vocab = corpus.vocab
+        self.vocab_dict = corpus.vocab_dict
+        self.vocab_size = len(self.vocab)
+        self.bs_size = corpus.bs_size
+        self.db_size = corpus.db_size
+        self.bos_id = self.vocab_dict[BOS]
+        self.eos_id = self.vocab_dict[EOS]
+        self.pad_id = self.vocab_dict[PAD]
+        self.config = config
+
+        self.use_gpu = config.use_gpu
+
+        self.embedding = None
+        self.y_size = config.y_size
+        self.k_size = config.k_size
+        # self.max_action = config.max_action
+        self.is_gauss = False
+        self.is_stochastic = config.is_stochastic
+
+        self.utt_encoder = copy.deepcopy(model.utt_encoder)
+
+        self.policy = copy.deepcopy(model.c2z)
+        if self.is_stochastic:
+            self.gumbel_connector = copy.deepcopy(model.gumbel_connector)
+
+    def forward(self, data_feed):
+        short_ctx_utts = np2var(extract_short_ctx(data_feed['contexts'], data_feed['context_lens']), LONG, self.use_gpu)
+        bs_label = np2var(data_feed['bs'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+        db_label = np2var(data_feed['db'], FLOAT, self.use_gpu)  # (batch_size, max_ctx_len, max_utt_len)
+
+        utt_summary, _, enc_outs = self.utt_encoder(short_ctx_utts.unsqueeze(1))
+        # create decoder initial states
+        enc_last = th.cat([bs_label, db_label, utt_summary.squeeze(1)], dim=1)
+
+
+        logits_qy, log_qy = self.policy(enc_last)
+        if self.is_stochastic:
+            z = self.gumbel_connector(logits_qy, hard=True)
+            soft_z = self.gumbel_connector(logits_qy, hard=False)
+        else:
+            z_idx = th.argmax(th.exp(log_qy), dim=1, keepdim=True)
+            z = cast_type(Variable(th.zeros(log_qy.size())), FLOAT, self.use_gpu)
+            z.scatter_(1, z_idx, 1.0)
+            soft_z = th.exp(log_qy)
+
+        return z, soft_z, log_qy
+
+
+
+class ReplayBuffer(object):
+    """
+    Buffer to store experiences, to be used in off-policy learning
+    """
+    def __init__(self, config): 
+        # true_responses = id2sent(next_state)
+        # pred_responses = model.z2x(action)
+
+        self.batch_size = config.batch_size
+        self.fix_episode = config.fix_episode
+        
+        self.experiences = namedtuple("Experience", field_names=["state", "action", "reward", "next_state", "next_action", "done", "Return"])
+        self.memory = deque()
+        self.seed = random.seed(config.random_seed)
+        # self.reinforce_data = config.reinforce_data
+
+    def add(self, states, actions, rewards, next_states, next_actions, dones, Returns):
+        if self.fix_episode:
+            self._add_episode(states, actions, rewards, next_states, next_actions, dones, Returns)
+        else:
+            for i in range(len(states)):
+                self._add(states[i], actions[i], rewards[i], next_states[i], next_actions[i], dones[i], Returns[i])
+
+    def _add(self, state, action, reward, next_state, next_action, done, Return):
+        e = self.experiences(state, action, reward, next_state, next_action, done, Return)
+        self.memory.append(e)
+
+    def _add_episode(self, states, actions, rewards, next_states, next_actions, dones, Returns):
+        ep = []
+        for s, a, r, s_, a_, d, R in zip(states, actions, rewards, next_states, next_actions, dones, Returns):
+            ep.append(self.experiences(s, a, r, s_, a_, d, R))
+        self.memory.append(ep)
+
+
+    def sample(self):
+        if self.fix_episode:
+            return self._sample_episode()
+        else:
+            return self._sample()
+
+    def _sample(self):
+        experiences = random.sample(self.memory, k = self.batch_size)
+
+        states = {}
+        states['contexts'] = np.asarray([e.state['contexts'] for e in experiences])
+        states['bs'] = np.asarray([e.state['bs'] for e in experiences])
+        states['db'] = np.asarray([e.state['db'] for e in experiences])
+        states['context_lens'] = np.asarray([e.state['context_lens'] for e in experiences]) 
+        states['goals'] = np.asarray([e.state['goals'] for e in experiences]) 
+
+        actions = np.asarray([e.action for e in experiences if e is not None])
+        rewards = np.asarray([e.reward for e in experiences if e is not None])
+        
+        next_states = {}
+        next_states['contexts'] = np.asarray([e.next_state['contexts'] for e in experiences])
+        next_states['bs'] = np.asarray([e.next_state['bs'] for e in experiences])
+        next_states['db'] = np.asarray([e.next_state['db'] for e in experiences])
+        next_states['context_lens'] = np.asarray([e.next_state['context_lens'] for e in experiences]) 
+        next_states['goals'] = np.asarray([e.next_state['goals'] for e in experiences])
+        
+        next_actions = np.asarray([e.next_action for e in experiences if e is not None])
+
+        dones = np.asarray([e.done for e in experiences if e is not None])
+        returns = np.asarray([e.Return for e in experiences if e is not None])
+        # if self.reinforce_data:
+            # rewards = dones * 10 + 1 # give positive rewards to all actions taken in the data
+
+        return (states, actions, rewards, next_states, next_actions, dones, returns)
+        # return experiences
+    
+    def _sample_episode(self):
+        # episodes = random.sample(self.memory, k = self.batch_size)
+        episodes = random.sample(self.memory, k = 1)
+
+        for experiences in episodes:
+            states = {}
+            states['contexts'] = np.asarray([e.state['contexts'] for e in experiences])
+            states['bs'] = np.asarray([e.state['bs'] for e in experiences])
+            states['db'] = np.asarray([e.state['db'] for e in experiences])
+            states['keys'] = [e.state['keys'] for e in experiences]
+            states['context_lens'] = np.asarray([e.state['context_lens'] for e in experiences]) 
+            states['goals'] = np.asarray([e.state['goals'] for e in experiences]) 
+
+            actions = np.asarray([e.action for e in experiences if e is not None])
+            rewards = np.asarray([e.reward for e in experiences if e is not None])
+            
+            next_states = {}
+            next_states['contexts'] = np.asarray([e.next_state['contexts'] for e in experiences])
+            next_states['bs'] = np.asarray([e.next_state['bs'] for e in experiences])
+            next_states['db'] = np.asarray([e.next_state['db'] for e in experiences])
+            next_states['keys'] = [e.next_state['keys'] for e in experiences]
+            next_states['context_lens'] = np.asarray([e.next_state['context_lens'] for e in experiences]) 
+            next_states['goals'] = np.asarray([e.next_state['goals'] for e in experiences])
+            
+            next_actions = np.asarray([e.next_action for e in experiences if e is not None])
+
+            dones = np.asarray([e.done for e in experiences if e is not None])
+            returns = np.asarray([e.Return for e in experiences if e is not None])
+            # if self.reinforce_data:
+                # rewards = dones * 10 + 1 # give positive rewards to all actions taken in the data
+
+        return (states, actions, rewards, next_states, next_actions, dones, returns)
+        # return experiences
+
+    def __len__(self):
+        return len(self.memory)
+
+    def save(self, path):
+        with open(path, 'wb') as f:
+            dill.dump(self.memory, f)
+
+    def load(self, path):
+        with open(path, 'rb') as f:
+            self.memory = dill.load(f)
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/record.py b/convlab/policy/lava/multiwoz/latent_dialog/record.py
new file mode 100644
index 0000000000000000000000000000000000000000..88a072b78670f92787b360ecf63e7fa9fe49c62a
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/record.py
@@ -0,0 +1,169 @@
+import numpy as np
+from convlab.policy.lava.multiwoz.latent_dialog.enc2dec.decoders import TEACH_FORCE, GEN, DecoderRNN, GEN_VALID
+from collections import Counter
+
+
+class UniquenessSentMetric(object):
+    """Metric that evaluates the number of unique sentences."""
+    def __init__(self):
+        self.seen = set()
+        self.all_sents = []
+
+    def record(self, sen):
+        self.seen.add(' '.join(sen))
+        self.all_sents.append(' '.join(sen))
+
+    def value(self):
+        return len(self.seen)
+
+    def top_n(self, n):
+        return Counter(self.all_sents).most_common(n)
+
+
+class UniquenessWordMetric(object):
+    """Metric that evaluates the number of unique sentences."""
+    def __init__(self):
+        self.seen = set()
+
+    def record(self, word_list):
+        self.seen.update(word_list)
+
+    def value(self):
+        return len(self.seen)
+
+
+def record_task(n_epsd, model, val_data, config, ppl_f, dialog, ctx_gen_eval, rl_f):
+    record_ppl(n_epsd, model, val_data, config, ppl_f)
+    record_rl_task(n_epsd, dialog, ctx_gen_eval, rl_f)
+
+
+def record(n_epsd, model, val_data, sv_config, lm_model, ppl_f, dialog, ctx_gen_eval, rl_f):
+    record_ppl_with_lm(n_epsd, model, val_data, sv_config, lm_model, ppl_f)
+    record_rl(n_epsd, dialog, ctx_gen_eval, rl_f)
+
+
+def record_ppl_with_lm(n_epsd, model, data, config, lm_model, ppl_f):
+    model.eval()
+    loss_list = []
+    data.epoch_init(config, shuffle=False, verbose=True)
+    while True:
+        batch = data.next_batch()
+        if batch is None:
+            break
+        for i in range(1):
+            loss = model(batch, mode=TEACH_FORCE, use_py=True)
+            loss_list.append(loss.nll.item())
+
+    # USE LM to test generation performance
+    data.epoch_init(config, shuffle=False, verbose=False)
+    gen_loss_list = []
+    # first generate
+    while True:
+        batch = data.next_batch()
+        if batch is None:
+            break
+
+        outputs, labels = model(batch, mode=GEN, gen_type=config.gen_type)
+        # move from GPU to CPU
+        labels = labels.cpu()
+        pred_labels = [t.cpu().data.numpy() for t in outputs[DecoderRNN.KEY_SEQUENCE]]
+        pred_labels = np.array(pred_labels, dtype=int).squeeze(-1).swapaxes(0, 1)  # (batch_size, max_dec_len)
+        # clean up the pred labels
+        clean_pred_labels = np.zeros((pred_labels.shape[0], pred_labels.shape[1]+1))
+        clean_pred_labels[:, 0] = model.sys_id
+        for b_id in range(pred_labels.shape[0]):
+            for t_id in range(pred_labels.shape[1]):
+                token = pred_labels[b_id, t_id]
+                clean_pred_labels[b_id, t_id + 1] = token
+                if token in [model.eos_id] or t_id == pred_labels.shape[1]-1:
+                    break
+
+        pred_out_lens = np.sum(np.sign(clean_pred_labels), axis=1)
+        max_pred_lens = np.max(pred_out_lens)
+        clean_pred_labels = clean_pred_labels[:, 0:int(max_pred_lens)]
+        batch['outputs'] = clean_pred_labels
+        batch['output_lens'] = pred_out_lens
+        loss = lm_model(batch, mode=TEACH_FORCE)
+        gen_loss_list.append(loss.nll.item())
+
+    avg_loss = np.average(loss_list)
+    avg_ppl = np.exp(avg_loss)
+    gen_avg_loss = np.average(gen_loss_list)
+    gen_avg_ppl = np.exp(gen_avg_loss)
+
+    ppl_f.write('{}\t{}\t{}\n'.format(n_epsd, avg_ppl, gen_avg_ppl))
+    ppl_f.flush()
+    model.train()
+
+
+def record_ppl(n_epsd, model, val_data, config, ppl_f):
+    model.eval()
+    loss_list = []
+    val_data.epoch_init(config, shuffle=False, verbose=True)
+    while True:
+        batch = val_data.next_batch()
+        if batch is None:
+            break
+        loss = model(batch, mode=TEACH_FORCE, use_py=True)
+        loss_list.append(loss.nll.item())
+    aver_loss = np.average(loss_list)
+    aver_ppl = np.exp(aver_loss)
+    ppl_f.write('{}\t{}\n'.format(n_epsd, aver_ppl))
+    ppl_f.flush()
+    model.train()
+
+
+def record_rl(n_epsd, dialog, ctx_gen, rl_f):
+    conv_list = []
+    reward_list = []
+    agree_list = []
+    sent_metric = UniquenessSentMetric()
+    word_metric = UniquenessWordMetric()
+
+    for ctxs in ctx_gen.ctxs:
+        conv, agree, rewards = dialog.run(ctxs)
+        true_reward = rewards[0] if agree else 0
+        reward_list.append(true_reward)
+        conv_list.append(conv)
+        agree_list.append(float(agree) if agree is not None else 0.0)
+        for turn in conv:
+            if turn[0] == 'Elder':
+                sent_metric.record(turn[1])
+                word_metric.record(turn[1])
+
+    # json.dump(conv_list, text_f, indent=4)
+    aver_reward = np.average(reward_list)
+    aver_agree = np.average(agree_list)
+    unique_sent_num = sent_metric.value()
+    unique_word_num = word_metric.value()
+    print(sent_metric.top_n(10))
+
+    rl_f.write('{}\t{}\t{}\t{}\t{}\n'.format(n_epsd, aver_reward, aver_agree, unique_sent_num, unique_word_num))
+    rl_f.flush()
+
+
+def record_rl_task(n_epsd, dialog, goal_gen, rl_f):
+    conv_list = []
+    reward_list = []
+    sent_metric = UniquenessSentMetric()
+    word_metric = UniquenessWordMetric()
+    print("Begin RL testing")
+    cnt = 0
+    for g_key, goal in goal_gen.iter(1):
+        cnt += 1
+        conv, success = dialog.run(g_key, goal)
+        true_reward = success
+        reward_list.append(true_reward)
+        conv_list.append(conv)
+        for turn in conv:
+            if turn[0] == 'Elder':
+                sent_metric.record(turn[1])
+                word_metric.record(turn[1])
+
+    # json.dump(conv_list, text_f, indent=4)
+    aver_reward = np.average(reward_list)
+    unique_sent_num = sent_metric.value()
+    unique_word_num = word_metric.value()
+    rl_f.write('{}\t{}\t{}\t{}\n'.format(n_epsd, aver_reward, unique_sent_num, unique_word_num))
+    rl_f.flush()
+    print("End RL testing")
diff --git a/convlab/policy/lava/multiwoz/latent_dialog/utils.py b/convlab/policy/lava/multiwoz/latent_dialog/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..764f95a1a5ce436781579b89f9fc8d6ce4bcf292
--- /dev/null
+++ b/convlab/policy/lava/multiwoz/latent_dialog/utils.py
@@ -0,0 +1,132 @@
+import os
+import numpy as np
+import random
+import torch as th
+from nltk import RegexpTokenizer
+from torch.autograd import Variable
+from nltk.tokenize.treebank import TreebankWordDetokenizer
+import logging
+import sys
+from collections import defaultdict
+
+INT = 0
+LONG = 1
+FLOAT = 2
+
+
+class Pack(dict):
+    def __getattr__(self, name):
+        return self[name]
+
+    def add(self, **kwargs):
+        for k, v in kwargs.items():
+            self[k] = v
+
+    def copy(self):
+        pack = Pack()
+        for k, v in self.items():
+            if type(v) is list:
+                pack[k] = list(v)
+            else:
+                pack[k] = v
+        return pack
+
+    @staticmethod
+    def msg_from_dict(dictionary, tokenize, speaker2id, bos_id, eos_id, include_domain=False):
+        pack = Pack()
+        for k, v in dictionary.items():
+            pack[k] = v
+        pack['speaker'] = speaker2id[pack.speaker]
+        pack['conf'] = dictionary.get('conf', 1.0)
+        utt = pack['utt']
+        if 'QUERY' in utt or "RET" in utt:
+            utt = str(utt)
+            # utt = utt.translate(None, ''.join([':', '"', "{", "}", "]", "["]))
+            utt = utt.translate(str.maketrans('', '', ''.join([':', '"', "{", "}", "]", "["])))
+            utt = str(utt)
+        if include_domain:
+            pack['utt'] = [bos_id, pack['speaker'], pack['domain']] + tokenize(utt) + [eos_id]
+        else:
+            pack['utt'] = [bos_id, pack['speaker']] + tokenize(utt) + [eos_id]
+        return pack
+
+def get_tokenize():
+    return RegexpTokenizer(r'\w+|#\w+|<\w+>|%\w+|[^\w\s]+').tokenize
+
+def get_detokenize():
+    return lambda x: TreebankWordDetokenizer().detokenize(x)
+
+def cast_type(var, dtype, use_gpu):
+    if use_gpu:
+        if dtype == INT:
+            var = var.type(th.cuda.IntTensor)
+        elif dtype == LONG:
+            var = var.type(th.cuda.LongTensor)
+        elif dtype == FLOAT:
+            var = var.type(th.cuda.FloatTensor)
+        else:
+            raise ValueError('Unknown dtype')
+    else:
+        if dtype == INT:
+            var = var.type(th.IntTensor)
+        elif dtype == LONG:
+            var = var.type(th.LongTensor)
+        elif dtype == FLOAT:
+            var = var.type(th.FloatTensor)
+        else:
+            raise ValueError('Unknown dtype')
+    return var
+
+def read_lines(file_name):
+    """Reads all the lines from the file."""
+    assert os.path.exists(file_name), 'file does not exists %s' % file_name
+    lines = []
+    with open(file_name, 'r') as f:
+        for line in f:
+            lines.append(line.strip())
+    return lines
+
+def set_seed(seed):
+    """Sets random seed everywhere."""
+    th.manual_seed(seed)
+    if th.cuda.is_available():
+        th.cuda.manual_seed(seed)
+    np.random.seed(seed)
+
+def prepare_dirs_loggers(config, script=""):
+    logFormatter = logging.Formatter("%(message)s")
+    rootLogger = logging.getLogger()
+    rootLogger.setLevel(logging.DEBUG)
+
+    consoleHandler = logging.StreamHandler(sys.stdout)
+    consoleHandler.setLevel(logging.DEBUG)
+    consoleHandler.setFormatter(logFormatter)
+    #rootLogger.addHandler(consoleHandler)
+
+    if hasattr(config, 'forward_only') and config.forward_only:
+        return
+
+    fileHandler = logging.FileHandler(os.path.join(config.saved_path,'session.log'))
+    fileHandler.setLevel(logging.DEBUG)
+    fileHandler.setFormatter(logFormatter)
+    rootLogger.addHandler(fileHandler)
+
+def get_chat_tokenize():
+    return nltk.RegexpTokenizer(r'\w+|<sil>|[^\w\s]+').tokenize
+
+class missingdict(defaultdict):
+    def __missing__(self, key):
+        return self.default_factory()
+
+def extract_short_ctx(context, context_lens, backward_size=1):
+    utts = []
+    for b_id in range(context.shape[0]):
+        utts.append(context[b_id, context_lens[b_id]-1])
+    return np.array(utts)
+
+def np2var(inputs, dtype, use_gpu):
+    if inputs is None:
+        return None
+    return cast_type(Variable(th.from_numpy(inputs)), 
+                     dtype, 
+                     use_gpu)
diff --git a/convlab/policy/lava/multiwoz/lava.py b/convlab/policy/lava/multiwoz/lava.py
index dad3f6b58a262f7ff6d6aa28c6b8842a8c14fd34..76d177396c4d9a9d4c7e20f18ce652f6f8852ad5 100755
--- a/convlab/policy/lava/multiwoz/lava.py
+++ b/convlab/policy/lava/multiwoz/lava.py
@@ -10,7 +10,8 @@ from convlab.policy.lava.multiwoz.latent_dialog.models_task import *
 from convlab.policy import Policy
 from convlab.util.file_util import cached_path
 from convlab.util.multiwoz.state import default_state
-from convlab.util.multiwoz.dbquery import Database
+# from convlab.util.multiwoz.dbquery import Database
+from data.unified_datasets.multiwoz21.database import Database
 from copy import deepcopy
 import json
 import os
@@ -156,7 +157,7 @@ def get_relevant_domains(state):
 
     for domain in state.keys():
         # print("--", domain, "--")
-        for slot, value in state[domain]['semi'].items():
+        for slot, value in state[domain].items():
             if len(value) > 0:
                 # print(slot, value)
                 domains.append(domain)
@@ -174,7 +175,8 @@ def addDBPointer(state, db):
     num_entities = {}
     for domain in domains:
         # entities = dbPointer.queryResultVenues(domain, {'metadata': state})
-        entities = db.query(domain, state[domain]['semi'].items())
+        constraints = [[slot, value] for slot, value in state[domain].items() if value] if domain in state else []
+        entities = db.query(domain, constraints, topk=10)
         num_entities[domain] = len(entities)
         if len(entities) > 0:
             # fields = dbPointer.table_schema(domain)
@@ -233,36 +235,37 @@ def delexicaliseReferenceNumber(sent, state):
     during data gathering was created randomly."""
     domains = ['restaurant', 'hotel', 'attraction',
                'train', 'taxi', 'hospital']  # , 'police']
-    for domain in domains:
-        if state[domain]['book']['booked']:
-            for slot in state[domain]['book']['booked'][0]:
-                if slot == 'reference':
-                    val = '[' + domain + '_' + slot + ']'
-                else:
+
+    if state['history'][-1][0]=="sys":
+        # print(state["booked"])
+        for domain in domains:
+            if state['booked'][domain]:
+                for slot in state['booked'][domain][0]:
                     val = '[' + domain + '_' + slot + ']'
-                key = normalize(state[domain]['book']['booked'][0][slot])
-                sent = (' ' + sent + ' ').replace(' ' +
-                                                  key + ' ', ' ' + val + ' ')
-
-                # try reference with hashtag
-                key = normalize("#" + state[domain]['book']['booked'][0][slot])
-                sent = (' ' + sent + ' ').replace(' ' +
-                                                  key + ' ', ' ' + val + ' ')
-
-                # try reference with ref#
-                key = normalize(
-                    "ref#" + state[domain]['book']['booked'][0][slot])
-                sent = (' ' + sent + ' ').replace(' ' +
-                                                  key + ' ', ' ' + val + ' ')
+                    key = normalize(state['booked'][domain][0][slot])
+                    sent = (' ' + sent + ' ').replace(' ' +
+                                                      key + ' ', ' ' + val + ' ')
+
+                    # try reference with hashtag
+                    key = normalize("#" + state['booked'][domain][0][slot])
+                    sent = (' ' + sent + ' ').replace(' ' +
+                                                      key + ' ', ' ' + val + ' ')
+
+                    # try reference with ref#
+                    key = normalize(
+                        "ref#" + state['booked'][domain][0][slot])
+                    sent = (' ' + sent + ' ').replace(' ' +
+                                                      key + ' ', ' ' + val + ' ')
+
     return sent
 
 def domain_mark_not_mentioned(state, active_domain):
-    if active_domain not in ['police', 'hospital', 'taxi', 'train', 'attraction', 'restaurant', 'hotel'] or active_domain is None:
+    if active_domain not in ['hospital', 'taxi', 'train', 'attraction', 'restaurant', 'hotel'] or active_domain is None:
         return
 
-    for s in state[active_domain]['semi']:
-        if state[active_domain]['semi'][s] == '':
-            state[active_domain]['semi'][s] = 'not mentioned'
+    for s in state[active_domain]:
+        if state[active_domain][s] == '':
+            state[active_domain][s] = 'not mentioned'
 
 def mark_not_mentioned(state):
     for domain in state:
@@ -274,9 +277,9 @@ def mark_not_mentioned(state):
             # for s in state[domain]['semi']:
             #     if s != 'book' and state[domain]['semi'][s] == '':
             #         state[domain]['semi'][s] = 'not mentioned'
-            for s in state[domain]['semi']:
-                if state[domain]['semi'][s] == '':
-                    state[domain]['semi'][s] = 'not mentioned'
+            for s in state[domain]:
+                if state[domain][s] == '':
+                    state[domain][s] = 'not mentioned'
         except Exception as e:
             # print(str(e))
             # pprint(state[domain])
@@ -331,22 +334,96 @@ def get_summary_bstate(bstate):
     assert len(summary_bstate) == 94
     return summary_bstate
 
+def get_summary_bstate_unifiedformat(state):
+    """Based on the mturk annotations we form multi-domain belief state"""
+    domains = [u'taxi', u'restaurant',  u'hospital',
+               u'hotel', u'attraction', u'train']#, u'police']
+    bstate = state['belief_state']
+    # booked = state['booked']
+    # how to make empty book this format instead of an empty dictionary?
+    #TODO fix booked info update in state!
+    booked = {
+            "taxi": [],
+            "hotel": [],
+            "restaurant": [],
+            "train": [],
+            "attraction": [],
+            "hospital": []
+            }
+
+    summary_bstate = []
+
+    for domain in domains:
+        domain_active = False
+
+        booking = []
+        if len(booked[domain]) > 0:
+            booking.append(1)
+        else:
+            booking.append(0)
+        if domain == 'train':
+            if not bstate[domain]['book people']:
+                booking.append(0)
+            else:
+                booking.append(1)
+            if booked[domain] and 'ticket' in booked[domain][0].keys():
+                booking.append(1)
+            else:
+                booking.append(0)
+        summary_bstate += booking
+
+        if domain == "restaurant":
+            book_slots = ['book day', 'book people', 'book time']
+        elif domain == "hotel":
+            book_slots = ['book day', 'book people', 'book stay']
+        else:
+            book_slots = []
+        for slot in book_slots:
+            if bstate[domain][slot] == '':
+                summary_bstate.append(0)
+            else:
+                summary_bstate.append(1)
+
+        for slot in [s for s in bstate[domain] if "book" not in s]:
+            slot_enc = [0, 0, 0]
+            if bstate[domain][slot] == 'not mentioned':
+                slot_enc[0] = 1
+            elif bstate[domain][slot] == 'dont care' or bstate[domain][slot] == 'dontcare' or bstate[domain][slot] == "don't care":
+                slot_enc[1] = 1
+            elif bstate[domain][slot]:
+                slot_enc[2] = 1
+            if slot_enc != [0, 0, 0]:
+                domain_active = True
+            summary_bstate += slot_enc
+
+        # quasi domain-tracker
+        if domain_active: # 7 domains
+            summary_bstate += [1]
+        else:
+            summary_bstate += [0]
+
+
+    # add manually from action as police is not tracked anymore in unified format
+    if "Police" in [act[1] for act in state['user_action']]:
+        summary_bstate += [0, 1]
+    else:
+        summary_bstate += [0, 0]
+
+    assert len(summary_bstate) == 94
+    return summary_bstate
+
 
 DEFAULT_CUDA_DEVICE = -1
 
 
 class LAVA(Policy):
     def __init__(self,
-                 model_file="/gpfs/project/lubis/public_code/LAVA/experiments_woz/sys_config_log_model/2020-05-12-14-51-49-actz_cat/rl-2020-05-18-10-50-48/reward_best.model", is_train=False):
+                 model_file="", is_train=False):
 
         if not model_file:
             raise Exception("No model for LAVA is specified!")
 
         temp_path = os.path.dirname(os.path.abspath(__file__))
-        # print(temp_path)
-        #zip_ref = zipfile.ZipFile(archive_file, 'r')
-        # zip_ref.extractall(temp_path)
-        # zip_ref.close()
 
         self.prev_state = default_state()
         self.prev_active_domain = None
@@ -354,24 +431,7 @@ class LAVA(Policy):
         domain_name = 'object_division'
         domain_info = domain.get_domain(domain_name)
         self.db=Database()
-        # data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data/data_2.1/')
-        # train_data_path = os.path.join(data_path,'train_dials.json')
-        # if not os.path.exists(train_data_path):
-            # zipped_file = os.path.join(data_path, 'norm-multi-woz.zip')
-            # archive = zipfile.ZipFile(zipped_file, 'r')
-            # archive.extractall(data_path)
-
-        # norm_multiwoz_path = data_path
-        # with open(os.path.join(norm_multiwoz_path, 'input_lang.index2word.json')) as f:
-            # self.input_lang_index2word = json.load(f)
-        # with open(os.path.join(norm_multiwoz_path, 'input_lang.word2index.json')) as f:
-            # self.input_lang_word2index = json.load(f)
-        # with open(os.path.join(norm_multiwoz_path, 'output_lang.index2word.json')) as f:
-            # self.output_lang_index2word = json.load(f)
-        # with open(os.path.join(norm_multiwoz_path, 'output_lang.word2index.json')) as f:
-            # self.output_lang_word2index = json.load(f)
-
-
+       
         path, _ = os.path.split(model_file)
         if "rl-" in model_file:
             rl_config_path = os.path.join(path, "rl_config.json")
@@ -386,13 +446,12 @@ class LAVA(Policy):
         try:
             self.corpus = corpora_inference.NormMultiWozCorpus(config)
         except (FileNotFoundError, PermissionError):
-            train_data_path = "/gpfs/project/lubis/LAVA_code/LAVA_dev/data/norm-multi-woz/train_dials.json"
+            train_data_path = "/gpfs/project/lubis/NeuralDialog-LaRL/data/norm-multi-woz/train_dials.json"
             config['train_path'] = train_data_path
             config['valid_path'] = train_data_path.replace("train", "val") 
             config['test_path'] = train_data_path.replace("train", "test") 
             self.corpus = corpora_inference.NormMultiWozCorpus(config)
 
-
         if "rl" in model_file:
             if "gauss" in model_file:
                 self.model = SysPerfectBD2Gauss(self.corpus, config)
@@ -429,7 +488,6 @@ class LAVA(Policy):
             self.rl_config["US_best_reward_model_path"] = model_file.replace(
                 ".model", "_US.model")
             if "lr_rl" not in config:
-                #config["lr_rl"] = config["init_lr"]
                 self.config["lr_rl"] = 0.01
                 self.config["gamma"] = 0.99
 
@@ -442,7 +500,6 @@ class LAVA(Policy):
                 lr=self.config.lr_rl,
                 momentum=self.config.momentum,
                 nesterov=False)
-            # nesterov=(self.config.nesterov and self.config.momentum > 0))
 
         if config.use_gpu:
             self.model.load_state_dict(torch.load(model_file))
@@ -504,39 +561,6 @@ class LAVA(Policy):
             utts.append(context[b_id, context_lens[b_id]-1])
         return np.array(utts)
 
-    def get_active_domain_test(self, prev_active_domain, prev_action, action):
-        domains = ['hotel', 'restaurant', 'attraction',
-                   'train', 'taxi', 'hospital', 'police']
-        active_domain = None
-        cur_action_keys = action.keys()
-        state = []
-        for act in cur_action_keys:
-            slots = act.split('-')
-            action = slots[0].lower()
-            state.append(action)
-
-        #  print('get_active_domain')
-        # for domain in domains:
-        """for domain in range(len(domains)):
-            domain = domains[i]
-            if domain not in prev_state and domain not in state:
-                continue
-            if domain in prev_state and domain not in state:
-                return domain
-            elif domain not in prev_state and domain in state:
-                return domain
-            elif prev_state[domain] != state[domain]:
-                active_domain = domain
-        if active_domain is None:
-            active_domain = prev_active_domain"""
-        if len(state) != 0:
-            active_domain = state[0]
-        if active_domain is None:
-            active_domain = prev_active_domain
-        elif active_domain == "general":
-            active_domain = prev_active_domain
-        return active_domain
-
     def is_masked_action(self, bs_label, db_label, response):
         """
         check if the generated response should be masked based on belief state and db result
@@ -563,15 +587,20 @@ class LAVA(Policy):
                         # print("MASK: inform no offer mentioning criteria")
                         # return True # always only inform "no match for your criteria" w/o mentioning them explicitly
                 elif any([p in response for p in REQ_TOKENS[domain]]) or "i have [value_count]" in response or "there are [value_count]" in response: # if requestable token is present
-                    # TODO also check for i have [value_count] match, not only the requestable tokens
                     if db_idx >= 0 and int(db_label[db_idx]) == 1: # and domain has a DB to be queried and there are no matches
                         # print("MASK: inform match when there are no DB match on domain {}".format(domain))
                         return True
 
         return False
 
+    def is_active(self, domain, state):
 
-    def get_active_domain(self, prev_active_domain, prev_state, state):
+        if domain in [act[1] for act in state['user_action']]:
+            return True
+        else:
+            return False
+
+    def get_active_domain_unified(self, prev_active_domain, prev_state, state):
         domains = ['hotel', 'restaurant', 'attraction',
                    'train', 'taxi', 'hospital', 'police']
         active_domain = None
@@ -580,30 +609,17 @@ class LAVA(Policy):
         # print("NEW_STATE",state)
         # print()
         for domain in domains:
-            if domain not in prev_state and domain not in state:
-                continue
-            if domain in prev_state and domain not in state:
+            if not self.is_active(domain, prev_state) and self.is_active(domain, state):
                 #print("case 1:",domain)
                 return domain
-            elif domain not in prev_state and domain in state:
-                #print("case 2:",domain)
+            elif self.is_active(domain, prev_state) and self.is_active(domain, state):
                 return domain
-            elif prev_state[domain] != state[domain]:
+            # elif self.is_active(domain, prev_state) and not self.is_active(domain, state):
+                #print("case 2:",domain)
+                # return domain
+            # elif prev_state['belief_state'][domain] != state['belief_state'][domain]:
                 #print("case 3:",domain)
-                active_domain = domain
-        if active_domain is None:
-            active_domain = prev_active_domain
-        return active_domain
-
-    def get_active_domain_new(self, prev_active_domain, prev_state, state):
-        domains = ['hotel', 'restaurant', 'attraction',
-                   'train', 'taxi', 'hospital', 'police']
-        active_domain = None
-        i = 0
-        for domain in domains:
-            if prev_state[domain] != state[domain]:
-                active_domain = domain
-                i += 1
+                # active_domain = domain
         if active_domain is None:
             active_domain = prev_active_domain
         return active_domain
@@ -642,7 +658,7 @@ class LAVA(Policy):
         else:
             return False
 
-    def predict_response(self, state):
+    def predict_response (self, state):
         # input state is in convlab format
         history = []
         for i in range(len(state['history'])):
@@ -675,8 +691,9 @@ class LAVA(Policy):
 
         # mark_not_mentioned(prev_state)
         #active_domain = self.get_active_domain_convlab(self.prev_active_domain, prev_bstate, bstate)
-        active_domain = self.get_active_domain(self.prev_active_domain, prev_bstate, bstate)
-        #print(active_domain)
+        active_domain = self.get_active_domain_unified(self.prev_active_domain, self.prev_state, state)
+        # print("---------")
+        # print("active domain: ", active_domain)
         # if active_domain is not None:
             # print(f"DST on {active_domain}: {bstate[active_domain]}")
 
@@ -685,16 +702,23 @@ class LAVA(Policy):
         top_results, num_results = None, None
         for t_id in range(len(context)):
             usr = context[t_id]
+            # print(usr)
 
             if t_id == 0: #system turns
                 if usr == "null":
                     usr = "<d>"
+                    # booked = {"taxi": [],
+                            # "restaurant": [],
+                            # "hospital": [],
+                            # "hotel": [],
+                            # "attraction": [],
+                            # "train": []}
             words = usr.split()
 
             usr = delexicalize.delexicalise(' '.join(words).lower(), self.dic)
 
             # parsing reference number GIVEN belief state
-            usr = delexicaliseReferenceNumber(usr, bstate)
+            usr = delexicaliseReferenceNumber(usr, state)
 
             # changes to numbers only here
             digitpat = re.compile('(^| )\d+( |$)')
@@ -702,11 +726,13 @@ class LAVA(Policy):
             
             # add database pointer
             pointer_vector, top_results, num_results = addDBPointer(bstate,self.db)
-           #print(top_results)
+            if state['history'][-1][0] == "sys":
+                booked = state['booked']
+            #print(top_results)
 
             # add booking pointer
             pointer_vector = addBookingPointer(bstate, pointer_vector)
-            belief_summary = get_summary_bstate(bstate)
+            belief_summary = get_summary_bstate_unifiedformat(state)
 
             usr_utt = [BOS] + usr.split() + [EOS]
             packed_val = {}
@@ -725,15 +751,13 @@ class LAVA(Policy):
 
         # data_feed is in LaRL format
         data_feed = prepare_batch_gen(results, self.config)
+        # print(belief_summary)
 
-        for i in range(10):
+        for i in range(1):
             outputs = self.model_predict(data_feed)
             self.prev_output = outputs
             mul = False
             
-            if self.is_masked_action(data_feed['bs'][0], data_feed['db'][0], outputs) and i < 9: # if it's the last try, accept masked action
-                continue
-
             # default lexicalization
             if active_domain is not None and active_domain in num_results:
                 num_results = num_results[active_domain]
@@ -748,26 +772,23 @@ class LAVA(Policy):
                     top_results = {active_domain: top_results[active_domain]}
                 else:
                     if active_domain == 'train': #special case, where we want the last match instead of the first
-                        if bstate['train']['semi']['arriveBy'] != "not mentioned" and len(bstate['train']['semi']['arriveBy']) > 0:
+                        if bstate['train']['arrive by'] != "not mentioned" and len(bstate['train']['arrive by']) > 0:
                             top_results = {active_domain: top_results[active_domain][-1]} # closest to arrive by
                         else:
                             top_results = {active_domain: top_results[active_domain][0]}
                     else:
-                        top_results = {active_domain: top_results[active_domain][0]}
+                        top_results = {active_domain: top_results[active_domain][0]} # if active domain is wrong, this becomes the wrong entity
             else:
                 top_results = {}
             state_with_history = deepcopy(bstate)
             state_with_history['history'] = deepcopy(state_history)
 
-            if active_domain in ["hotel", "attraction", "train", "restaurant"] and len(top_results.keys()) == 0: # no db match for active domain
+            if active_domain in ["hotel", "attraction", "train", "restaurant"] and active_domain not in top_results.keys(): # no db match for active domain
                 if any([p in outputs for p in REQ_TOKENS[active_domain]]):
-                    # self.fail_info_penalty += 1
-                    # response = "I am sorry there are no matches."
-                    # print(outputs)
                     response = "I am sorry, can you say that again?"
                     database_results = {}
                 else:
-                    response = self.populate_template(
+                    response = self.populate_template_unified(
                             outputs, top_results, num_results, state_with_history, active_domain)
                     # print(response)
 
@@ -776,60 +797,22 @@ class LAVA(Policy):
                     response = self.populate_template_options(outputs, top_results, num_results, state_with_history)
                 else:
                     try:
-                        response = self.populate_template(
+                        response = self.populate_template_unified(
                         outputs, top_results, num_results, state_with_history, active_domain)
                     except:
-                        print(outputs)
-
+                        print("can not lexicalize: ", outputs)
+                        response = "I am sorry, can you say that again?"
                        
-
-            for domain in DOMAIN_REQ_TOKEN:
-                """
-                mask out of domain action
-                """
-                if domain != active_domain and any([p in outputs for p in REQ_TOKENS[domain]]):
-                    # print(f"MASK: illegal action for {active_domain}: {outputs}")
-                    response = "Can I help you with anything else?"
-                    self.wrong_domain_penalty += 1
-
-                 
-        # if active_domain is not None:
-            # print ("===========================")
-
-            # print(active_domain)
-            # print ("BS: ")
-            # for k, v in bstate[active_domain].items():
-            # print(k, ": ", v)
-            # print ("DB: ")
-            # if len(database_results.keys()) > 0:
-            # for k, v in database_results[active_domain][1][0 % database_results[active_domain][0]].items():
-            # print(k, ": ", v)
-            # print ("===========================")
-            # print ("input: ")
-            # for turn in data_feed['contexts'][0]:
-            # print(" ".join(self.corpus.id2sent(turn)))
-            # print ("system delex: ", outputs)
-            # print ("system: ",response)
-            # print ("===========================\n")
-            self.num_generated_response += 1
-            break
-
         response = response.replace("free pounds", "free")
         response = response.replace("pounds pounds", "pounds")
         if any([p in response for  p in ["not mentioned", "dontcare", "[", "]"]]):
-            # response = "I am sorry there are no matches."
-            # print(usr)
-            # print(outputs)
-            # print(response)
-            # print(active_domain, len(top_results[active_domain]), delex_bstate[active_domain])
-            # pdb.set_trace()
-            # response = "I am sorry can you repeat that?"
             response = "I am sorry, can you say that again?"
 
 
         return response, active_domain
 
-    def populate_template(self, template, top_results, num_results, state, active_domain):
+
+    def populate_template_unified(self, template, top_results, num_results, state, active_domain):
         # print("template:",template)
         # print("top_results:",top_results)
         # active_domain = None if len(
@@ -848,7 +831,7 @@ class LAVA(Policy):
                 if domain == 'train' and slot == 'id':
                     slot = 'trainID'
                 elif active_domain != 'train' and slot == 'price':
-                    slot = 'pricerange'
+                    slot = 'price range'
                 elif slot == 'reference':
                     slot = 'Ref'
                 if domain in top_results and len(top_results[domain]) > 0 and slot in top_results[domain]:
@@ -864,27 +847,24 @@ class LAVA(Policy):
                         elif active_domain == "restaurant":
                             if "people" in tokens[index:index+1] or "table" in tokens[index-2:index]:
                                 response.append(
-                                    state[active_domain]["book"]["people"])
+                                    state[active_domain]["book people"])
                         elif active_domain == "train":
                             if "ticket" in " ".join(tokens[index-2:index+1]) or "people" in tokens[index:]:
                                 response.append(
-                                    state[active_domain]["book"]["people"])
+                                    state[active_domain]["book people"])
                             elif index+1 < len(tokens) and "minute" in tokens[index+1]:
                                 response.append(
                                     top_results['train']['duration'].split()[0])
                         elif active_domain == "hotel":
                             if index+1 < len(tokens):
                                 if "star" in tokens[index+1]:
-                                    try:
-                                        response.append(top_results['hotel']['stars'])
-                                    except:
-                                        response.append(state['hotel']['semi']['stars'])
+                                    response.append(top_results['hotel']['stars'])
                                 elif "nights" in tokens[index+1]:
                                     response.append(
-                                        state[active_domain]["book"]["stay"])
+                                        state[active_domain]["book stay"])
                                 elif "people" in tokens[index+1]:
                                     response.append(
-                                        state[active_domain]["book"]["people"])
+                                        state[active_domain]["book people"])
                         elif active_domain == "attraction":
                             if index + 1 < len(tokens):
                                 if "pounds" in tokens[index+1] and "entrance fee" in " ".join(tokens[index-3:index]):
@@ -912,14 +892,14 @@ class LAVA(Policy):
                                     top_results[active_domain]["destination"])
                             elif active_domain == "taxi":
                                 response.append(
-                                    state[active_domain]['semi']["destination"])
+                                    state[active_domain]["destination"])
                         elif 'leav' in " ".join(tokens[index-2:index]) or "from" in tokens[index-2:index] or "depart" in " ".join(tokens[index-2:index]):
                             if active_domain == "train":
                                 response.append(
                                     top_results[active_domain]["departure"])
                             elif active_domain == "taxi":
                                 response.append(
-                                    state[active_domain]['semi']["departure"])
+                                    state[active_domain]["departure"])
                         elif "hospital" in template:
                             response.append("Cambridge")
                         else:
@@ -928,9 +908,9 @@ class LAVA(Policy):
                                     if d == 'history':
                                         continue
                                     for s in ['destination', 'departure']:
-                                        if s in state[d]['semi']:
+                                        if s in state[d]:
                                             response.append(
-                                                state[d]['semi'][s])
+                                                state[d][s])
                                             raise
                             except:
                                 pass
@@ -938,7 +918,7 @@ class LAVA(Policy):
                                 response.append(token)
                     elif slot == 'time':
                         if 'arrive' in ' '.join(response[-5:]) or 'arrival' in ' '.join(response[-5:]) or 'arriving' in ' '.join(response[-3:]):
-                            if active_domain is "train" and 'arriveBy' in top_results[active_domain]:
+                            if active_domain == "train" and 'arriveBy' in top_results[active_domain]:
                                 # print('{} -> {}'.format(token, top_results[active_domain]['arriveBy']))
                                 response.append(
                                     top_results[active_domain]['arriveBy'])
@@ -946,12 +926,12 @@ class LAVA(Policy):
                             for d in state:
                                 if d == 'history':
                                     continue
-                                if 'arriveBy' in state[d]['semi']:
+                                if 'arrive by' in state[d]:
                                     response.append(
-                                        state[d]['semi']['arriveBy'])
+                                        state[d]['arrive by'])
                                     break
                         elif 'leave' in ' '.join(response[-5:]) or 'leaving' in ' '.join(response[-5:]) or 'departure' in ' '.join(response[-3:]):
-                            if active_domain is "train" and 'leaveAt' in top_results[active_domain]:
+                            if active_domain == "train" and 'leaveAt' in top_results[active_domain]:
                                 # print('{} -> {}'.format(token, top_results[active_domain]['leaveAt']))
                                 response.append(
                                     top_results[active_domain]['leaveAt'])
@@ -959,23 +939,23 @@ class LAVA(Policy):
                             for d in state:
                                 if d == 'history':
                                     continue
-                                if 'leaveAt' in state[d]['semi']:
+                                if 'leave at' in state[d]:
                                     response.append(
-                                        state[d]['semi']['leaveAt'])
+                                        state[d]['leave at'])
                                     break
                         elif 'book' in response or "booked" in response:
-                            if state['restaurant']['book']['time'] != "":
+                            if state['restaurant']['book time'] != "":
                                 response.append(
-                                    state['restaurant']['book']['time'])
+                                    state['restaurant']['book time'])
                         else:
                             try:
                                 for d in state:
                                     if d == 'history':
                                         continue
-                                    for s in ['arriveBy', 'leaveAt']:
-                                        if s in state[d]['semi']:
+                                    for s in ['arrive by', 'leave at']:
+                                        if s in state[d]:
                                             response.append(
-                                                state[d]['semi'][s])
+                                                state[d][s])
                                             raise
                             except:
                                 pass
@@ -999,9 +979,9 @@ class LAVA(Policy):
                             response.append(
                                 top_results[active_domain][slot].split()[0])
                     elif slot == "day" and active_domain in ["restaurant", "hotel"]:
-                        if state[active_domain]['book']['day'] != "":
+                        if state[active_domain]['book day'] != "":
                             response.append(
-                                state[active_domain]['book']['day'])
+                                state[active_domain]['book day'])
 
                     else:
                         # slot-filling based on query results
@@ -1014,8 +994,8 @@ class LAVA(Policy):
                             for d in state:
                                 if d == 'history':
                                     continue
-                                if slot in state[d]['semi']:
-                                    response.append(state[d]['semi'][slot])
+                                if slot in state[d]:
+                                    response.append(state[d][slot])
                                     break
                             else:
                                 response.append(token)
@@ -1028,7 +1008,7 @@ class LAVA(Policy):
                         elif slot == 'address':
                             response.append("56 Lincoln street")
                         elif slot == "postcode":
-                            response.append('533421')
+                            response.append('cb1p3')
                     elif domain == 'police':
                         if slot == 'phone':
                             response.append('01223358966')
@@ -1037,7 +1017,7 @@ class LAVA(Policy):
                         elif slot == 'address':
                             response.append('Parkside, Cambridge')
                         elif slot == 'postcode':
-                            response.append('533420')
+                            response.append('cb3l3')
                     elif domain == 'taxi':
                         if slot == 'phone':
                             response.append('01223358966')
@@ -1068,6 +1048,7 @@ class LAVA(Policy):
 
         # if "not mentioned" in response:
         #    pdb.set_trace()
+        # print("lexicalized: ", response)
 
         return response
 
@@ -1158,9 +1139,9 @@ class LAVA(Policy):
                                     if d == 'history':
                                         continue
                                     for s in ['destination', 'departure']:
-                                        if s in state[d]['semi']:
+                                        if s in state[d]:
                                             response.append(
-                                                state[d]['semi'][s])
+                                                state[d][s])
                                             raise
                             except:
                                 pass
@@ -1176,9 +1157,9 @@ class LAVA(Policy):
                             for d in state:
                                 if d == 'history':
                                     continue
-                                if 'arriveBy' in state[d]['semi']:
+                                if 'arriveBy' in state[d]:
                                     response.append(
-                                        state[d]['semi']['arriveBy'])
+                                        state[d]['arrive by'])
                                     break
                         elif 'leav' in ' '.join(response[-7:]) or 'depart' in ' '.join(response[-7:]):
                             if active_domain is not None and 'leaveAt' in top_results[active_domain][result_idx]:
@@ -1189,23 +1170,23 @@ class LAVA(Policy):
                             for d in state:
                                 if d == 'history':
                                     continue
-                                if 'leaveAt' in state[d]['semi']:
+                                if 'leave at' in state[d]:
                                     response.append(
-                                        state[d]['semi']['leaveAt'])
+                                        state[d]['leave at'])
                                     break
                         elif 'book' in response or "booked" in response:
-                            if state['restaurant']['book']['time'] != "":
+                            if state['restaurant']['book time'] != "":
                                 response.append(
-                                    state['restaurant']['book']['time'])
+                                    state['restaurant']['book time'])
                         else:
                             try:
                                 for d in state:
                                     if d == 'history':
                                         continue
-                                    for s in ['arriveBy', 'leaveAt']:
-                                        if s in state[d]['semi']:
+                                    for s in ['arrive by', 'leave at']:
+                                        if s in state[d]:
                                             response.append(
-                                                state[d]['semi'][s])
+                                                state[d][s])
                                             raise
                             except:
                                 pass
@@ -1227,9 +1208,9 @@ class LAVA(Policy):
                             response.append(
                                 top_results[active_domain][result_idx][slot].split()[0])
                     elif slot == "day" and active_domain in ["restaurant", "hotel"]:
-                        if state[active_domain]['book']['day'] != "":
+                        if state[active_domain]['book day'] != "":
                             response.append(
-                                state[active_domain]['book']['day'])
+                                state[active_domain]['book day'])
 
                     else:
                         # slot-filling based on query results
@@ -1243,8 +1224,8 @@ class LAVA(Policy):
                             for d in state:
                                 if d == 'history':
                                     continue
-                                if slot in state[d]['semi']:
-                                    response.append(state[d]['semi'][slot])
+                                if slot in state[d]:
+                                    response.append(state[d][slot])
                                     break
                             else:
                                 response.append(token)
@@ -1294,26 +1275,16 @@ class LAVA(Policy):
         return response
 
     def model_predict(self, data_feed):
-        # TODO use model's forward function, add null vector for the target response
         self.logprobs = []
         logprobs, pred_labels, joint_logpz, sample_y = self.model.forward_rl(
             data_feed, self.model.config.max_dec_len)
-        # if len(data_feed['bs']) == 1:
-        #    logprobs = [logprobs]
 
-        # for log_prob in logprobs:
-        #    self.logprobs.extend(log_prob)
         self.logprobs.extend(joint_logpz)
 
         pred_labels = np.array(
-            [pred_labels], dtype=int)  # .squeeze(-1).swapaxes(0, 1)
+            [pred_labels], dtype=int)
         de_tknize = get_detokenize()
-        # if pred_labels.shape[1] == self.model.config.max_utt_len:
-            # pdb.set_trace()
         pred_str = get_sent(self.model.vocab, de_tknize, pred_labels, 0)
-        #for b_id in range(pred_labels.shape[0]):
-            # only one val for pred_str now
-            # pred_str = get_sent(self.model.vocab, de_tknize, pred_labels, b_id)
 
         return pred_str
 
@@ -1336,11 +1307,8 @@ class LAVA(Policy):
 
         loss = 0
         # estimate the loss using one MonteCarlo rollout
-        # TODO better loss estimation?
-        # TODO better update, instead of reinforce?
         for lp, re in zip(logprobs, rewards):
             loss -= lp * re
-        #tmp = self.model.state_dict()['c2z.p_h.weight'].clone()
         self.opt.zero_grad()
         if "fp16" in self.config and self.config.fp16:
             with amp.scale_loss(loss, self.opt) as scaled_loss:
@@ -1350,10 +1318,7 @@ class LAVA(Policy):
             loss.backward()
             nn.utils.clip_grad_norm_(self.model.parameters(), self.config.grad_clip)
         
-        #self._print_grad()
         self.opt.step()
-        #tmp2 = self.model.state_dict()['c2z.p_h.weight'].clone()
-        #print(tmp==tmp2)
         
     def _print_grad(self):
         for name, p in self.model.named_parameters():
@@ -1529,7 +1494,8 @@ if __name__ == '__main__':
              'history': [['sys', ''],
                          ['user', 'Could you book a 4 stars hotel east of town for one night, 1 person?']]}
 
-    cur_model = LAVA()
+    model_file="path/to/model" # points to model from lava repo
+    cur_model = LAVA(model_file)
 
     response = cur_model.predict(state)
     # print(response)
diff --git a/convlab/policy/mle/README.md b/convlab/policy/mle/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..c13140497508d79ec6faedfc588fa4f6af7043f7
--- /dev/null
+++ b/convlab/policy/mle/README.md
@@ -0,0 +1,22 @@
+# Maximum Likelihood Estimator (MLE)
+
+MLE learns a MLP model in a supervised way using a provided dataset. The trained model can be used as intialisation point for running RL trainings with PPO or GDPL for instance.
+
+## Supervised Training
+
+Starting a training is as easy as executing
+
+```sh
+$ python train.py --dataset_name=DATASET_NAME --seed=SEED --eval_freq=FREQ
+```
+
+The dataset name can be "multiwoz21" or "sgd" for instance. The first time you run that command, it will take longer as the dataset needs to be pre-processed. The evaluation frequency decides after how many epochs should be evaluated.
+
+Other hyperparameters such as learning rate or number of epochs can be set in the config.json file.
+
+We provide a model trained on multiwoz21 on hugging-face: https://huggingface.co/ConvLab/mle-policy-multiwoz21
+
+
+## Evaluation
+
+Evaluation on the validation data set takes place during training.
\ No newline at end of file
diff --git a/convlab/policy/mle/loader.py b/convlab/policy/mle/loader.py
index ebc01a0149dbeabf872da4ceb4ba184bfb54fd22..9c10d2a7a3841efb55ba418e5cf56708e4e94b7d 100755
--- a/convlab/policy/mle/loader.py
+++ b/convlab/policy/mle/loader.py
@@ -2,6 +2,9 @@ import os
 import pickle
 import torch
 import torch.utils.data as data
+from copy import deepcopy
+
+from tqdm import tqdm
 
 from convlab.policy.vector.vector_binary import VectorBinary
 from convlab.util import load_policy_data, load_dataset
@@ -12,18 +15,20 @@ from convlab.policy.vector.dataset import ActDataset
 
 class PolicyDataVectorizer:
     
-    def __init__(self, dataset_name='multiwoz21', vector=None):
+    def __init__(self, dataset_name='multiwoz21', vector=None, dst=None):
         self.dataset_name = dataset_name
         if vector is None:
             self.vector = VectorBinary(dataset_name)
         else:
             self.vector = vector
+        self.dst = dst
         self.process_data()
 
     def process_data(self):
-
-        processed_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                                     f'processed_data/{self.dataset_name}_{type(self.vector).__name__}')
+        name = f"{self.dataset_name}_"
+        name += f"{type(self.dst).__name__}_" if self.dst is not None else ""
+        name += f"{type(self.vector).__name__}"
+        processed_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), name)
         if os.path.exists(processed_dir):
             print('Load processed data file')
             self._load_data(processed_dir)
@@ -42,15 +47,64 @@ class PolicyDataVectorizer:
             self.data[split] = []
             raw_data = data_split[split]
 
-            for data_point in raw_data:
-                state = default_state()
+            if self.dst is not None:
+                self.dst.init_session()
+
+            for data_point in tqdm(raw_data):
+                if self.dst is None:
+                    state = default_state()
+
+                    state['belief_state'] = data_point['context'][-1]['state']
+                    state['user_action'] = flatten_acts(data_point['context'][-1]['dialogue_acts'])
+                elif "setsumbt" in str(self.dst):
+                    last_system_utt = data_point['context'][-2]['utterance'] if len(data_point['context']) > 1 else ''
+                    self.dst.state['history'].append(['sys', last_system_utt])
+
+                    usr_utt = data_point['context'][-1]['utterance']
+                    state = deepcopy(self.dst.update(usr_utt))
+                    self.dst.state['history'].append(['usr', usr_utt])
+                elif "trippy" in str(self.dst):
+                    # Get last system acts and text.
+                    # System acts are used to fill the inform memory.
+                    last_system_acts = []
+                    last_system_utt = ''
+                    if len(data_point['context']) > 1:
+                        last_system_acts = []
+                        for act_type in data_point['context'][-2]['dialogue_acts']:
+                            for act in data_point['context'][-2]['dialogue_acts'][act_type]:
+                                value = ''
+                                if 'value' not in act:
+                                    if act['intent'] == 'request':
+                                        value = '?'
+                                    elif act['intent'] == 'inform':
+                                        value = 'yes'
+                                else:
+                                    value = act['value']
+                                last_system_acts.append([act['intent'], act['domain'], act['slot'], value])
+                        last_system_utt = data_point['context'][-2]['utterance']
+
+                    # Get current user acts and text.
+                    # User acts are used for internal evaluation.
+                    usr_acts = []
+                    for act_type in data_point['context'][-1]['dialogue_acts']:
+                        for act in data_point['context'][-1]['dialogue_acts'][act_type]:
+                            usr_acts.append([act['intent'], act['domain'], act['slot'], act['value'] if 'value' in act else ''])
+                    usr_utt = data_point['context'][-1]['utterance']
+
+                    # Update the state for DST, then update the state via DST.
+                    self.dst.state['system_action'] = last_system_acts
+                    self.dst.state['user_action'] = usr_acts
+                    self.dst.state['history'].append(['sys', last_system_utt])
+                    self.dst.state['history'].append(['usr', usr_utt])
+                    state = deepcopy(self.dst.update(usr_utt))
+                else:
+                    raise NameError(f"Tracker: {self.dst} not implemented.")
 
-                state['belief_state'] = data_point['context'][-1]['state']
-                state['user_action'] = flatten_acts(data_point['context'][-1]['dialogue_acts'])
-                last_system_act = data_point['context'][-2]['dialogue_acts'] \
-                    if len(data_point['context']) > 1 else {}
+                last_system_act = data_point['context'][-2]['dialogue_acts'] if len(data_point['context']) > 1 else {}
                 state['system_action'] = flatten_acts(last_system_act)
                 state['terminated'] = data_point['terminated']
+                if self.dst is not None and state['terminated']:
+                    self.dst.init_session()
                 state['booked'] = data_point['booked']
                 dialogue_act = flatten_acts(data_point['dialogue_acts'])
 
diff --git a/convlab/policy/mle/train.py b/convlab/policy/mle/train.py
index 2b82a476e5db119fe80b44517fd1b0d5e21fcfa6..5253f95d9709fe19071ab7e36ed5a7339597bbe9 100755
--- a/convlab/policy/mle/train.py
+++ b/convlab/policy/mle/train.py
@@ -12,6 +12,7 @@ from convlab.util.custom_util import set_seed, init_logging, save_config
 from convlab.util.train_util import to_device
 from convlab.policy.rlmodule import MultiDiscretePolicy
 from convlab.policy.vector.vector_binary import VectorBinary
+from convlab.policy.vector.vector_binary_fuzzy import VectorBinaryFuzzy
 
 root_dir = os.path.dirname(
     os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
@@ -137,15 +138,6 @@ class MLE_Trainer(MLE_Trainer_Abstract):
     def __init__(self, manager, vector, cfg):
         self._init_data(manager, cfg)
 
-        try:
-            self.use_entropy = manager.use_entropy
-            self.use_mutual_info = manager.use_mutual_info
-            self.use_confidence_scores = manager.use_confidence_scores
-        except:
-            self.use_entropy = False
-            self.use_mutual_info = False
-            self.use_confidence_scores = False
-
         # override the loss defined in the MLE_Trainer_Abstract to support pos_weight
         pos_weight = cfg['pos_weight'] * torch.ones(vector.da_dim).to(device=DEVICE)
         self.multi_entropy_loss = nn.BCEWithLogitsLoss(pos_weight=pos_weight)
@@ -161,6 +153,10 @@ def arg_parser():
     parser.add_argument("--seed", type=int, default=0)
     parser.add_argument("--eval_freq", type=int, default=1)
     parser.add_argument("--dataset_name", type=str, default="multiwoz21")
+    parser.add_argument("--use_masking", action='store_true')
+
+    parser.add_argument("--dst", type=str, default=None)
+    parser.add_argument("--dst_args", type=str, default=None)
 
     args = parser.parse_args()
     return args
@@ -181,8 +177,35 @@ if __name__ == '__main__':
     set_seed(args.seed)
     logging.info(f"Seed used: {args.seed}")
 
-    vector = VectorBinary(dataset_name=args.dataset_name, use_masking=False)
-    manager = PolicyDataVectorizer(dataset_name=args.dataset_name, vector=vector)
+    if args.dst is None:
+        vector = VectorBinary(dataset_name=args.dataset_name, use_masking=args.use_masking)
+        dst = None
+    elif args.dst == "setsumbt":
+        dst_args = [arg.split('=', 1) for arg in args.dst_args.split(', ')
+                    if '=' in arg] if args.dst_args is not None else []
+        dst_args = {key: eval(value) for key, value in dst_args}
+        from convlab.dst.setsumbt import SetSUMBTTracker
+        dst = SetSUMBTTracker(**dst_args)
+        if dst.return_confidence_scores:
+            from convlab.policy.vector.vector_uncertainty import VectorUncertainty
+            vector = VectorUncertainty(dataset_name=args.dataset_name, use_masking=args.use_masking,
+                                       manually_add_entity_names=False,
+                                       use_confidence_scores=dst.return_confidence_scores,
+                                       confidence_thresholds=dst.confidence_thresholds,
+                                       use_state_total_uncertainty=dst.return_belief_state_entropy,
+                                       use_state_knowledge_uncertainty=dst.return_belief_state_mutual_info)
+        else:
+            vector = VectorBinary(dataset_name=args.dataset_name, use_masking=args.use_masking)
+    elif args.dst == "trippy":
+        dst_args = [arg.split('=', 1) for arg in args.dst_args.split(', ')
+                    if '=' in arg] if args.dst_args is not None else []
+        dst_args = {key: eval(value) for key, value in dst_args}
+        from convlab.dst.trippy import TRIPPY
+        dst = TRIPPY(**dst_args)
+        vector = VectorBinaryFuzzy(dataset_name=args.dataset_name, use_masking=args.use_masking)
+    else:
+        raise NameError(f"Tracker: {args.dst} not implemented.")
+    manager = PolicyDataVectorizer(dataset_name=args.dataset_name, vector=vector, dst=dst)
     agent = MLE_Trainer(manager, vector, cfg)
 
     logging.info('Start training')
diff --git a/convlab/policy/pg/README.md b/convlab/policy/pg/README.md
index d61365276fe2d8e932ea41f32ad663742a17ca15..23032f827237d2364f28933f44631a6e981d7a60 100755
--- a/convlab/policy/pg/README.md
+++ b/convlab/policy/pg/README.md
@@ -1,36 +1,49 @@
-# REINFORCE
+# Policy Gradient (PG)
 
-A simple stochastic gradient algorithm for policy gradient reinforcement learning. We adapt REINFORCE to the dialog policy.
+PG is an on-policy reinforcement learning algorithm that uses the policy gradient theorem to perform policy updates, using directly the return as value estimation
+. 
+## Supervised pre-training
 
-## Train
+If you want to obtain a supervised model for pre-training, please have a look in the MLE policy folder.
 
-Run `train.py` in the `pg` directory:
+## RL training
 
-```bash
-python train.py
+Starting a RL training is as easy as executing
+
+```sh
+$ python train.py --path=your_environment_config --seed=SEED
 ```
 
-For better performance, we can do immitating learning before reinforcement learning. The immitating learning is implemented in the `mle` directory.
+One example for the environment-config is **semantic_level_config.json**, where parameters for the training are specified, for instance
 
-For example, if the trained model of immitating learning is saved at FOLDER_OF_MODEL/best_mle.pol.mdl, then you can run
+- load_path: provide a path to initialise the model with a pre-trained model, skip the ending .pol.mdl
+- process_num: the number of processes to use during evaluation to speed it up
+- num_eval_dialogues: how many evaluation dialogues should be used
+- epoch: how many training epochs to run. One epoch consists of collecting dialogues + performing an update
+- eval_frequency: after how many epochs perform an evaluation
+- batchsz: the number of training dialogues collected before doing an update
 
-```bash
-python train.py --load_path FOLDER_OF_MODEL/best_mle
-```
+Moreover, you can specify the full dialogue pipeline here, such as the user policy, NLU for system and user, etc.
+
+Parameters that are tied to the RL algorithm and the model architecture can be changed in config.json.
+
+
+## Evaluation
 
-Note that the *.pol.mdl* suffix should not appear in the --load_path argument.
+For creating evaluation plots and running evaluation dialogues, please have a look in the README of the policy folder.
 
-## Reference
+## References
 
 ```
-@article{williams1992simple,
-  title={Simple statistical gradient-following algorithms for connectionist reinforcement learning},
-  author={Williams, Ronald J},
-  journal={Machine learning},
-  volume={8},
-  number={3-4},
-  pages={229--256},
-  year={1992},
-  publisher={Springer}
+@inproceedings{NIPS1999_464d828b,
+ author = {Sutton, Richard S and McAllester, David and Singh, Satinder and Mansour, Yishay},
+ booktitle = {Advances in Neural Information Processing Systems},
+ editor = {S. Solla and T. Leen and K. M\"{u}ller},
+ pages = {},
+ publisher = {MIT Press},
+ title = {Policy Gradient Methods for Reinforcement Learning with Function Approximation},
+ url = {https://proceedings.neurips.cc/paper/1999/file/464d828b85b0bed98e80ade0a5c43b0f-Paper.pdf},
+ volume = {12},
+ year = {1999}
 }
 ```
\ No newline at end of file
diff --git a/convlab/policy/pg/train.py b/convlab/policy/pg/train.py
index 05d51015b38072ffc00b72ae11f9d1ba0ee0ade0..72a52e26bf54f558ca5fc24e0fa7fbbf8f25608e 100755
--- a/convlab/policy/pg/train.py
+++ b/convlab/policy/pg/train.py
@@ -184,7 +184,7 @@ if __name__ == '__main__':
     parser = ArgumentParser()
     parser.add_argument("--path", type=str, default='convlab/policy/pg/semantic_level_config.json',
                         help="Load path for config file")
-    parser.add_argument("--seed", type=int, default=0,
+    parser.add_argument("--seed", type=int, default=None,
                         help="Seed for the policy parameter initialization")
     parser.add_argument("--mode", type=str, default='info',
                         help="Set level for logger")
@@ -199,7 +199,7 @@ if __name__ == '__main__':
     logger, tb_writer, current_time, save_path, config_save_path, dir_path, log_save_path = \
         init_logging(os.path.dirname(os.path.abspath(__file__)), mode)
 
-    args = [('model', 'seed', seed)]
+    args = [('model', 'seed', seed)] if seed is not None else list()
 
     environment_config = load_config_file(path)
     save_config(vars(parser.parse_args()), environment_config, config_save_path)
@@ -261,7 +261,7 @@ if __name__ == '__main__':
 
         if idx % conf['model']['eval_frequency'] == 0 and idx != 0:
             time_now = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
-            logging.info(f"Evaluating at Epoch: {idx} - {time_now}" + '-'*60)
+            logging.info(f"Evaluating after Dialogues: {idx * conf['model']['batchsz']} - {time_now}" + '-' * 60)
 
             eval_dict = eval_policy(conf, policy_sys, env, sess, save_eval, log_save_path)
 
diff --git a/convlab/policy/plot_results/README.md b/convlab/policy/plot_results/README.md
index 724dda6dfdeac7ca4cc103c445fde531b062df06..216d3a2df783ad647625134599c9c0df1fcc26cf 100644
--- a/convlab/policy/plot_results/README.md
+++ b/convlab/policy/plot_results/README.md
@@ -38,3 +38,5 @@ The file structure of the exp_dir is like this:
                     └── events.* 
 
 If you want to truncate the figure to a certain number of training dialogues on the x-axis, use the argument `--max-dialogues`.
+
+This script will automatically generate plots in the folder **--out-file** showing several performance metrics such as success rate and return, but also additional information such as the action distributions.
\ No newline at end of file
diff --git a/convlab/policy/plot_results/example_map.json b/convlab/policy/plot_results/example_map.json
index 72b5c6cc721b487cd313d2af7731b39607230a1a..7f5cd6242d93a70dafed36262a375c90e3702511 100644
--- a/convlab/policy/plot_results/example_map.json
+++ b/convlab/policy/plot_results/example_map.json
@@ -1,10 +1,18 @@
 [
   {
-    "dir": "ppo",
-    "legend": "PPO"
+    "dir": "scratch",
+    "legend": "scratch"
+  },
+    {
+    "dir": "sgd",
+    "legend": "SGD"
   },
   {
-    "dir": "pg",
-    "legend": "PG"
+    "dir": "mwoz",
+    "legend": "1%MWOZ"
+  },
+    {
+    "dir": "sgd_mw",
+    "legend": "SGD->1%MWOZ"
   }
 ]
\ No newline at end of file
diff --git a/convlab/policy/plot_results/plot.py b/convlab/policy/plot_results/plot.py
index 8dca2b84b79ff2c150ec4cb7389bc6e6ae43d5f4..c4032da663ec7a27af455c498b5243fb70500208 100644
--- a/convlab/policy/plot_results/plot.py
+++ b/convlab/policy/plot_results/plot.py
@@ -7,9 +7,12 @@ import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 import seaborn as sns
+import sys
 from tensorboard.backend.event_processing import event_accumulator
 from tqdm import tqdm
 
+from convlab.policy.plot_results.plot_action_distributions import plot_distributions
+
 
 def get_args():
     parser = argparse.ArgumentParser(description='Export tensorboard data')
@@ -22,8 +25,14 @@ def get_args():
     parser.add_argument("--max-dialogues", type=int, default=0)
     parser.add_argument("--fill-between", type=float, default=0.3,
                         help="the transparency of the std err area")
+    parser.add_argument("--fontsize", type=int, default=18)
+    parser.add_argument("--font", type=str, default="Times New Roman")
+    parser.add_argument("--figure-size", type=str, help="Format 'width,height', eg '6,5'", default='6,5')
+    parser.add_argument("--figure-face-color", type=str, default='#E6E6E6')
 
     args = parser.parse_args()
+    args.figure_size = eval(args.figure_size)
+    plt.rcParams["font.family"] = args.font
     return args
 
 
@@ -56,53 +65,60 @@ def read_tb_data(in_path):
     return df
 
 
-def plot(data, out_file, plot_type="complete_rate", show_image=False, fill_between=0.3, max_dialogues=0, y_label=''):
+def plot(data, out_file, plot_type="complete_rate", show_image=False, fill_between=0.3, max_dialogues=0, y_label='',
+         fontsize=16, figsize=(12, 8), facecolor='#E6E6E6'):
 
     legends = [alg for alg in data]
     clrs = sns.color_palette("husl", len(legends))
-    plt.figure(plot_type)
-
-    with sns.axes_style("darkgrid"):
-        for i, alg in enumerate(legends):
-
-            max_step = min([len(d[plot_type]) for d in data[alg]])
-            if max_dialogues > 0:
-                max_length = min([len([s for s in d['steps'] if s <= max_dialogues]) for d in data[alg]])
-                max_step = min([max_length, max_step])
-            print("max_step: ", max_step)
-
-            value = np.array([d[plot_type][:max_step] for d in data[alg]])
-            step = np.array([d['steps'][:max_step] for d in data[alg]][0])
-            mean, err = np.mean(value, axis=0), np.std(value, axis=0)
-            plt.plot(
-                step, mean, c=clrs[i], label=alg)
-
-            plt.fill_between(
-                step, mean - err,
-                mean + err, alpha=fill_between, facecolor=clrs[i])
-        # locs, labels = plt.xticks()
-        # plt.xticks(locs, labels)
-        #plt.yticks(np.arange(10) / 10)
-        #plt.yticks([0.5, 0.6, 0.7])
-        plt.xlabel('Training dialogues')
-        if len(y_label) > 0:
-            plt.ylabel(y_label)
-        else:
-            plt.ylabel(plot_type)
-        plt.legend(fancybox=True, shadow=False, ncol=1, loc='lower left')
-        plt.savefig(out_file, bbox_inches='tight')
-
-        if show_image:
-            plt.show()
+    plt.figure(plot_type, figsize=figsize)
+    plt.gca().patch.set_facecolor(facecolor)
+    plt.grid(color='w', linestyle='solid', alpha=0.5)
+
+    largest_max = -sys.maxsize
+    smallest_min = sys.maxsize
+    for i, alg in enumerate(legends):
+
+        max_step = min([len(d[plot_type]) for d in data[alg]])
+        if max_dialogues > 0:
+            max_length = min([len([s for s in d['steps'] if s <= max_dialogues]) for d in data[alg]])
+            max_step = min([max_length, max_step])
+
+        value = np.array([d[plot_type][:max_step] for d in data[alg]])
+        step = np.array([d['steps'][:max_step] for d in data[alg]][0])
+        seeds_used = value.shape[0]
+        mean, err = np.mean(value, axis=0), np.std(value, axis=0)
+        err = err / np.sqrt(seeds_used)
+        plt.plot(
+            step, mean, c=clrs[i], label=alg)
+        plt.fill_between(
+            step, mean - err,
+            mean + err, alpha=fill_between, facecolor=clrs[i])
+        largest_max = mean.max() if mean.max() > largest_max else largest_max
+        smallest_min = mean.min() if mean.min() < smallest_min else smallest_min
+
+    plt.xlabel('Training Dialogues', fontsize=fontsize)
+    #plt.gca().yaxis.set_major_locator(plt.MultipleLocator(round((largest_max - smallest_min) / 10.0, 2)))
+    if len(y_label) > 0:
+        plt.ylabel(y_label.title(), fontsize=fontsize)
+    else:
+        plt.ylabel(plot_type.title(), fontsize=fontsize)
+    plt.xticks(fontsize=fontsize-4)
+    plt.yticks(fontsize=fontsize-4)
+    plt.legend(fancybox=True, shadow=False, ncol=1, loc='best', fontsize=fontsize)
+    plt.savefig(out_file + ".pdf", bbox_inches='tight', dpi=400, pad_inches=0)
+
+    if show_image:
+        plt.show()
 
 
 if __name__ == "__main__":
     args = get_args()
 
-    y_label_dict = {"complete_rate": 'Complete rate', "success_rate": 'Success rate', 'turns': 'Average turns',
-                    'avg_return': 'Average Return'}
+    y_label_dict = {"complete_rate": 'Complete Rate', "success_rate": 'Success Rate', 'turns': 'Average Turns',
+                    'avg_return': 'Average Return', "success_rate_strict": 'Strict Success Rate',
+                    "avg_actions": "Average Actions"}
 
-    for plot_type in ["complete_rate", "success_rate", 'turns', 'avg_return']:
+    for plot_type in ["complete_rate", "success_rate", "success_rate_strict", 'turns', 'avg_return', 'avg_actions']:
         file_name, file_extension = os.path.splitext(args.out_file)
         os.makedirs(file_name, exist_ok=True)
         fig_name = f"{file_name}_{plot_type}{file_extension}"
@@ -114,4 +130,11 @@ if __name__ == "__main__":
              plot_type=plot_type,
              fill_between=args.fill_between,
              max_dialogues=args.max_dialogues,
-             y_label=y_label_dict[plot_type])
+             y_label=y_label_dict[plot_type],
+             fontsize=args.fontsize,
+             figsize=args.figure_size,
+             facecolor=args.figure_face_color)
+
+    plot_distributions(args.dir, json.load(open(args.map_file)), args.out_file, fontsize=args.fontsize, font=args.font,
+                       figsize=args.figure_size, facecolor=args.figure_face_color)
+
diff --git a/convlab/policy/plot_results/plot_action_distributions.py b/convlab/policy/plot_results/plot_action_distributions.py
new file mode 100644
index 0000000000000000000000000000000000000000..8a8f923a594718f011f3ba742d7b964a0da11029
--- /dev/null
+++ b/convlab/policy/plot_results/plot_action_distributions.py
@@ -0,0 +1,158 @@
+import numpy as np
+import matplotlib.pyplot as plt
+import os
+import seaborn as sns
+import pandas as pd
+
+
+def extract_action_distributions_across_seeds(algorithm_dir_path):
+    '''
+    We extract the information directly from the train_INFO.log file. An evaluation step has either of the two forms:
+
+    Evaluating at start - 2022-11-03-08-53-38------------------------------------------------------------
+    Complete: 0.636+-0.02, Success: 0.51+-0.02, Success strict: 0.432+-0.02, Average......
+    **OR**
+    Evaluating after Dialogues: 1000 - 2022-11-03-09-18-42------------------------------------------------------------
+    Complete: 0.786+-0.02, Success: 0.686+-0.02, Success strict: 0.634+-0.02, Average Return: 24.42.......
+    '''
+
+    seed_dir_paths = [f.path for f in os.scandir(
+        algorithm_dir_path) if f.is_dir()]
+    seed_dir_names = [f.name for f in os.scandir(
+        algorithm_dir_path) if f.is_dir()]
+
+    # dict below will have the form {0: {book: [], inform: [], ..}, 1000: {book: [], inform: [], ..}, ...}
+    # where 0 and 1000 are evaluation steps and the list will be as long as the number of seeds used
+    distribution_per_step_dict = {}
+
+    for seed_dir_name, seed_dir_path in zip(seed_dir_names, seed_dir_paths):
+
+        with open(os.path.join(seed_dir_path, 'logs', 'train_INFO.log'), "r") as f:
+            evaluation_found = False
+            num_dialogues = 0
+            for line in f:
+                line = line.strip()
+                if "evaluating at" in line.lower() or "evaluating after" in line.lower():
+                    evaluation_found = True
+                    if not "at start" in line.lower():
+                        num_dialogues = int(line.split(" ")[3])
+                    continue
+                if evaluation_found:
+                    # extracts the strings "book action: 0.3", "inform action: 0.4" ....
+                    action_distribution_string = [a for a in line.split(", ")
+                                                  if "actions" in a.lower() and "average actions" not in a.lower()]
+
+                    if num_dialogues in distribution_per_step_dict:
+                        for action_string in action_distribution_string:
+                            action = action_string.lower().split(" ")[0]
+                            distribution = float(
+                                action_string.lower().split(": ")[-1])
+                            if action in distribution_per_step_dict[num_dialogues]:
+                                distribution_per_step_dict[num_dialogues][action].append(
+                                    distribution)
+                            else:
+                                distribution_per_step_dict[num_dialogues][action] = [
+                                    distribution]
+                    else:
+                        distribution_per_step_dict[num_dialogues] = {}
+                        for action_string in action_distribution_string:
+                            action = action_string.lower().split(" ")[0]
+                            distribution = float(
+                                action_string.lower().split(": ")[-1])
+                            distribution_per_step_dict[num_dialogues][action] = [
+                                distribution]
+
+                    evaluation_found = False
+
+    return distribution_per_step_dict
+
+
+def plot_distributions(dir_path, alg_maps, output_dir, fill_between=0.3, fontsize=16, font="Times New Roman",
+                       figsize=(12, 8), facecolor='#E6E6E6'):
+    plt.rcParams["font.family"] = font
+    clrs = sns.color_palette("husl", len(alg_maps))
+
+    alg_paths = [os.path.join(dir_path, alg_map['dir'])
+                 for alg_map in alg_maps]
+    action_distributions = [
+        extract_action_distributions_across_seeds(path) for path in alg_paths]
+    possible_actions = action_distributions[0][0].keys()
+
+    create_bar_plots(action_distributions, alg_maps,
+                     possible_actions, output_dir,
+                     fontsize, figsize, facecolor)
+
+    for action in possible_actions:
+        plt.clf()
+        plt.figure(figsize=figsize)
+        plt.gca().patch.set_facecolor(facecolor)
+        plt.grid(color='w', linestyle='solid', alpha=0.5)
+
+        largest_max = 0
+        smallest_min = 1
+        for i, alg_distribution in enumerate(action_distributions):
+            steps = alg_distribution.keys()
+            try:
+                distributions = np.array(
+                    [alg_distribution[step][action] for step in steps])
+                # length = num_Evaluations * num_seeds
+                mean, std_dev = np.mean(distributions, axis=1), np.std(
+                    distributions, axis=1)
+                seeds_used = distributions.shape[1]
+                std_error = std_dev / np.sqrt(seeds_used)
+
+                # with sns.axes_style("darkgrid"):
+                plt.plot(steps, mean, c=clrs[i],
+                         label=f"{alg_maps[i]['legend']}")
+                plt.fill_between(
+                    steps, mean - std_error,
+                    mean + std_error, alpha=fill_between, facecolor=clrs[i])
+
+                largest_max = mean.max() if mean.max() > largest_max else largest_max
+                smallest_min = mean.min() if mean.min() < smallest_min else smallest_min
+
+            except Exception as e:
+                # catch if an algorithm does not have a specific action
+                print(e)
+        print(action)
+        if round((largest_max - smallest_min) / 10.0, 2) > 0:
+            plt.gca().yaxis.set_major_locator(plt.MultipleLocator(
+                round((largest_max - smallest_min) / 10.0, 2)))
+        plt.xticks(fontsize=fontsize-4, rotation=0)
+        plt.yticks(fontsize=fontsize-4)
+        plt.xlabel('Training Dialogues', fontsize=fontsize)
+        plt.ylabel(f"{action.title()} Intent Probability", fontsize=fontsize)
+        plt.legend(fancybox=True, shadow=False, ncol=1, loc='best')
+        plt.savefig(
+            output_dir + f'/{action}_probability.pdf', bbox_inches='tight',
+            dpi=400, pad_inches=0)
+
+
+def create_bar_plots(action_distributions, alg_maps, possible_actions, output_dir, fontsize, figsize, facecolor):
+
+    max_step = max(action_distributions[0].keys())
+    final_distributions = [distribution[max_step]
+                           for distribution in action_distributions]
+
+    df_list = []
+    for action in possible_actions:
+        action_list = [action.title()]
+        for distribution in final_distributions:
+            action_list.append(np.mean(distribution[action]))
+        df_list.append(action_list)
+
+    df = pd.DataFrame(df_list, columns=[
+                      'Probabilities'] + [alg_map["legend"] for alg_map in alg_maps])
+    plt.figure(figsize = figsize)
+    plt.rcParams.update({'font.size': fontsize})
+    fig = df.plot(x='Probabilities', kind='bar', stacked=False,
+                  rot=0, grid=True, color=sns.color_palette("husl", len(alg_maps)),
+                  fontsize=fontsize, figsize=figsize).get_figure()
+    plt.gca().patch.set_facecolor(facecolor)
+    plt.grid(color='w', linestyle='solid', alpha=0.5)
+    plt.yticks(np.arange(0, 1, 0.1), fontsize=fontsize-4)
+    plt.xticks(fontsize=fontsize-4)
+    plt.xlabel('Intents', fontsize=fontsize)
+    plt.ylabel('Probability', fontsize=fontsize)
+    fig.savefig(os.path.join(output_dir, "final_action_probabilities.pdf"),
+                dpi=400, bbox_inches='tight', pad_inches=0)
diff --git a/convlab/policy/ppo/README.md b/convlab/policy/ppo/README.md
index 6bf87252ffbba511d32e275a079fc7cba2e954a9..c762253ce4fe769bb2c540b39d39df713881a7f3 100755
--- a/convlab/policy/ppo/README.md
+++ b/convlab/policy/ppo/README.md
@@ -1,34 +1,55 @@
-# PPO
+# Proximal Policy Optimization (PPO)
 
-A policy optimization method in policy based reinforcement learning that uses
-multiple epochs of stochastic gradient ascent and a constant
-clipping mechanism as the soft constraint to perform each policy update. We adapt PPO to the dialog policy.
+Proximal Policy Optimization (Schulmann et. al. 2017) is an on-policy reinforcement learning algorithm. The architecture used is a simple MLP and thus not transferable to new ontologies.
 
-## Train
+## Supervised pre-training
 
-Run `train.py` in the `ppo` directory:
+If you want to obtain a supervised model for pre-training, please have a look in the MLE policy folder.
 
-```bash
-python train.py
+## RL training
+
+Starting a RL training is as easy as executing
+
+```sh
+$ python train.py --path=your_environment_config --seed=SEED
 ```
 
-For better performance, we can do immitating learning before reinforcement learning. The immitating learning is implemented in the `mle` directory.
+One example for the environment-config is **semantic_level_config.json**, where parameters for the training are specified, for instance
 
-For example, if the trained model of immitating learning is saved at FOLDER_OF_MODEL/best_mle.pol.mdl, then you can run
+- load_path: provide a path to initialise the model with a pre-trained model, skip the ending .pol.mdl
+- process_num: the number of processes to use during evaluation to speed it up
+- num_eval_dialogues: how many evaluation dialogues should be used
+- epoch: how many training epochs to run. One epoch consists of collecting dialogues + performing an update
+- eval_frequency: after how many epochs perform an evaluation
+- batchsz: the number of training dialogues collected before doing an update
 
-```bash
-python train.py --load_path FOLDER_OF_MODEL/best_mle
-```
+Moreover, you can specify the full dialogue pipeline here, such as the user policy, NLU for system and user, etc.
+
+Parameters that are tied to the RL algorithm and the model architecture can be changed in config.json.
+
+
+## Evaluation
 
-Note that the *.pol.mdl* suffix should not appear in the --load_path argument.
+For creating evaluation plots and running evaluation dialogues, please have a look in the README of the policy folder.
 
-## Reference
+## References
 
 ```
-@article{schulman2017proximal,
-  title={Proximal policy optimization algorithms},
-  author={Schulman, John and Wolski, Filip and Dhariwal, Prafulla and Radford, Alec and Klimov, Oleg},
-  journal={arXiv preprint arXiv:1707.06347},
-  year={2017}
+@article{DBLP:journals/corr/SchulmanWDRK17,
+  author    = {John Schulman and
+               Filip Wolski and
+               Prafulla Dhariwal and
+               Alec Radford and
+               Oleg Klimov},
+  title     = {Proximal Policy Optimization Algorithms},
+  journal   = {CoRR},
+  volume    = {abs/1707.06347},
+  year      = {2017},
+  url       = {http://arxiv.org/abs/1707.06347},
+  eprinttype = {arXiv},
+  eprint    = {1707.06347},
+  timestamp = {Mon, 13 Aug 2018 16:47:34 +0200},
+  biburl    = {https://dblp.org/rec/journals/corr/SchulmanWDRK17.bib},
+  bibsource = {dblp computer science bibliography, https://dblp.org}
 }
 ```
\ No newline at end of file
diff --git a/convlab/policy/ppo/semanticGenTUS-RuleDST-PPOPolicy.json b/convlab/policy/ppo/semanticGenTUS-RuleDST-PPOPolicy.json
new file mode 100644
index 0000000000000000000000000000000000000000..0e8774e20898c2855f368127f8e14e193ac2c21d
--- /dev/null
+++ b/convlab/policy/ppo/semanticGenTUS-RuleDST-PPOPolicy.json
@@ -0,0 +1,44 @@
+{
+	"model": {
+		"load_path": "convlab/policy/ppo/pretrained_models/mle",
+		"pretrained_load_path": "",
+		"use_pretrained_initialisation": false,
+		"batchsz": 500,
+		"seed": 0,
+		"epoch": 50,
+		"eval_frequency": 5,
+		"process_num": 1,
+		"num_eval_dialogues": 500,
+		"sys_semantic_to_usr": false
+	},
+	"vectorizer_sys": {
+		"uncertainty_vector_mul": {
+			"class_path": "convlab.policy.vector.vector_binary.VectorBinary",
+			"ini_params": {
+				"use_masking": true,
+				"manually_add_entity_names": true,
+				"seed": 0
+			}
+		}
+	},
+	"nlu_sys": {},
+	"dst_sys": {
+		"RuleDST": {
+			"class_path": "convlab.dst.rule.multiwoz.dst.RuleDST",
+			"ini_params": {}
+		}
+	},
+	"sys_nlg": {},
+	"nlu_usr": {},
+	"dst_usr": {},
+	"policy_usr": {
+		"RulePolicy": {
+			"class_path": "convlab.policy.genTUS.stepGenTUS.UserPolicy",
+			"ini_params": {
+				"model_checkpoint": "convlab/policy/genTUS/unify/experiments/multiwoz21_0_1.0",
+				"character": "usr"
+			}
+		}
+	},
+	"usr_nlg": {}
+}
diff --git a/convlab/policy/ppo/semantic_level_config.json b/convlab/policy/ppo/semantic_level_config.json
index 381acac72bd2c28e870e091e7c36d499c1bbd76c..b9908c9cb7717515775221227f3fba19636d20dc 100644
--- a/convlab/policy/ppo/semantic_level_config.json
+++ b/convlab/policy/ppo/semantic_level_config.json
@@ -1,12 +1,12 @@
 {
 	"model": {
-		"load_path": "convlab/policy/mle/experiments/experiment_2022-05-23-14-08-43/save/supervised",
+		"load_path": "",
 		"use_pretrained_initialisation": false,
 		"pretrained_load_path": "",
 		"batchsz": 1000,
 		"seed": 0,
-		"epoch": 50,
-		"eval_frequency": 5,
+		"epoch": 10,
+		"eval_frequency": 1,
 		"process_num": 4,
 		"sys_semantic_to_usr": false,
 		"num_eval_dialogues": 500
@@ -40,4 +40,4 @@
 		}
 	},
 	"usr_nlg": {}
-}
\ No newline at end of file
+}
diff --git a/convlab/policy/ppo/setsumbt_end_baseline_config.json b/convlab/policy/ppo/setsumbt_config.json
similarity index 53%
rename from convlab/policy/ppo/setsumbt_end_baseline_config.json
rename to convlab/policy/ppo/setsumbt_config.json
index ea84dd7670a3a27dcc267e476c894bf939c9ba10..31a8ac6d275166e4163e416e0dbef6f742cddb7f 100644
--- a/convlab/policy/ppo/setsumbt_end_baseline_config.json
+++ b/convlab/policy/ppo/setsumbt_config.json
@@ -1,22 +1,22 @@
 {
 	"model": {
-		"load_path": "supervised",
+		"load_path": "",
 		"pretrained_load_path": "",
 		"use_pretrained_initialisation": false,
 		"batchsz": 1000,
 		"seed": 0,
 		"epoch": 50,
 		"eval_frequency": 5,
-		"process_num": 4,
+		"process_num": 2,
 		"num_eval_dialogues": 500,
-		"sys_semantic_to_usr": false
+		"sys_semantic_to_usr": true
 	},
 	"vectorizer_sys": {
 		"uncertainty_vector_mul": {
-			"class_path": "convlab.policy.vector.vector_multiwoz_uncertainty.MultiWozVector",
+			"class_path": "convlab.policy.vector.vector_binary.VectorBinary",
 			"ini_params": {
 				"use_masking": false,
-				"manually_add_entity_names": false,
+				"manually_add_entity_names": true,
 				"seed": 0
 			}
 		}
@@ -24,12 +24,9 @@
 	"nlu_sys": {},
 	"dst_sys": {
 		"setsumbt-mul": {
-			"class_path": "convlab.dst.setsumbt.multiwoz.Tracker.SetSUMBTTracker",
+			"class_path": "convlab.dst.setsumbt.SetSUMBTTracker",
 			"ini_params": {
-				"model_path": "https://zenodo.org/record/5497808/files/setsumbt_end.zip",
-				"get_confidence_scores": true,
-				"return_mutual_info": false,
-				"return_entropy": true
+				"model_path": "/gpfs/project/niekerk/models/setsumbt_models/SetSUMBT+ActPrediction-multiwoz21-roberta-gru-cosine-distribution_distillation-Seed0-30-08-22-15-00"
 			}
 		}
 	},
@@ -41,16 +38,7 @@
 			}
 		}
 	},
-	"nlu_usr": {
-		"BERTNLU": {
-			"class_path": "convlab.nlu.jointBERT.multiwoz.BERTNLU",
-			"ini_params": {
-				"mode": "sys",
-				"config_file": "multiwoz_sys_context.json",
-				"model_file": "https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_sys_context.zip"
-			}
-		}
-	},
+	"nlu_usr": {},
 	"dst_usr": {},
 	"policy_usr": {
 		"RulePolicy": {
@@ -65,9 +53,9 @@
 			"class_path": "convlab.nlg.template.multiwoz.TemplateNLG",
 			"ini_params": {
 				"is_user": true,
-				"label_noise": 0.0,
+				"label_noise": 0.05,
 				"text_noise": 0.0
 			}
 		}
 	}
-}
\ No newline at end of file
+}
diff --git a/convlab/policy/ppo/setsumbt_unc_config.json b/convlab/policy/ppo/setsumbt_unc_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..6b7d115aafa53a2bfc5c58672e67086f04d5884d
--- /dev/null
+++ b/convlab/policy/ppo/setsumbt_unc_config.json
@@ -0,0 +1,65 @@
+{
+	"model": {
+		"load_path": "/gpfs/project/niekerk/src/ConvLab3/convlab/policy/mle/experiments/experiment_2022-11-10-10-37-30/save/supervised",
+		"pretrained_load_path": "",
+		"use_pretrained_initialisation": false,
+		"batchsz": 1000,
+		"seed": 0,
+		"epoch": 50,
+		"eval_frequency": 5,
+		"process_num": 2,
+		"num_eval_dialogues": 500,
+		"sys_semantic_to_usr": true
+	},
+	"vectorizer_sys": {
+		"uncertainty_vector_mul": {
+			"class_path": "convlab.policy.vector.vector_uncertainty.VectorUncertainty",
+			"ini_params": {
+				"use_masking": false,
+				"manually_add_entity_names": true,
+				"seed": 0,
+				"use_confidence_scores": true,
+				"use_state_knowledge_uncertainty": true
+			}
+		}
+	},
+	"nlu_sys": {},
+	"dst_sys": {
+		"setsumbt-mul": {
+			"class_path": "convlab.dst.setsumbt.SetSUMBTTracker",
+			"ini_params": {
+				"model_path": "/gpfs/project/niekerk/models/setsumbt_models/SetSUMBT+ActPrediction-multiwoz21-roberta-gru-cosine-distribution_distillation-Seed0-30-08-22-15-00",
+				"return_confidence_scores": true,
+				"return_belief_state_mutual_info": true
+			}
+		}
+	},
+	"sys_nlg": {
+		"TemplateNLG": {
+			"class_path": "convlab.nlg.template.multiwoz.TemplateNLG",
+			"ini_params": {
+				"is_user": false
+			}
+		}
+	},
+	"nlu_usr": {},
+	"dst_usr": {},
+	"policy_usr": {
+		"RulePolicy": {
+			"class_path": "convlab.policy.rule.multiwoz.RulePolicy",
+			"ini_params": {
+				"character": "usr"
+			}
+		}
+	},
+	"usr_nlg": {
+		"TemplateNLG": {
+			"class_path": "convlab.nlg.template.multiwoz.TemplateNLG",
+			"ini_params": {
+				"is_user": true,
+				"label_noise": 0.05,
+				"text_noise": 0.0
+			}
+		}
+	}
+}
\ No newline at end of file
diff --git a/convlab/policy/ppo/train.py b/convlab/policy/ppo/train.py
index 899e91525ba67de7015b1b7fafeab516e4b02f2b..703a55005b8c07578b85765a626d9871deebf26e 100755
--- a/convlab/policy/ppo/train.py
+++ b/convlab/policy/ppo/train.py
@@ -184,7 +184,7 @@ if __name__ == '__main__':
     parser = ArgumentParser()
     parser.add_argument("--path", type=str, default='convlab/policy/ppo/semantic_level_config.json',
                         help="Load path for config file")
-    parser.add_argument("--seed", type=int, default=0,
+    parser.add_argument("--seed", type=int, default=None,
                         help="Seed for the policy parameter initialization")
     parser.add_argument("--mode", type=str, default='info',
                         help="Set level for logger")
@@ -199,7 +199,7 @@ if __name__ == '__main__':
     logger, tb_writer, current_time, save_path, config_save_path, dir_path, log_save_path = \
         init_logging(os.path.dirname(os.path.abspath(__file__)), mode)
 
-    args = [('model', 'seed', seed)]
+    args = [('model', 'seed', seed)] if seed is not None else list()
 
     environment_config = load_config_file(path)
     save_config(vars(parser.parse_args()), environment_config, config_save_path)
@@ -228,13 +228,6 @@ if __name__ == '__main__':
 
     env, sess = env_config(conf, policy_sys)
 
-    # Setup uncertainty thresholding
-    if env.sys_dst:
-        try:
-            if env.sys_dst.use_confidence_scores:
-                policy_sys.vector.setup_uncertain_query(env.sys_dst.thresholds)
-        except:
-            logging.info('Uncertainty threshold not set.')
 
     policy_sys.current_time = current_time
     policy_sys.log_dir = config_save_path.replace('configs', 'logs')
@@ -261,7 +254,7 @@ if __name__ == '__main__':
 
         if idx % conf['model']['eval_frequency'] == 0 and idx != 0:
             time_now = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
-            logging.info(f"Evaluating at Epoch: {idx} - {time_now}" + '-'*60)
+            logging.info(f"Evaluating after Dialogues: {idx * conf['model']['batchsz']} - {time_now}" + '-' * 60)
 
             eval_dict = eval_policy(conf, policy_sys, env, sess, save_eval, log_save_path)
 
diff --git a/convlab/policy/ppo/trippy_config.json b/convlab/policy/ppo/trippy_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..41b1c3623aca944312c6389e55e34d72422fb6e0
--- /dev/null
+++ b/convlab/policy/ppo/trippy_config.json
@@ -0,0 +1,73 @@
+{
+	"model": {
+		"load_path": "/path/to/model/checkpoint",
+		"pretrained_load_path": "",
+		"use_pretrained_initialisation": false,
+		"batchsz": 1000,
+		"seed": 0,
+		"epoch": 50,
+		"eval_frequency": 5,
+		"process_num": 2,
+		"num_eval_dialogues": 500,
+		"sys_semantic_to_usr": false
+	},
+        "vectorizer_sys": {
+	        "fuzzy_vector_mul": {
+		        "class_path": "convlab.policy.vector.vector_binary_fuzzy.VectorBinaryFuzzy",
+		        "ini_params": {
+			        "use_masking": true,
+			        "manually_add_entity_names": true,
+			        "seed": 0
+			}
+		}
+	},
+	"nlu_sys": {},
+	"dst_sys": {
+		"TripPy": {
+			"class_path": "convlab.dst.trippy.TRIPPY",
+		        "ini_params": {
+			        "model_type": "roberta",
+			        "model_name": "roberta-base",
+			        "model_path": "/path/to/model/checkpoint",
+			        "dataset_name": "multiwoz21"
+			}
+		}
+	},
+	"sys_nlg": {
+		"TemplateNLG": {
+			"class_path": "convlab.nlg.template.multiwoz.TemplateNLG",
+			"ini_params": {
+				"is_user": false
+			}
+		}
+	},
+	"nlu_usr": {
+		"BERTNLU": {
+			"class_path": "convlab.nlu.jointBERT.unified_datasets.BERTNLU",
+			"ini_params": {
+				"mode": "sys",
+                                "config_file": "multiwoz21_sys_context3.json",
+				"model_file": "/path/to/model/checkpoint.zip"
+			}
+		}
+	},
+	"dst_usr": {},
+	"policy_usr": {
+		"RulePolicy": {
+			"class_path": "convlab.policy.rule.multiwoz.RulePolicy",
+			"ini_params": {
+				"character": "usr"
+			}
+		}
+	},
+	"usr_nlg": {
+		"TemplateNLG": {
+			"class_path": "convlab.nlg.template.multiwoz.TemplateNLG",
+			"ini_params": {
+				"is_user": true,
+				"label_noise": 0.0,
+				"text_noise": 0.0
+			}
+		}
+	}
+}
diff --git a/convlab/policy/ppo/tus_semantic_level_config.json b/convlab/policy/ppo/tus_semantic_level_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..9d56646cc857de85b47a1f9925a0e4bf89d8b524
--- /dev/null
+++ b/convlab/policy/ppo/tus_semantic_level_config.json
@@ -0,0 +1,43 @@
+{
+	"model": {
+		"load_path": "convlab/policy/ppo/pretrained_models/mle",
+		"use_pretrained_initialisation": false,
+		"pretrained_load_path": "",
+		"batchsz": 1000,
+		"seed": 0,
+		"epoch": 50,
+		"eval_frequency": 5,
+		"process_num": 4,
+		"sys_semantic_to_usr": false,
+		"num_eval_dialogues": 500
+	},
+	"vectorizer_sys": {
+		"uncertainty_vector_mul": {
+			"class_path": "convlab.policy.vector.vector_binary.VectorBinary",
+			"ini_params": {
+				"use_masking": true,
+				"manually_add_entity_names": false,
+				"seed": 0
+			}
+		}
+	},
+	"nlu_sys": {},
+	"dst_sys": {
+		"RuleDST": {
+			"class_path": "convlab.dst.rule.multiwoz.dst.RuleDST",
+			"ini_params": {}
+		}
+	},
+	"sys_nlg": {},
+	"nlu_usr": {},
+	"dst_usr": {},
+	"policy_usr": {
+		"TUSPolicy": {
+			"class_path": "convlab.policy.tus.unify.TUS.UserPolicy",
+			"ini_params": {
+				"config": "convlab/policy/tus/unify/exp/multiwoz.json"
+			}
+		}
+	},
+	"usr_nlg": {}
+}
diff --git a/convlab/policy/rlmodule.py b/convlab/policy/rlmodule.py
index db46026656d908b2453a9143b3f482ce7378e382..18844ca375967cb9524dacd3a81a4f85a2cf9041 100755
--- a/convlab/policy/rlmodule.py
+++ b/convlab/policy/rlmodule.py
@@ -287,7 +287,7 @@ class Value(nn.Module):
 Transition_evaluator = namedtuple('Transition_evaluator',
                                   ('complete', 'success', 'success_strict', 'total_return_complete', 'total_return_success', 'turns',
                                    'avg_actions', 'task_success', 'book_actions', 'inform_actions', 'request_actions', 'select_actions',
-                                   'offer_actions'))
+                                   'offer_actions', 'recommend_actions'))
 
 
 class Memory_evaluator(object):
diff --git a/convlab/policy/tus/README.md b/convlab/policy/tus/README.md
index 2976e17bcbd5662ff4802eef9efb2219172f95af..9424edcf3795220a582809e1455c942f300c7740 100644
--- a/convlab/policy/tus/README.md
+++ b/convlab/policy/tus/README.md
@@ -1,30 +1,48 @@
-**TUS** is a domain-independent user simulator with transformers for task-oriented dialogue systems. It is based on the [ConvLab-2](https://github.com/thu-coai/ConvLab-2) framework. Therefore, you should follow their instruction to install the package.
+**TUS** is a domain-independent user simulator with transformers for task-oriented dialogue systems.
 
 ## Introduction
-Our model is a domain-independent user simulator, which means it is not based on any domain-dependent freatures and the output representation is also domain-independent. Therefore, it can easily adapt to a new domain, without additional feature engineering and model retraining.
+Our model is a domain-independent user simulator, which means its input and output representations are domain agnostic. Therefore, it can easily adapt to a new domain, without additional feature engineering and model retraining.
 
-The code of TUS is in `convlab/policy/tus` and a rule-based DST of user is also created in `convlab/dst/rule/multiwoz/dst.py` based on the rule-based DST in `convlab/dst/rule/multiwoz/dst.py`.
+The code of TUS is in `convlab/policy/tus`.
 
-## How to run the model
-### Train the user simulator
-`python3 convlab/policy/tus/multiwoz/train.py --user_config convlab/policy/tus/multiwoz/exp/default.json`
+## Usage
+### Train TUS from scratch
 
-One default configuration is placed in `convlab/policy/tus/multiwoz/exp/default.json`. They can be modified based on your requirements. For example, the output directory can be specified in the configuration (`model_dir`).
+```
+python3 convlab/policy/tus/unify/train.py --dataset $dataset --dial-ids-order $dial_ids_order --split2ratio $split2ratio --user-config $config
+```
+
+`dataset` can be `multiwoz21`, `sgd`, `tm`, `sgd+tm`, or `all`.
+`dial_ids_order` can be 0, 1 or 2
+`split2ratio` can be 0.01, 0.1 or 1
+Default configurations are placed in `convlab/policy/tus/unify/exp`. They can be modified based on your requirements. 
+
+For example, you can train TUS for multiwoz21 by 
+`python3 convlab/policy/tus/unify/train.py --dataset multiwoz21 --dial-ids-order 0 --split2ratio 1 --user-config "convlab/policy/tus/unify/exp/multiwoz.json"`
+
+### Evaluate TUS
 
 ### Train a dialogue policy with TUS
 You can use it as a normal user simulator by `PipelineAgent`. For example,
 ```python
 import json
 from convlab.dialog_agent.agent import PipelineAgent
-from convlab.dst.rule.multiwoz.usr_dst import UserRuleDST
-from convlab.policy.tus.multiwoz.TUS import UserPolicy
+from convlab.policy.tus.unify.TUS import UserPolicy
 
-user_config_file = "convlab/policy/tus/multiwoz/exp/default.json"
-dst_usr = UserRuleDST()
+user_config_file = "convlab/policy/tus/unify/exp/multiwoz.json"
 user_config = json.load(open(user_config_file))
 policy_usr = UserPolicy(user_config)
-simulator = PipelineAgent(None, dst_usr, policy_usr, None, 'user')
+simulator = PipelineAgent(None, None, policy_usr, None, 'user')
+```
+then you can train your system with this simulator.
+
+There is an example config, which trains a PPO policy with TUS in semantic level, in `convlab/policy/ppo/tus_semantic_level_config.json`.
+You can train a PPO policy as following, 
+```
+config="convlab/policy/ppo/tus_semantic_level_config.json"
+python3 convlab/policy/ppo/train.py --path $config
 ```
+notice: You should name your pretrained policy as `convlab/policy/ppo/pretrained_models/mle` or modify the `load_path` of `model` in the config `convlab/policy/ppo/tus_semantic_level_config.json`.
 
 
 <!---citation--->
diff --git a/convlab/policy/tus/multiwoz/Goal.py b/convlab/policy/tus/multiwoz/Goal.py
index e337a1bd3b5668589f130ae2e2ec5a1615fdd985..c26e942723fd9e76901572dd2a85fd7433c4b275 100644
--- a/convlab/policy/tus/multiwoz/Goal.py
+++ b/convlab/policy/tus/multiwoz/Goal.py
@@ -44,10 +44,10 @@ class Goal(object):
         elif goal_generator is None and goal is not None:
             self.domains = []
             self.domain_goals = {}
-            for domain in goal:
-                if domain in SysDa2Goal and goal[domain]:  # TODO check order
+            for domain in goal.domains:
+                if domain in SysDa2Goal and goal.domain_goals[domain]:  # TODO check order
                     self.domains.append(domain)
-                    self.domain_goals[domain] = goal[domain]
+                    self.domain_goals[domain] = goal.domain_goals[domain]
         else:
             print("Warning!!! One of goal_generator or goal should not be None!!!")
 
diff --git a/convlab/policy/tus/multiwoz/TUS.py b/convlab/policy/tus/multiwoz/TUS.py
index 725098d9162d6900746aa7398c14a8aafe49d786..f8f70ab3e918073079dec850dac9ad78ff79f36a 100644
--- a/convlab/policy/tus/multiwoz/TUS.py
+++ b/convlab/policy/tus/multiwoz/TUS.py
@@ -18,7 +18,8 @@ from convlab.util.custom_util import model_downloader
 from convlab.policy.rule.multiwoz.policy_agenda_multiwoz import unified_format, act_dict_to_flat_tuple
 from convlab.util import relative_import_module_from_unified_datasets
 
-reverse_da, normalize_domain_slot_value = relative_import_module_from_unified_datasets('multiwoz21', 'preprocess.py', ['reverse_da', 'normalize_domain_slot_value'])
+reverse_da, normalize_domain_slot_value = relative_import_module_from_unified_datasets(
+    'multiwoz21', 'preprocess.py', ['reverse_da', 'normalize_domain_slot_value'])
 
 
 DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
diff --git a/convlab/policy/tus/unify/Goal.py b/convlab/policy/tus/unify/Goal.py
new file mode 100644
index 0000000000000000000000000000000000000000..610469bb4246cf6c16ef4271c89827cab6421437
--- /dev/null
+++ b/convlab/policy/tus/unify/Goal.py
@@ -0,0 +1,302 @@
+import time
+import json
+from convlab.policy.tus.unify.util import split_slot_name, slot_name_map
+from convlab.util.custom_util import slot_mapping
+
+from random import sample, shuffle
+from pprint import pprint
+DEF_VAL_UNK = '?'  # Unknown
+DEF_VAL_DNC = 'dontcare'  # Do not care
+DEF_VAL_NUL = 'none'  # for none
+DEF_VAL_BOOKED = 'yes'  # for booked
+DEF_VAL_NOBOOK = 'no'  # for booked
+NOT_SURE_VALS = [DEF_VAL_UNK, DEF_VAL_DNC, DEF_VAL_NUL, DEF_VAL_NOBOOK, ""]
+
+
+# only support user goal from dataset
+
+
+def is_time(goal, status):
+    if isTimeFormat(goal) and isTimeFormat(status):
+        return True
+    return False
+
+
+def isTimeFormat(input):
+    try:
+        time.strptime(input, '%H:%M')
+        return True
+    except ValueError:
+        return False
+
+
+def old_goal2list(goal: dict, reorder=False) -> list:
+    goal_list = []
+    for domain in goal:
+        for slot_type in ['info', 'book', 'reqt']:
+            if slot_type not in goal[domain]:
+                continue
+            temp = []
+            for slot in goal[domain][slot_type]:
+                s = slot
+                if slot in slot_name_map:
+                    s = slot_name_map[slot]
+                elif slot in slot_name_map[domain]:
+                    s = slot_name_map[domain][slot]
+                # domain, intent, slot, value
+                if slot_type in ['info', 'book']:
+                    i = "inform"
+                    v = goal[domain][slot_type][slot]
+                else:
+                    i = "request"
+                    v = DEF_VAL_UNK
+                s = slot_mapping.get(s, s)
+                temp.append([domain, i, s, v])
+            shuffle(temp)
+            goal_list = goal_list + temp
+    # shuffle_goal = goal_list[:1] + sample(goal_list[1:], len(goal_list)-1)
+    # return shuffle_goal
+    return goal_list
+
+
+class Goal(object):
+    """ User Goal Model Class. """
+
+    def __init__(self, goal: list = None):
+        """
+        create new Goal by random
+        Args:
+            goal (list): user goal built from user history
+            ontology (dict): domains, slots, values
+        """
+        self.goal = goal
+        self.max_domain_len = 6
+        self.max_slot_len = 20
+        self.local_id = {}
+
+        self.domains = []
+        # goal: {domain: {"info": {slot: value}, "reqt": {slot:?}}, ...}
+        self.domain_goals = {}
+        # status: {domain: {slot: value}}
+        self.status = {}
+        self.user_history = {}
+        self.init_goal_status(goal)
+        self.init_local_id()
+
+    def __str__(self):
+        return '-----Goal-----\n' + \
+               json.dumps(self.domain_goals, indent=4) + \
+               '\n-----Goal-----'
+
+    def init_goal_status(self, goal):
+        for domain, intent, slot, value in goal:  # check this order
+            if domain not in self.domains:
+                self.domains.append(domain)
+                self.domain_goals[domain] = {}
+
+            # "book" domain is not clear for unify data format
+
+            if "request" in intent.lower():
+                if "reqt" not in self.domain_goals[domain]:
+                    self.domain_goals[domain]["reqt"] = {}
+                self.domain_goals[domain]["reqt"][slot] = DEF_VAL_UNK
+
+            elif "info" in intent.lower():
+                if "info" not in self.domain_goals[domain]:
+                    self.domain_goals[domain]["info"] = {}
+                self.domain_goals[domain]["info"][slot] = value
+
+            self.user_history[f"{domain}-{slot}"] = value
+
+    def task_complete(self):
+        """
+        Check that all requests have been met
+        Returns:
+            (boolean): True to accomplish.
+        """
+        for domain in self.domain_goals:
+            if domain not in self.status:
+                # print(f"{domain} is not mentioned")
+                return False
+            if "info" in self.domain_goals[domain]:
+                for slot in self.domain_goals[domain]["info"]:
+                    if slot not in self.status[domain]:
+                        # print(f"{slot} is not mentioned")
+                        return False
+                    goal = self.domain_goals[domain]["info"][slot].lower()
+                    status = self.status[domain][slot].lower()
+                    if goal != status and not is_time(goal, status):
+                        # print(f"conflict slot {slot}: {goal} <-> {status}")
+                        return False
+            if "reqt" in self.domain_goals[domain]:
+                for slot in self.domain_goals[domain]["reqt"]:
+                    if self.domain_goals[domain]["reqt"][slot] == DEF_VAL_UNK:
+                        # print(f"not fulfilled request{domain}-{slot}")
+                        return False
+
+        return True
+
+    def init_local_id(self):
+        # local_id = {
+        #     "domain 1": {
+        #         "ID": [1, 0, 0],
+        #         "SLOT": {
+        #             "slot 1": [1, 0, 0],
+        #             "slot 2": [0, 1, 0]}}}
+
+        for domain_id, domain in enumerate(self.domains):
+            self._init_domain_id(domain)
+            self._update_domain_id(domain, domain_id)
+            slot_id = 0
+            for slot_type in ["info", "book", "reqt"]:
+                for slot in self.domain_goals[domain].get(slot_type, {}):
+                    self._init_slot_id(domain, slot)
+                    self._update_slot_id(domain, slot, slot_id)
+                    slot_id += 1
+
+    def insert_local_id(self, new_slot_name):
+        # domain, slot = new_slot_name.split('-')
+        domain, slot = split_slot_name(new_slot_name)
+        if domain not in self.local_id:
+            self._init_domain_id(domain)
+            domain_id = len(self.domains) + 1
+            self._update_domain_id(domain, domain_id)
+            self._init_slot_id(domain, slot)
+            # the first slot for a new domain
+            self._update_slot_id(domain, slot, 0)
+
+        else:
+            slot_id = len(self.local_id[domain]["SLOT"]) + 1
+            self._init_slot_id(domain, slot)
+            self._update_slot_id(domain, slot, slot_id)
+
+    def get_slot_id(self, slot_name):
+        # print(slot_name)
+        # domain, slot = slot_name.split('-')
+        domain, slot = split_slot_name(slot_name)
+        if domain in self.local_id and slot in self.local_id[domain]["SLOT"]:
+            return self.local_id[domain]["ID"], self.local_id[domain]["SLOT"][slot]
+        else:  # a slot not in original user goal
+            self.insert_local_id(slot_name)
+            domain_id, slot_id = self.get_slot_id(slot_name)
+            return domain_id, slot_id
+
+    def action_list(self, sys_act=None):
+        priority_action = [x for x in self.user_history]
+
+        if sys_act:
+            for _, domain, slot, _ in sys_act:
+                slot_name = f"{domain}-{slot}"
+                if slot_name and slot_name not in priority_action:
+                    priority_action.insert(0, slot_name)
+
+        return priority_action
+
+    def update(self, action: list = None, char: str = "system"):
+        # update request and booked
+        if char not in ["user", "system"]:
+            print(f"unknown role: {char}")
+        self._update_status(action, char)
+        self._update_goal(action, char)
+        return self.status
+
+    def _update_status(self, action: list, char: str):
+        for intent, domain, slot, value in action:
+            if slot == "arrive by":
+                slot = "arriveBy"
+            elif slot == "leave at":
+                slot = "leaveAt"
+            if domain not in self.status:
+                self.status[domain] = {}
+            # update info
+            if "info" in intent:
+                self.status[domain][slot] = value
+            elif "request" in intent:
+                self.status[domain][slot] = DEF_VAL_UNK
+
+    def _update_goal(self, action: list, char: str):
+        # update requt slots in goal
+        for intent, domain, slot, value in action:
+            if slot == "arrive by":
+                slot = "arriveBy"
+            elif slot == "leave at":
+                slot = "leaveAt"
+            if "info" not in intent:
+                continue
+            if self._check_update_request(domain, slot) and value != "?":
+                self.domain_goals[domain]['reqt'][slot] = value
+                # print(f"update reqt {slot} = {value} from system action")
+
+    def _update_slot(self, domain, slot, value):
+        self.domain_goals[domain]['reqt'][slot] = value
+
+    def _check_update_request(self, domain, slot):
+        # check whether one slot is a request slot
+        if domain not in self.domain_goals:
+            return False
+        if 'reqt' not in self.domain_goals[domain]:
+            return False
+        if slot not in self.domain_goals[domain]['reqt']:
+            return False
+        return True
+
+    def _check_value(self, value=None):
+        if not value:
+            return False
+        if value in NOT_SURE_VALS:
+            return False
+        return True
+
+    def _init_domain_id(self, domain):
+        self.local_id[domain] = {"ID": [0] * self.max_domain_len, "SLOT": {}}
+
+    def _init_slot_id(self, domain, slot):
+        self.local_id[domain]["SLOT"][slot] = [0] * self.max_slot_len
+
+    def _update_domain_id(self, domain, domain_id):
+        if domain_id < self.max_domain_len:
+            self.local_id[domain]["ID"][domain_id] = 1
+        else:
+            print(
+                f"too many doamins: {domain_id} > {self.max_domain_len}")
+
+    def _update_slot_id(self, domain, slot, slot_id):
+        if slot_id < self.max_slot_len:
+            self.local_id[domain]["SLOT"][slot][slot_id] = 1
+        else:
+            print(
+                f"too many slots, {slot_id} > {self.max_slot_len}")
+
+
+if __name__ == "__main__":
+    data_goal = [["restaurant", "inform", "cuisine", "punjabi"],
+                 ["restaurant", "inform", "city", "milpitas"],
+                 ["restaurant", "request", "price_range", ""],
+                 ["restaurant", "request", "street_address", ""]]
+    goal = Goal(data_goal)
+    print(goal)
+    action = {"char": "system",
+              "action": [["request", "restaurant", "cuisine", "?"], ["request", "restaurant", "city", "?"]]}
+    goal.update(action["action"], action["char"])
+    print(goal.status)
+    print("complete:", goal.task_complete())
+    action = {"char": "user",
+              "action": [["inform", "restaurant", "cuisine", "punjabi"], ["inform", "restaurant", "city", "milpitas"]]}
+    goal.update(action["action"], action["char"])
+    print(goal.status)
+    print("complete:", goal.task_complete())
+    action = {"char": "system",
+              "action": [["inform", "restaurant", "price_range", "cheap"]]}
+    goal.update(action["action"], action["char"])
+    print(goal.status)
+    print("complete:", goal.task_complete())
+    action = {"char": "user",
+              "action": [["request", "restaurant", "street_address", ""]]}
+    goal.update(action["action"], action["char"])
+    print(goal.status)
+    print("complete:", goal.task_complete())
+    action = {"char": "system",
+              "action": [["inform", "restaurant", "street_address", "ABCD"]]}
+    goal.update(action["action"], action["char"])
+    print(goal.status)
+    print("complete:", goal.task_complete())
diff --git a/convlab/policy/tus/unify/TUS.py b/convlab/policy/tus/unify/TUS.py
new file mode 100644
index 0000000000000000000000000000000000000000..be931eca16b03164fea094d55c08b0c85acdbd64
--- /dev/null
+++ b/convlab/policy/tus/unify/TUS.py
@@ -0,0 +1,456 @@
+import json
+import os
+import random
+from copy import deepcopy
+
+import torch
+from convlab.policy.policy import Policy
+from convlab.policy.tus.multiwoz.transformer import TransformerActionPrediction
+from convlab.policy.tus.unify.Goal import Goal
+from convlab.policy.tus.unify.usermanager import BinaryFeature
+from convlab.policy.tus.unify.util import create_goal, split_slot_name
+from convlab.util import (load_dataset,
+                          relative_import_module_from_unified_datasets)
+from convlab.util.custom_util import model_downloader
+from convlab.task.multiwoz.goal_generator import GoalGenerator
+from convlab.policy.tus.unify.Goal import old_goal2list
+from convlab.policy.rule.multiwoz.policy_agenda_multiwoz import Goal as ABUS_Goal
+
+
+reverse_da, normalize_domain_slot_value = relative_import_module_from_unified_datasets(
+    'multiwoz21', 'preprocess.py', ['reverse_da', 'normalize_domain_slot_value'])
+
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+DEF_VAL_UNK = '?'  # Unknown
+DEF_VAL_DNC = 'dontcare'  # Do not care
+DEF_VAL_NUL = 'none'  # for none
+DEF_VAL_BOOKED = 'yes'  # for booked
+DEF_VAL_NOBOOK = 'no'  # for booked
+Inform = "inform"
+Request = "request"
+NOT_SURE_VALS = [DEF_VAL_UNK, DEF_VAL_DNC, DEF_VAL_NUL, DEF_VAL_NOBOOK, ""]
+
+
+# TODO not ready for unify dataformat now
+class UserActionPolicy(Policy):
+    def __init__(self, config, pretrain=True, dataset="multiwoz21"):
+        Policy.__init__(self)
+        self.dataset = dataset
+        if isinstance(config, str):
+            self.config = json.load(open(config))
+        else:
+            self.config = config
+
+        feat_type = self.config.get("feat_type", "binary")
+        # print("feat_type", feat_type)
+        self.feat_handler = BinaryFeature(self.config)
+
+        self.config["num_token"] = config["num_token"]
+        self.user = TransformerActionPrediction(self.config).to(device=DEVICE)
+        if pretrain:
+            model_path = os.path.join(
+                self.config["model_dir"], "model-non-zero")  # self.config["model_name"])
+            print(f"loading model from {model_path}...")
+            self.load(model_path)
+        self.user.eval()
+        self.use_domain_mask = self.config.get("domain_mask", False)
+        self.max_turn = 40
+        self.mentioned_domain = []
+        self.reward = {"success": 40,
+                       "fail": -20}
+        self.sys_acts = []
+        self.goal_gen = GoalGenerator()
+        self.raw_goal = None
+
+    def _no_offer(self, system_in):
+        for intent, domain, slot, value in system_in:
+            if intent.lower() == "nooffer":
+                self.terminated = True
+                return True
+            else:
+                return False
+
+    def predict(self, sys_dialog_act, mode="max"):
+        # update goal
+        self.predict_action_list = self.goal.action_list(sys_dialog_act)
+        cur_state = self.goal.update(action=sys_dialog_act, char="system")
+        self.sys_acts.append(sys_dialog_act)
+
+        # need better way to handle this
+        if self._no_offer(sys_dialog_act):
+            return [["bye", "general", "none", "none"]]
+
+        # update constraint
+        self.time_step += 2
+
+        feature, mask = self.feat_handler.get_feature(
+            all_slot=self.predict_action_list,
+            user_goal=self.goal,
+            cur_state=cur_state,
+            pre_state=self.sys_history_state,
+            sys_action=sys_dialog_act,
+            usr_action=self.pre_usr_act)
+        feature = torch.tensor([feature], dtype=torch.float).to(DEVICE)
+        mask = torch.tensor([mask], dtype=torch.bool).to(DEVICE)
+
+        self.sys_history_state = cur_state
+
+        usr_output = self.user.forward(feature, mask)
+        usr_action = self.transform_usr_act(
+            usr_output, self.predict_action_list, mode)
+        domains = [act[1] for act in usr_action]
+        none_slot_acts = self._add_none_slot_act(domains)
+        usr_action = none_slot_acts + usr_action
+
+        self.pre_usr_act = deepcopy(usr_action)
+
+        if len(usr_action) < 1:
+            print("EMPTY ACTION")
+
+        # convert user action to unify data format
+        norm_usr_action = []
+        for intent, domain, slot, value in usr_action:
+            intent = intent
+            # domain, slot, value = normalize_domain_slot_value(
+            #     domain, slot, value)
+            norm_usr_action.append([intent, domain, slot, value])
+
+        cur_state = self.goal.update(action=norm_usr_action, char="user")
+
+        return norm_usr_action
+
+        # return usr_action
+
+    def init_session(self, goal=None):
+        self.mentioned_domain = []
+        self.time_step = 0
+        self.topic = 'NONE'
+        remove_domain = "police"  # remove police domain in inference
+
+        if type(goal) == ABUS_Goal:
+            self.raw_goal = goal.domain_goals
+            goal_list = old_goal2list(goal.domain_goals)
+            goal = Goal(goal_list)
+        elif type(goal) == Goal:
+            self.raw_goal = goal.domain_goals
+        else:
+            goal = ABUS_Goal(self.goal_gen)
+            self.raw_goal = goal.domain_goals
+            goal_list = old_goal2list(goal.domain_goals)
+            goal = Goal(goal_list)
+
+        self.read_goal(goal)
+        self.feat_handler.initFeatureHandeler(self.goal)
+
+        # print(self.goal)
+        if self.config.get("reorder", False):
+            self.predict_action_list = self.goal.action_list()
+        else:
+            self.predict_action_list = self.action_list
+        self.sys_history_state = None  # to save sys history
+        self.terminated = False
+
+        self.pre_usr_act = None
+        self.sys_acts = []
+
+    def read_goal(self, data_goal):
+        if type(data_goal) == Goal:
+            self.goal = data_goal
+        else:
+            self.goal = Goal(goal=data_goal)
+
+    # def new_goal(self, remove_domain="police", domain_len=None):
+    #     keep_generate_goal = True
+    #     while keep_generate_goal:
+    #         self.goal = Goal(goal_generator=self.goal_gen)
+    #         if (domain_len and len(self.goal.domains) != domain_len) or \
+    #                 (remove_domain and remove_domain in self.goal.domains):
+    #             keep_generate_goal = True
+    #         else:
+    #             keep_generate_goal = False
+
+    def load(self, model_path=None):
+        self.user.load_state_dict(torch.load(model_path, map_location=DEVICE))
+
+    def load_state_dict(self, model=None):
+        self.user.load_state_dict(model)
+
+    def _get_goal(self):
+        # internal usage
+        return self.goal.domain_goals
+
+    def get_goal(self):
+        # for outside usage, e.g. evaluator
+        return self.raw_goal
+
+    def get_reward(self):
+        if self.goal.task_complete():
+            # reward = 2 * self.max_turn
+            reward = self.reward["success"]
+            # reward = 1
+
+        elif self.time_step >= self.max_turn:
+            # reward = -1 * self.max_turn
+            reward = self.reward["fail"]
+            # reward = -1
+
+        else:
+            # reward = -1.0
+            reward = 0
+        return reward
+
+    def _add_none_slot_act(self, domains):
+        actions = []
+        for domain in domains:
+            domain = domain.lower()
+            if domain not in self.mentioned_domain and domain != 'general':
+                actions.append([Inform, domain, "none", "none"])
+                self.mentioned_domain.append(domain)
+        return actions
+
+    def _finish_conversation(self):
+
+        if self.goal.task_complete():
+            return True, [['thank', 'general', 'none', 'none']]
+
+        if self.time_step > self.max_turn:
+            return True, [["bye", "general", "none", "none"]]
+
+        if len(self.sys_acts) >= 3:
+            if self.sys_acts[-1] == self.sys_acts[-2] and self.sys_acts[-2] == self.sys_acts[-3]:
+                return True, [["bye", "general", "none", "none"]]
+
+        return False, [[]]
+
+    def transform_usr_act(self, usr_output, action_list, mode="max"):
+        is_finish, usr_action = self._finish_conversation()
+        if is_finish:
+            self.terminated = True
+            # if "bye" == usr_action[0][0]:
+            #     print("fail")
+            #     pprint(self.goal.domain_goals)
+            #     pprint(self.goal.status)
+            return usr_action
+
+        usr_action = self._get_acts(
+            usr_output, action_list, mode)
+
+        # if usr_action is empty, sample at least one
+        while not usr_action:
+            usr_action = self._get_acts(
+                usr_output, action_list, mode="pick-one")
+
+        if self.use_domain_mask:
+            domain_mask = self._get_prediction_domain(torch.round(
+                torch.sigmoid(usr_output[0, 0, :])).tolist())
+            usr_action = self._mask_user_action(usr_action, domain_mask)
+
+        return usr_action
+
+    def _get_acts(self, usr_output, action_list, mode="max"):
+        score = {}
+        for index, slot_name in enumerate(action_list):
+            weights = self.user.softmax(usr_output[0, index + 1, :])
+            if mode == "max":
+                o = torch.argmax(usr_output[0, index + 1, :]).item()
+
+            elif mode == "sample" or mode == "pick-one":
+                o = random.choices(
+                    range(self.config["out_dim"]),
+                    weights=weights,
+                    k=1)
+                o = o[0]
+            else:
+                print("(BUG) unknown mode")
+            v = weights[o]
+            score[slot_name] = {"output": o, "weight": v}
+
+        usr_action = self._append_actions(action_list, score)
+
+        if mode == "sample" and len(usr_action) > 3:
+            slot_names = []
+            outputs = []
+            scores = []
+            for index, slot_name in enumerate(action_list):
+                weights = self.user.softmax(usr_output[0, index + 1, :])
+                o = torch.argmax(usr_output[0, index + 1, 1:]).item() + 1
+                slot_names.append(slot_name)
+                outputs.append(o)
+                scores.append(weights[o].item())
+            slot_name = random.choices(
+                slot_names,
+                weights=scores,
+                k=3)
+            slot_name = slot_name[0]
+            score[slot_name]["output"] = outputs[slot_names.index(slot_name)]
+            score[slot_name]["weight"] = scores[slot_names.index(slot_name)]
+            # print(score)
+            usr_action = self._append_actions(action_list, score)
+
+        if mode == "pick-one" and not usr_action:
+            # print("pick-one")
+            slot_names = []
+            outputs = []
+            scores = []
+            for index, slot_name in enumerate(action_list):
+                weights = self.user.softmax(usr_output[0, index + 1, :])
+                o = torch.argmax(usr_output[0, index + 1, 1:]).item() + 1
+                slot_names.append(slot_name)
+                outputs.append(o)
+                scores.append(weights[o].item())
+            slot_name = random.choices(
+                slot_names,
+                weights=scores,
+                k=1)
+            slot_name = slot_name[0]
+            score[slot_name]["output"] = outputs[slot_names.index(slot_name)]
+            score[slot_name]["weight"] = scores[slot_names.index(slot_name)]
+            # print(score)
+            usr_action = self._append_actions(action_list, score)
+
+        return usr_action
+
+    def _append_actions(self, action_list, score):
+        usr_action = []
+        for index, slot_name in enumerate(action_list):
+            domain, slot = split_slot_name(slot_name)
+            is_action, act = self._add_user_action(
+                output=score[slot_name]["output"],
+                domain=domain,
+                slot=slot)
+            if is_action:
+                usr_action += act
+        return usr_action
+
+    def _mask_user_action(self, usr_action, mask):
+        mask_action = []
+        for intent, domain, slot, value in usr_action:
+            if domain.lower() in mask:
+                mask_action += [[intent, domain, slot, value]]
+        return mask_action
+
+    def _get_prediction_domain(self, domain_output):
+        predict_domain = []
+        if domain_output[0] > 0:
+            predict_domain.append('general')
+        for index, value in enumerate(domain_output[1:]):
+            if value > 0 and index < len(self.goal.domains):
+                predict_domain.append(self.goal.domains[index])
+        return predict_domain
+
+    def _add_user_action(self, output, domain, slot):
+        goal = self._get_goal()
+        is_action = False
+        act = [[]]
+        value = None
+
+        # get intent
+        if output == 1:
+            intent = Request
+        else:
+            intent = Inform
+
+        # "?"
+        if output == 1:  # "?"
+            value = DEF_VAL_UNK
+
+        # "dontcare"
+        elif output == 2:
+            value = DEF_VAL_DNC
+
+        # system
+        elif output == 3 and domain in self.sys_history_state:
+            value = self.sys_history_state[domain].get(
+                slot, "")
+
+        elif output == 4 and domain in goal:  # usr
+            for slot_type in ["info"]:
+                if slot_type in goal[domain] and slot in goal[domain][slot_type]:
+                    value = goal[domain][slot_type][slot]
+
+        # elif output == 5 and domain.lower() in goal:
+        #     if domain.lower() not in self.all_values["all_value"]:
+        #         value = None
+        #     elif slot.lower() not in self.all_values["all_value"][domain.lower()]:
+        #         value = None
+        #     else:
+        #         value = random.choice(
+        #             list(self.all_values["all_value"][domain.lower()][slot.lower()].keys()))
+
+        if value:
+            is_action, act = self._form_action(
+                intent, domain, slot, value)
+
+        return is_action, act
+
+    def _get_action_slot(self, domain, slot):
+        return slot
+
+    def _form_action(self, intent, domain, slot, value):
+        action_slot = self._get_action_slot(domain, slot)
+        if action_slot:
+            return True, [[intent, domain, action_slot, value]]
+        return False, [[]]
+
+    def is_terminated(self):
+        # Is there any action to say?
+        return self.terminated
+
+    def _slot_type(self, domain, slot):
+        slot_type = ""
+        if slot in self.sys_history_state[domain]["book"]:
+            slot_type = "book"
+        elif slot in self.sys_history_state[domain]["semi"]:
+            slot_type = "semi"
+
+        return slot_type
+
+
+class UserPolicy(Policy):
+    def __init__(self, config, dial_ids_order=0):
+        if isinstance(config, str):
+            self.config = json.load(open(config))
+        else:
+            self.config = config
+        self.config["model_dir"] = f'{self.config["model_dir"]}_{dial_ids_order}'
+        print("model_dir", self.config['model_dir'])
+        if not os.path.exists(self.config["model_dir"]):
+            # os.mkdir(self.config["model_dir"])
+            model_downloader(os.path.dirname(self.config["model_dir"]),
+                             "https://zenodo.org/record/7369429/files/multiwoz_0.zip")
+        self.slot2dbattr = {
+            'open hours': 'openhours',
+            'price range': 'pricerange',
+            'arrive by': 'arriveBy',
+            'leave at': 'leaveAt',
+            'train id': 'trainID'
+        }
+        self.dbattr2slot = {}
+        for k, v in self.slot2dbattr.items():
+            self.dbattr2slot[v] = k
+
+        self.policy = UserActionPolicy(self.config)
+
+    def predict(self, state):
+        raw_act = self.policy.predict(state)
+        act = []
+        for intent, domain, slot, value in raw_act:
+            if slot in self.dbattr2slot:
+                slot = self.dbattr2slot[slot]
+            act.append([intent, domain, slot, value])
+        return act
+
+    def init_session(self, goal=None):
+        self.policy.init_session(goal)
+
+    def is_terminated(self):
+        return self.policy.is_terminated()
+
+    def get_reward(self):
+        return self.policy.get_reward()
+
+    def get_goal(self):
+        if hasattr(self.policy, 'get_goal'):
+            return self.policy.get_goal()
+        return None
diff --git a/convlab/policy/tus/unify/analysis.py b/convlab/policy/tus/unify/analysis.py
new file mode 100644
index 0000000000000000000000000000000000000000..f2461e3a024b0c83209521f3627ea66cf3a0ffd3
--- /dev/null
+++ b/convlab/policy/tus/unify/analysis.py
@@ -0,0 +1,260 @@
+import argparse
+import json
+import os
+
+import pandas as pd
+import torch
+from torch.utils.data import DataLoader
+from tqdm import tqdm
+
+from convlab.policy.rule.multiwoz import RulePolicy
+from convlab.policy.tus.unify.Goal import Goal
+from convlab.policy.tus.unify.TUS import UserPolicy
+from convlab.policy.tus.unify.usermanager import TUSDataManager
+from convlab.policy.tus.unify.util import create_goal, parse_dialogue_act
+from convlab.util import load_dataset
+
+
+def check_device():
+    if torch.cuda.is_available():
+        print("using GPU")
+        return torch.device('cuda')
+    else:
+        print("using CPU")
+        return torch.device('cpu')
+
+
+class Analysis:
+    def __init__(self, config, analysis_dir='user-analysis-result', show_dialog=False, save_dialog=True):
+        if not os.path.exists(analysis_dir):
+            os.makedirs(analysis_dir)
+        self.dialog_dir = os.path.join(analysis_dir, 'dialog')
+        if not os.path.exists(self.dialog_dir):
+            os.makedirs(self.dialog_dir)
+        self.dir = analysis_dir
+        self.config = config
+        self.device = check_device()
+        self.show_dialog = show_dialog
+        self.save_dialog = save_dialog
+        self.max_turn = 40
+
+    def get_usr(self, usr="tus", load_path=None):
+        # if using "tus", we read config
+        # for the other user simulators, we read load_path
+        usr = usr.lower()
+        if usr == "tus":
+            policy_usr = UserPolicy(self.config)
+        else:
+            print(f"Unsupport user type: {usr}")
+        # TODO VHUS
+
+        return policy_usr
+
+    def data_interact_test(self, test_data, usr="tus", user_mode=None, load_path=None):
+        if user_mode:
+            # origin_model_name = "-".join(self.config["model_name"].split('-')[:-1])
+            self.config["model_name"] = f"model-{user_mode}"
+
+        result = []
+        label = []
+        policy_usr = self.get_usr(usr=usr, load_path=load_path)
+
+        for dialog in tqdm(test_data):
+            if self.show_dialog:
+                print(f"dialog_id: {dialog['dialog_id']}")
+            goal = Goal(create_goal(dialog))
+
+            sys_act = []
+            policy_usr.init_session(goal=goal)
+            if not policy_usr.get_goal():
+                continue
+            turn_num = len(dialog["turns"])
+            start = 0
+            if dialog["turns"][0]["speaker"] == "system":
+                start = 1
+            for turn_id in range(start, turn_num, 2):
+                if turn_id > 0:
+                    sys_act = parse_dialogue_act(
+                        dialog["turns"][turn_id - 1]["dialogue_acts"])
+                usr_act = policy_usr.predict(sys_act)
+                golden_usr = parse_dialogue_act(
+                    dialog["turns"][turn_id]["dialogue_acts"])
+                result.append(usr_act)
+                label.append(golden_usr)
+
+        for domain in [None]:
+
+            statistic = self._data_f1(result, label, domain)
+            ana_result = {}
+            for stat_type in statistic:
+                s = statistic[stat_type]["success"] / \
+                    statistic[stat_type]["count"]
+                ana_result[stat_type] = s
+            ana_result["f1"] = 2/((1/ana_result["precision"])
+                                  * (1/ana_result["recall"]))
+
+            print(user_mode)
+            for stat_type in ana_result:
+                print(f'{stat_type}: {ana_result[stat_type]}')
+            col = [c for c in ana_result]
+            df_f1 = pd.DataFrame([ana_result[c] for c in col], col)
+            print(df_f1)
+            if domain:
+                df_f1.to_csv(os.path.join(
+                    self.dir, f'{domain}-{user_mode}_data_scores.csv'))
+            else:
+                df_f1.to_csv(os.path.join(
+                    self.config["model_dir"], f'{user_mode}_data_scores.csv'))
+
+    def _extract_domain_related_actions(self, actions, select_domain):
+        #
+        domain_related_acts = []
+        for act in actions:
+            domain = act[1].lower()
+            if domain == select_domain:
+                domain_related_acts.append(act)
+        return domain_related_acts
+
+    def _data_f1(self, result, label, domain=None):
+        #
+        statistic = {}
+        for stat_type in ["precision", "recall", "turn_acc"]:
+            statistic[stat_type] = {"success": 0, "count": 0}
+
+        for r, l in zip(result, label):
+            if domain:
+                r = self._extract_domain_related_actions(r, domain)
+                l = self._extract_domain_related_actions(l, domain)
+
+            if self._skip(l, r, domain):
+                continue
+            # check turn accuracy
+            turn_acc, tp, fp, fn = self._check(r, l)
+            if self.show_dialog:
+                print(r, l)
+                print(turn_acc, tp, fp, fn)
+            if turn_acc:
+                statistic["turn_acc"]["success"] += 1
+            statistic["turn_acc"]["count"] += 1
+            statistic["precision"]["success"] += tp
+            statistic["precision"]["count"] += tp + fp
+            statistic["recall"]["success"] += tp
+            statistic["recall"]["count"] += tp + fn
+
+        return statistic
+
+    @staticmethod
+    def _skip(label, result, domain=None):
+        #
+        ignore = False
+        if domain:
+            if not label and not result:
+                ignore = True
+        else:
+            if not label:
+                ignore = True
+            for intent, domain, slot, value in label:
+                if intent.lower() in ["thank", "bye"]:
+                    ignore = True
+
+        return ignore
+
+    def _check(self, r, l):
+        #
+        # TODO domain check
+        # [['Inform', 'Attraction', 'Addr', 'dontcare']] [['thank', 'general', 'none', 'none']]
+        # skip this one
+        turn_acc = True
+        tp = 0
+        fp = 0
+        fn = 0
+        for a in r:
+            is_none_slot, is_in = self._is_in(a, l)
+            if is_none_slot:
+                continue
+
+            if is_in:
+                tp += 1
+            else:
+                fp += 1
+                turn_acc = False
+
+        for a in l:
+            is_none_slot, is_in = self._is_in(a, r)
+            if is_none_slot:
+                continue
+
+            if is_in:
+                tp += 1
+            else:
+                fn += 1
+                turn_acc = False
+
+        return turn_acc, tp/2, fp, fn
+
+    @staticmethod
+    def _is_in(a, acts):
+        #
+        is_none_slot = False
+        intent, domain, slot, value = a
+        if slot.lower() == "none" or domain.lower() == "general":
+            is_none_slot = True
+            return is_none_slot, True
+        if a in acts:
+            return is_none_slot, True
+        else:
+            for i, d, s, v in acts:
+                if i == intent and d == domain and s == slot:
+                    return is_none_slot, True
+            return is_none_slot, False
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--analysis_dir", type=str,
+                        default="user-analysis-result")
+    parser.add_argument("--user_config", type=str,
+                        default="convlab/policy/tus/multiwoz/exp/multiwoz.json")
+    parser.add_argument("--user_mode", type=str, default="")
+    parser.add_argument("--do_data", action="store_true")
+    parser.add_argument("--usr", type=str, default="tus")
+    parser.add_argument("--domain", type=str, default="",
+                        help="the user goal must contain a specific domain")
+    parser.add_argument("--load_path", type=str, default="",
+                        help="load path for certain models")
+    parser.add_argument("--dataset", type=str, default="multiwoz21",
+                        help="data type")
+    parser.add_argument("--dial_ids_order", type=int, default=0)
+
+    args = parser.parse_args()
+
+    analysis_dir = os.path.join(f"{args.analysis_dir}-{args.usr}")
+
+    if not os.path.exists(os.path.join(analysis_dir)):
+        os.makedirs(analysis_dir)
+
+    config = json.load(open(args.user_config))
+    if args.user_mode:
+        config["model_name"] = config["model_name"] + '-' + args.user_mode
+
+    # config["model_dir"] = f'{config["model_dir"]}_{args.dial_ids_order}'
+    # with open(config["all_slot"]) as f:
+    #     action_list = [line.strip() for line in f]
+    # config["num_token"] = len(action_list)
+
+    ana = Analysis(config, analysis_dir=analysis_dir)
+
+    if args.usr == "tus" and args.do_data:
+        test_data = load_dataset(args.dataset,
+                                 dial_ids_order=args.dial_ids_order)["test"]
+        if args.user_mode:
+            ana.data_interact_test(test_data=test_data,
+                                   usr=args.usr,
+                                   user_mode=args.user_mode,
+                                   load_path=args.load_path)
+        else:
+            for user_mode in ["loss", "total", "turn", "non-zero"]:
+                ana.data_interact_test(test_data=test_data,
+                                       usr=args.usr,
+                                       user_mode=user_mode,
+                                       load_path=args.load_path)
diff --git a/convlab/policy/tus/unify/exp/all.json b/convlab/policy/tus/unify/exp/all.json
new file mode 100644
index 0000000000000000000000000000000000000000..b45b1b80725eeb8eb82cf582794cebfee09b3cd2
--- /dev/null
+++ b/convlab/policy/tus/unify/exp/all.json
@@ -0,0 +1,32 @@
+{
+    "model_dir": "convlab/policy/tus/unify/all",
+    "model_name": "model",
+    "num_epoch": 50,
+    "batch_size": 128,
+    "learning_rate": 0.0001,
+    "num_token": 65,
+    "debug": false,
+    "gelu": false,
+    "dropout": 0.1,
+    "embed_dim": 79,
+    "out_dim": 6,
+    "hidden": 200,
+    "num_transformer": 2,
+    "weight_factor": [
+        1,
+        1,
+        1,
+        1,
+        1,
+        5
+    ],
+    "window": 3,
+    "nhead": 4,
+    "dim_feedforward": 200,
+    "num_transform_layer": 2,
+    "turn-pos": false,
+    "reorder": true,
+    "conflict": false,
+    "remove_domain": "police",
+    "domain_feat": true
+}
\ No newline at end of file
diff --git a/convlab/policy/tus/unify/exp/default.json b/convlab/policy/tus/unify/exp/default.json
new file mode 100644
index 0000000000000000000000000000000000000000..5d681849f8b478f8d6b3efe4e86d937e90277f56
--- /dev/null
+++ b/convlab/policy/tus/unify/exp/default.json
@@ -0,0 +1,32 @@
+{
+    "model_dir": "convlab/policy/tus/multiwoz/default",
+    "model_name": "model",
+    "num_epoch": 50,
+    "batch_size": 128,
+    "learning_rate": 0.0001,
+    "num_token": 65,
+    "debug": false,
+    "gelu": false,
+    "dropout": 0.1,
+    "embed_dim": 79,
+    "out_dim": 6,
+    "hidden": 200,
+    "num_transformer": 2,
+    "weight_factor": [
+        1,
+        1,
+        1,
+        1,
+        1,
+        5
+    ],
+    "window": 3,
+    "nhead": 4,
+    "dim_feedforward": 200,
+    "num_transform_layer": 2,
+    "turn-pos": false,
+    "reorder": true,
+    "conflict": false,
+    "remove_domain": "police",
+    "domain_feat": true
+}
\ No newline at end of file
diff --git a/convlab/policy/tus/unify/exp/multiwoz.json b/convlab/policy/tus/unify/exp/multiwoz.json
new file mode 100644
index 0000000000000000000000000000000000000000..46d90c8b4e76d43f74491b4fd9259cab849c2299
--- /dev/null
+++ b/convlab/policy/tus/unify/exp/multiwoz.json
@@ -0,0 +1,32 @@
+{
+    "model_dir": "convlab/policy/tus/unify/multiwoz",
+    "model_name": "model",
+    "num_epoch": 50,
+    "batch_size": 128,
+    "learning_rate": 0.0001,
+    "num_token": 65,
+    "debug": false,
+    "gelu": false,
+    "dropout": 0.1,
+    "embed_dim": 79,
+    "out_dim": 6,
+    "hidden": 200,
+    "num_transformer": 2,
+    "weight_factor": [
+        1,
+        1,
+        1,
+        1,
+        1,
+        5
+    ],
+    "window": 3,
+    "nhead": 4,
+    "dim_feedforward": 200,
+    "num_transform_layer": 2,
+    "turn-pos": false,
+    "reorder": true,
+    "conflict": false,
+    "remove_domain": "police",
+    "domain_feat": true
+}
\ No newline at end of file
diff --git a/convlab/policy/tus/unify/exp/multiwoz_001.json b/convlab/policy/tus/unify/exp/multiwoz_001.json
new file mode 100644
index 0000000000000000000000000000000000000000..5aa3ec7665b82102ef1979e857f8318e5ad09e9a
--- /dev/null
+++ b/convlab/policy/tus/unify/exp/multiwoz_001.json
@@ -0,0 +1,32 @@
+{
+    "model_dir": "convlab/policy/tus/unify/multiwoz001",
+    "model_name": "model",
+    "num_epoch": 50,
+    "batch_size": 128,
+    "learning_rate": 0.0001,
+    "num_token": 65,
+    "debug": false,
+    "gelu": false,
+    "dropout": 0.1,
+    "embed_dim": 79,
+    "out_dim": 6,
+    "hidden": 200,
+    "num_transformer": 2,
+    "weight_factor": [
+        1,
+        1,
+        1,
+        1,
+        1,
+        5
+    ],
+    "window": 3,
+    "nhead": 4,
+    "dim_feedforward": 200,
+    "num_transform_layer": 2,
+    "turn-pos": false,
+    "reorder": true,
+    "conflict": false,
+    "remove_domain": "police",
+    "domain_feat": true
+}
\ No newline at end of file
diff --git a/convlab/policy/tus/unify/exp/multiwoz_01.json b/convlab/policy/tus/unify/exp/multiwoz_01.json
new file mode 100644
index 0000000000000000000000000000000000000000..34fc3587ed0e037c11e910ee42511b583bd6adf8
--- /dev/null
+++ b/convlab/policy/tus/unify/exp/multiwoz_01.json
@@ -0,0 +1,32 @@
+{
+    "model_dir": "convlab/policy/tus/unify/multiwoz01",
+    "model_name": "model",
+    "num_epoch": 50,
+    "batch_size": 128,
+    "learning_rate": 0.0001,
+    "num_token": 65,
+    "debug": false,
+    "gelu": false,
+    "dropout": 0.1,
+    "embed_dim": 79,
+    "out_dim": 6,
+    "hidden": 200,
+    "num_transformer": 2,
+    "weight_factor": [
+        1,
+        1,
+        1,
+        1,
+        1,
+        5
+    ],
+    "window": 3,
+    "nhead": 4,
+    "dim_feedforward": 200,
+    "num_transform_layer": 2,
+    "turn-pos": false,
+    "reorder": true,
+    "conflict": false,
+    "remove_domain": "police",
+    "domain_feat": true
+}
\ No newline at end of file
diff --git a/convlab/policy/tus/unify/exp/pretrain-multiwoz.json b/convlab/policy/tus/unify/exp/pretrain-multiwoz.json
new file mode 100644
index 0000000000000000000000000000000000000000..a101658d85b23dc5d1634cc454c9de36f9769ae3
--- /dev/null
+++ b/convlab/policy/tus/unify/exp/pretrain-multiwoz.json
@@ -0,0 +1,33 @@
+{
+    "model_dir": "convlab/policy/tus/unify/pretrain-multiwoz",
+    "pretrain": "convlab/policy/tus/unify/sgd+tm",
+    "model_name": "model",
+    "num_epoch": 50,
+    "batch_size": 128,
+    "learning_rate": 0.0001,
+    "num_token": 65,
+    "debug": false,
+    "gelu": false,
+    "dropout": 0.1,
+    "embed_dim": 79,
+    "out_dim": 6,
+    "hidden": 200,
+    "num_transformer": 2,
+    "weight_factor": [
+        1,
+        1,
+        1,
+        1,
+        1,
+        5
+    ],
+    "window": 3,
+    "nhead": 4,
+    "dim_feedforward": 200,
+    "num_transform_layer": 2,
+    "turn-pos": false,
+    "reorder": true,
+    "conflict": false,
+    "remove_domain": "police",
+    "domain_feat": true
+}
\ No newline at end of file
diff --git a/convlab/policy/tus/unify/exp/pretrain-multiwoz_001.json b/convlab/policy/tus/unify/exp/pretrain-multiwoz_001.json
new file mode 100644
index 0000000000000000000000000000000000000000..cca2bef73572c6932cf74a4fc540079bb60327ac
--- /dev/null
+++ b/convlab/policy/tus/unify/exp/pretrain-multiwoz_001.json
@@ -0,0 +1,33 @@
+{
+    "model_dir": "convlab/policy/tus/unify/pretrain-multiwoz001",
+    "pretrain": "convlab/policy/tus/unify/sgd+tm",
+    "model_name": "model",
+    "num_epoch": 50,
+    "batch_size": 128,
+    "learning_rate": 0.0001,
+    "num_token": 65,
+    "debug": false,
+    "gelu": false,
+    "dropout": 0.1,
+    "embed_dim": 79,
+    "out_dim": 6,
+    "hidden": 200,
+    "num_transformer": 2,
+    "weight_factor": [
+        1,
+        1,
+        1,
+        1,
+        1,
+        5
+    ],
+    "window": 3,
+    "nhead": 4,
+    "dim_feedforward": 200,
+    "num_transform_layer": 2,
+    "turn-pos": false,
+    "reorder": true,
+    "conflict": false,
+    "remove_domain": "police",
+    "domain_feat": true
+}
\ No newline at end of file
diff --git a/convlab/policy/tus/unify/exp/pretrain-multiwoz_01.json b/convlab/policy/tus/unify/exp/pretrain-multiwoz_01.json
new file mode 100644
index 0000000000000000000000000000000000000000..a662dfc5df459819076e58f680388abeb919137c
--- /dev/null
+++ b/convlab/policy/tus/unify/exp/pretrain-multiwoz_01.json
@@ -0,0 +1,33 @@
+{
+    "model_dir": "convlab/policy/tus/unify/multiwoz01",
+    "pretrain": "convlab/policy/tus/unify/sgd+tm",
+    "model_name": "model",
+    "num_epoch": 50,
+    "batch_size": 128,
+    "learning_rate": 0.0001,
+    "num_token": 65,
+    "debug": false,
+    "gelu": false,
+    "dropout": 0.1,
+    "embed_dim": 79,
+    "out_dim": 6,
+    "hidden": 200,
+    "num_transformer": 2,
+    "weight_factor": [
+        1,
+        1,
+        1,
+        1,
+        1,
+        5
+    ],
+    "window": 3,
+    "nhead": 4,
+    "dim_feedforward": 200,
+    "num_transform_layer": 2,
+    "turn-pos": false,
+    "reorder": true,
+    "conflict": false,
+    "remove_domain": "police",
+    "domain_feat": true
+}
\ No newline at end of file
diff --git a/convlab/policy/tus/unify/exp/sgd+tm.json b/convlab/policy/tus/unify/exp/sgd+tm.json
new file mode 100644
index 0000000000000000000000000000000000000000..d16e23aba23a1342f0ef5bca4e07933fa998a065
--- /dev/null
+++ b/convlab/policy/tus/unify/exp/sgd+tm.json
@@ -0,0 +1,32 @@
+{
+    "model_dir": "convlab/policy/tus/unify/sgd+tm",
+    "model_name": "model",
+    "num_epoch": 50,
+    "batch_size": 128,
+    "learning_rate": 0.0001,
+    "num_token": 65,
+    "debug": false,
+    "gelu": false,
+    "dropout": 0.1,
+    "embed_dim": 79,
+    "out_dim": 6,
+    "hidden": 200,
+    "num_transformer": 2,
+    "weight_factor": [
+        1,
+        1,
+        1,
+        1,
+        1,
+        5
+    ],
+    "window": 3,
+    "nhead": 4,
+    "dim_feedforward": 200,
+    "num_transform_layer": 2,
+    "turn-pos": false,
+    "reorder": true,
+    "conflict": false,
+    "remove_domain": "police",
+    "domain_feat": true
+}
\ No newline at end of file
diff --git a/convlab/policy/tus/unify/exp/sgd.json b/convlab/policy/tus/unify/exp/sgd.json
new file mode 100644
index 0000000000000000000000000000000000000000..8961ee35bcbf9e90ed4ba80761e1548e9be970bc
--- /dev/null
+++ b/convlab/policy/tus/unify/exp/sgd.json
@@ -0,0 +1,32 @@
+{
+    "model_dir": "convlab/policy/tus/unify/sgd",
+    "model_name": "model",
+    "num_epoch": 50,
+    "batch_size": 128,
+    "learning_rate": 0.0001,
+    "num_token": 65,
+    "debug": false,
+    "gelu": false,
+    "dropout": 0.1,
+    "embed_dim": 79,
+    "out_dim": 6,
+    "hidden": 200,
+    "num_transformer": 2,
+    "weight_factor": [
+        1,
+        1,
+        1,
+        1,
+        1,
+        5
+    ],
+    "window": 3,
+    "nhead": 4,
+    "dim_feedforward": 200,
+    "num_transform_layer": 2,
+    "turn-pos": false,
+    "reorder": true,
+    "conflict": false,
+    "remove_domain": "police",
+    "domain_feat": true
+}
\ No newline at end of file
diff --git a/convlab/policy/tus/unify/exp/tm1.json b/convlab/policy/tus/unify/exp/tm1.json
new file mode 100644
index 0000000000000000000000000000000000000000..7ab472b7fdc5795b0452f3d53709842d8a0a7df5
--- /dev/null
+++ b/convlab/policy/tus/unify/exp/tm1.json
@@ -0,0 +1,32 @@
+{
+    "model_dir": "convlab/policy/tus/unify/tm1",
+    "model_name": "model",
+    "num_epoch": 50,
+    "batch_size": 128,
+    "learning_rate": 0.0001,
+    "num_token": 65,
+    "debug": false,
+    "gelu": false,
+    "dropout": 0.1,
+    "embed_dim": 79,
+    "out_dim": 6,
+    "hidden": 200,
+    "num_transformer": 2,
+    "weight_factor": [
+        1,
+        1,
+        1,
+        1,
+        1,
+        5
+    ],
+    "window": 3,
+    "nhead": 4,
+    "dim_feedforward": 200,
+    "num_transform_layer": 2,
+    "turn-pos": false,
+    "reorder": true,
+    "conflict": false,
+    "remove_domain": "police",
+    "domain_feat": true
+}
\ No newline at end of file
diff --git a/convlab/policy/tus/unify/exp/tm2.json b/convlab/policy/tus/unify/exp/tm2.json
new file mode 100644
index 0000000000000000000000000000000000000000..47430090f570afffa5e6b3f07d7383986efe29d8
--- /dev/null
+++ b/convlab/policy/tus/unify/exp/tm2.json
@@ -0,0 +1,32 @@
+{
+    "model_dir": "convlab/policy/tus/unify/tm2",
+    "model_name": "model",
+    "num_epoch": 50,
+    "batch_size": 128,
+    "learning_rate": 0.0001,
+    "num_token": 65,
+    "debug": false,
+    "gelu": false,
+    "dropout": 0.1,
+    "embed_dim": 79,
+    "out_dim": 6,
+    "hidden": 200,
+    "num_transformer": 2,
+    "weight_factor": [
+        1,
+        1,
+        1,
+        1,
+        1,
+        5
+    ],
+    "window": 3,
+    "nhead": 4,
+    "dim_feedforward": 200,
+    "num_transform_layer": 2,
+    "turn-pos": false,
+    "reorder": true,
+    "conflict": false,
+    "remove_domain": "police",
+    "domain_feat": true
+}
\ No newline at end of file
diff --git a/convlab/policy/tus/unify/exp/tm3.json b/convlab/policy/tus/unify/exp/tm3.json
new file mode 100644
index 0000000000000000000000000000000000000000..fb1857cacbae8006c14a9b616d58aed5312aa303
--- /dev/null
+++ b/convlab/policy/tus/unify/exp/tm3.json
@@ -0,0 +1,32 @@
+{
+    "model_dir": "convlab/policy/tus/unify/tm3",
+    "model_name": "model",
+    "num_epoch": 50,
+    "batch_size": 128,
+    "learning_rate": 0.0001,
+    "num_token": 65,
+    "debug": false,
+    "gelu": false,
+    "dropout": 0.1,
+    "embed_dim": 79,
+    "out_dim": 6,
+    "hidden": 200,
+    "num_transformer": 2,
+    "weight_factor": [
+        1,
+        1,
+        1,
+        1,
+        1,
+        5
+    ],
+    "window": 3,
+    "nhead": 4,
+    "dim_feedforward": 200,
+    "num_transform_layer": 2,
+    "turn-pos": false,
+    "reorder": true,
+    "conflict": false,
+    "remove_domain": "police",
+    "domain_feat": true
+}
\ No newline at end of file
diff --git a/convlab/policy/tus/unify/train.py b/convlab/policy/tus/unify/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..dbaac0ab6772eca7cc2d0481ebb5c88afbe74197
--- /dev/null
+++ b/convlab/policy/tus/unify/train.py
@@ -0,0 +1,285 @@
+import json
+import os
+
+import torch
+import torch.optim as optim
+from tqdm import tqdm
+from convlab.policy.tus.multiwoz.analysis import Analysis
+from convlab.util import load_dataset, load_ontology
+
+
+def check_device():
+    if torch.cuda.is_available():
+        print("using GPU")
+        return torch.device('cuda')
+    else:
+        print("using CPU")
+        return torch.device('cpu')
+
+
+class Trainer:
+    def __init__(self, model, config):
+        self.model = model
+        self.config = config
+        self.num_epoch = self.config["num_epoch"]
+        self.batch_size = self.config["batch_size"]
+        self.device = check_device()
+        print(self.device)
+        self.optimizer = optim.Adam(
+            model.parameters(), lr=self.config["learning_rate"])
+
+        self.ana = Analysis(config)
+
+    def training(self, train_data, test_data=None):
+
+        self.model = self.model.to(self.device)
+        if not os.path.exists(self.config["model_dir"]):
+            os.makedirs(self.config["model_dir"])
+
+        save_path = os.path.join(
+            self.config["model_dir"], self.config["model_name"])
+
+        # best = [0, 0, 0]
+        best = {"loss": 100}
+        lowest_loss = 100
+        for epoch in range(self.num_epoch):
+            print("epoch {}".format(epoch))
+            total_loss = self.train_epoch(train_data)
+            print("loss: {}".format(total_loss))
+            if test_data is not None:
+                acc = self.eval(test_data)
+
+            if total_loss < lowest_loss:
+                best["loss"] = total_loss
+                print(f"save model in {save_path}-loss")
+                torch.save(self.model.state_dict(), f"{save_path}-loss")
+
+            for acc_type in acc:
+                if acc_type not in best:
+                    best[acc_type] = 0
+                temp = acc[acc_type]["correct"] / acc[acc_type]["total"]
+                if best[acc_type] < temp:
+                    best[acc_type] = temp
+                    print(f"save model in {save_path}-{acc_type}")
+                    torch.save(self.model.state_dict(),
+                               f"{save_path}-{acc_type}")
+            if epoch < 10 and epoch > 5:
+                print(f"save model in {save_path}-{epoch}")
+                torch.save(self.model.state_dict(),
+                           f"{save_path}-{epoch}")
+            print(f"save latest model in {save_path}")
+            torch.save(self.model.state_dict(), save_path)
+
+    def train_epoch(self, data_loader):
+        self.model.train()
+        total_loss = 0
+        result = {}
+        # result = {"id": {"slot": {"prediction": [],"label": []}}}
+        count = 0
+        for i, data in enumerate(tqdm(data_loader, ascii=True, desc="Training"), 0):
+            input_feature = data["input"].to(self.device)
+            mask = data["mask"].to(self.device)
+            label = data["label"].to(self.device)
+            if self.config.get("domain_traget", True):
+                domain = data["domain"].to(self.device)
+            else:
+                domain = None
+            self.optimizer.zero_grad()
+
+            loss, output = self.model(input_feature, mask, label, domain)
+
+            loss.backward()
+            self.optimizer.step()
+            total_loss += float(loss)
+            count += 1
+
+        return total_loss / count
+
+    def eval(self, test_data):
+        self.model.zero_grad()
+        self.model.eval()
+
+        result = {}
+
+        with torch.no_grad():
+            correct, total, non_zero_correct, non_zero_total = 0, 0, 0, 0
+            for i, data in enumerate(tqdm(test_data, ascii=True, desc="Evaluation"), 0):
+                input_feature = data["input"].to(self.device)
+                mask = data["mask"].to(self.device)
+                label = data["label"].to(self.device)
+                output = self.model(input_feature, mask)
+                r = parse_result(output, label)
+                for r_type in r:
+                    if r_type not in result:
+                        result[r_type] = {"correct": 0, "total": 0}
+                    for n in result[r_type]:
+                        result[r_type][n] += float(r[r_type][n])
+
+        for r_type in result:
+            temp = result[r_type]['correct'] / result[r_type]['total']
+            print(f"{r_type}: {temp}")
+
+        return result
+
+
+def parse_result(prediction, label):
+    # result = {"id": {"slot": {"prediction": [],"label": []}}}
+    # dialog_index = ["dialog-id"_"slot-name", "dialog-id"_"slot-name", ...]
+    # prdiction = [0, 1, 0, ...] # after max
+
+    _, arg_prediction = torch.max(prediction.data, -1)
+    batch_size, token_num = label.shape
+    result = {
+        "non-zero": {"correct": 0, "total": 0},
+        "total": {"correct": 0, "total": 0},
+        "turn": {"correct": 0, "total": 0}
+    }
+
+    for batch_num in range(batch_size):
+        turn_acc = True
+        for element in range(token_num):
+            result["total"]["total"] += 1
+            if label[batch_num][element] > 0:
+                result["non-zero"]["total"] += 1
+
+            if arg_prediction[batch_num][element + 1] == label[batch_num][element]:
+                if label[batch_num][element] > 0:
+                    result["non-zero"]["correct"] += 1
+                result["total"]["correct"] += 1
+
+            elif arg_prediction[batch_num][element + 1] == 0 and label[batch_num][element] < 0:
+                result["total"]["correct"] += 1
+
+            else:
+                if label[batch_num][element] >= 0:
+                    turn_acc = False
+
+        result["turn"]["total"] += 1
+        if turn_acc:
+            result["turn"]["correct"] += 1
+
+    return result
+
+
+def f1(target, result):
+    target_len = 0
+    result_len = 0
+    tp = 0
+    for t, r in zip(target, result):
+        if t:
+            target_len += 1
+        if r:
+            result_len += 1
+        if r == t and t:
+            tp += 1
+    precision = 0
+    recall = 0
+    if result_len:
+        precision = tp / result_len
+    if target_len:
+        recall = tp / target_len
+    f1_score = 2 / (1 / precision + 1 / recall)
+    return f1_score, precision, recall
+
+
+def save_data(data, file_name, file_dir):
+    if not os.path.exists(file_dir):
+        os.makedirs(file_dir)
+    f_name = os.path.join(file_dir, file_name)
+    torch.save(data, f_name)
+    # with open(f_name, 'wb') as data_obj:
+    #     pickle.dump(data, data_obj, pickle.HIGHEST_PROTOCOL)
+    print(f"save data to {f_name}")
+
+
+if __name__ == "__main__":
+    import argparse
+    import os
+    from convlab.policy.tus.multiwoz.transformer import \
+        TransformerActionPrediction
+    from convlab.policy.tus.unify.usermanager import \
+        TUSDataManager
+    from torch.utils.data import DataLoader
+    from convlab.policy.tus.unify.util import update_config_file
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--user-config", type=str,
+                        default="convlab/policy/tus/unify/exp/default.json")
+    parser.add_argument("--force-read-data", '-f', action='store_true',
+                        help="Force to read data from scratch")
+    parser.add_argument("--dataset", type=str, default="multiwoz21")
+    parser.add_argument("--dial-ids-order", type=int, default=0)
+    parser.add_argument("--split2ratio", type=float, default=1)
+
+    args = parser.parse_args()
+    config_file = open(args.user_config)
+    config = json.load(config_file)
+    config_file.close()
+    if args.dataset == "all":
+        print("merge all datasets...")
+        all_dataset = ["multiwoz21", "sgd", "tm1", "tm2", "tm3"]
+        datasets = {}
+        for dataset in all_dataset:
+            datasets[dataset] = load_dataset(dataset,
+                                             dial_ids_order=args.dial_ids_order,
+                                             split2ratio={'train': args.split2ratio})
+        # merge dataset
+        raw_data = {}
+        for data_type in ["train", "test"]:
+            raw_data[data_type] = []
+            for dataset in all_dataset:
+                raw_data[data_type] += datasets[dataset][data_type]
+
+    elif args.dataset == "sgd+tm":
+        print("merge multiple datasets...")
+        all_dataset = ["sgd", "tm1", "tm2", "tm3"]
+        datasets = {}
+        for dataset in all_dataset:
+            datasets[dataset] = load_dataset(dataset,
+                                             dial_ids_order=args.dial_ids_order,
+                                             split2ratio={'train': args.split2ratio})
+        # merge dataset
+        raw_data = {}
+        for data_type in ["train", "test"]:
+            raw_data[data_type] = []
+            for dataset in all_dataset:
+                raw_data[data_type] += datasets[dataset][data_type]
+
+    else:
+        print(f"load single dataset {args.dataset}/{args.split2ratio}")
+        raw_data = load_dataset(args.dataset,
+                                dial_ids_order=args.dial_ids_order,
+                                split2ratio={'train': args.split2ratio})
+
+    batch_size = config["batch_size"]
+
+    # load data with "load_data"
+
+    # check train/test data
+    data = {"train": {}, "test": {}}
+    for data_type in data:
+        data[data_type]["data"] = TUSDataManager(
+            config, raw_data[data_type])
+
+    # check the embed_dim and update it
+    embed_dim = data["train"]["data"].features["input"].shape[-1]
+    if embed_dim != config["embed_dim"]:
+        config["embed_dim"] = embed_dim
+        update_config_file(file_name=args.user_config,
+                           attribute="embed_dim", value=embed_dim)
+
+    train_data = DataLoader(data["train"]["data"],
+                            batch_size=batch_size, shuffle=True)
+    test_data = DataLoader(data["test"]["data"],
+                           batch_size=batch_size, shuffle=True)
+
+    model = TransformerActionPrediction(config)
+
+    if "pretrain" in config:
+        pretrain_weight = os.path.join(
+            f'{config["pretrain"]}_{args.dial_ids_order}', f"model-loss")
+        print(f"fine tune based on {pretrain_weight}...")
+        model.load_state_dict(torch.load(
+            pretrain_weight, map_location=check_device()))
+    trainer = Trainer(model, config)
+    trainer.training(train_data, test_data)
diff --git a/convlab/policy/tus/unify/usermanager.py b/convlab/policy/tus/unify/usermanager.py
new file mode 100644
index 0000000000000000000000000000000000000000..3192d3d578ca3698424b16f47933d6863208f77c
--- /dev/null
+++ b/convlab/policy/tus/unify/usermanager.py
@@ -0,0 +1,515 @@
+import json
+import os
+from collections import Counter
+
+import torch
+from convlab.policy.tus.unify.Goal import Goal
+from convlab.policy.tus.unify.util import parse_dialogue_act, parse_user_goal, metadata2state, int2onehot, create_goal, split_slot_name
+from torch.utils.data import Dataset
+from tqdm import tqdm
+
+NOT_MENTIONED = "not mentioned"
+
+
+def dic2list(da2goal):
+    action_list = []
+    for domain in da2goal:
+        for slot in da2goal[domain]:
+            if da2goal[domain][slot] is None:
+                continue
+            act = f"{domain}-{da2goal[domain][slot]}"
+            if act not in action_list:
+                action_list.append(act)
+    return action_list
+
+
+class TUSDataManager(Dataset):
+    def __init__(self,
+                 config,
+                 data,
+                 max_turns=12):
+
+        self.config = config
+        self.feature_handler = BinaryFeature(self.config)
+        self.features = self.process(data, max_turns)
+
+    def __getitem__(self, index):
+        return {label: self.features[label][index] if self.features[label] is not None else None
+                for label in self.features}
+
+    def __len__(self):
+        return self.features['input'].size(0)
+
+    def resample(self, size=None):
+        n_dialogues = self.__len__()
+        if not size:
+            size = n_dialogues
+
+        dialogues = torch.randint(low=0, high=n_dialogues, size=(size,))
+        self.features = {
+            label: self.features[label][dialogues] for label in self.features}
+
+    def to(self, device):
+        self.device = device
+        self.features = {label: self.features[label].to(
+            device) for label in self.features}
+
+    def process(self, data, max_turns):
+
+        feature = {"id": [], "input": [],
+                   "label": [], "mask": [], "domain": []}
+        # TODO remove dst. trace in user goal
+        for dialog in tqdm(data, ascii=True, desc="Processing"):
+            # TODO build user goal from history
+            user_goal = Goal(create_goal(dialog))
+            if not user_goal.domain_goals:
+                continue
+
+            # if one domain is removed, we skip all data related to this domain
+            # remove police at default
+            if "police" in user_goal.domains:
+                continue
+
+            turn_num = len(dialog["turns"])
+            pre_state = {}
+            sys_act = []
+            self.feature_handler.initFeatureHandeler(user_goal)
+
+            start = 0
+            if dialog["turns"][0]["speaker"] == "system":
+                start = 1
+
+            for turn_id in range(start, turn_num, 2):
+                # dialog start from user
+                action_list = user_goal.action_list(sys_act)
+                if turn_id > 0:
+                    # cur_state = data[dialog_id]["log"][turn_id-1]["metadata"]
+                    sys_act = parse_dialogue_act(
+                        dialog["turns"][turn_id - 1]["dialogue_acts"])
+                cur_state = user_goal.update(action=sys_act, char="system")
+
+                usr_act = parse_dialogue_act(
+                    dialog["turns"][turn_id]["dialogue_acts"])
+
+                input_feature, mask = self.feature_handler.get_feature(
+                    action_list, user_goal, cur_state, pre_state, sys_act)  # TODO why
+                label = self.feature_handler.generate_label(
+                    action_list, user_goal, cur_state, usr_act)
+                domain_label = self.feature_handler.domain_label(
+                    user_goal, usr_act)
+                # pre_state = user_goal.update(action=usr_act, char="user") # trick?
+                feature["id"].append(dialog["dialogue_id"])
+                feature["input"].append(input_feature)
+                feature["mask"].append(mask)
+                feature["label"].append(label)
+                feature["domain"].append(domain_label)
+
+        print("label distribution")
+        label_distribution = Counter()
+        for label in feature["label"]:
+            label_distribution += Counter(label)
+        print(label_distribution)
+        feature["input"] = torch.tensor(feature["input"], dtype=torch.float)
+        feature["label"] = torch.tensor(feature["label"], dtype=torch.long)
+        feature["mask"] = torch.tensor(feature["mask"], dtype=torch.bool)
+        feature["domain"] = torch.tensor(feature["domain"], dtype=torch.float)
+        for feat_type in ["input", "label", "mask", "domain"]:
+            print("{}: {}".format(feat_type, feature[feat_type].shape))
+        return feature
+
+
+class Feature:
+    def __init__(self, config):
+        self.config = config
+        self.intents = ["inform", "request", "recommend", "select",
+                        "book", "nobook", "offerbook", "offerbooked", "nooffer"]
+        self.general_intent = ["reqmore", "bye", "thank", "welcome", "greet"]
+        self.default_values = ["none", "?", "dontcare"]
+        # path = os.path.dirname(
+        #     os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
+        # path = os.path.join(path, 'data/multiwoz/all_value.json')
+        # self.all_values = json.load(open(path))
+
+    def initFeatureHandeler(self, goal: Goal):
+        self.goal = goal
+        self.domain_list = goal.domains
+        usr = parse_user_goal(goal)
+        self.constrains = {}  # slot: fulfill
+        self.requirements = {}  # slot: fulfill
+        self.pre_usr = []
+        self.all_slot = None
+        self.user_feat_hist = []
+        for slot in usr:
+            if usr[slot] != "?":
+                self.constrains[slot] = NOT_MENTIONED
+
+    def get_feature(self, all_slot, user_goal, cur_state, pre_state=None, sys_action=None, usr_action=None, state_vectorize=False):
+        """ 
+        given current dialog information and return the input feature 
+        user_goal: Goal()
+        cur_state: dict, {domain: "semi": {slot: value}, "book": {slot: value, "booked": []}}("metadata" in the data set)
+        sys_action: [[intent, domain, slot, value]]
+        """
+
+        feature = []
+        usr = parse_user_goal(user_goal)
+
+        if sys_action and not state_vectorize:
+            self.update_constrain(sys_action)
+
+        cur = metadata2state(cur_state)
+        pre = {}
+        if pre_state != None:
+            pre = metadata2state(pre_state)
+        if not self.pre_usr and not state_vectorize:
+            self.pre_usr = [0] * len(all_slot)
+
+        usr_act_feat = self.get_user_action_feat(
+            all_slot, user_goal, usr_action)
+
+        for slot in all_slot:
+            feat = self.slot_feature(
+                slot, usr, cur, pre, sys_action, usr_act_feat)
+            feature.append(feat)
+
+        if not state_vectorize:
+            self.user_feat_hist.append(feature)
+
+        feature, mask = self.pad_feature(max_memory=self.config["window"])
+
+        return feature, mask
+
+    def slot_feature(self, slot, user_goal, current_state, previous_state, sys_action, usr_action):
+        pass
+
+    def pad_feature(self, max_memory=5):
+        feature = []
+        num_feat = len(self.user_feat_hist)
+
+        feat_dim = len(self.user_feat_hist[0][0])
+
+        for feat_index in range(num_feat - 1, max(num_feat - 1 - max_memory, -1), -1):
+            if feat_index == num_feat - 1:
+                special_token = self.slot_feature(
+                    "CLS", {}, {}, {}, [], [])
+            else:
+                special_token = self.slot_feature(
+                    "SEP", {}, {}, {}, [], [])
+            feature += [special_token]
+            feature += self.user_feat_hist[feat_index]
+
+        max_len = max_memory * self.config["num_token"]
+        if len(feature) < max_len:
+            padding = [[0] * feat_dim] * (max_len - len(feature))
+            feature += padding
+            mask = [False] * len(feature) + [True] * (max_len - len(feature))
+        else:
+            mask = [False] * max_len
+
+        return feature[:max_len], mask[:max_len]
+
+    def domain_label(self, user_goal, dialog_act):
+        labels = [0] * self.config["out_dim"]
+        goal_domains = user_goal.domains
+        no_domain = True
+
+        for intent, domain, slot, value in dialog_act:
+            # domain = domain.lower()
+            if domain in goal_domains:
+                index = goal_domains.index(domain)
+                labels[index + 1] = 1
+                no_domain = False
+        if no_domain:
+            labels[0] = 1
+        return labels
+
+    def generate_label(self, action_list: list, user_goal, cur_state, dialog_act):
+        # label = "none", "?", "dontcare", "system", "user", "change"
+
+        labels = [-1] * self.config["num_token"]
+
+        usr = parse_user_goal(user_goal)
+        cur = metadata2state(cur_state)
+        # print("usr", usr)
+        # print("cur", cur)
+        # print(action_list)
+        for intent, domain, slot, value in dialog_act:
+            # domain = domain.lower()
+            # value = value.lower()
+            # slot = slot.lower()
+            # name = util.act2slot(intent, domain, slot, value, self.all_values)
+            name = f"{domain}-{slot}"
+
+            if name not in action_list:
+                # print(f"Not handle name {name} in getting label")
+                continue
+            name_id = action_list.index(name)
+            if name_id >= self.config["num_token"]:
+                continue
+            if value == "?":
+                labels[name_id] = 1
+            elif value == "dontcare":
+                labels[name_id] = 2
+            elif name in cur and value == cur[name]:
+                labels[name_id] = 3
+            elif name in usr and value == usr[name]:
+                labels[name_id] = 4
+            elif (name in cur or name in usr) and value not in [cur.get(name), usr.get(name)]:
+                labels[name_id] = 5
+
+        for name in action_list:
+            domain = name.split('-')[0]
+            name_id = action_list.index(name)
+            if name_id < len(labels):
+                if labels[name_id] < 0 and domain in self.domain_list:
+                    labels[name_id] = 0
+
+        self.pre_usr = labels
+
+        return labels
+
+    def get_user_action_feat(self, all_slot, user_goal, usr_act):
+        pass
+
+    def update_constrain(self, action):
+        """ 
+        update constrain status by system actions
+        action = [[intent, domain, slot, name]]
+        """
+        for intent, domain, slot, value in action:
+            # domain = domain.lower()
+            # slot = slot.lower()
+            if domain in self.domain_list:
+                # slot = SysDa2Goal[domain].get(slot, "none")
+                slot_name = f"{domain}-{slot}"
+            elif domain == "booking":
+                if slot.lower() == "ref":
+                    continue
+                # slot = SysDa2Goal[domain].get(slot, "none")
+                # domain = util.get_booking_domain(
+                #     slot, value, self.all_values, self.domain_list)
+                if not domain:
+                    continue  # work around
+                slot_name = f"{domain}-{slot}"
+
+            else:
+                continue
+            if value != "?":
+                self.constrains[slot_name] = value
+
+    @staticmethod
+    def concatenate_subvectors(vec_list):
+        vec = []
+        for sub_vec in vec_list:
+            vec += sub_vec
+        return vec
+
+
+class BinaryFeature(Feature):
+    def __init__(self, config):
+        super().__init__(config)
+
+    def slot_feature(self, slot, user_goal, current_state, previous_state, sys_action, usr_action):
+        feat = []
+        feat += self._special_token(slot)
+        feat += self._value_representation(
+            slot, current_state.get(slot, NOT_MENTIONED))
+        feat += self._value_representation(
+            slot, user_goal.get(slot, NOT_MENTIONED))
+        feat += self._is_constrain_request(slot, user_goal)
+        feat += self._is_fulfill(slot, user_goal)
+        if self.config.get("conflict", True):
+            feat += self._conflict_check(user_goal, current_state, slot)
+        if self.config.get("domain_feat", False):
+            # feat += self.domain_feat(slot)
+            if slot in ["CLS", "SEP"]:
+                feat += [0] * (self.goal.max_domain_len +
+                               self.goal.max_slot_len)
+            else:
+                domain_feat, slot_feat = self.goal.get_slot_id(slot)
+                feat += domain_feat + slot_feat
+        feat += self._first_mention_detection(
+            previous_state, current_state, slot)
+        feat += self._just_mention(slot, sys_action)
+        feat += self._action_representation(slot, sys_action)
+        # need change from 0 to domain predictor
+        if slot in ["CLS", "SEP"]:
+            feat += [0] * self.config["out_dim"]
+        else:
+            feat += usr_action[slot]
+        return feat
+
+    def get_user_action_feat(self, all_slot, user_goal, usr_act):
+        if usr_act:
+            usr_label = self.generate_label(
+                all_slot, user_goal, {}, usr_act)
+            self.pre_usr = usr_label
+        usr_act_feat = {}
+        for index, slot in zip(self.pre_usr, all_slot):
+            usr_act_feat[slot] = int2onehot(index, self.config["out_dim"])
+        return usr_act_feat
+
+    def _special_token(self, slot):
+        special_token = ["CLS", "SEP"]
+        feat = [0] * len(special_token)
+        if slot in special_token:
+            feat[special_token.index(slot)] = 1
+        return feat
+
+    def _is_constrain_request(self, feature_slot, user_goal):
+        if feature_slot in ["CLS", "SEP"]:
+            return [0, 0]
+        # [is_constrain, is_request]
+        value = user_goal.get(feature_slot, NOT_MENTIONED)
+        if value == "?":
+            return [0, 1]
+        elif value == NOT_MENTIONED:
+            return [0, 0]
+        else:
+            return [1, 0]
+
+    def _is_fulfill(self, feature_slot, user_goal):
+        if feature_slot in ["CLS", "SEP"]:
+            return [0]
+
+        if feature_slot in user_goal and user_goal.get(feature_slot) == self.constrains.get(feature_slot):
+            return [1]
+        return [0]
+
+    def _just_mention(self, feature_slot, sys_action):
+        """
+        the system action just mentioned this slot
+        """
+        if feature_slot in ["CLS", "SEP"]:
+            return [0]
+        if not sys_action:
+            return [0]
+        sys_action_slot = []
+        for intent, domain, slot, value in sys_action:
+            # domain = domain.lower()
+            # slot = slot.lower()
+            # value = value.lower()
+            # if domain == "booking":
+            #     domain = util.get_booking_domain(
+            #         slot, value, self.all_values, self.domain_list)
+            if domain in sys_action:
+                action = f"{domain}-{slot}"
+                sys_action_slot.append(action)
+        if feature_slot in sys_action_slot:
+            return [1]
+        return [0]
+
+    def _action_representation(self, feature_slot, action):
+
+        gen_vec = [0] * len(self.general_intent)
+        # ["none", "?", other]
+        intent2act = {intent: [0] * 3 for intent in self.intents}
+
+        if action is None or feature_slot in ["CLS", "SEP"]:
+            return self._concatenate_action_vector(intent2act, gen_vec)
+        for intent, domain, slot, value in action:
+            # domain = domain.lower()
+            # slot = slot.lower()
+            # value = value.lower()
+
+            # general
+            if domain == "general":
+                self._update_general_action(gen_vec, intent)
+            else:
+                # if domain == "booking":
+                #     domain = util.get_booking_domain(
+                #         slot, value, self.all_values, self.domain_list)
+                self._update_intent2act(
+                    feature_slot, intent2act,
+                    domain, intent, slot, value)
+                # TODO special slots, "choice, ref, none"
+
+        return self._concatenate_action_vector(intent2act, gen_vec)
+
+    def _update_general_action(self, vec, intent):
+        if intent in self.general_intent:
+            vec[self.general_intent.index(intent)] = 1
+
+    def _update_intent2act(self, feature_slot, intent2act, domain, intent, slot, value):
+        feature_domain, feature_slot = split_slot_name(
+            feature_slot)  # .split('-')
+        # intent = intent.lower()
+        # slot = slot.lower()
+        # value = value.lower()
+        if slot == "none" and feature_domain == domain:  # None slot
+            intent2act[intent][2] = 1
+        elif feature_domain == domain and slot == feature_slot and intent in intent2act:
+            if value == "none":
+                intent2act[intent][0] = 1
+            elif value == "?":
+                intent2act[intent][1] = 1
+            else:
+                intent2act[intent][2] = 1
+
+    def _concatenate_action_vector(self, intent2act, general):
+        feat = []
+        for intent in intent2act:
+            feat += intent2act[intent]
+        feat += general
+        return feat
+
+    def _value_representation(self, slot, value):
+        if slot in ["CLS", "SEP"]:
+            return [0, 0, 0, 0]
+        if value == NOT_MENTIONED:
+            return [1, 0, 0, 0]
+        else:
+            temp_vector = [0] * (len(self.default_values) + 1)
+            if value in self.default_values:
+                temp_vector[self.default_values.index(value)] = 1
+            else:
+                temp_vector[-1] = 1
+
+            return temp_vector
+
+    def _conflict_check(self, user_goal, system_state, slot):
+        # conflict = [1] else [0]
+        if slot in ["CLS", "SEP"]:
+            return [0]
+        usr = user_goal.get(slot, NOT_MENTIONED)
+        sys = system_state.get(slot, NOT_MENTIONED)
+        if usr in [NOT_MENTIONED, "none", ""] and sys in [NOT_MENTIONED, "none", ""]:
+            return [0]
+
+        if usr != sys or (usr == "?" and sys == "?"):
+            # print(f"{slot} has different value: {usr} and {sys}.")
+            # conflict = uniform(0.2, 1)
+            conflict = 1
+            return [conflict]
+        return [0]
+
+    def _first_mention_detection(self, pre_state, cur_state, slot):
+        if slot in ["CLS", "SEP"]:
+            return [0]
+
+        first_mention = [1]
+        not_first_mention = [0]
+        cur = cur_state.get(slot, NOT_MENTIONED)
+        if pre_state is None:
+            if cur not in [NOT_MENTIONED, "none"]:
+                return first_mention
+            else:
+                return not_first_mention
+
+        pre = pre_state.get(slot, NOT_MENTIONED)
+
+        if pre in [NOT_MENTIONED, "none"] and cur not in [NOT_MENTIONED, "none"]:
+            return first_mention
+
+        return not_first_mention  # hasn't been mentioned
+
+
+if __name__ == "__main__":
+    from convlab.util import load_dataset, load_ontology
+    data = load_dataset("multiwoz21")
+    config = json.load(open("convlab/policy/tus/unify/exp/default.json"))
+    # test = TUSDataManager(config, data["test"])
+    train = TUSDataManager(config, data["train"])
+    # data = load_dataset("sgd")
+    # config = json.load(open("convlab/policy/tus/unify/exp/default.json"))
+    # data_manager = TUSDataManager(config, data["test"])
diff --git a/convlab/policy/tus/unify/util.py b/convlab/policy/tus/unify/util.py
new file mode 100644
index 0000000000000000000000000000000000000000..d65f72a06e181e66bfe0d7ac0c60f0c03a56ad43
--- /dev/null
+++ b/convlab/policy/tus/unify/util.py
@@ -0,0 +1,245 @@
+from convlab.policy.tus.multiwoz.Da2Goal import SysDa2Goal, UsrDa2Goal
+from convlab.util import load_dataset
+
+import json
+
+NOT_MENTIONED = "not mentioned"
+
+
+def load_experiment_dataset(data_name="multiwoz21", dial_ids_order=0, split2ratio=1):
+    ratio = {'train': split2ratio, 'validation': split2ratio}
+    if data_name == "all" or data_name == "sgd+tm" or data_name == "tm":
+        print("merge all datasets...")
+        if data_name == "all":
+            all_dataset = ["multiwoz21", "sgd", "tm1", "tm2", "tm3"]
+        if data_name == "sgd+tm":
+            all_dataset = ["sgd", "tm1", "tm2", "tm3"]
+        if data_name == "tm":
+            all_dataset = ["tm1", "tm2", "tm3"]
+
+        datasets = {}
+        for name in all_dataset:
+            datasets[name] = load_dataset(
+                name,
+                dial_ids_order=dial_ids_order,
+                split2ratio=ratio)
+        raw_data = merge_dataset(datasets, all_dataset[0])
+
+    else:
+        print(f"load single dataset {data_name}/{split2ratio}")
+        raw_data = load_dataset(data_name,
+                                dial_ids_order=dial_ids_order,
+                                split2ratio=ratio)
+    return raw_data
+
+
+def merge_dataset(datasets, data_name):
+    data_split = [x for x in datasets[data_name]]
+    raw_data = {}
+    for data_type in data_split:
+        raw_data[data_type] = []
+        for dataname, dataset in datasets.items():
+            print(f"merge {dataname}...")
+            raw_data[data_type] += dataset[data_type]
+    return raw_data
+
+
+def int2onehot(index, output_dim=6, remove_zero=False):
+    one_hot = [0] * output_dim
+    if remove_zero:
+        if index == 0:
+            one_hot[index] = 1
+    else:
+        if index >= 0:
+            one_hot[index] = 1
+
+    return one_hot
+
+
+def parse_user_goal(raw_goal):
+    """flatten user goal structure"""
+    goal = raw_goal.domain_goals
+    user_goal = {}
+    for domain in goal:
+        # if domain not in UsrDa2Goal:
+        #     continue
+        for slot_type in goal[domain]:
+            if slot_type in ["fail_info", "fail_book", "booked"]:
+                continue  # TODO [fail_info] fix in the future
+            if slot_type in ["info", "book", "reqt"]:
+                for slot in goal[domain][slot_type]:
+                    slot_name = f"{domain}-{slot}"
+                    user_goal[slot_name] = goal[domain][slot_type][slot]
+
+    return user_goal
+
+
+def parse_dialogue_act(dialogue_act):
+    """ transfer action from dict to list """
+    actions = []
+    for action_type in dialogue_act:
+        for act in dialogue_act[action_type]:
+            domain = act["domain"]
+            if "value" in act:
+                actions.append(
+                    [act["intent"], domain, act["slot"], act["value"]])
+            else:
+                if act["intent"] == "request":
+                    actions.append(
+                        [act["intent"], domain, act["slot"], "?"])
+                else:
+                    slot = act.get("slot", "none")
+                    value = act.get("value", "none")
+                    actions.append(
+                        [act["intent"], domain, slot, value])
+
+    return actions
+
+
+def metadata2state(metadata):
+    """
+    parse metadata in the data set or dst
+    """
+    slot_value = {}
+
+    for domain in metadata:
+        for slot in metadata[domain]:
+            slot_name = f"{domain}-{slot}"
+            value = metadata[domain][slot]
+            if not value or value == NOT_MENTIONED:
+                value = "none"
+            slot_value[slot_name] = value
+
+    return slot_value
+
+
+def get_booking_domain(slot, value, all_values, domain_list):
+    """ 
+    find the domain for domain booking, excluding slot "ref"
+    """
+    found = ""
+    if not slot:
+        return found
+    slot = slot.lower()
+    value = value.lower()
+    for domain in domain_list:
+        if slot in all_values["all_value"][domain] \
+                and value in all_values["all_value"][domain][slot]:
+            found = domain
+    return found
+
+
+def update_config_file(file_name, attribute, value):
+    with open(file_name, 'r') as config_file:
+        config = json.load(config_file)
+
+    config[attribute] = value
+    print(config)
+    with open(file_name, 'w') as config_file:
+        json.dump(config, config_file)
+    print(f"update {attribute} = {value}")
+
+
+def create_goal(dialog) -> list:
+    # a list of {'intent': ..., 'domain': ..., 'slot': ..., 'value': ...}
+    dicts = []
+    for turn in dialog['turns']:
+        # print(turn['speaker'])
+        # assert (i % 2 == 0) == (turn['speaker'] == 'user')
+        # if i % 2 == 0:
+        if turn['speaker'] == 'user':
+            dicts += turn['dialogue_acts']['categorical']
+            dicts += turn['dialogue_acts']['binary']
+            dicts += turn['dialogue_acts']['non-categorical']
+    tuples = []
+    for d in dicts:
+        if "value" not in d:
+            if d['intent'] == "request":
+                value = "?"
+            else:
+                value = "none"
+        else:
+            value = d["value"]
+        slot = d.get("slot", "none")
+        domain = d['domain']  # .split('_')[0]
+        tuples.append(
+            (d['domain'], d['intent'], slot, value)
+        )
+
+    user_goal = []  # a list of (domain, intent, slot, value)
+    for domain, intent, slot, value in tuples:
+        # if slot == "intent":
+        #     continue
+        if not slot:
+            continue
+        if intent == "inform" and value == "":
+            continue
+        # if intent == "request" and value != "":
+        #     intent = "inform"
+        user_goal.append((domain, intent, slot, value))
+    user_goal = unique_list(user_goal)
+    inform_slots = {(domain, slot) for (domain, intent, slot,
+                                        value) in user_goal if intent == "inform"}
+    user_goal = [(domain, intent, slot, value) for (domain, intent, slot, value)
+                 in user_goal if not (intent == "request" and (domain, slot) in inform_slots)]
+    return user_goal
+
+
+def unique_list(list_):
+    r = []
+    for x in list_:
+        if x not in r:
+            r.append(x)
+    return r
+
+
+def split_slot_name(slot_name):
+    tokens = slot_name.split('-')
+    if len(tokens) == 2:
+        return tokens[0], tokens[1]
+    else:
+        return tokens[0], '-'.join(tokens[1:])
+
+
+# copy from data.unified_datasets.multiwoz21
+slot_name_map = {
+    'addr': "address",
+    'post': "postcode",
+    'pricerange': "price range",
+    'arrive': "arrive by",
+    'arriveby': "arrive by",
+    'leave': "leave at",
+    'leaveat': "leave at",
+    'depart': "departure",
+    'dest': "destination",
+    'fee': "entrance fee",
+    'open': 'open hours',
+    'car': "type",
+    'car type': "type",
+    'ticket': 'price',
+    'trainid': 'train id',
+    'id': 'train id',
+    'people': 'book people',
+    'stay': 'book stay',
+    'none': '',
+    'attraction': {
+        'price': 'entrance fee'
+    },
+    'hospital': {},
+    'hotel': {
+        'day': 'book day', 'price': "price range"
+    },
+    'restaurant': {
+        'day': 'book day', 'time': 'book time', 'price': "price range"
+    },
+    'taxi': {},
+    'train': {
+        'day': 'day', 'time': "duration"
+    },
+    'police': {},
+    'booking': {}
+}
+
+if __name__ == "__main__":
+    print(split_slot_name("restaurant-search-location"))
+    print(split_slot_name("sports-day.match"))
diff --git a/convlab/policy/vector/README.md b/convlab/policy/vector/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..107ba733f2075204ed47c098a046d2f7805fa7b3
--- /dev/null
+++ b/convlab/policy/vector/README.md
@@ -0,0 +1,18 @@
+# Vectoriser
+
+The vectoriser is a module used by the policy network and has several functionalities
+
+1. it translates the semantic dialogue act into a vector representation usable for the policy network
+2. it translates the policy network output back into a lexicalized semantic act
+3. it creates an action masking that the policy can use to forbid illegal actions
+
+There is a **vector_base** class that has many functionalities already implemented. All other vector classes are inherited from the base class.
+
+If you build a new vectoriser, you need at least the following method:
+
+    
+    def state_vectorize(self, state):
+        # translates the semantic dialogue state into vector representation
+        # will be used by the policy module
+    
+See the implemented vector classes for examples.
\ No newline at end of file
diff --git a/convlab/policy/vector/dataset.py b/convlab/policy/vector/dataset.py
index 0aa1b7ad879f2d814cece3a98bb457b71ad99033..5b233e6659abc18da69a3efe6bb2d52185aa30fd 100755
--- a/convlab/policy/vector/dataset.py
+++ b/convlab/policy/vector/dataset.py
@@ -18,6 +18,26 @@ class ActDataset(data.Dataset):
         return self.num_total
 
 
+class ActDatasetKG(data.Dataset):
+    def __init__(self, action_batch, a_masks, current_domain_mask_batch, non_current_domain_mask_batch):
+        self.action_batch = action_batch
+        self.a_masks = a_masks
+        self.current_domain_mask_batch = current_domain_mask_batch
+        self.non_current_domain_mask_batch = non_current_domain_mask_batch
+        self.num_total = len(action_batch)
+
+    def __getitem__(self, index):
+        action = self.action_batch[index]
+        action_mask = self.a_masks[index]
+        current_domain_mask = self.current_domain_mask_batch[index]
+        non_current_domain_mask = self.non_current_domain_mask_batch[index]
+
+        return action, action_mask, current_domain_mask, non_current_domain_mask, index
+
+    def __len__(self):
+        return self.num_total
+
+
 class ActStateDataset(data.Dataset):
     def __init__(self, s_s, a_s, next_s):
         self.s_s = s_s
@@ -32,4 +52,4 @@ class ActStateDataset(data.Dataset):
         return s, a, next_s
     
     def __len__(self):
-        return self.num_total
\ No newline at end of file
+        return self.num_total
diff --git a/convlab/policy/vector/vector_base.py b/convlab/policy/vector/vector_base.py
index 8b7d8ff0ddafed41efc91b249003ae55c525bc93..8f72144ce37a970fe4855a19d5bc8002fc2b4034 100644
--- a/convlab/policy/vector/vector_base.py
+++ b/convlab/policy/vector/vector_base.py
@@ -2,10 +2,11 @@
 import os
 import sys
 import numpy as np
+import logging
 
 from copy import deepcopy
 from convlab.policy.vec import Vector
-from convlab.util.custom_util import flatten_acts
+from convlab.util.custom_util import flatten_acts, timeout
 from convlab.util.multiwoz.lexicalize import delexicalize_da, flat_da, deflat_da, lexicalize_da
 from convlab.util import load_ontology, load_database, load_dataset
 
@@ -22,18 +23,20 @@ class VectorBase(Vector):
 
         super().__init__()
 
+        logging.info(f"Vectorizer: Data set used is {dataset_name}")
         self.set_seed(seed)
         self.ontology = load_ontology(dataset_name)
         try:
             # execute to make sure that the database exists or is downloaded otherwise
-            load_database(dataset_name)
+            if dataset_name == "multiwoz21":
+                load_database(dataset_name)
             # the following two lines are needed for pickling correctly during multi-processing
             exec(f'from data.unified_datasets.{dataset_name}.database import Database')
             self.db = eval('Database()')
             self.db_domains = self.db.domains
         except Exception as e:
             self.db = None
-            self.db_domains = None
+            self.db_domains = []
             print(f"VectorBase: {e}")
 
         self.dataset_name = dataset_name
@@ -272,6 +275,8 @@ class VectorBase(Vector):
         2. If there is an entity available, can not say NoOffer or NoBook
         '''
         mask_list = np.zeros(self.da_dim)
+        if number_entities_dict is None:
+            return mask_list
         for i in range(self.da_dim):
             action = self.vec2act[i]
             domain, intent, slot, value = action.split('-')
diff --git a/convlab/policy/vector/vector_binary.py b/convlab/policy/vector/vector_binary.py
index 9efde57562c30d2f41aecf2331d23f4669f4ae14..e780dc645043f4775b208479abf022dccce649a5 100755
--- a/convlab/policy/vector/vector_binary.py
+++ b/convlab/policy/vector/vector_binary.py
@@ -8,7 +8,7 @@ from .vector_base import VectorBase
 class VectorBinary(VectorBase):
 
     def __init__(self, dataset_name='multiwoz21', character='sys', use_masking=False, manually_add_entity_names=True,
-                 seed=0):
+                 seed=0, **kwargs):
 
         super().__init__(dataset_name, character, use_masking, manually_add_entity_names, seed)
 
@@ -65,7 +65,7 @@ class VectorBinary(VectorBase):
         return state_vec, mask
 
     def get_mask(self, domain_active_dict, number_entities_dict):
-        domain_mask = self.compute_domain_mask(domain_active_dict)
+        #domain_mask = self.compute_domain_mask(domain_active_dict)
         entity_mask = self.compute_entity_mask(number_entities_dict)
         general_mask = self.compute_general_mask()
         mask = entity_mask + general_mask
diff --git a/convlab/policy/vector/vector_binary_fuzzy.py b/convlab/policy/vector/vector_binary_fuzzy.py
new file mode 100755
index 0000000000000000000000000000000000000000..5314cf1303f99f0ac2d0fbc5a2bca34c4e3d74c4
--- /dev/null
+++ b/convlab/policy/vector/vector_binary_fuzzy.py
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+import sys
+import numpy as np
+from convlab.util.multiwoz.lexicalize import delexicalize_da, flat_da
+from .vector_binary import VectorBinary
+
+
+class VectorBinaryFuzzy(VectorBinary):
+
+    def __init__(self, dataset_name='multiwoz21', character='sys', use_masking=False, manually_add_entity_names=True,
+                 seed=0):
+
+        super().__init__(dataset_name, character, use_masking, manually_add_entity_names, seed)
+
+    def dbquery_domain(self, domain):
+        """
+        query entities of specified domain
+        Args:
+            domain string:
+                domain to query
+        Returns:
+            entities list:
+                list of entities of the specified domain
+        """
+        # Get all user constraints
+        constraints = [[slot, value] for slot, value in self.state[domain].items() if value] \
+            if domain in self.state else []
+        xx = self.db.query(domain=domain, state=[], soft_contraints=constraints, fuzzy_match_ratio=100, topk=10)
+        yy = self.db.query(domain=domain, state=constraints, topk=10)
+        #print("STRICT:", yy)
+        #print("FUZZY :", xx)
+        #if len(yy) == 1 and len(xx) > 1:
+        #    import pdb
+        #    pdb.set_trace()
+        return xx
+        #return self.db.query(domain=domain, state=[], soft_contraints=constraints, fuzzy_match_ratio=100, topk=10)
diff --git a/convlab/policy/vector/vector_multiwoz_uncertainty.py b/convlab/policy/vector/vector_multiwoz_uncertainty.py
deleted file mode 100644
index 6a0850f4ce7c791f7da99c9c8b9e632eac543ba2..0000000000000000000000000000000000000000
--- a/convlab/policy/vector/vector_multiwoz_uncertainty.py
+++ /dev/null
@@ -1,238 +0,0 @@
-# -*- coding: utf-8 -*-
-import sys
-import os
-import numpy as np
-import logging
-from convlab.util.multiwoz.lexicalize import delexicalize_da, flat_da
-from convlab.util.multiwoz.state import default_state
-from convlab.util.multiwoz.multiwoz_slot_trans import REF_SYS_DA
-from .vector_binary import VectorBinary as VectorBase
-
-DEFAULT_INTENT_FILEPATH = os.path.join(
-    os.path.dirname(os.path.dirname(os.path.dirname(
-        os.path.dirname(os.path.abspath(__file__))))),
-    'data/multiwoz/trackable_intent.json'
-)
-
-
-SLOT_MAP = {'taxi_types': 'car type'}
-
-
-class MultiWozVector(VectorBase):
-
-    def __init__(self, voc_file=None, voc_opp_file=None, character='sys',
-                 intent_file=DEFAULT_INTENT_FILEPATH,
-                 use_confidence_scores=False,
-                 use_entropy=False,
-                 use_mutual_info=False,
-                 use_masking=False,
-                 manually_add_entity_names=False,
-                 seed=0,
-                 shrink=False):
-
-        self.use_confidence_scores = use_confidence_scores
-        self.use_entropy = use_entropy
-        self.use_mutual_info = use_mutual_info
-        self.thresholds = None
-
-        super().__init__(voc_file, voc_opp_file, character, intent_file, use_masking, manually_add_entity_names, seed)
-
-    def get_state_dim(self):
-        self.belief_state_dim = 0
-        for domain in self.belief_domains:
-            for slot in default_state()['belief_state'][domain.lower()]['semi']:
-                # Dim 1 - indicator/confidence score
-                # Dim 2 - Entropy (Total uncertainty) / Mutual information (knowledge unc)
-                slot_dim = 1 if not self.use_entropy else 2
-                slot_dim += 1 if self.use_mutual_info else 0
-                self.belief_state_dim += slot_dim
-
-        self.state_dim = self.da_opp_dim + self.da_dim + self.belief_state_dim + \
-            len(self.db_domains) + 6 * len(self.db_domains) + 1
-
-    def dbquery_domain(self, domain):
-        """
-        query entities of specified domain
-        Args:
-            domain string:
-                domain to query
-        Returns:
-            entities list:
-                list of entities of the specified domain
-        """
-        # Get all user constraints
-        constraint = self.state[domain.lower()]['semi']
-        constraint = {k: i for k, i in constraint.items() if i and i not in ['dontcare', "do n't care", "do not care"]}
-
-        # Remove constraints for which the uncertainty is high
-        if self.confidence_scores is not None and self.use_confidence_scores and self.thresholds != None:
-            # Collect threshold values for each domain-slot pair
-            thres = self.thresholds.get(domain.lower(), {})
-            thres = {k: thres.get(k, 0.05) for k in constraint}
-            # Get confidence scores for each constraint
-            probs = self.confidence_scores.get(domain.lower(), {})
-            probs = {k: probs.get(k, {}).get('inform', 1.0)
-                     for k in constraint}
-
-            # Filter out constraints for which confidence is lower than threshold
-            constraint = {k: i for k, i in constraint.items()
-                          if probs[k] >= thres[k]}
-
-        return self.db.query(domain.lower(), constraint.items())
-
-    # Add thresholds for db_queries
-    def setup_uncertain_query(self, thresholds):
-        self.use_confidence_scores = True
-        self.thresholds = thresholds
-        logging.info('DB Search uncertainty activated.')
-
-    def vectorize_user_act_confidence_scores(self, state, opp_action):
-        """Return confidence scores for the user actions"""
-        opp_act_vec = np.zeros(self.da_opp_dim)
-        for da in self.opp2vec:
-            domain, intent, slot, value = da.split('-')
-            if domain.lower() in state['belief_state_probs']:
-                # Map slot name to match user actions
-                slot = REF_SYS_DA[domain].get(
-                    slot, slot) if domain in REF_SYS_DA else slot
-                slot = slot if slot else 'none'
-                slot = SLOT_MAP.get(slot, slot)
-                domain = domain.lower()
-
-                if slot in state['belief_state_probs'][domain]:
-                    prob = state['belief_state_probs'][domain][slot]
-                elif slot.lower() in state['belief_state_probs'][domain]:
-                    prob = state['belief_state_probs'][domain][slot.lower()]
-                else:
-                    prob = {}
-
-                intent = intent.lower()
-                if intent in prob:
-                    prob = float(prob[intent])
-                elif da in opp_action:
-                    prob = 1.0
-                else:
-                    prob = 0.0
-            elif da in opp_action:
-                prob = 1.0
-            else:
-                prob = 0.0
-            opp_act_vec[self.opp2vec[da]] = prob
-
-        return opp_act_vec
-
-    def state_vectorize(self, state):
-        """vectorize a state
-
-        Args:
-            state (dict):
-                Dialog state
-            action (tuple):
-                Dialog act
-        Returns:
-            state_vec (np.array):
-                Dialog state vector
-        """
-        self.state = state['belief_state']
-        self.confidence_scores = state['belief_state_probs'] if 'belief_state_probs' in state else None
-        domain_active_dict = {}
-        for domain in self.belief_domains:
-            domain_active_dict[domain] = False
-
-        # when character is sys, to help query database when da is booking-book
-        # update current domain according to user action
-        if self.character == 'sys':
-            action = state['user_action']
-            for intent, domain, slot, value in action:
-                domain_active_dict[domain] = True
-
-        action = state['user_action'] if self.character == 'sys' else state['system_action']
-        opp_action = delexicalize_da(action, self.requestable)
-        opp_action = flat_da(opp_action)
-        if 'belief_state_probs' in state and self.use_confidence_scores:
-            opp_act_vec = self.vectorize_user_act_confidence_scores(
-                state, opp_action)
-        else:
-            opp_act_vec = np.zeros(self.da_opp_dim)
-            for da in opp_action:
-                if da in self.opp2vec:
-                    prob = 1.0
-                    opp_act_vec[self.opp2vec[da]] = prob
-
-        action = state['system_action'] if self.character == 'sys' else state['user_action']
-        action = delexicalize_da(action, self.requestable)
-        action = flat_da(action)
-        last_act_vec = np.zeros(self.da_dim)
-        for da in action:
-            if da in self.act2vec:
-                last_act_vec[self.act2vec[da]] = 1.
-
-        belief_state = np.zeros(self.belief_state_dim)
-        i = 0
-        for domain in self.belief_domains:
-            if self.use_confidence_scores and 'belief_state_probs' in state:
-                for slot in state['belief_state'][domain.lower()]['semi']:
-                    if slot in state['belief_state_probs'][domain.lower()]:
-                        prob = state['belief_state_probs'][domain.lower()
-                                                           ][slot]
-                        prob = prob['inform'] if 'inform' in prob else None
-                    if prob:
-                        belief_state[i] = float(prob)
-                    i += 1
-            else:
-                for slot, value in state['belief_state'][domain.lower()]['semi'].items():
-                    if value and value != 'not mentioned':
-                        belief_state[i] = 1.
-                    i += 1
-            if 'active_domains' in state:
-                domain_active = state['active_domains'][domain.lower()]
-                domain_active_dict[domain] = domain_active
-            else:
-                if [slot for slot, value in state['belief_state'][domain.lower()]['semi'].items() if value]:
-                    domain_active_dict[domain] = True
-
-        # Add knowledge and/or total uncertainty to the belief state
-        if self.use_entropy and 'entropy' in state:
-            for domain in self.belief_domains:
-                for slot in state['belief_state'][domain.lower()]['semi']:
-                    if slot in state['entropy'][domain.lower()]:
-                        belief_state[i] = float(
-                            state['entropy'][domain.lower()][slot])
-                    i += 1
-
-        if self.use_mutual_info and 'mutual_information' in state:
-            for domain in self.belief_domains:
-                for slot in state['belief_state'][domain.lower()]['semi']:
-                    if slot in state['mutual_information'][domain.lower()]:
-                        belief_state[i] = float(
-                            state['mutual_information'][domain.lower()][slot])
-                    i += 1
-
-        book = np.zeros(len(self.db_domains))
-        for i, domain in enumerate(self.db_domains):
-            if state['belief_state'][domain.lower()]['book']['booked']:
-                book[i] = 1.
-
-        degree, number_entities_dict = self.pointer()
-
-        final = 1. if state['terminated'] else 0.
-
-        state_vec = np.r_[opp_act_vec, last_act_vec,
-                          belief_state, book, degree, final]
-        assert len(state_vec) == self.state_dim
-
-        if self.use_mask is not None:
-            # None covers the case for policies that don't use masking at all, so do not expect an output "state_vec, mask"
-            if self.use_mask:
-                domain_mask = self.compute_domain_mask(domain_active_dict)
-                entity_mask = self.compute_entity_mask(number_entities_dict)
-                general_mask = self.compute_general_mask()
-                mask = domain_mask + entity_mask + general_mask
-                for i in range(self.da_dim):
-                    mask[i] = -int(bool(mask[i])) * sys.maxsize
-            else:
-                mask = np.zeros(self.da_dim)
-
-            return state_vec, mask
-        else:
-            return state_vec
diff --git a/convlab/policy/vector/vector_nodes.py b/convlab/policy/vector/vector_nodes.py
new file mode 100644
index 0000000000000000000000000000000000000000..2e073669effc518cee4efd1f03d25bbd501b65af
--- /dev/null
+++ b/convlab/policy/vector/vector_nodes.py
@@ -0,0 +1,170 @@
+# -*- coding: utf-8 -*-
+import sys
+import numpy as np
+import logging
+
+from convlab.util.multiwoz.lexicalize import delexicalize_da, flat_da
+from .vector_base import VectorBase
+
+
+class VectorNodes(VectorBase):
+
+    def __init__(self, dataset_name='multiwoz21', character='sys', use_masking=False, manually_add_entity_names=True,
+                 seed=0, filter_state=True):
+
+        super().__init__(dataset_name, character, use_masking, manually_add_entity_names, seed)
+        self.filter_state = filter_state
+        logging.info(f"We filter state by active domains: {self.filter_state}")
+
+    def get_state_dim(self):
+        self.belief_state_dim = 0
+
+        for domain in self.ontology['state']:
+            for slot in self.ontology['state'][domain]:
+                self.belief_state_dim += 1
+
+        self.state_dim = self.da_opp_dim + self.da_dim + self.belief_state_dim + \
+            len(self.db_domains) + 6 * len(self.db_domains) + 1
+
+    def init_kg_graph(self):
+        self.kg_info = []
+
+    def add_graph_node(self, domain, node_type, description, value):
+
+        node = {"domain": domain, "node_type": node_type, "description": description, "value": value}
+        self.kg_info.append(node)
+
+    def state_vectorize(self, state):
+        """vectorize a state
+
+        Args:
+            state (dict):
+                Dialog state
+            action (tuple):
+                Dialog act
+        Returns:
+            state_vec (np.array):
+                Dialog state vector
+        """
+        self.state = state['belief_state']
+        domain_active_dict = self.init_domain_active_dict()
+        self.init_kg_graph()
+
+        # when character is sys, to help query database when da is booking-book
+        # update current domain according to user action
+        if self.character == 'sys':
+            action = state['user_action']
+            for intent, domain, slot, value in action:
+                domain_active_dict[domain] = True
+
+        self.get_user_act_feature(state)
+        self.get_sys_act_feature(state)
+        domain_active_dict = self.get_user_goal_feature(state, domain_active_dict)
+        self.get_general_features(state, domain_active_dict)
+
+        if self.db is not None:
+            number_entities_dict = self.get_db_features()
+        else:
+            number_entities_dict = None
+
+        if self.filter_state:
+            self.kg_info = self.filter_inactive_domains(domain_active_dict)
+
+        if self.use_mask:
+            mask = self.get_mask(domain_active_dict, number_entities_dict)
+            for i in range(self.da_dim):
+                mask[i] = -int(bool(mask[i])) * sys.maxsize
+        else:
+            mask = np.zeros(self.da_dim)
+
+        return np.zeros(1), mask
+
+    def get_mask(self, domain_active_dict, number_entities_dict):
+        #domain_mask = self.compute_domain_mask(domain_active_dict)
+        entity_mask = self.compute_entity_mask(number_entities_dict)
+        general_mask = self.compute_general_mask()
+        mask = entity_mask + general_mask
+        return mask
+
+    def get_db_features(self):
+
+        degree, number_entities_dict = self.pointer()
+        feature_type = 'db'
+        for domain, num_entities in number_entities_dict.items():
+            description = f"db-{domain}-entities".lower()
+            # self.add_graph_node(domain, feature_type, description, int(num_entities > 0))
+            self.add_graph_node(domain, feature_type, description, min(num_entities, 5) / 5)
+        return number_entities_dict
+
+    def get_user_goal_feature(self, state, domain_active_dict):
+
+        feature_type = 'user goal'
+        for domain in self.belief_domains:
+            # the if case is needed because SGD only saves the dialogue state info for active domains
+            if domain in state['belief_state']:
+                for slot, value in state['belief_state'][domain].items():
+                    description = f"user goal-{domain}-{slot}".lower()
+                    value = 1.0 if (value and value != "not mentioned") else 0.0
+                    self.add_graph_node(domain, feature_type, description, value)
+
+                if [slot for slot, value in state['belief_state'][domain].items() if value]:
+                    domain_active_dict[domain] = True
+        return domain_active_dict
+
+    def get_sys_act_feature(self, state):
+
+        feature_type = 'last system act'
+        action = state['system_action'] if self.character == 'sys' else state['user_action']
+        action = delexicalize_da(action, self.requestable)
+        action = flat_da(action)
+        for da in action:
+            if da in self.act2vec:
+                domain = da.split('-')[0]
+                description = "system-" + da
+                value = 1.0
+                self.add_graph_node(domain, feature_type, description.lower(), value)
+
+    def get_user_act_feature(self, state):
+        # user-act feature
+        feature_type = 'user act'
+        action = state['user_action'] if self.character == 'sys' else state['system_action']
+        opp_action = delexicalize_da(action, self.requestable)
+        opp_action = flat_da(opp_action)
+
+        for da in opp_action:
+            if da in self.opp2vec:
+                domain = da.split('-')[0]
+                description = "user-" + da
+                value = 1.0
+                self.add_graph_node(domain, feature_type, description.lower(), value)
+
+    def get_general_features(self, state, domain_active_dict):
+
+        feature_type = 'general'
+        if 'booked' in state:
+            for i, domain in enumerate(self.db_domains):
+                if domain in state['booked']:
+                    description = f"general-{domain}-booked".lower()
+                    value = 1.0 if state['booked'][domain] else 0.0
+                    self.add_graph_node(domain, feature_type, description, value)
+
+        for domain in self.domains:
+            if domain == 'general':
+                continue
+            value = 1.0 if domain_active_dict[domain] else 0
+            description = f"general-{domain}".lower()
+            self.add_graph_node(domain, feature_type, description, value)
+
+    def filter_inactive_domains(self, domain_active_dict):
+
+        kg_filtered = []
+        for node in self.kg_info:
+            domain = node['domain']
+            if domain in domain_active_dict:
+                if domain_active_dict[domain]:
+                    kg_filtered.append(node)
+            else:
+                kg_filtered.append(node)
+
+        return kg_filtered
+
diff --git a/convlab/policy/vector/vector_uncertainty.py b/convlab/policy/vector/vector_uncertainty.py
new file mode 100644
index 0000000000000000000000000000000000000000..7da05449dbe247cacf01bac4970ee669cd670c44
--- /dev/null
+++ b/convlab/policy/vector/vector_uncertainty.py
@@ -0,0 +1,166 @@
+# -*- coding: utf-8 -*-
+import sys
+import numpy as np
+import logging
+
+from convlab.util.multiwoz.lexicalize import delexicalize_da, flat_da
+from convlab.policy.vector.vector_binary import VectorBinary
+
+
+class VectorUncertainty(VectorBinary):
+    """Vectorise state and state uncertainty predictions"""
+
+    def __init__(self,
+                 dataset_name: str = 'multiwoz21',
+                 character: str = 'sys',
+                 use_masking: bool = False,
+                 manually_add_entity_names: bool = True,
+                 seed: str = 0,
+                 use_confidence_scores: bool = True,
+                 confidence_thresholds: dict = None,
+                 use_state_total_uncertainty: bool = False,
+                 use_state_knowledge_uncertainty: bool = False):
+        """
+        Args:
+            dataset_name: Name of environment dataset
+            character: Character of the agent (sys/usr)
+            use_masking: If true certain actions are masked during devectorisation
+            manually_add_entity_names: If true inform entity name actions are manually added
+            seed: Seed
+            use_confidence_scores: If true confidence scores are used in state vectorisation
+            confidence_thresholds: If true confidence thresholds are used in database querying
+            use_state_total_uncertainty: If true state entropy is added to the state vector
+            use_state_knowledge_uncertainty: If true state mutual information is added to the state vector
+        """
+
+        self.use_confidence_scores = use_confidence_scores
+        self.use_state_total_uncertainty = use_state_total_uncertainty
+        self.use_state_knowledge_uncertainty = use_state_knowledge_uncertainty
+        if confidence_thresholds is not None:
+            self.setup_uncertain_query(confidence_thresholds)
+
+        super().__init__(dataset_name, character, use_masking, manually_add_entity_names, seed)
+
+    def get_state_dim(self):
+        self.belief_state_dim = 0
+
+        for domain in self.ontology['state']:
+            for slot in self.ontology['state'][domain]:
+                # Dim 1 - indicator/confidence score
+                # Dim 2 - Entropy (Total uncertainty) / Mutual information (knowledge unc)
+                slot_dim = 1 if not self.use_state_total_uncertainty else 2
+                slot_dim += 1 if self.use_state_knowledge_uncertainty else 0
+                self.belief_state_dim += slot_dim
+
+        self.state_dim = self.da_opp_dim + self.da_dim + self.belief_state_dim + \
+            len(self.db_domains) + 6 * len(self.db_domains) + 1
+
+    # Add thresholds for db_queries
+    def setup_uncertain_query(self, confidence_thresholds):
+        self.use_confidence_scores = True
+        self.confidence_thresholds = confidence_thresholds
+        logging.info('DB Search uncertainty activated.')
+
+    def dbquery_domain(self, domain):
+        """
+        query entities of specified domain
+        Args:
+            domain string:
+                domain to query
+        Returns:
+            entities list:
+                list of entities of the specified domain
+        """
+        # Get all user constraints
+        constraints = {slot: value for slot, value in self.state[domain].items()
+                       if slot and value not in ['dontcare',
+                                                 "do n't care", "do not care"]} if domain in self.state else dict()
+
+        # Remove constraints for which the uncertainty is high
+        if self.confidence_scores is not None and self.use_confidence_scores and self.confidence_thresholds is not None:
+            # Collect threshold values for each domain-slot pair
+            threshold = self.confidence_thresholds.get(domain, dict())
+            threshold = {slot: threshold.get(slot, 0.05) for slot in constraints}
+            # Get confidence scores for each constraint
+            probs = self.confidence_scores.get(domain, dict())
+            probs = {slot: probs.get(slot, {}).get('inform', 1.0) for slot in constraints}
+
+            # Filter out constraints for which confidence is lower than threshold
+            constraints = {slot: value for slot, value in constraints.items() if probs[slot] >= threshold[slot]}
+
+        return self.db.query(domain, constraints.items(), topk=10)
+
+    def vectorize_user_act(self, state):
+        """Return confidence scores for the user actions"""
+        self.confidence_scores = state['belief_state_probs'] if 'belief_state_probs' in state else None
+        action = state['user_action'] if self.character == 'sys' else state['system_action']
+        opp_action = delexicalize_da(action, self.requestable)
+        opp_action = flat_da(opp_action)
+        opp_act_vec = np.zeros(self.da_opp_dim)
+        for da in opp_action:
+            if da in self.opp2vec:
+                if 'belief_state_probs' in state and self.use_confidence_scores:
+                    domain, intent, slot, value = da.split('-')
+                    if domain in state['belief_state_probs']:
+                        slot = slot if slot else 'none'
+                        if slot in state['belief_state_probs'][domain]:
+                            prob = state['belief_state_probs'][domain][slot]
+                        elif slot.lower() in state['belief_state_probs'][domain]:
+                            prob = state['belief_state_probs'][domain][slot.lower()]
+                        else:
+                            prob = dict()
+
+                        if intent in prob:
+                            prob = float(prob[intent])
+                        else:
+                            prob = 1.0
+                    else:
+                        prob = 1.0
+                else:
+                    prob = 1.0
+                opp_act_vec[self.opp2vec[da]] = prob
+
+        return opp_act_vec
+
+    def vectorize_belief_state(self, state, domain_active_dict):
+        belief_state = np.zeros(self.belief_state_dim)
+        i = 0
+        for domain in self.belief_domains:
+            if self.use_confidence_scores and 'belief_state_probs' in state:
+                for slot in state['belief_state'][domain]:
+                    prob = None
+                    if slot in state['belief_state_probs'][domain]:
+                        prob = state['belief_state_probs'][domain][slot]
+                        prob = prob['inform'] if 'inform' in prob else None
+                    if prob:
+                        belief_state[i] = float(prob)
+                    i += 1
+            else:
+                for slot, value in state['belief_state'][domain].items():
+                    if value and value != 'not mentioned':
+                        belief_state[i] = 1.
+                    i += 1
+
+            if 'active_domains' in state:
+                domain_active = state['active_domains'][domain]
+                domain_active_dict[domain] = domain_active
+            else:
+                if [slot for slot, value in state['belief_state'][domain].items() if value]:
+                    domain_active_dict[domain] = True
+
+        # Add knowledge and/or total uncertainty to the belief state
+        if self.use_state_total_uncertainty and 'entropy' in state:
+            for domain in self.belief_domains:
+                for slot in state['belief_state'][domain]:
+                    if slot in state['entropy'][domain]:
+                        belief_state[i] = float(state['entropy'][domain][slot])
+                    i += 1
+
+        if self.use_state_knowledge_uncertainty and 'mutual_information' in state:
+            for domain in self.belief_domains:
+                for slot in state['belief_state'][domain]:
+                    if slot in state['mutual_information'][domain]:
+                        belief_state[i] = float(state['mutual_information'][domain][slot])
+                    i += 1
+
+        return belief_state, domain_active_dict
diff --git a/convlab/policy/vtrace_DPT/README.md b/convlab/policy/vtrace_DPT/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..002a8a050cc8bf573761a1b5ba2276d844a6db7d
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/README.md
@@ -0,0 +1,105 @@
+# Dynamic Dialogue Policy Transformer (DDPT)
+
+The dynamic dialogue policy transformer (Geishauser et. al. 2022) is a model built for continual reinforcement learning. It uses a pre-trained RoBERTa language model to construct embeddings for each state information and domain, slot and value in the action set. As a consequence, it can be used for different ontologies and is able to deal with new state information as well as actions. The backbone architecture is a transformer encoder-decoder.
+
+It uses the CLEAR algorithm (Rolnick et. al. 2019) for continual reinforcement learning that builds on top of VTRACE (Espheholt et. al. 2018). The current folder supports only training in a stationary environment and no continual learning, which uses VTRACE as algorithm.
+
+## Supervised pre-training
+
+If you want to pre-train the model on a dataset, use the command
+
+```sh
+$ python supervised/train_supervised.py --dataset_name=DATASET_NAME --seed=SEED --model_path=""
+```
+
+The first time you run that command, it will take longer as the dataset needs to be pre-processed.
+
+This will create a corresponding experiments folder under supervised/experiments, where the model is saved in /save.
+
+You can specify the dataset that you would like to use, e.g. "multiwoz21" or "sgd". You can also specify a model_path if you have already a pre-trained model, for instance when you first train on SGD before you fine-tune on multiwoz21 data.
+
+You can specify hyperparamters such as epoch, supervised_lr and data_percentage (how much of the data you want to use) in the config.json file.
+
+We provide several supervised trained models on hugging-face to reproduce the results:
+
+- pre-trained on SGD: https://huggingface.co/ConvLab/ddpt-policy-sgd
+- pre-trained on 1% multiwoz21: https://huggingface.co/ConvLab/ddpt-policy-0.01multiwoz21
+- pre-trained on SGD and afterwards on 1% multiwoz21: https://huggingface.co/ConvLab/ddpt-policy-sgd_0.01multiwoz21
+
+## RL training
+
+Starting a RL training is as easy as executing
+
+```sh
+$ python train.py --path=your_environment_config --seed=SEED
+```
+
+One example for the environment-config is **semantic_level_config.json**, where parameters for the training are specified, for instance
+
+- load_path: provide a path to initialise the model with a pre-trained model, skip the ending .pol.mdl
+- process_num: the number of processes to use during evaluation to speed it up
+- num_eval_dialogues: how many evaluation dialogues should be used
+- eval_frequency: after how many training dialogues an evaluation should be performed
+- total_dialogues: how many training dialogues should be done in total
+- new_dialogues: how many new dialogues should be collected before a policy update
+
+Moreover, you can specify the full dialogue pipeline here, such as the user policy, NLU for system and user, etc.
+
+Parameters that are tied to the RL algorithm and the model architecture can be changed in config.json.
+
+
+## Evaluation
+
+For creating evaluation plots and running evaluation dialogues, please have a look in the README of the policy folder.
+
+## References
+
+```
+@inproceedings{geishauser-etal-2022-dynamic,
+    title = "Dynamic Dialogue Policy for Continual Reinforcement Learning",
+    author = "Geishauser, Christian  and
+      van Niekerk, Carel  and
+      Lin, Hsien-chin  and
+      Lubis, Nurul  and
+      Heck, Michael  and
+      Feng, Shutong  and
+      Ga{\v{s}}i{\'c}, Milica",
+    booktitle = "Proceedings of the 29th International Conference on Computational Linguistics",
+    month = oct,
+    year = "2022",
+    address = "Gyeongju, Republic of Korea",
+    publisher = "International Committee on Computational Linguistics",
+    url = "https://aclanthology.org/2022.coling-1.21",
+    pages = "266--284",
+    abstract = "Continual learning is one of the key components of human learning and a necessary requirement of artificial intelligence. As dialogue can potentially span infinitely many topics and tasks, a task-oriented dialogue system must have the capability to continually learn, dynamically adapting to new challenges while preserving the knowledge it already acquired. Despite the importance, continual reinforcement learning of the dialogue policy has remained largely unaddressed. The lack of a framework with training protocols, baseline models and suitable metrics, has so far hindered research in this direction. In this work we fill precisely this gap, enabling research in dialogue policy optimisation to go from static to dynamic learning. We provide a continual learning algorithm, baseline architectures and metrics for assessing continual learning models. Moreover, we propose the dynamic dialogue policy transformer (DDPT), a novel dynamic architecture that can integrate new knowledge seamlessly, is capable of handling large state spaces and obtains significant zero-shot performance when being exposed to unseen domains, without any growth in network parameter size. We validate the strengths of DDPT in simulation with two user simulators as well as with humans.",
+}
+
+@inproceedings{NEURIPS2019_fa7cdfad,
+ author = {Rolnick, David and Ahuja, Arun and Schwarz, Jonathan and Lillicrap, Timothy and Wayne, Gregory},
+ booktitle = {Advances in Neural Information Processing Systems},
+ editor = {H. Wallach and H. Larochelle and A. Beygelzimer and F. d\textquotesingle Alch\'{e}-Buc and E. Fox and R. Garnett},
+ pages = {},
+ publisher = {Curran Associates, Inc.},
+ title = {Experience Replay for Continual Learning},
+ url = {https://proceedings.neurips.cc/paper/2019/file/fa7cdfad1a5aaf8370ebeda47a1ff1c3-Paper.pdf},
+ volume = {32},
+ year = {2019}
+}
+
+@InProceedings{pmlr-v80-espeholt18a,
+  title = 	 {{IMPALA}: Scalable Distributed Deep-{RL} with Importance Weighted Actor-Learner Architectures},
+  author =       {Espeholt, Lasse and Soyer, Hubert and Munos, Remi and Simonyan, Karen and Mnih, Vlad and Ward, Tom and Doron, Yotam and Firoiu, Vlad and Harley, Tim and Dunning, Iain and Legg, Shane and Kavukcuoglu, Koray},
+  booktitle = 	 {Proceedings of the 35th International Conference on Machine Learning},
+  pages = 	 {1407--1416},
+  year = 	 {2018},
+  editor = 	 {Dy, Jennifer and Krause, Andreas},
+  volume = 	 {80},
+  series = 	 {Proceedings of Machine Learning Research},
+  month = 	 {10--15 Jul},
+  publisher =    {PMLR},
+  pdf = 	 {http://proceedings.mlr.press/v80/espeholt18a/espeholt18a.pdf},
+  url = 	 {https://proceedings.mlr.press/v80/espeholt18a.html},
+  abstract = 	 {In this work we aim to solve a large collection of tasks using a single reinforcement learning agent with a single set of parameters. A key challenge is to handle the increased amount of data and extended training time. We have developed a new distributed agent IMPALA (Importance Weighted Actor-Learner Architecture) that not only uses resources more efficiently in single-machine training but also scales to thousands of machines without sacrificing data efficiency or resource utilisation. We achieve stable learning at high throughput by combining decoupled acting and learning with a novel off-policy correction method called V-trace. We demonstrate the effectiveness of IMPALA for multi-task reinforcement learning on DMLab-30 (a set of 30 tasks from the DeepMind Lab environment (Beattie et al., 2016)) and Atari57 (all available Atari games in Arcade Learning Environment (Bellemare et al., 2013a)). Our results show that IMPALA is able to achieve better performance than previous agents with less data, and crucially exhibits positive transfer between tasks as a result of its multi-task approach.}
+}
+
+```
\ No newline at end of file
diff --git a/convlab/policy/vtrace_DPT/__init__.py b/convlab/policy/vtrace_DPT/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..13469e5bc59921487499878f90bc146bc0a43e4c
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/__init__.py
@@ -0,0 +1 @@
+from convlab.policy.vtrace_DPT.vtrace import VTRACE
\ No newline at end of file
diff --git a/convlab/policy/vtrace_DPT/config.json b/convlab/policy/vtrace_DPT/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..13362e74f11fd3705820028366b145c7d66ea4d9
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/config.json
@@ -0,0 +1,76 @@
+{
+	"batchsz": 64,
+	"epoch": 40,
+	"gamma": 0.99,
+	"policy_lr": 5e-06,
+	"supervised_lr": 1e-05,
+	"entropy_weight": 0.01,
+	"value_lr": 0.0001,
+	"save_dir": "save",
+	"log_dir": "log",
+	"save_per_epoch": 5000,
+	"hidden_size": 256,
+	"load": "save/best",
+	"logging_mode": "INFO",
+	"use_cer": true,
+	"memory_size": 5000,
+	"behaviour_cloning_weight": 0.1,
+	"supervised_weight": 0.0,
+	"online_offline_ratio": 0.20,
+	"smoothed_value_function": false,
+	"use_reservoir_sampling": false,
+	"seed": 0,
+	"lambda": 1,
+	"tau": 0.001,
+	"policy_freq": 1,
+	"print_per_batch": 400,
+	"c": 1.0,
+	"rho_bar": 1,
+	"max_length": 10,
+	"noisy_linear": false,
+	"dataset_name": "multiwoz21",
+	"data_percentage": 1.0,
+	"dialogue_order": 0,
+	"multiwoz_like": false,
+	"regularization_weight": 0.0,
+
+	"enc_input_dim": 128,
+	"enc_nhead": 2,
+	"enc_d_hid": 128,
+	"enc_nlayers": 4,
+	"enc_dropout": 0.1,
+
+	"dec_input_dim": 128,
+	"dec_nhead": 2,
+	"dec_d_hid": 128,
+	"dec_nlayers": 2,
+	"dec_dropout": 0.0,
+
+	"action_embedding_dim": 128,
+	"domain_embedding_dim": 64,
+	"value_embedding_dim": 12,
+	"node_embedding_dim": 128,
+	"roberta_path": "",
+	"node_attention": true,
+	"semantic_descriptions": true,
+	"freeze_roberta": true,
+	"use_pooled": false,
+	"mean": true,
+	"roberta_actions": true,
+	"independent_descriptions": true,
+	"random_matrix": false,
+	"distance_metric": false,
+
+	"verbose": false,
+	"ignore_features": [],
+	"domains_removed": ["hospital", "police", "train", "hotel", "attraction", "taxi"],
+	"only_active_values": false,
+	"permuted_data": false,
+	"need_weights": false,
+
+	"cls_dim": 128,
+	"independent": true,
+	"old_critic": false,
+	"pos_weight": 5,
+	"weight_decay": 0.00001
+}
\ No newline at end of file
diff --git a/convlab/policy/vtrace_DPT/create_descriptions.py b/convlab/policy/vtrace_DPT/create_descriptions.py
new file mode 100644
index 0000000000000000000000000000000000000000..5357fafafcf4972aa8eebaecf8c56e9fba79afe7
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/create_descriptions.py
@@ -0,0 +1,66 @@
+import os
+import json
+
+from convlab.policy.vector.vector_binary import VectorBinary
+from convlab.util import load_ontology, load_database
+from convlab.util.custom_util import timeout
+
+
+def create_description_dicts(name='multiwoz21'):
+
+    vector = VectorBinary(name)
+    ontology = load_ontology(name)
+    default_state = ontology['state']
+    domains = list(ontology['domains'].keys())
+
+    if name == "multiwoz21":
+        db = load_database(name)
+        db_domains = db.domains
+    else:
+        db = None
+        db_domains = []
+
+    root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    voc_file = os.path.join(root_dir, f'vector/action_dicts/{name}_VectorBinary/sys_da_voc.txt')
+    voc_opp_file = os.path.join(root_dir, f'vector/action_dicts/{name}_VectorBinary/user_da_voc.txt')
+
+    with open(voc_file) as f:
+        da_voc = f.read().splitlines()
+    with open(voc_opp_file) as f:
+        da_voc_opp = f.read().splitlines()
+
+    description_dict_semantic = {}
+
+    for domain in default_state:
+        for slot in default_state[domain]:
+            domain = domain.lower()
+            description_dict_semantic[f"user goal-{domain}-{slot.lower()}"] = f"user goal {domain} {slot}"
+
+    if db_domains:
+        for domain in db_domains:
+            domain = domain.lower()
+            description_dict_semantic[f"db-{domain}-entities"] = f"data base {domain} number of entities"
+            description_dict_semantic[f"general-{domain}-booked"] = f"general {domain} booked"
+
+    for domain in domains:
+        domain = domain.lower()
+        description_dict_semantic[f"general-{domain}"] = f"domain {domain}"
+
+    for act in da_voc:
+        domain, intent, slot, value = act.split("-")
+        domain = domain.lower()
+        description_dict_semantic["system-"+act.lower()] = f"last system act {domain} {intent} {slot} {value}"
+
+    for act in da_voc_opp:
+        domain, intent, slot, value = [item.lower() for item in act.split("-")]
+        domain = domain.lower()
+        description_dict_semantic["user-"+act.lower()] = f"user act {domain} {intent} {slot} {value}"
+
+    root_dir = os.path.dirname(os.path.abspath(__file__))
+    os.makedirs(os.path.join(root_dir, "descriptions"), exist_ok=True)
+    with open(os.path.join(root_dir, 'descriptions', f'semantic_information_descriptions_{name}.json'), "w") as f:
+        json.dump(description_dict_semantic, f)
+
+
+if __name__ == '__main__':
+    create_description_dicts()
diff --git a/convlab/policy/vtrace_DPT/descriptions/semantic_information_descriptions_multiwoz21.json b/convlab/policy/vtrace_DPT/descriptions/semantic_information_descriptions_multiwoz21.json
new file mode 100644
index 0000000000000000000000000000000000000000..f0757293c3f05fbd7b77dbf356f3e7c4fc578371
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/descriptions/semantic_information_descriptions_multiwoz21.json
@@ -0,0 +1 @@
+{"user goal-attraction-type": "user goal attraction type", "user goal-attraction-name": "user goal attraction name", "user goal-attraction-area": "user goal attraction area", "user goal-hotel-name": "user goal hotel name", "user goal-hotel-area": "user goal hotel area", "user goal-hotel-parking": "user goal hotel parking", "user goal-hotel-price range": "user goal hotel price range", "user goal-hotel-stars": "user goal hotel stars", "user goal-hotel-internet": "user goal hotel internet", "user goal-hotel-type": "user goal hotel type", "user goal-hotel-book stay": "user goal hotel book stay", "user goal-hotel-book day": "user goal hotel book day", "user goal-hotel-book people": "user goal hotel book people", "user goal-restaurant-food": "user goal restaurant food", "user goal-restaurant-price range": "user goal restaurant price range", "user goal-restaurant-name": "user goal restaurant name", "user goal-restaurant-area": "user goal restaurant area", "user goal-restaurant-book time": "user goal restaurant book time", "user goal-restaurant-book day": "user goal restaurant book day", "user goal-restaurant-book people": "user goal restaurant book people", "user goal-taxi-leave at": "user goal taxi leave at", "user goal-taxi-destination": "user goal taxi destination", "user goal-taxi-departure": "user goal taxi departure", "user goal-taxi-arrive by": "user goal taxi arrive by", "user goal-train-leave at": "user goal train leave at", "user goal-train-destination": "user goal train destination", "user goal-train-day": "user goal train day", "user goal-train-arrive by": "user goal train arrive by", "user goal-train-departure": "user goal train departure", "user goal-train-book people": "user goal train book people", "user goal-hospital-department": "user goal hospital department", "db-restaurant-entities": "data base restaurant number of entities", "general-restaurant-booked": "general restaurant booked", "db-hotel-entities": "data base hotel number of entities", "general-hotel-booked": "general hotel booked", "db-attraction-entities": "data base attraction number of entities", "general-attraction-booked": "general attraction booked", "db-train-entities": "data base train number of entities", "general-train-booked": "general train booked", "db-hospital-entities": "data base hospital number of entities", "general-hospital-booked": "general hospital booked", "db-police-entities": "data base police number of entities", "general-police-booked": "general police booked", "general-attraction": "domain attraction", "general-hotel": "domain hotel", "general-taxi": "domain taxi", "general-restaurant": "domain restaurant", "general-train": "domain train", "general-police": "domain police", "general-hospital": "domain hospital", "general-general": "domain general", "system-attraction-inform-address-1": "last system act attraction inform address 1", "system-attraction-inform-address-2": "last system act attraction inform address 2", "system-attraction-inform-address-3": "last system act attraction inform address 3", "system-attraction-inform-area-1": "last system act attraction inform area 1", "system-attraction-inform-area-2": "last system act attraction inform area 2", "system-attraction-inform-area-3": "last system act attraction inform area 3", "system-attraction-inform-choice-1": "last system act attraction inform choice 1", "system-attraction-inform-choice-2": "last system act attraction inform choice 2", "system-attraction-inform-choice-3": "last system act attraction inform choice 3", "system-attraction-inform-entrance fee-1": "last system act attraction inform entrance fee 1", "system-attraction-inform-entrance fee-2": "last system act attraction inform entrance fee 2", "system-attraction-inform-name-1": "last system act attraction inform name 1", "system-attraction-inform-name-2": "last system act attraction inform name 2", "system-attraction-inform-name-3": "last system act attraction inform name 3", "system-attraction-inform-name-4": "last system act attraction inform name 4", "system-attraction-inform-phone-1": "last system act attraction inform phone 1", "system-attraction-inform-postcode-1": "last system act attraction inform postcode 1", "system-attraction-inform-type-1": "last system act attraction inform type 1", "system-attraction-inform-type-2": "last system act attraction inform type 2", "system-attraction-inform-type-3": "last system act attraction inform type 3", "system-attraction-inform-type-4": "last system act attraction inform type 4", "system-attraction-inform-type-5": "last system act attraction inform type 5", "system-attraction-nooffer-area-1": "last system act attraction nooffer area 1", "system-attraction-nooffer-none-none": "last system act attraction nooffer none none", "system-attraction-nooffer-type-1": "last system act attraction nooffer type 1", "system-attraction-recommend-address-1": "last system act attraction recommend address 1", "system-attraction-recommend-address-2": "last system act attraction recommend address 2", "system-attraction-recommend-area-1": "last system act attraction recommend area 1", "system-attraction-recommend-entrance fee-1": "last system act attraction recommend entrance fee 1", "system-attraction-recommend-name-1": "last system act attraction recommend name 1", "system-attraction-recommend-phone-1": "last system act attraction recommend phone 1", "system-attraction-recommend-postcode-1": "last system act attraction recommend postcode 1", "system-attraction-recommend-type-1": "last system act attraction recommend type 1", "system-attraction-request-area-?": "last system act attraction request area ?", "system-attraction-request-entrance fee-?": "last system act attraction request entrance fee ?", "system-attraction-request-name-?": "last system act attraction request name ?", "system-attraction-request-type-?": "last system act attraction request type ?", "system-attraction-select-none-none": "last system act attraction select none none", "system-attraction-select-type-1": "last system act attraction select type 1", "system-attraction-select-type-2": "last system act attraction select type 2", "system-attraction-select-type-3": "last system act attraction select type 3", "system-general-bye-none-none": "last system act general bye none none", "system-general-greet-none-none": "last system act general greet none none", "system-general-reqmore-none-none": "last system act general reqmore none none", "system-general-welcome-none-none": "last system act general welcome none none", "system-hospital-inform-address-1": "last system act hospital inform address 1", "system-hospital-inform-department-1": "last system act hospital inform department 1", "system-hospital-inform-phone-1": "last system act hospital inform phone 1", "system-hospital-inform-postcode-1": "last system act hospital inform postcode 1", "system-hospital-request-department-?": "last system act hospital request department ?", "system-hotel-book-none-none": "last system act hotel book none none", "system-hotel-inform-address-1": "last system act hotel inform address 1", "system-hotel-inform-address-2": "last system act hotel inform address 2", "system-hotel-inform-area-1": "last system act hotel inform area 1", "system-hotel-inform-area-2": "last system act hotel inform area 2", "system-hotel-inform-book day-1": "last system act hotel inform book day 1", "system-hotel-inform-book people-1": "last system act hotel inform book people 1", "system-hotel-inform-book stay-1": "last system act hotel inform book stay 1", "system-hotel-inform-choice-1": "last system act hotel inform choice 1", "system-hotel-inform-choice-2": "last system act hotel inform choice 2", "system-hotel-inform-choice-3": "last system act hotel inform choice 3", "system-hotel-inform-internet-1": "last system act hotel inform internet 1", "system-hotel-inform-name-1": "last system act hotel inform name 1", "system-hotel-inform-name-2": "last system act hotel inform name 2", "system-hotel-inform-name-3": "last system act hotel inform name 3", "system-hotel-inform-parking-1": "last system act hotel inform parking 1", "system-hotel-inform-phone-1": "last system act hotel inform phone 1", "system-hotel-inform-postcode-1": "last system act hotel inform postcode 1", "system-hotel-inform-price range-1": "last system act hotel inform price range 1", "system-hotel-inform-price range-2": "last system act hotel inform price range 2", "system-hotel-inform-ref-1": "last system act hotel inform ref 1", "system-hotel-inform-stars-1": "last system act hotel inform stars 1", "system-hotel-inform-stars-2": "last system act hotel inform stars 2", "system-hotel-inform-type-1": "last system act hotel inform type 1", "system-hotel-inform-type-2": "last system act hotel inform type 2", "system-hotel-nooffer-area-1": "last system act hotel nooffer area 1", "system-hotel-nooffer-none-none": "last system act hotel nooffer none none", "system-hotel-nooffer-price range-1": "last system act hotel nooffer price range 1", "system-hotel-nooffer-stars-1": "last system act hotel nooffer stars 1", "system-hotel-nooffer-type-1": "last system act hotel nooffer type 1", "system-hotel-offerbook-name-1": "last system act hotel offerbook name 1", "system-hotel-recommend-address-1": "last system act hotel recommend address 1", "system-hotel-recommend-area-1": "last system act hotel recommend area 1", "system-hotel-recommend-internet-1": "last system act hotel recommend internet 1", "system-hotel-recommend-name-1": "last system act hotel recommend name 1", "system-hotel-recommend-parking-1": "last system act hotel recommend parking 1", "system-hotel-recommend-price range-1": "last system act hotel recommend price range 1", "system-hotel-recommend-stars-1": "last system act hotel recommend stars 1", "system-hotel-recommend-type-1": "last system act hotel recommend type 1", "system-hotel-request-area-?": "last system act hotel request area ?", "system-hotel-request-book day-?": "last system act hotel request book day ?", "system-hotel-request-book people-?": "last system act hotel request book people ?", "system-hotel-request-book stay-?": "last system act hotel request book stay ?", "system-hotel-request-internet-?": "last system act hotel request internet ?", "system-hotel-request-name-?": "last system act hotel request name ?", "system-hotel-request-parking-?": "last system act hotel request parking ?", "system-hotel-request-price range-?": "last system act hotel request price range ?", "system-hotel-request-stars-?": "last system act hotel request stars ?", "system-hotel-request-type-?": "last system act hotel request type ?", "system-hotel-select-area-1": "last system act hotel select area 1", "system-hotel-select-area-2": "last system act hotel select area 2", "system-hotel-select-name-1": "last system act hotel select name 1", "system-hotel-select-none-none": "last system act hotel select none none", "system-hotel-select-price range-1": "last system act hotel select price range 1", "system-hotel-select-price range-2": "last system act hotel select price range 2", "system-hotel-select-stars-1": "last system act hotel select stars 1", "system-hotel-select-type-1": "last system act hotel select type 1", "system-hotel-select-type-2": "last system act hotel select type 2", "system-police-inform-address-1": "last system act police inform address 1", "system-police-inform-name-1": "last system act police inform name 1", "system-police-inform-phone-1": "last system act police inform phone 1", "system-police-inform-postcode-1": "last system act police inform postcode 1", "system-restaurant-book-none-none": "last system act restaurant book none none", "system-restaurant-inform-address-1": "last system act restaurant inform address 1", "system-restaurant-inform-address-2": "last system act restaurant inform address 2", "system-restaurant-inform-area-1": "last system act restaurant inform area 1", "system-restaurant-inform-area-2": "last system act restaurant inform area 2", "system-restaurant-inform-book day-1": "last system act restaurant inform book day 1", "system-restaurant-inform-book people-1": "last system act restaurant inform book people 1", "system-restaurant-inform-book time-1": "last system act restaurant inform book time 1", "system-restaurant-inform-choice-1": "last system act restaurant inform choice 1", "system-restaurant-inform-choice-2": "last system act restaurant inform choice 2", "system-restaurant-inform-choice-3": "last system act restaurant inform choice 3", "system-restaurant-inform-food-1": "last system act restaurant inform food 1", "system-restaurant-inform-food-2": "last system act restaurant inform food 2", "system-restaurant-inform-food-3": "last system act restaurant inform food 3", "system-restaurant-inform-food-4": "last system act restaurant inform food 4", "system-restaurant-inform-name-1": "last system act restaurant inform name 1", "system-restaurant-inform-name-2": "last system act restaurant inform name 2", "system-restaurant-inform-name-3": "last system act restaurant inform name 3", "system-restaurant-inform-name-4": "last system act restaurant inform name 4", "system-restaurant-inform-phone-1": "last system act restaurant inform phone 1", "system-restaurant-inform-postcode-1": "last system act restaurant inform postcode 1", "system-restaurant-inform-postcode-2": "last system act restaurant inform postcode 2", "system-restaurant-inform-price range-1": "last system act restaurant inform price range 1", "system-restaurant-inform-price range-2": "last system act restaurant inform price range 2", "system-restaurant-inform-ref-1": "last system act restaurant inform ref 1", "system-restaurant-nobook-book time-1": "last system act restaurant nobook book time 1", "system-restaurant-nooffer-area-1": "last system act restaurant nooffer area 1", "system-restaurant-nooffer-food-1": "last system act restaurant nooffer food 1", "system-restaurant-nooffer-none-none": "last system act restaurant nooffer none none", "system-restaurant-nooffer-price range-1": "last system act restaurant nooffer price range 1", "system-restaurant-offerbook-name-1": "last system act restaurant offerbook name 1", "system-restaurant-recommend-address-1": "last system act restaurant recommend address 1", "system-restaurant-recommend-area-1": "last system act restaurant recommend area 1", "system-restaurant-recommend-food-1": "last system act restaurant recommend food 1", "system-restaurant-recommend-name-1": "last system act restaurant recommend name 1", "system-restaurant-recommend-phone-1": "last system act restaurant recommend phone 1", "system-restaurant-recommend-postcode-1": "last system act restaurant recommend postcode 1", "system-restaurant-recommend-price range-1": "last system act restaurant recommend price range 1", "system-restaurant-request-area-?": "last system act restaurant request area ?", "system-restaurant-request-book day-?": "last system act restaurant request book day ?", "system-restaurant-request-book people-?": "last system act restaurant request book people ?", "system-restaurant-request-book time-?": "last system act restaurant request book time ?", "system-restaurant-request-food-?": "last system act restaurant request food ?", "system-restaurant-request-name-?": "last system act restaurant request name ?", "system-restaurant-request-price range-?": "last system act restaurant request price range ?", "system-restaurant-select-area-1": "last system act restaurant select area 1", "system-restaurant-select-area-2": "last system act restaurant select area 2", "system-restaurant-select-food-1": "last system act restaurant select food 1", "system-restaurant-select-food-2": "last system act restaurant select food 2", "system-restaurant-select-food-3": "last system act restaurant select food 3", "system-restaurant-select-name-1": "last system act restaurant select name 1", "system-restaurant-select-none-none": "last system act restaurant select none none", "system-restaurant-select-price range-1": "last system act restaurant select price range 1", "system-restaurant-select-price range-2": "last system act restaurant select price range 2", "system-taxi-book-none-none": "last system act taxi book none none", "system-taxi-inform-arrive by-1": "last system act taxi inform arrive by 1", "system-taxi-inform-departure-1": "last system act taxi inform departure 1", "system-taxi-inform-destination-1": "last system act taxi inform destination 1", "system-taxi-inform-leave at-1": "last system act taxi inform leave at 1", "system-taxi-inform-none-none": "last system act taxi inform none none", "system-taxi-inform-phone-1": "last system act taxi inform phone 1", "system-taxi-inform-type-1": "last system act taxi inform type 1", "system-taxi-request-arrive by-?": "last system act taxi request arrive by ?", "system-taxi-request-departure-?": "last system act taxi request departure ?", "system-taxi-request-destination-?": "last system act taxi request destination ?", "system-taxi-request-leave at-?": "last system act taxi request leave at ?", "system-train-book-none-none": "last system act train book none none", "system-train-inform-arrive by-1": "last system act train inform arrive by 1", "system-train-inform-arrive by-2": "last system act train inform arrive by 2", "system-train-inform-book people-1": "last system act train inform book people 1", "system-train-inform-choice-1": "last system act train inform choice 1", "system-train-inform-choice-2": "last system act train inform choice 2", "system-train-inform-day-1": "last system act train inform day 1", "system-train-inform-departure-1": "last system act train inform departure 1", "system-train-inform-destination-1": "last system act train inform destination 1", "system-train-inform-duration-1": "last system act train inform duration 1", "system-train-inform-leave at-1": "last system act train inform leave at 1", "system-train-inform-leave at-2": "last system act train inform leave at 2", "system-train-inform-leave at-3": "last system act train inform leave at 3", "system-train-inform-none-none": "last system act train inform none none", "system-train-inform-price-1": "last system act train inform price 1", "system-train-inform-ref-1": "last system act train inform ref 1", "system-train-inform-train id-1": "last system act train inform train id 1", "system-train-offerbook-arrive by-1": "last system act train offerbook arrive by 1", "system-train-offerbook-destination-1": "last system act train offerbook destination 1", "system-train-offerbook-leave at-1": "last system act train offerbook leave at 1", "system-train-offerbook-none-none": "last system act train offerbook none none", "system-train-offerbook-train id-1": "last system act train offerbook train id 1", "system-train-request-arrive by-?": "last system act train request arrive by ?", "system-train-request-book people-?": "last system act train request book people ?", "system-train-request-day-?": "last system act train request day ?", "system-train-request-departure-?": "last system act train request departure ?", "system-train-request-destination-?": "last system act train request destination ?", "system-train-request-leave at-?": "last system act train request leave at ?", "system-train-select-leave at-1": "last system act train select leave at 1", "system-train-select-none-none": "last system act train select none none", "user-attraction-inform-area-1": "user act attraction inform area 1", "user-attraction-inform-name-1": "user act attraction inform name 1", "user-attraction-inform-none-none": "user act attraction inform none none", "user-attraction-inform-type-1": "user act attraction inform type 1", "user-attraction-request-address-?": "user act attraction request address ?", "user-attraction-request-area-?": "user act attraction request area ?", "user-attraction-request-entrance fee-?": "user act attraction request entrance fee ?", "user-attraction-request-phone-?": "user act attraction request phone ?", "user-attraction-request-postcode-?": "user act attraction request postcode ?", "user-attraction-request-type-?": "user act attraction request type ?", "user-general-bye-none-none": "user act general bye none none", "user-general-greet-none-none": "user act general greet none none", "user-general-thank-none-none": "user act general thank none none", "user-hospital-inform-department-1": "user act hospital inform department 1", "user-hospital-inform-none-none": "user act hospital inform none none", "user-hospital-request-address-?": "user act hospital request address ?", "user-hospital-request-phone-?": "user act hospital request phone ?", "user-hospital-request-postcode-?": "user act hospital request postcode ?", "user-hotel-inform-area-1": "user act hotel inform area 1", "user-hotel-inform-book day-1": "user act hotel inform book day 1", "user-hotel-inform-book people-1": "user act hotel inform book people 1", "user-hotel-inform-book stay-1": "user act hotel inform book stay 1", "user-hotel-inform-internet-1": "user act hotel inform internet 1", "user-hotel-inform-name-1": "user act hotel inform name 1", "user-hotel-inform-none-none": "user act hotel inform none none", "user-hotel-inform-parking-1": "user act hotel inform parking 1", "user-hotel-inform-price range-1": "user act hotel inform price range 1", "user-hotel-inform-stars-1": "user act hotel inform stars 1", "user-hotel-inform-type-1": "user act hotel inform type 1", "user-hotel-request-address-?": "user act hotel request address ?", "user-hotel-request-area-?": "user act hotel request area ?", "user-hotel-request-internet-?": "user act hotel request internet ?", "user-hotel-request-parking-?": "user act hotel request parking ?", "user-hotel-request-phone-?": "user act hotel request phone ?", "user-hotel-request-postcode-?": "user act hotel request postcode ?", "user-hotel-request-price range-?": "user act hotel request price range ?", "user-hotel-request-ref-?": "user act hotel request ref ?", "user-hotel-request-stars-?": "user act hotel request stars ?", "user-hotel-request-type-?": "user act hotel request type ?", "user-police-inform-name-1": "user act police inform name 1", "user-police-inform-none-none": "user act police inform none none", "user-police-request-address-?": "user act police request address ?", "user-police-request-phone-?": "user act police request phone ?", "user-police-request-postcode-?": "user act police request postcode ?", "user-restaurant-inform-area-1": "user act restaurant inform area 1", "user-restaurant-inform-book day-1": "user act restaurant inform book day 1", "user-restaurant-inform-book people-1": "user act restaurant inform book people 1", "user-restaurant-inform-book time-1": "user act restaurant inform book time 1", "user-restaurant-inform-food-1": "user act restaurant inform food 1", "user-restaurant-inform-name-1": "user act restaurant inform name 1", "user-restaurant-inform-none-none": "user act restaurant inform none none", "user-restaurant-inform-price range-1": "user act restaurant inform price range 1", "user-restaurant-request-address-?": "user act restaurant request address ?", "user-restaurant-request-area-?": "user act restaurant request area ?", "user-restaurant-request-food-?": "user act restaurant request food ?", "user-restaurant-request-phone-?": "user act restaurant request phone ?", "user-restaurant-request-postcode-?": "user act restaurant request postcode ?", "user-restaurant-request-price range-?": "user act restaurant request price range ?", "user-restaurant-request-ref-?": "user act restaurant request ref ?", "user-taxi-inform-arrive by-1": "user act taxi inform arrive by 1", "user-taxi-inform-departure-1": "user act taxi inform departure 1", "user-taxi-inform-destination-1": "user act taxi inform destination 1", "user-taxi-inform-leave at-1": "user act taxi inform leave at 1", "user-taxi-inform-none-none": "user act taxi inform none none", "user-taxi-request-phone-?": "user act taxi request phone ?", "user-taxi-request-type-?": "user act taxi request type ?", "user-train-inform-arrive by-1": "user act train inform arrive by 1", "user-train-inform-book people-1": "user act train inform book people 1", "user-train-inform-day-1": "user act train inform day 1", "user-train-inform-departure-1": "user act train inform departure 1", "user-train-inform-destination-1": "user act train inform destination 1", "user-train-inform-leave at-1": "user act train inform leave at 1", "user-train-inform-none-none": "user act train inform none none", "user-train-request-arrive by-?": "user act train request arrive by ?", "user-train-request-duration-?": "user act train request duration ?", "user-train-request-leave at-?": "user act train request leave at ?", "user-train-request-price-?": "user act train request price ?", "user-train-request-ref-?": "user act train request ref ?", "user-train-request-train id-?": "user act train request train id ?"}
\ No newline at end of file
diff --git a/convlab/policy/vtrace_DPT/descriptions/semantic_information_descriptions_sgd.json b/convlab/policy/vtrace_DPT/descriptions/semantic_information_descriptions_sgd.json
new file mode 100644
index 0000000000000000000000000000000000000000..2539067dfd07e25e35359218ed4487a6da057910
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/descriptions/semantic_information_descriptions_sgd.json
@@ -0,0 +1 @@
+{"user goal-banks_1-account_type": "user goal banks_1 account_type", "user goal-banks_1-recipient_account_type": "user goal banks_1 recipient_account_type", "user goal-banks_1-balance": "user goal banks_1 balance", "user goal-banks_1-amount": "user goal banks_1 amount", "user goal-banks_1-recipient_account_name": "user goal banks_1 recipient_account_name", "user goal-buses_1-from_location": "user goal buses_1 from_location", "user goal-buses_1-to_location": "user goal buses_1 to_location", "user goal-buses_1-from_station": "user goal buses_1 from_station", "user goal-buses_1-to_station": "user goal buses_1 to_station", "user goal-buses_1-leaving_date": "user goal buses_1 leaving_date", "user goal-buses_1-leaving_time": "user goal buses_1 leaving_time", "user goal-buses_1-fare": "user goal buses_1 fare", "user goal-buses_1-travelers": "user goal buses_1 travelers", "user goal-buses_1-transfers": "user goal buses_1 transfers", "user goal-buses_2-origin": "user goal buses_2 origin", "user goal-buses_2-destination": "user goal buses_2 destination", "user goal-buses_2-origin_station_name": "user goal buses_2 origin_station_name", "user goal-buses_2-destination_station_name": "user goal buses_2 destination_station_name", "user goal-buses_2-departure_date": "user goal buses_2 departure_date", "user goal-buses_2-price": "user goal buses_2 price", "user goal-buses_2-departure_time": "user goal buses_2 departure_time", "user goal-buses_2-group_size": "user goal buses_2 group_size", "user goal-buses_2-fare_type": "user goal buses_2 fare_type", "user goal-calendar_1-event_date": "user goal calendar_1 event_date", "user goal-calendar_1-event_time": "user goal calendar_1 event_time", "user goal-calendar_1-event_location": "user goal calendar_1 event_location", "user goal-calendar_1-event_name": "user goal calendar_1 event_name", "user goal-calendar_1-available_start_time": "user goal calendar_1 available_start_time", "user goal-calendar_1-available_end_time": "user goal calendar_1 available_end_time", "user goal-events_1-category": "user goal events_1 category", "user goal-events_1-subcategory": "user goal events_1 subcategory", "user goal-events_1-event_name": "user goal events_1 event_name", "user goal-events_1-date": "user goal events_1 date", "user goal-events_1-time": "user goal events_1 time", "user goal-events_1-number_of_seats": "user goal events_1 number_of_seats", "user goal-events_1-city_of_event": "user goal events_1 city_of_event", "user goal-events_1-event_location": "user goal events_1 event_location", "user goal-events_1-address_of_location": "user goal events_1 address_of_location", "user goal-events_2-event_type": "user goal events_2 event_type", "user goal-events_2-category": "user goal events_2 category", "user goal-events_2-event_name": "user goal events_2 event_name", "user goal-events_2-date": "user goal events_2 date", "user goal-events_2-time": "user goal events_2 time", "user goal-events_2-number_of_tickets": "user goal events_2 number_of_tickets", "user goal-events_2-city": "user goal events_2 city", "user goal-events_2-venue": "user goal events_2 venue", "user goal-events_2-venue_address": "user goal events_2 venue_address", "user goal-flights_1-passengers": "user goal flights_1 passengers", "user goal-flights_1-seating_class": "user goal flights_1 seating_class", "user goal-flights_1-origin_city": "user goal flights_1 origin_city", "user goal-flights_1-destination_city": "user goal flights_1 destination_city", "user goal-flights_1-origin_airport": "user goal flights_1 origin_airport", "user goal-flights_1-destination_airport": "user goal flights_1 destination_airport", "user goal-flights_1-departure_date": "user goal flights_1 departure_date", "user goal-flights_1-return_date": "user goal flights_1 return_date", "user goal-flights_1-number_stops": "user goal flights_1 number_stops", "user goal-flights_1-outbound_departure_time": "user goal flights_1 outbound_departure_time", "user goal-flights_1-outbound_arrival_time": "user goal flights_1 outbound_arrival_time", "user goal-flights_1-inbound_arrival_time": "user goal flights_1 inbound_arrival_time", "user goal-flights_1-inbound_departure_time": "user goal flights_1 inbound_departure_time", "user goal-flights_1-price": "user goal flights_1 price", "user goal-flights_1-refundable": "user goal flights_1 refundable", "user goal-flights_1-airlines": "user goal flights_1 airlines", "user goal-flights_2-passengers": "user goal flights_2 passengers", "user goal-flights_2-seating_class": "user goal flights_2 seating_class", "user goal-flights_2-origin": "user goal flights_2 origin", "user goal-flights_2-destination": "user goal flights_2 destination", "user goal-flights_2-origin_airport": "user goal flights_2 origin_airport", "user goal-flights_2-destination_airport": "user goal flights_2 destination_airport", "user goal-flights_2-departure_date": "user goal flights_2 departure_date", "user goal-flights_2-return_date": "user goal flights_2 return_date", "user goal-flights_2-number_stops": "user goal flights_2 number_stops", "user goal-flights_2-outbound_departure_time": "user goal flights_2 outbound_departure_time", "user goal-flights_2-outbound_arrival_time": "user goal flights_2 outbound_arrival_time", "user goal-flights_2-inbound_arrival_time": "user goal flights_2 inbound_arrival_time", "user goal-flights_2-inbound_departure_time": "user goal flights_2 inbound_departure_time", "user goal-flights_2-fare": "user goal flights_2 fare", "user goal-flights_2-is_redeye": "user goal flights_2 is_redeye", "user goal-flights_2-airlines": "user goal flights_2 airlines", "user goal-homes_1-area": "user goal homes_1 area", "user goal-homes_1-address": "user goal homes_1 address", "user goal-homes_1-property_name": "user goal homes_1 property_name", "user goal-homes_1-phone_number": "user goal homes_1 phone_number", "user goal-homes_1-furnished": "user goal homes_1 furnished", "user goal-homes_1-pets_allowed": "user goal homes_1 pets_allowed", "user goal-homes_1-rent": "user goal homes_1 rent", "user goal-homes_1-visit_date": "user goal homes_1 visit_date", "user goal-homes_1-number_of_beds": "user goal homes_1 number_of_beds", "user goal-homes_1-number_of_baths": "user goal homes_1 number_of_baths", "user goal-hotels_1-destination": "user goal hotels_1 destination", "user goal-hotels_1-number_of_rooms": "user goal hotels_1 number_of_rooms", "user goal-hotels_1-check_in_date": "user goal hotels_1 check_in_date", "user goal-hotels_1-number_of_days": "user goal hotels_1 number_of_days", "user goal-hotels_1-star_rating": "user goal hotels_1 star_rating", "user goal-hotels_1-hotel_name": "user goal hotels_1 hotel_name", "user goal-hotels_1-street_address": "user goal hotels_1 street_address", "user goal-hotels_1-phone_number": "user goal hotels_1 phone_number", "user goal-hotels_1-price_per_night": "user goal hotels_1 price_per_night", "user goal-hotels_1-has_wifi": "user goal hotels_1 has_wifi", "user goal-hotels_2-where_to": "user goal hotels_2 where_to", "user goal-hotels_2-number_of_adults": "user goal hotels_2 number_of_adults", "user goal-hotels_2-check_in_date": "user goal hotels_2 check_in_date", "user goal-hotels_2-check_out_date": "user goal hotels_2 check_out_date", "user goal-hotels_2-rating": "user goal hotels_2 rating", "user goal-hotels_2-address": "user goal hotels_2 address", "user goal-hotels_2-phone_number": "user goal hotels_2 phone_number", "user goal-hotels_2-total_price": "user goal hotels_2 total_price", "user goal-hotels_2-has_laundry_service": "user goal hotels_2 has_laundry_service", "user goal-hotels_3-location": "user goal hotels_3 location", "user goal-hotels_3-number_of_rooms": "user goal hotels_3 number_of_rooms", "user goal-hotels_3-check_in_date": "user goal hotels_3 check_in_date", "user goal-hotels_3-check_out_date": "user goal hotels_3 check_out_date", "user goal-hotels_3-average_rating": "user goal hotels_3 average_rating", "user goal-hotels_3-hotel_name": "user goal hotels_3 hotel_name", "user goal-hotels_3-street_address": "user goal hotels_3 street_address", "user goal-hotels_3-phone_number": "user goal hotels_3 phone_number", "user goal-hotels_3-price": "user goal hotels_3 price", "user goal-hotels_3-pets_welcome": "user goal hotels_3 pets_welcome", "user goal-media_1-title": "user goal media_1 title", "user goal-media_1-genre": "user goal media_1 genre", "user goal-media_1-subtitles": "user goal media_1 subtitles", "user goal-media_1-directed_by": "user goal media_1 directed_by", "user goal-movies_1-price": "user goal movies_1 price", "user goal-movies_1-number_of_tickets": "user goal movies_1 number_of_tickets", "user goal-movies_1-show_type": "user goal movies_1 show_type", "user goal-movies_1-theater_name": "user goal movies_1 theater_name", "user goal-movies_1-show_time": "user goal movies_1 show_time", "user goal-movies_1-show_date": "user goal movies_1 show_date", "user goal-movies_1-genre": "user goal movies_1 genre", "user goal-movies_1-street_address": "user goal movies_1 street_address", "user goal-movies_1-location": "user goal movies_1 location", "user goal-movies_1-movie_name": "user goal movies_1 movie_name", "user goal-music_1-song_name": "user goal music_1 song_name", "user goal-music_1-artist": "user goal music_1 artist", "user goal-music_1-album": "user goal music_1 album", "user goal-music_1-genre": "user goal music_1 genre", "user goal-music_1-year": "user goal music_1 year", "user goal-music_1-playback_device": "user goal music_1 playback_device", "user goal-music_2-song_name": "user goal music_2 song_name", "user goal-music_2-artist": "user goal music_2 artist", "user goal-music_2-album": "user goal music_2 album", "user goal-music_2-genre": "user goal music_2 genre", "user goal-music_2-playback_device": "user goal music_2 playback_device", "user goal-rentalcars_1-type": "user goal rentalcars_1 type", "user goal-rentalcars_1-car_name": "user goal rentalcars_1 car_name", "user goal-rentalcars_1-pickup_location": "user goal rentalcars_1 pickup_location", "user goal-rentalcars_1-pickup_date": "user goal rentalcars_1 pickup_date", "user goal-rentalcars_1-pickup_time": "user goal rentalcars_1 pickup_time", "user goal-rentalcars_1-pickup_city": "user goal rentalcars_1 pickup_city", "user goal-rentalcars_1-dropoff_date": "user goal rentalcars_1 dropoff_date", "user goal-rentalcars_1-total_price": "user goal rentalcars_1 total_price", "user goal-rentalcars_2-car_type": "user goal rentalcars_2 car_type", "user goal-rentalcars_2-car_name": "user goal rentalcars_2 car_name", "user goal-rentalcars_2-pickup_location": "user goal rentalcars_2 pickup_location", "user goal-rentalcars_2-pickup_date": "user goal rentalcars_2 pickup_date", "user goal-rentalcars_2-pickup_time": "user goal rentalcars_2 pickup_time", "user goal-rentalcars_2-pickup_city": "user goal rentalcars_2 pickup_city", "user goal-rentalcars_2-dropoff_date": "user goal rentalcars_2 dropoff_date", "user goal-rentalcars_2-total_price": "user goal rentalcars_2 total_price", "user goal-restaurants_1-restaurant_name": "user goal restaurants_1 restaurant_name", "user goal-restaurants_1-date": "user goal restaurants_1 date", "user goal-restaurants_1-time": "user goal restaurants_1 time", "user goal-restaurants_1-serves_alcohol": "user goal restaurants_1 serves_alcohol", "user goal-restaurants_1-has_live_music": "user goal restaurants_1 has_live_music", "user goal-restaurants_1-phone_number": "user goal restaurants_1 phone_number", "user goal-restaurants_1-street_address": "user goal restaurants_1 street_address", "user goal-restaurants_1-party_size": "user goal restaurants_1 party_size", "user goal-restaurants_1-price_range": "user goal restaurants_1 price_range", "user goal-restaurants_1-city": "user goal restaurants_1 city", "user goal-restaurants_1-cuisine": "user goal restaurants_1 cuisine", "user goal-ridesharing_1-destination": "user goal ridesharing_1 destination", "user goal-ridesharing_1-shared_ride": "user goal ridesharing_1 shared_ride", "user goal-ridesharing_1-ride_fare": "user goal ridesharing_1 ride_fare", "user goal-ridesharing_1-approximate_ride_duration": "user goal ridesharing_1 approximate_ride_duration", "user goal-ridesharing_1-number_of_riders": "user goal ridesharing_1 number_of_riders", "user goal-ridesharing_2-destination": "user goal ridesharing_2 destination", "user goal-ridesharing_2-ride_type": "user goal ridesharing_2 ride_type", "user goal-ridesharing_2-ride_fare": "user goal ridesharing_2 ride_fare", "user goal-ridesharing_2-wait_time": "user goal ridesharing_2 wait_time", "user goal-ridesharing_2-number_of_seats": "user goal ridesharing_2 number_of_seats", "user goal-services_1-stylist_name": "user goal services_1 stylist_name", "user goal-services_1-phone_number": "user goal services_1 phone_number", "user goal-services_1-average_rating": "user goal services_1 average_rating", "user goal-services_1-is_unisex": "user goal services_1 is_unisex", "user goal-services_1-street_address": "user goal services_1 street_address", "user goal-services_1-city": "user goal services_1 city", "user goal-services_1-appointment_date": "user goal services_1 appointment_date", "user goal-services_1-appointment_time": "user goal services_1 appointment_time", "user goal-services_2-dentist_name": "user goal services_2 dentist_name", "user goal-services_2-phone_number": "user goal services_2 phone_number", "user goal-services_2-address": "user goal services_2 address", "user goal-services_2-city": "user goal services_2 city", "user goal-services_2-appointment_date": "user goal services_2 appointment_date", "user goal-services_2-appointment_time": "user goal services_2 appointment_time", "user goal-services_2-offers_cosmetic_services": "user goal services_2 offers_cosmetic_services", "user goal-services_3-doctor_name": "user goal services_3 doctor_name", "user goal-services_3-phone_number": "user goal services_3 phone_number", "user goal-services_3-average_rating": "user goal services_3 average_rating", "user goal-services_3-street_address": "user goal services_3 street_address", "user goal-services_3-city": "user goal services_3 city", "user goal-services_3-appointment_date": "user goal services_3 appointment_date", "user goal-services_3-appointment_time": "user goal services_3 appointment_time", "user goal-services_3-type": "user goal services_3 type", "user goal-travel_1-location": "user goal travel_1 location", "user goal-travel_1-attraction_name": "user goal travel_1 attraction_name", "user goal-travel_1-category": "user goal travel_1 category", "user goal-travel_1-phone_number": "user goal travel_1 phone_number", "user goal-travel_1-free_entry": "user goal travel_1 free_entry", "user goal-travel_1-good_for_kids": "user goal travel_1 good_for_kids", "user goal-weather_1-precipitation": "user goal weather_1 precipitation", "user goal-weather_1-humidity": "user goal weather_1 humidity", "user goal-weather_1-wind": "user goal weather_1 wind", "user goal-weather_1-temperature": "user goal weather_1 temperature", "user goal-weather_1-city": "user goal weather_1 city", "user goal-weather_1-date": "user goal weather_1 date", "user goal-alarm_1-alarm_time": "user goal alarm_1 alarm_time", "user goal-alarm_1-alarm_name": "user goal alarm_1 alarm_name", "user goal-alarm_1-new_alarm_time": "user goal alarm_1 new_alarm_time", "user goal-alarm_1-new_alarm_name": "user goal alarm_1 new_alarm_name", "user goal-banks_2-account_type": "user goal banks_2 account_type", "user goal-banks_2-recipient_account_type": "user goal banks_2 recipient_account_type", "user goal-banks_2-account_balance": "user goal banks_2 account_balance", "user goal-banks_2-transfer_amount": "user goal banks_2 transfer_amount", "user goal-banks_2-recipient_name": "user goal banks_2 recipient_name", "user goal-banks_2-transfer_time": "user goal banks_2 transfer_time", "user goal-flights_3-passengers": "user goal flights_3 passengers", "user goal-flights_3-flight_class": "user goal flights_3 flight_class", "user goal-flights_3-origin_city": "user goal flights_3 origin_city", "user goal-flights_3-destination_city": "user goal flights_3 destination_city", "user goal-flights_3-origin_airport_name": "user goal flights_3 origin_airport_name", "user goal-flights_3-destination_airport_name": "user goal flights_3 destination_airport_name", "user goal-flights_3-departure_date": "user goal flights_3 departure_date", "user goal-flights_3-return_date": "user goal flights_3 return_date", "user goal-flights_3-number_stops": "user goal flights_3 number_stops", "user goal-flights_3-outbound_departure_time": "user goal flights_3 outbound_departure_time", "user goal-flights_3-outbound_arrival_time": "user goal flights_3 outbound_arrival_time", "user goal-flights_3-inbound_arrival_time": "user goal flights_3 inbound_arrival_time", "user goal-flights_3-inbound_departure_time": "user goal flights_3 inbound_departure_time", "user goal-flights_3-price": "user goal flights_3 price", "user goal-flights_3-number_checked_bags": "user goal flights_3 number_checked_bags", "user goal-flights_3-airlines": "user goal flights_3 airlines", "user goal-flights_3-arrives_next_day": "user goal flights_3 arrives_next_day", "user goal-hotels_4-location": "user goal hotels_4 location", "user goal-hotels_4-number_of_rooms": "user goal hotels_4 number_of_rooms", "user goal-hotels_4-check_in_date": "user goal hotels_4 check_in_date", "user goal-hotels_4-stay_length": "user goal hotels_4 stay_length", "user goal-hotels_4-star_rating": "user goal hotels_4 star_rating", "user goal-hotels_4-place_name": "user goal hotels_4 place_name", "user goal-hotels_4-street_address": "user goal hotels_4 street_address", "user goal-hotels_4-phone_number": "user goal hotels_4 phone_number", "user goal-hotels_4-price_per_night": "user goal hotels_4 price_per_night", "user goal-hotels_4-smoking_allowed": "user goal hotels_4 smoking_allowed", "user goal-media_2-movie_name": "user goal media_2 movie_name", "user goal-media_2-genre": "user goal media_2 genre", "user goal-media_2-subtitle_language": "user goal media_2 subtitle_language", "user goal-media_2-director": "user goal media_2 director", "user goal-media_2-actors": "user goal media_2 actors", "user goal-media_2-price": "user goal media_2 price", "user goal-movies_2-title": "user goal movies_2 title", "user goal-movies_2-genre": "user goal movies_2 genre", "user goal-movies_2-aggregate_rating": "user goal movies_2 aggregate_rating", "user goal-movies_2-starring": "user goal movies_2 starring", "user goal-movies_2-director": "user goal movies_2 director", "user goal-restaurants_2-restaurant_name": "user goal restaurants_2 restaurant_name", "user goal-restaurants_2-date": "user goal restaurants_2 date", "user goal-restaurants_2-time": "user goal restaurants_2 time", "user goal-restaurants_2-has_seating_outdoors": "user goal restaurants_2 has_seating_outdoors", "user goal-restaurants_2-has_vegetarian_options": "user goal restaurants_2 has_vegetarian_options", "user goal-restaurants_2-phone_number": "user goal restaurants_2 phone_number", "user goal-restaurants_2-rating": "user goal restaurants_2 rating", "user goal-restaurants_2-address": "user goal restaurants_2 address", "user goal-restaurants_2-number_of_seats": "user goal restaurants_2 number_of_seats", "user goal-restaurants_2-price_range": "user goal restaurants_2 price_range", "user goal-restaurants_2-location": "user goal restaurants_2 location", "user goal-restaurants_2-category": "user goal restaurants_2 category", "user goal-services_4-therapist_name": "user goal services_4 therapist_name", "user goal-services_4-phone_number": "user goal services_4 phone_number", "user goal-services_4-address": "user goal services_4 address", "user goal-services_4-city": "user goal services_4 city", "user goal-services_4-appointment_date": "user goal services_4 appointment_date", "user goal-services_4-appointment_time": "user goal services_4 appointment_time", "user goal-services_4-type": "user goal services_4 type", "user goal-buses_3-from_city": "user goal buses_3 from_city", "user goal-buses_3-to_city": "user goal buses_3 to_city", "user goal-buses_3-from_station": "user goal buses_3 from_station", "user goal-buses_3-to_station": "user goal buses_3 to_station", "user goal-buses_3-departure_date": "user goal buses_3 departure_date", "user goal-buses_3-departure_time": "user goal buses_3 departure_time", "user goal-buses_3-price": "user goal buses_3 price", "user goal-buses_3-additional_luggage": "user goal buses_3 additional_luggage", "user goal-buses_3-num_passengers": "user goal buses_3 num_passengers", "user goal-buses_3-category": "user goal buses_3 category", "user goal-events_3-event_type": "user goal events_3 event_type", "user goal-events_3-event_name": "user goal events_3 event_name", "user goal-events_3-date": "user goal events_3 date", "user goal-events_3-time": "user goal events_3 time", "user goal-events_3-number_of_tickets": "user goal events_3 number_of_tickets", "user goal-events_3-price_per_ticket": "user goal events_3 price_per_ticket", "user goal-events_3-city": "user goal events_3 city", "user goal-events_3-venue": "user goal events_3 venue", "user goal-events_3-venue_address": "user goal events_3 venue_address", "user goal-flights_4-number_of_tickets": "user goal flights_4 number_of_tickets", "user goal-flights_4-seating_class": "user goal flights_4 seating_class", "user goal-flights_4-origin_airport": "user goal flights_4 origin_airport", "user goal-flights_4-destination_airport": "user goal flights_4 destination_airport", "user goal-flights_4-departure_date": "user goal flights_4 departure_date", "user goal-flights_4-return_date": "user goal flights_4 return_date", "user goal-flights_4-is_nonstop": "user goal flights_4 is_nonstop", "user goal-flights_4-outbound_departure_time": "user goal flights_4 outbound_departure_time", "user goal-flights_4-outbound_arrival_time": "user goal flights_4 outbound_arrival_time", "user goal-flights_4-inbound_arrival_time": "user goal flights_4 inbound_arrival_time", "user goal-flights_4-inbound_departure_time": "user goal flights_4 inbound_departure_time", "user goal-flights_4-price": "user goal flights_4 price", "user goal-flights_4-airlines": "user goal flights_4 airlines", "user goal-homes_2-intent": "user goal homes_2 intent", "user goal-homes_2-area": "user goal homes_2 area", "user goal-homes_2-address": "user goal homes_2 address", "user goal-homes_2-property_name": "user goal homes_2 property_name", "user goal-homes_2-phone_number": "user goal homes_2 phone_number", "user goal-homes_2-has_garage": "user goal homes_2 has_garage", "user goal-homes_2-in_unit_laundry": "user goal homes_2 in_unit_laundry", "user goal-homes_2-price": "user goal homes_2 price", "user goal-homes_2-visit_date": "user goal homes_2 visit_date", "user goal-homes_2-number_of_beds": "user goal homes_2 number_of_beds", "user goal-homes_2-number_of_baths": "user goal homes_2 number_of_baths", "user goal-media_3-title": "user goal media_3 title", "user goal-media_3-genre": "user goal media_3 genre", "user goal-media_3-subtitle_language": "user goal media_3 subtitle_language", "user goal-media_3-starring": "user goal media_3 starring", "user goal-messaging_1-location": "user goal messaging_1 location", "user goal-messaging_1-contact_name": "user goal messaging_1 contact_name", "user goal-movies_3-movie_title": "user goal movies_3 movie_title", "user goal-movies_3-genre": "user goal movies_3 genre", "user goal-movies_3-percent_rating": "user goal movies_3 percent_rating", "user goal-movies_3-cast": "user goal movies_3 cast", "user goal-movies_3-directed_by": "user goal movies_3 directed_by", "user goal-music_3-track": "user goal music_3 track", "user goal-music_3-artist": "user goal music_3 artist", "user goal-music_3-album": "user goal music_3 album", "user goal-music_3-genre": "user goal music_3 genre", "user goal-music_3-year": "user goal music_3 year", "user goal-music_3-device": "user goal music_3 device", "user goal-payment_1-payment_method": "user goal payment_1 payment_method", "user goal-payment_1-amount": "user goal payment_1 amount", "user goal-payment_1-receiver": "user goal payment_1 receiver", "user goal-payment_1-private_visibility": "user goal payment_1 private_visibility", "user goal-rentalcars_3-car_type": "user goal rentalcars_3 car_type", "user goal-rentalcars_3-car_name": "user goal rentalcars_3 car_name", "user goal-rentalcars_3-pickup_location": "user goal rentalcars_3 pickup_location", "user goal-rentalcars_3-start_date": "user goal rentalcars_3 start_date", "user goal-rentalcars_3-pickup_time": "user goal rentalcars_3 pickup_time", "user goal-rentalcars_3-city": "user goal rentalcars_3 city", "user goal-rentalcars_3-end_date": "user goal rentalcars_3 end_date", "user goal-rentalcars_3-price_per_day": "user goal rentalcars_3 price_per_day", "user goal-rentalcars_3-add_insurance": "user goal rentalcars_3 add_insurance", "user goal-trains_1-from": "user goal trains_1 from", "user goal-trains_1-to": "user goal trains_1 to", "user goal-trains_1-from_station": "user goal trains_1 from_station", "user goal-trains_1-to_station": "user goal trains_1 to_station", "user goal-trains_1-date_of_journey": "user goal trains_1 date_of_journey", "user goal-trains_1-journey_start_time": "user goal trains_1 journey_start_time", "user goal-trains_1-total": "user goal trains_1 total", "user goal-trains_1-number_of_adults": "user goal trains_1 number_of_adults", "user goal-trains_1-class": "user goal trains_1 class", "user goal-trains_1-trip_protection": "user goal trains_1 trip_protection", "general-banks_1": "domain banks_1", "general-buses_1": "domain buses_1", "general-buses_2": "domain buses_2", "general-calendar_1": "domain calendar_1", "general-events_1": "domain events_1", "general-events_2": "domain events_2", "general-flights_1": "domain flights_1", "general-flights_2": "domain flights_2", "general-homes_1": "domain homes_1", "general-hotels_1": "domain hotels_1", "general-hotels_2": "domain hotels_2", "general-hotels_3": "domain hotels_3", "general-media_1": "domain media_1", "general-movies_1": "domain movies_1", "general-music_1": "domain music_1", "general-music_2": "domain music_2", "general-rentalcars_1": "domain rentalcars_1", "general-rentalcars_2": "domain rentalcars_2", "general-restaurants_1": "domain restaurants_1", "general-ridesharing_1": "domain ridesharing_1", "general-ridesharing_2": "domain ridesharing_2", "general-services_1": "domain services_1", "general-services_2": "domain services_2", "general-services_3": "domain services_3", "general-travel_1": "domain travel_1", "general-weather_1": "domain weather_1", "general-alarm_1": "domain alarm_1", "general-banks_2": "domain banks_2", "general-flights_3": "domain flights_3", "general-hotels_4": "domain hotels_4", "general-media_2": "domain media_2", "general-movies_2": "domain movies_2", "general-restaurants_2": "domain restaurants_2", "general-services_4": "domain services_4", "general-buses_3": "domain buses_3", "general-events_3": "domain events_3", "general-flights_4": "domain flights_4", "general-homes_2": "domain homes_2", "general-media_3": "domain media_3", "general-messaging_1": "domain messaging_1", "general-movies_3": "domain movies_3", "general-music_3": "domain music_3", "general-payment_1": "domain payment_1", "general-rentalcars_3": "domain rentalcars_3", "general-trains_1": "domain trains_1", "system--goodbye-none-none": "last system act  goodbye none none", "system--req_more-none-none": "last system act  req_more none none", "system-alarm_1-confirm-new_alarm_name-1": "last system act alarm_1 confirm new_alarm_name 1", "system-alarm_1-confirm-new_alarm_time-1": "last system act alarm_1 confirm new_alarm_time 1", "system-alarm_1-inform_count-count-1": "last system act alarm_1 inform_count count 1", "system-alarm_1-notify_success-none-none": "last system act alarm_1 notify_success none none", "system-alarm_1-offer-alarm_name-1": "last system act alarm_1 offer alarm_name 1", "system-alarm_1-offer-alarm_time-1": "last system act alarm_1 offer alarm_time 1", "system-alarm_1-offer_intent-addalarm-1": "last system act alarm_1 offer_intent AddAlarm 1", "system-alarm_1-request-new_alarm_time-?": "last system act alarm_1 request new_alarm_time ?", "system-banks_1-confirm-account_type-1": "last system act banks_1 confirm account_type 1", "system-banks_1-confirm-amount-1": "last system act banks_1 confirm amount 1", "system-banks_1-confirm-recipient_account_name-1": "last system act banks_1 confirm recipient_account_name 1", "system-banks_1-confirm-recipient_account_type-1": "last system act banks_1 confirm recipient_account_type 1", "system-banks_1-notify_success-none-none": "last system act banks_1 notify_success none none", "system-banks_1-offer-account_type-1": "last system act banks_1 offer account_type 1", "system-banks_1-offer-balance-1": "last system act banks_1 offer balance 1", "system-banks_1-offer_intent-transfermoney-1": "last system act banks_1 offer_intent TransferMoney 1", "system-banks_1-request-account_type-?": "last system act banks_1 request account_type ?", "system-banks_1-request-amount-?": "last system act banks_1 request amount ?", "system-banks_1-request-recipient_account_name-?": "last system act banks_1 request recipient_account_name ?", "system-banks_2-confirm-account_type-1": "last system act banks_2 confirm account_type 1", "system-banks_2-confirm-recipient_account_type-1": "last system act banks_2 confirm recipient_account_type 1", "system-banks_2-confirm-recipient_name-1": "last system act banks_2 confirm recipient_name 1", "system-banks_2-confirm-transfer_amount-1": "last system act banks_2 confirm transfer_amount 1", "system-banks_2-inform-transfer_time-1": "last system act banks_2 inform transfer_time 1", "system-banks_2-notify_success-none-none": "last system act banks_2 notify_success none none", "system-banks_2-offer-account_balance-1": "last system act banks_2 offer account_balance 1", "system-banks_2-offer-account_type-1": "last system act banks_2 offer account_type 1", "system-banks_2-offer_intent-transfermoney-1": "last system act banks_2 offer_intent TransferMoney 1", "system-banks_2-request-account_type-?": "last system act banks_2 request account_type ?", "system-banks_2-request-recipient_name-?": "last system act banks_2 request recipient_name ?", "system-banks_2-request-transfer_amount-?": "last system act banks_2 request transfer_amount ?", "system-buses_1-confirm-from_location-1": "last system act buses_1 confirm from_location 1", "system-buses_1-confirm-leaving_date-1": "last system act buses_1 confirm leaving_date 1", "system-buses_1-confirm-leaving_time-1": "last system act buses_1 confirm leaving_time 1", "system-buses_1-confirm-to_location-1": "last system act buses_1 confirm to_location 1", "system-buses_1-confirm-travelers-1": "last system act buses_1 confirm travelers 1", "system-buses_1-inform-from_station-1": "last system act buses_1 inform from_station 1", "system-buses_1-inform-to_station-1": "last system act buses_1 inform to_station 1", "system-buses_1-inform-transfers-1": "last system act buses_1 inform transfers 1", "system-buses_1-inform_count-count-1": "last system act buses_1 inform_count count 1", "system-buses_1-notify_failure-none-none": "last system act buses_1 notify_failure none none", "system-buses_1-notify_success-none-none": "last system act buses_1 notify_success none none", "system-buses_1-offer-fare-1": "last system act buses_1 offer fare 1", "system-buses_1-offer-leaving_time-1": "last system act buses_1 offer leaving_time 1", "system-buses_1-offer-transfers-1": "last system act buses_1 offer transfers 1", "system-buses_1-offer_intent-buybusticket-1": "last system act buses_1 offer_intent BuyBusTicket 1", "system-buses_1-request-from_location-?": "last system act buses_1 request from_location ?", "system-buses_1-request-leaving_date-?": "last system act buses_1 request leaving_date ?", "system-buses_1-request-leaving_time-?": "last system act buses_1 request leaving_time ?", "system-buses_1-request-to_location-?": "last system act buses_1 request to_location ?", "system-buses_1-request-travelers-?": "last system act buses_1 request travelers ?", "system-buses_2-confirm-departure_date-1": "last system act buses_2 confirm departure_date 1", "system-buses_2-confirm-departure_time-1": "last system act buses_2 confirm departure_time 1", "system-buses_2-confirm-destination-1": "last system act buses_2 confirm destination 1", "system-buses_2-confirm-fare_type-1": "last system act buses_2 confirm fare_type 1", "system-buses_2-confirm-group_size-1": "last system act buses_2 confirm group_size 1", "system-buses_2-confirm-origin-1": "last system act buses_2 confirm origin 1", "system-buses_2-inform-destination_station_name-1": "last system act buses_2 inform destination_station_name 1", "system-buses_2-inform-origin_station_name-1": "last system act buses_2 inform origin_station_name 1", "system-buses_2-inform_count-count-1": "last system act buses_2 inform_count count 1", "system-buses_2-notify_failure-none-none": "last system act buses_2 notify_failure none none", "system-buses_2-notify_success-none-none": "last system act buses_2 notify_success none none", "system-buses_2-offer-departure_time-1": "last system act buses_2 offer departure_time 1", "system-buses_2-offer-fare_type-1": "last system act buses_2 offer fare_type 1", "system-buses_2-offer-price-1": "last system act buses_2 offer price 1", "system-buses_2-offer_intent-buybusticket-1": "last system act buses_2 offer_intent BuyBusTicket 1", "system-buses_2-request-departure_date-?": "last system act buses_2 request departure_date ?", "system-buses_2-request-departure_time-?": "last system act buses_2 request departure_time ?", "system-buses_2-request-destination-?": "last system act buses_2 request destination ?", "system-buses_2-request-group_size-?": "last system act buses_2 request group_size ?", "system-buses_2-request-origin-?": "last system act buses_2 request origin ?", "system-buses_3-confirm-additional_luggage-1": "last system act buses_3 confirm additional_luggage 1", "system-buses_3-confirm-departure_date-1": "last system act buses_3 confirm departure_date 1", "system-buses_3-confirm-departure_time-1": "last system act buses_3 confirm departure_time 1", "system-buses_3-confirm-from_city-1": "last system act buses_3 confirm from_city 1", "system-buses_3-confirm-num_passengers-1": "last system act buses_3 confirm num_passengers 1", "system-buses_3-confirm-to_city-1": "last system act buses_3 confirm to_city 1", "system-buses_3-inform-category-1": "last system act buses_3 inform category 1", "system-buses_3-inform-from_station-1": "last system act buses_3 inform from_station 1", "system-buses_3-inform-to_station-1": "last system act buses_3 inform to_station 1", "system-buses_3-inform_count-count-1": "last system act buses_3 inform_count count 1", "system-buses_3-notify_failure-none-none": "last system act buses_3 notify_failure none none", "system-buses_3-notify_success-none-none": "last system act buses_3 notify_success none none", "system-buses_3-offer-departure_time-1": "last system act buses_3 offer departure_time 1", "system-buses_3-offer-price-1": "last system act buses_3 offer price 1", "system-buses_3-offer_intent-buybusticket-1": "last system act buses_3 offer_intent BuyBusTicket 1", "system-buses_3-request-departure_date-?": "last system act buses_3 request departure_date ?", "system-buses_3-request-departure_time-?": "last system act buses_3 request departure_time ?", "system-buses_3-request-from_city-?": "last system act buses_3 request from_city ?", "system-buses_3-request-num_passengers-?": "last system act buses_3 request num_passengers ?", "system-buses_3-request-to_city-?": "last system act buses_3 request to_city ?", "system-calendar_1-confirm-event_date-1": "last system act calendar_1 confirm event_date 1", "system-calendar_1-confirm-event_location-1": "last system act calendar_1 confirm event_location 1", "system-calendar_1-confirm-event_name-1": "last system act calendar_1 confirm event_name 1", "system-calendar_1-confirm-event_time-1": "last system act calendar_1 confirm event_time 1", "system-calendar_1-inform_count-count-1": "last system act calendar_1 inform_count count 1", "system-calendar_1-notify_success-none-none": "last system act calendar_1 notify_success none none", "system-calendar_1-offer-available_end_time-1": "last system act calendar_1 offer available_end_time 1", "system-calendar_1-offer-available_start_time-1": "last system act calendar_1 offer available_start_time 1", "system-calendar_1-offer-event_date-1": "last system act calendar_1 offer event_date 1", "system-calendar_1-offer-event_name-1": "last system act calendar_1 offer event_name 1", "system-calendar_1-offer-event_time-1": "last system act calendar_1 offer event_time 1", "system-calendar_1-offer_intent-addevent-1": "last system act calendar_1 offer_intent AddEvent 1", "system-calendar_1-request-event_date-?": "last system act calendar_1 request event_date ?", "system-calendar_1-request-event_location-?": "last system act calendar_1 request event_location ?", "system-calendar_1-request-event_name-?": "last system act calendar_1 request event_name ?", "system-calendar_1-request-event_time-?": "last system act calendar_1 request event_time ?", "system-events_1-confirm-city_of_event-1": "last system act events_1 confirm city_of_event 1", "system-events_1-confirm-date-1": "last system act events_1 confirm date 1", "system-events_1-confirm-event_name-1": "last system act events_1 confirm event_name 1", "system-events_1-confirm-number_of_seats-1": "last system act events_1 confirm number_of_seats 1", "system-events_1-inform-address_of_location-1": "last system act events_1 inform address_of_location 1", "system-events_1-inform-event_location-1": "last system act events_1 inform event_location 1", "system-events_1-inform-subcategory-1": "last system act events_1 inform subcategory 1", "system-events_1-inform-time-1": "last system act events_1 inform time 1", "system-events_1-inform_count-count-1": "last system act events_1 inform_count count 1", "system-events_1-notify_failure-none-none": "last system act events_1 notify_failure none none", "system-events_1-notify_success-none-none": "last system act events_1 notify_success none none", "system-events_1-offer-date-1": "last system act events_1 offer date 1", "system-events_1-offer-event_location-1": "last system act events_1 offer event_location 1", "system-events_1-offer-event_name-1": "last system act events_1 offer event_name 1", "system-events_1-offer-time-1": "last system act events_1 offer time 1", "system-events_1-offer_intent-buyeventtickets-1": "last system act events_1 offer_intent BuyEventTickets 1", "system-events_1-request-category-?": "last system act events_1 request category ?", "system-events_1-request-city_of_event-?": "last system act events_1 request city_of_event ?", "system-events_1-request-date-?": "last system act events_1 request date ?", "system-events_1-request-event_name-?": "last system act events_1 request event_name ?", "system-events_1-request-number_of_seats-?": "last system act events_1 request number_of_seats ?", "system-events_2-confirm-city-1": "last system act events_2 confirm city 1", "system-events_2-confirm-date-1": "last system act events_2 confirm date 1", "system-events_2-confirm-event_name-1": "last system act events_2 confirm event_name 1", "system-events_2-confirm-number_of_tickets-1": "last system act events_2 confirm number_of_tickets 1", "system-events_2-inform-category-1": "last system act events_2 inform category 1", "system-events_2-inform-time-1": "last system act events_2 inform time 1", "system-events_2-inform-venue-1": "last system act events_2 inform venue 1", "system-events_2-inform-venue_address-1": "last system act events_2 inform venue_address 1", "system-events_2-inform_count-count-1": "last system act events_2 inform_count count 1", "system-events_2-notify_success-none-none": "last system act events_2 notify_success none none", "system-events_2-offer-date-1": "last system act events_2 offer date 1", "system-events_2-offer-event_name-1": "last system act events_2 offer event_name 1", "system-events_2-offer-venue-1": "last system act events_2 offer venue 1", "system-events_2-offer_intent-buyeventtickets-1": "last system act events_2 offer_intent BuyEventTickets 1", "system-events_2-request-city-?": "last system act events_2 request city ?", "system-events_2-request-date-?": "last system act events_2 request date ?", "system-events_2-request-event_name-?": "last system act events_2 request event_name ?", "system-events_2-request-event_type-?": "last system act events_2 request event_type ?", "system-events_2-request-number_of_tickets-?": "last system act events_2 request number_of_tickets ?", "system-events_3-confirm-city-1": "last system act events_3 confirm city 1", "system-events_3-confirm-date-1": "last system act events_3 confirm date 1", "system-events_3-confirm-event_name-1": "last system act events_3 confirm event_name 1", "system-events_3-confirm-number_of_tickets-1": "last system act events_3 confirm number_of_tickets 1", "system-events_3-inform-price_per_ticket-1": "last system act events_3 inform price_per_ticket 1", "system-events_3-inform-venue_address-1": "last system act events_3 inform venue_address 1", "system-events_3-inform_count-count-1": "last system act events_3 inform_count count 1", "system-events_3-notify_success-none-none": "last system act events_3 notify_success none none", "system-events_3-offer-date-1": "last system act events_3 offer date 1", "system-events_3-offer-event_name-1": "last system act events_3 offer event_name 1", "system-events_3-offer-time-1": "last system act events_3 offer time 1", "system-events_3-offer-venue-1": "last system act events_3 offer venue 1", "system-events_3-offer_intent-buyeventtickets-1": "last system act events_3 offer_intent BuyEventTickets 1", "system-events_3-request-city-?": "last system act events_3 request city ?", "system-events_3-request-date-?": "last system act events_3 request date ?", "system-events_3-request-event_name-?": "last system act events_3 request event_name ?", "system-events_3-request-event_type-?": "last system act events_3 request event_type ?", "system-events_3-request-number_of_tickets-?": "last system act events_3 request number_of_tickets ?", "system-flights_1-confirm-airlines-1": "last system act flights_1 confirm airlines 1", "system-flights_1-confirm-departure_date-1": "last system act flights_1 confirm departure_date 1", "system-flights_1-confirm-destination_city-1": "last system act flights_1 confirm destination_city 1", "system-flights_1-confirm-inbound_departure_time-1": "last system act flights_1 confirm inbound_departure_time 1", "system-flights_1-confirm-origin_city-1": "last system act flights_1 confirm origin_city 1", "system-flights_1-confirm-outbound_departure_time-1": "last system act flights_1 confirm outbound_departure_time 1", "system-flights_1-confirm-passengers-1": "last system act flights_1 confirm passengers 1", "system-flights_1-confirm-return_date-1": "last system act flights_1 confirm return_date 1", "system-flights_1-confirm-seating_class-1": "last system act flights_1 confirm seating_class 1", "system-flights_1-inform-destination_airport-1": "last system act flights_1 inform destination_airport 1", "system-flights_1-inform-inbound_arrival_time-1": "last system act flights_1 inform inbound_arrival_time 1", "system-flights_1-inform-number_stops-1": "last system act flights_1 inform number_stops 1", "system-flights_1-inform-origin_airport-1": "last system act flights_1 inform origin_airport 1", "system-flights_1-inform-outbound_arrival_time-1": "last system act flights_1 inform outbound_arrival_time 1", "system-flights_1-inform-refundable-1": "last system act flights_1 inform refundable 1", "system-flights_1-inform_count-count-1": "last system act flights_1 inform_count count 1", "system-flights_1-notify_failure-none-none": "last system act flights_1 notify_failure none none", "system-flights_1-notify_success-none-none": "last system act flights_1 notify_success none none", "system-flights_1-offer-airlines-1": "last system act flights_1 offer airlines 1", "system-flights_1-offer-inbound_departure_time-1": "last system act flights_1 offer inbound_departure_time 1", "system-flights_1-offer-number_stops-1": "last system act flights_1 offer number_stops 1", "system-flights_1-offer-outbound_departure_time-1": "last system act flights_1 offer outbound_departure_time 1", "system-flights_1-offer-price-1": "last system act flights_1 offer price 1", "system-flights_1-offer_intent-reserveonewayflight-1": "last system act flights_1 offer_intent ReserveOnewayFlight 1", "system-flights_1-offer_intent-reserveroundtripflights-1": "last system act flights_1 offer_intent ReserveRoundtripFlights 1", "system-flights_1-request-airlines-?": "last system act flights_1 request airlines ?", "system-flights_1-request-departure_date-?": "last system act flights_1 request departure_date ?", "system-flights_1-request-destination_city-?": "last system act flights_1 request destination_city ?", "system-flights_1-request-inbound_departure_time-?": "last system act flights_1 request inbound_departure_time ?", "system-flights_1-request-origin_city-?": "last system act flights_1 request origin_city ?", "system-flights_1-request-outbound_departure_time-?": "last system act flights_1 request outbound_departure_time ?", "system-flights_1-request-return_date-?": "last system act flights_1 request return_date ?", "system-flights_2-inform-destination_airport-1": "last system act flights_2 inform destination_airport 1", "system-flights_2-inform-is_redeye-1": "last system act flights_2 inform is_redeye 1", "system-flights_2-inform-origin_airport-1": "last system act flights_2 inform origin_airport 1", "system-flights_2-inform-outbound_arrival_time-1": "last system act flights_2 inform outbound_arrival_time 1", "system-flights_2-inform_count-count-1": "last system act flights_2 inform_count count 1", "system-flights_2-offer-airlines-1": "last system act flights_2 offer airlines 1", "system-flights_2-offer-fare-1": "last system act flights_2 offer fare 1", "system-flights_2-offer-inbound_departure_time-1": "last system act flights_2 offer inbound_departure_time 1", "system-flights_2-offer-number_stops-1": "last system act flights_2 offer number_stops 1", "system-flights_2-offer-outbound_departure_time-1": "last system act flights_2 offer outbound_departure_time 1", "system-flights_2-request-departure_date-?": "last system act flights_2 request departure_date ?", "system-flights_2-request-destination-?": "last system act flights_2 request destination ?", "system-flights_2-request-origin-?": "last system act flights_2 request origin ?", "system-flights_2-request-return_date-?": "last system act flights_2 request return_date ?", "system-flights_3-inform-arrives_next_day-1": "last system act flights_3 inform arrives_next_day 1", "system-flights_3-inform-destination_airport_name-1": "last system act flights_3 inform destination_airport_name 1", "system-flights_3-inform-origin_airport_name-1": "last system act flights_3 inform origin_airport_name 1", "system-flights_3-inform-outbound_arrival_time-1": "last system act flights_3 inform outbound_arrival_time 1", "system-flights_3-inform_count-count-1": "last system act flights_3 inform_count count 1", "system-flights_3-offer-airlines-1": "last system act flights_3 offer airlines 1", "system-flights_3-offer-inbound_departure_time-1": "last system act flights_3 offer inbound_departure_time 1", "system-flights_3-offer-number_stops-1": "last system act flights_3 offer number_stops 1", "system-flights_3-offer-outbound_departure_time-1": "last system act flights_3 offer outbound_departure_time 1", "system-flights_3-offer-price-1": "last system act flights_3 offer price 1", "system-flights_3-request-departure_date-?": "last system act flights_3 request departure_date ?", "system-flights_3-request-destination_city-?": "last system act flights_3 request destination_city ?", "system-flights_3-request-origin_city-?": "last system act flights_3 request origin_city ?", "system-flights_3-request-return_date-?": "last system act flights_3 request return_date ?", "system-flights_4-inform-inbound_arrival_time-1": "last system act flights_4 inform inbound_arrival_time 1", "system-flights_4-inform-number_of_tickets-1": "last system act flights_4 inform number_of_tickets 1", "system-flights_4-inform-outbound_arrival_time-1": "last system act flights_4 inform outbound_arrival_time 1", "system-flights_4-inform-seating_class-1": "last system act flights_4 inform seating_class 1", "system-flights_4-inform_count-count-1": "last system act flights_4 inform_count count 1", "system-flights_4-offer-airlines-1": "last system act flights_4 offer airlines 1", "system-flights_4-offer-inbound_departure_time-1": "last system act flights_4 offer inbound_departure_time 1", "system-flights_4-offer-is_nonstop-1": "last system act flights_4 offer is_nonstop 1", "system-flights_4-offer-outbound_departure_time-1": "last system act flights_4 offer outbound_departure_time 1", "system-flights_4-offer-price-1": "last system act flights_4 offer price 1", "system-flights_4-request-departure_date-?": "last system act flights_4 request departure_date ?", "system-flights_4-request-destination_airport-?": "last system act flights_4 request destination_airport ?", "system-flights_4-request-origin_airport-?": "last system act flights_4 request origin_airport ?", "system-flights_4-request-return_date-?": "last system act flights_4 request return_date ?", "system-homes_1-confirm-property_name-1": "last system act homes_1 confirm property_name 1", "system-homes_1-confirm-visit_date-1": "last system act homes_1 confirm visit_date 1", "system-homes_1-inform-furnished-1": "last system act homes_1 inform furnished 1", "system-homes_1-inform-pets_allowed-1": "last system act homes_1 inform pets_allowed 1", "system-homes_1-inform-phone_number-1": "last system act homes_1 inform phone_number 1", "system-homes_1-inform_count-count-1": "last system act homes_1 inform_count count 1", "system-homes_1-notify_failure-none-none": "last system act homes_1 notify_failure none none", "system-homes_1-notify_success-none-none": "last system act homes_1 notify_success none none", "system-homes_1-offer-address-1": "last system act homes_1 offer address 1", "system-homes_1-offer-number_of_baths-1": "last system act homes_1 offer number_of_baths 1", "system-homes_1-offer-number_of_beds-1": "last system act homes_1 offer number_of_beds 1", "system-homes_1-offer-property_name-1": "last system act homes_1 offer property_name 1", "system-homes_1-offer-rent-1": "last system act homes_1 offer rent 1", "system-homes_1-offer_intent-schedulevisit-1": "last system act homes_1 offer_intent ScheduleVisit 1", "system-homes_1-request-area-?": "last system act homes_1 request area ?", "system-homes_1-request-number_of_beds-?": "last system act homes_1 request number_of_beds ?", "system-homes_1-request-visit_date-?": "last system act homes_1 request visit_date ?", "system-homes_2-confirm-property_name-1": "last system act homes_2 confirm property_name 1", "system-homes_2-confirm-visit_date-1": "last system act homes_2 confirm visit_date 1", "system-homes_2-inform-has_garage-1": "last system act homes_2 inform has_garage 1", "system-homes_2-inform-in_unit_laundry-1": "last system act homes_2 inform in_unit_laundry 1", "system-homes_2-inform-phone_number-1": "last system act homes_2 inform phone_number 1", "system-homes_2-inform_count-count-1": "last system act homes_2 inform_count count 1", "system-homes_2-notify_success-none-none": "last system act homes_2 notify_success none none", "system-homes_2-offer-address-1": "last system act homes_2 offer address 1", "system-homes_2-offer-price-1": "last system act homes_2 offer price 1", "system-homes_2-offer-property_name-1": "last system act homes_2 offer property_name 1", "system-homes_2-offer_intent-schedulevisit-1": "last system act homes_2 offer_intent ScheduleVisit 1", "system-homes_2-request-area-?": "last system act homes_2 request area ?", "system-homes_2-request-intent-?": "last system act homes_2 request intent ?", "system-homes_2-request-number_of_baths-?": "last system act homes_2 request number_of_baths ?", "system-homes_2-request-number_of_beds-?": "last system act homes_2 request number_of_beds ?", "system-homes_2-request-visit_date-?": "last system act homes_2 request visit_date ?", "system-hotels_1-confirm-check_in_date-1": "last system act hotels_1 confirm check_in_date 1", "system-hotels_1-confirm-destination-1": "last system act hotels_1 confirm destination 1", "system-hotels_1-confirm-hotel_name-1": "last system act hotels_1 confirm hotel_name 1", "system-hotels_1-confirm-number_of_days-1": "last system act hotels_1 confirm number_of_days 1", "system-hotels_1-confirm-number_of_rooms-1": "last system act hotels_1 confirm number_of_rooms 1", "system-hotels_1-inform-has_wifi-1": "last system act hotels_1 inform has_wifi 1", "system-hotels_1-inform-phone_number-1": "last system act hotels_1 inform phone_number 1", "system-hotels_1-inform-price_per_night-1": "last system act hotels_1 inform price_per_night 1", "system-hotels_1-inform-street_address-1": "last system act hotels_1 inform street_address 1", "system-hotels_1-inform_count-count-1": "last system act hotels_1 inform_count count 1", "system-hotels_1-notify_success-none-none": "last system act hotels_1 notify_success none none", "system-hotels_1-offer-hotel_name-1": "last system act hotels_1 offer hotel_name 1", "system-hotels_1-offer-star_rating-1": "last system act hotels_1 offer star_rating 1", "system-hotels_1-offer_intent-reservehotel-1": "last system act hotels_1 offer_intent ReserveHotel 1", "system-hotels_1-request-check_in_date-?": "last system act hotels_1 request check_in_date ?", "system-hotels_1-request-destination-?": "last system act hotels_1 request destination ?", "system-hotels_1-request-hotel_name-?": "last system act hotels_1 request hotel_name ?", "system-hotels_1-request-number_of_days-?": "last system act hotels_1 request number_of_days ?", "system-hotels_2-confirm-check_in_date-1": "last system act hotels_2 confirm check_in_date 1", "system-hotels_2-confirm-check_out_date-1": "last system act hotels_2 confirm check_out_date 1", "system-hotels_2-confirm-number_of_adults-1": "last system act hotels_2 confirm number_of_adults 1", "system-hotels_2-confirm-where_to-1": "last system act hotels_2 confirm where_to 1", "system-hotels_2-inform-has_laundry_service-1": "last system act hotels_2 inform has_laundry_service 1", "system-hotels_2-inform-phone_number-1": "last system act hotels_2 inform phone_number 1", "system-hotels_2-inform-total_price-1": "last system act hotels_2 inform total_price 1", "system-hotels_2-inform_count-count-1": "last system act hotels_2 inform_count count 1", "system-hotels_2-notify_success-none-none": "last system act hotels_2 notify_success none none", "system-hotels_2-offer-address-1": "last system act hotels_2 offer address 1", "system-hotels_2-offer-rating-1": "last system act hotels_2 offer rating 1", "system-hotels_2-offer_intent-bookhouse-1": "last system act hotels_2 offer_intent BookHouse 1", "system-hotels_2-request-check_in_date-?": "last system act hotels_2 request check_in_date ?", "system-hotels_2-request-check_out_date-?": "last system act hotels_2 request check_out_date ?", "system-hotels_2-request-number_of_adults-?": "last system act hotels_2 request number_of_adults ?", "system-hotels_2-request-where_to-?": "last system act hotels_2 request where_to ?", "system-hotels_3-confirm-check_in_date-1": "last system act hotels_3 confirm check_in_date 1", "system-hotels_3-confirm-check_out_date-1": "last system act hotels_3 confirm check_out_date 1", "system-hotels_3-confirm-hotel_name-1": "last system act hotels_3 confirm hotel_name 1", "system-hotels_3-confirm-location-1": "last system act hotels_3 confirm location 1", "system-hotels_3-confirm-number_of_rooms-1": "last system act hotels_3 confirm number_of_rooms 1", "system-hotels_3-inform-pets_welcome-1": "last system act hotels_3 inform pets_welcome 1", "system-hotels_3-inform-phone_number-1": "last system act hotels_3 inform phone_number 1", "system-hotels_3-inform-price-1": "last system act hotels_3 inform price 1", "system-hotels_3-inform-street_address-1": "last system act hotels_3 inform street_address 1", "system-hotels_3-inform_count-count-1": "last system act hotels_3 inform_count count 1", "system-hotels_3-notify_success-none-none": "last system act hotels_3 notify_success none none", "system-hotels_3-offer-average_rating-1": "last system act hotels_3 offer average_rating 1", "system-hotels_3-offer-hotel_name-1": "last system act hotels_3 offer hotel_name 1", "system-hotels_3-offer_intent-reservehotel-1": "last system act hotels_3 offer_intent ReserveHotel 1", "system-hotels_3-request-check_in_date-?": "last system act hotels_3 request check_in_date ?", "system-hotels_3-request-check_out_date-?": "last system act hotels_3 request check_out_date ?", "system-hotels_3-request-hotel_name-?": "last system act hotels_3 request hotel_name ?", "system-hotels_3-request-location-?": "last system act hotels_3 request location ?", "system-hotels_4-confirm-check_in_date-1": "last system act hotels_4 confirm check_in_date 1", "system-hotels_4-confirm-location-1": "last system act hotels_4 confirm location 1", "system-hotels_4-confirm-number_of_rooms-1": "last system act hotels_4 confirm number_of_rooms 1", "system-hotels_4-confirm-place_name-1": "last system act hotels_4 confirm place_name 1", "system-hotels_4-confirm-stay_length-1": "last system act hotels_4 confirm stay_length 1", "system-hotels_4-inform-phone_number-1": "last system act hotels_4 inform phone_number 1", "system-hotels_4-inform-price_per_night-1": "last system act hotels_4 inform price_per_night 1", "system-hotels_4-inform-smoking_allowed-1": "last system act hotels_4 inform smoking_allowed 1", "system-hotels_4-inform-street_address-1": "last system act hotels_4 inform street_address 1", "system-hotels_4-inform_count-count-1": "last system act hotels_4 inform_count count 1", "system-hotels_4-notify_success-none-none": "last system act hotels_4 notify_success none none", "system-hotels_4-offer-place_name-1": "last system act hotels_4 offer place_name 1", "system-hotels_4-offer-star_rating-1": "last system act hotels_4 offer star_rating 1", "system-hotels_4-offer_intent-reservehotel-1": "last system act hotels_4 offer_intent ReserveHotel 1", "system-hotels_4-request-check_in_date-?": "last system act hotels_4 request check_in_date ?", "system-hotels_4-request-location-?": "last system act hotels_4 request location ?", "system-hotels_4-request-stay_length-?": "last system act hotels_4 request stay_length ?", "system-media_1-confirm-subtitles-1": "last system act media_1 confirm subtitles 1", "system-media_1-confirm-title-1": "last system act media_1 confirm title 1", "system-media_1-inform-directed_by-1": "last system act media_1 inform directed_by 1", "system-media_1-inform-genre-1": "last system act media_1 inform genre 1", "system-media_1-inform_count-count-1": "last system act media_1 inform_count count 1", "system-media_1-notify_failure-none-none": "last system act media_1 notify_failure none none", "system-media_1-notify_success-none-none": "last system act media_1 notify_success none none", "system-media_1-offer-title-1": "last system act media_1 offer title 1", "system-media_1-offer-title-2": "last system act media_1 offer title 2", "system-media_1-offer-title-3": "last system act media_1 offer title 3", "system-media_1-offer_intent-playmovie-1": "last system act media_1 offer_intent PlayMovie 1", "system-media_1-request-genre-?": "last system act media_1 request genre ?", "system-media_1-request-title-?": "last system act media_1 request title ?", "system-media_2-confirm-movie_name-1": "last system act media_2 confirm movie_name 1", "system-media_2-confirm-subtitle_language-1": "last system act media_2 confirm subtitle_language 1", "system-media_2-inform-price-1": "last system act media_2 inform price 1", "system-media_2-inform_count-count-1": "last system act media_2 inform_count count 1", "system-media_2-notify_success-none-none": "last system act media_2 notify_success none none", "system-media_2-offer-movie_name-1": "last system act media_2 offer movie_name 1", "system-media_2-offer-movie_name-2": "last system act media_2 offer movie_name 2", "system-media_2-offer-movie_name-3": "last system act media_2 offer movie_name 3", "system-media_2-offer_intent-rentmovie-1": "last system act media_2 offer_intent RentMovie 1", "system-media_2-request-genre-?": "last system act media_2 request genre ?", "system-media_3-confirm-subtitle_language-1": "last system act media_3 confirm subtitle_language 1", "system-media_3-confirm-title-1": "last system act media_3 confirm title 1", "system-media_3-inform-starring-1": "last system act media_3 inform starring 1", "system-media_3-inform_count-count-1": "last system act media_3 inform_count count 1", "system-media_3-notify_success-none-none": "last system act media_3 notify_success none none", "system-media_3-offer-title-1": "last system act media_3 offer title 1", "system-media_3-offer-title-2": "last system act media_3 offer title 2", "system-media_3-offer-title-3": "last system act media_3 offer title 3", "system-media_3-offer_intent-playmovie-1": "last system act media_3 offer_intent PlayMovie 1", "system-media_3-request-genre-?": "last system act media_3 request genre ?", "system-messaging_1-confirm-contact_name-1": "last system act messaging_1 confirm contact_name 1", "system-messaging_1-confirm-location-1": "last system act messaging_1 confirm location 1", "system-messaging_1-notify_success-none-none": "last system act messaging_1 notify_success none none", "system-messaging_1-request-contact_name-?": "last system act messaging_1 request contact_name ?", "system-messaging_1-request-location-?": "last system act messaging_1 request location ?", "system-movies_1-confirm-location-1": "last system act movies_1 confirm location 1", "system-movies_1-confirm-movie_name-1": "last system act movies_1 confirm movie_name 1", "system-movies_1-confirm-number_of_tickets-1": "last system act movies_1 confirm number_of_tickets 1", "system-movies_1-confirm-show_date-1": "last system act movies_1 confirm show_date 1", "system-movies_1-confirm-show_time-1": "last system act movies_1 confirm show_time 1", "system-movies_1-confirm-show_type-1": "last system act movies_1 confirm show_type 1", "system-movies_1-inform-genre-1": "last system act movies_1 inform genre 1", "system-movies_1-inform-price-1": "last system act movies_1 inform price 1", "system-movies_1-inform-street_address-1": "last system act movies_1 inform street_address 1", "system-movies_1-inform_count-count-1": "last system act movies_1 inform_count count 1", "system-movies_1-notify_failure-none-none": "last system act movies_1 notify_failure none none", "system-movies_1-notify_success-none-none": "last system act movies_1 notify_success none none", "system-movies_1-offer-movie_name-1": "last system act movies_1 offer movie_name 1", "system-movies_1-offer-movie_name-2": "last system act movies_1 offer movie_name 2", "system-movies_1-offer-movie_name-3": "last system act movies_1 offer movie_name 3", "system-movies_1-offer-show_time-1": "last system act movies_1 offer show_time 1", "system-movies_1-offer-theater_name-1": "last system act movies_1 offer theater_name 1", "system-movies_1-offer_intent-buymovietickets-1": "last system act movies_1 offer_intent BuyMovieTickets 1", "system-movies_1-request-location-?": "last system act movies_1 request location ?", "system-movies_1-request-movie_name-?": "last system act movies_1 request movie_name ?", "system-movies_1-request-number_of_tickets-?": "last system act movies_1 request number_of_tickets ?", "system-movies_1-request-show_date-?": "last system act movies_1 request show_date ?", "system-movies_1-request-show_time-?": "last system act movies_1 request show_time ?", "system-movies_1-request-show_type-?": "last system act movies_1 request show_type ?", "system-movies_2-inform_count-count-1": "last system act movies_2 inform_count count 1", "system-movies_2-offer-aggregate_rating-1": "last system act movies_2 offer aggregate_rating 1", "system-movies_2-offer-title-1": "last system act movies_2 offer title 1", "system-movies_3-inform-cast-1": "last system act movies_3 inform cast 1", "system-movies_3-inform-directed_by-1": "last system act movies_3 inform directed_by 1", "system-movies_3-inform-genre-1": "last system act movies_3 inform genre 1", "system-movies_3-inform_count-count-1": "last system act movies_3 inform_count count 1", "system-movies_3-offer-movie_title-1": "last system act movies_3 offer movie_title 1", "system-movies_3-offer-percent_rating-1": "last system act movies_3 offer percent_rating 1", "system-music_1-confirm-playback_device-1": "last system act music_1 confirm playback_device 1", "system-music_1-confirm-song_name-1": "last system act music_1 confirm song_name 1", "system-music_1-inform-album-1": "last system act music_1 inform album 1", "system-music_1-inform-genre-1": "last system act music_1 inform genre 1", "system-music_1-inform-year-1": "last system act music_1 inform year 1", "system-music_1-inform_count-count-1": "last system act music_1 inform_count count 1", "system-music_1-notify_success-none-none": "last system act music_1 notify_success none none", "system-music_1-offer-album-1": "last system act music_1 offer album 1", "system-music_1-offer-artist-1": "last system act music_1 offer artist 1", "system-music_1-offer-song_name-1": "last system act music_1 offer song_name 1", "system-music_1-offer_intent-playsong-1": "last system act music_1 offer_intent PlaySong 1", "system-music_1-request-song_name-?": "last system act music_1 request song_name ?", "system-music_2-confirm-playback_device-1": "last system act music_2 confirm playback_device 1", "system-music_2-confirm-song_name-1": "last system act music_2 confirm song_name 1", "system-music_2-inform-genre-1": "last system act music_2 inform genre 1", "system-music_2-inform_count-count-1": "last system act music_2 inform_count count 1", "system-music_2-notify_success-none-none": "last system act music_2 notify_success none none", "system-music_2-offer-album-1": "last system act music_2 offer album 1", "system-music_2-offer-artist-1": "last system act music_2 offer artist 1", "system-music_2-offer-song_name-1": "last system act music_2 offer song_name 1", "system-music_2-offer_intent-playmedia-1": "last system act music_2 offer_intent PlayMedia 1", "system-music_2-request-song_name-?": "last system act music_2 request song_name ?", "system-music_3-confirm-device-1": "last system act music_3 confirm device 1", "system-music_3-confirm-track-1": "last system act music_3 confirm track 1", "system-music_3-inform-genre-1": "last system act music_3 inform genre 1", "system-music_3-inform-year-1": "last system act music_3 inform year 1", "system-music_3-inform_count-count-1": "last system act music_3 inform_count count 1", "system-music_3-notify_success-none-none": "last system act music_3 notify_success none none", "system-music_3-offer-album-1": "last system act music_3 offer album 1", "system-music_3-offer-artist-1": "last system act music_3 offer artist 1", "system-music_3-offer-track-1": "last system act music_3 offer track 1", "system-music_3-offer_intent-playmedia-1": "last system act music_3 offer_intent PlayMedia 1", "system-payment_1-confirm-amount-1": "last system act payment_1 confirm amount 1", "system-payment_1-confirm-payment_method-1": "last system act payment_1 confirm payment_method 1", "system-payment_1-confirm-private_visibility-1": "last system act payment_1 confirm private_visibility 1", "system-payment_1-confirm-receiver-1": "last system act payment_1 confirm receiver 1", "system-payment_1-notify_success-none-none": "last system act payment_1 notify_success none none", "system-payment_1-request-amount-?": "last system act payment_1 request amount ?", "system-payment_1-request-payment_method-?": "last system act payment_1 request payment_method ?", "system-payment_1-request-receiver-?": "last system act payment_1 request receiver ?", "system-rentalcars_1-confirm-dropoff_date-1": "last system act rentalcars_1 confirm dropoff_date 1", "system-rentalcars_1-confirm-pickup_date-1": "last system act rentalcars_1 confirm pickup_date 1", "system-rentalcars_1-confirm-pickup_location-1": "last system act rentalcars_1 confirm pickup_location 1", "system-rentalcars_1-confirm-pickup_time-1": "last system act rentalcars_1 confirm pickup_time 1", "system-rentalcars_1-confirm-type-1": "last system act rentalcars_1 confirm type 1", "system-rentalcars_1-inform-car_name-1": "last system act rentalcars_1 inform car_name 1", "system-rentalcars_1-inform-total_price-1": "last system act rentalcars_1 inform total_price 1", "system-rentalcars_1-inform_count-count-1": "last system act rentalcars_1 inform_count count 1", "system-rentalcars_1-notify_success-none-none": "last system act rentalcars_1 notify_success none none", "system-rentalcars_1-offer-car_name-1": "last system act rentalcars_1 offer car_name 1", "system-rentalcars_1-offer-pickup_date-1": "last system act rentalcars_1 offer pickup_date 1", "system-rentalcars_1-offer-pickup_location-1": "last system act rentalcars_1 offer pickup_location 1", "system-rentalcars_1-offer-type-1": "last system act rentalcars_1 offer type 1", "system-rentalcars_1-offer_intent-reservecar-1": "last system act rentalcars_1 offer_intent ReserveCar 1", "system-rentalcars_1-request-dropoff_date-?": "last system act rentalcars_1 request dropoff_date ?", "system-rentalcars_1-request-pickup_city-?": "last system act rentalcars_1 request pickup_city ?", "system-rentalcars_1-request-pickup_date-?": "last system act rentalcars_1 request pickup_date ?", "system-rentalcars_1-request-pickup_location-?": "last system act rentalcars_1 request pickup_location ?", "system-rentalcars_1-request-pickup_time-?": "last system act rentalcars_1 request pickup_time ?", "system-rentalcars_1-request-type-?": "last system act rentalcars_1 request type ?", "system-rentalcars_2-confirm-car_type-1": "last system act rentalcars_2 confirm car_type 1", "system-rentalcars_2-confirm-dropoff_date-1": "last system act rentalcars_2 confirm dropoff_date 1", "system-rentalcars_2-confirm-pickup_date-1": "last system act rentalcars_2 confirm pickup_date 1", "system-rentalcars_2-confirm-pickup_location-1": "last system act rentalcars_2 confirm pickup_location 1", "system-rentalcars_2-confirm-pickup_time-1": "last system act rentalcars_2 confirm pickup_time 1", "system-rentalcars_2-inform-car_name-1": "last system act rentalcars_2 inform car_name 1", "system-rentalcars_2-inform-total_price-1": "last system act rentalcars_2 inform total_price 1", "system-rentalcars_2-inform_count-count-1": "last system act rentalcars_2 inform_count count 1", "system-rentalcars_2-notify_success-none-none": "last system act rentalcars_2 notify_success none none", "system-rentalcars_2-offer-car_name-1": "last system act rentalcars_2 offer car_name 1", "system-rentalcars_2-offer-car_type-1": "last system act rentalcars_2 offer car_type 1", "system-rentalcars_2-offer-pickup_date-1": "last system act rentalcars_2 offer pickup_date 1", "system-rentalcars_2-offer-pickup_location-1": "last system act rentalcars_2 offer pickup_location 1", "system-rentalcars_2-offer_intent-reservecar-1": "last system act rentalcars_2 offer_intent ReserveCar 1", "system-rentalcars_2-request-car_type-?": "last system act rentalcars_2 request car_type ?", "system-rentalcars_2-request-dropoff_date-?": "last system act rentalcars_2 request dropoff_date ?", "system-rentalcars_2-request-pickup_city-?": "last system act rentalcars_2 request pickup_city ?", "system-rentalcars_2-request-pickup_date-?": "last system act rentalcars_2 request pickup_date ?", "system-rentalcars_2-request-pickup_location-?": "last system act rentalcars_2 request pickup_location ?", "system-rentalcars_2-request-pickup_time-?": "last system act rentalcars_2 request pickup_time ?", "system-rentalcars_3-confirm-add_insurance-1": "last system act rentalcars_3 confirm add_insurance 1", "system-rentalcars_3-confirm-car_type-1": "last system act rentalcars_3 confirm car_type 1", "system-rentalcars_3-confirm-end_date-1": "last system act rentalcars_3 confirm end_date 1", "system-rentalcars_3-confirm-pickup_location-1": "last system act rentalcars_3 confirm pickup_location 1", "system-rentalcars_3-confirm-pickup_time-1": "last system act rentalcars_3 confirm pickup_time 1", "system-rentalcars_3-confirm-start_date-1": "last system act rentalcars_3 confirm start_date 1", "system-rentalcars_3-inform-car_name-1": "last system act rentalcars_3 inform car_name 1", "system-rentalcars_3-inform-price_per_day-1": "last system act rentalcars_3 inform price_per_day 1", "system-rentalcars_3-inform_count-count-1": "last system act rentalcars_3 inform_count count 1", "system-rentalcars_3-notify_success-none-none": "last system act rentalcars_3 notify_success none none", "system-rentalcars_3-offer-car_name-1": "last system act rentalcars_3 offer car_name 1", "system-rentalcars_3-offer-car_type-1": "last system act rentalcars_3 offer car_type 1", "system-rentalcars_3-offer-pickup_location-1": "last system act rentalcars_3 offer pickup_location 1", "system-rentalcars_3-offer_intent-reservecar-1": "last system act rentalcars_3 offer_intent ReserveCar 1", "system-rentalcars_3-request-add_insurance-?": "last system act rentalcars_3 request add_insurance ?", "system-rentalcars_3-request-car_type-?": "last system act rentalcars_3 request car_type ?", "system-rentalcars_3-request-city-?": "last system act rentalcars_3 request city ?", "system-rentalcars_3-request-end_date-?": "last system act rentalcars_3 request end_date ?", "system-rentalcars_3-request-pickup_location-?": "last system act rentalcars_3 request pickup_location ?", "system-rentalcars_3-request-pickup_time-?": "last system act rentalcars_3 request pickup_time ?", "system-rentalcars_3-request-start_date-?": "last system act rentalcars_3 request start_date ?", "system-restaurants_1-confirm-city-1": "last system act restaurants_1 confirm city 1", "system-restaurants_1-confirm-date-1": "last system act restaurants_1 confirm date 1", "system-restaurants_1-confirm-party_size-1": "last system act restaurants_1 confirm party_size 1", "system-restaurants_1-confirm-restaurant_name-1": "last system act restaurants_1 confirm restaurant_name 1", "system-restaurants_1-confirm-time-1": "last system act restaurants_1 confirm time 1", "system-restaurants_1-inform-cuisine-1": "last system act restaurants_1 inform cuisine 1", "system-restaurants_1-inform-has_live_music-1": "last system act restaurants_1 inform has_live_music 1", "system-restaurants_1-inform-phone_number-1": "last system act restaurants_1 inform phone_number 1", "system-restaurants_1-inform-price_range-1": "last system act restaurants_1 inform price_range 1", "system-restaurants_1-inform-serves_alcohol-1": "last system act restaurants_1 inform serves_alcohol 1", "system-restaurants_1-inform-street_address-1": "last system act restaurants_1 inform street_address 1", "system-restaurants_1-inform_count-count-1": "last system act restaurants_1 inform_count count 1", "system-restaurants_1-notify_failure-none-none": "last system act restaurants_1 notify_failure none none", "system-restaurants_1-notify_success-none-none": "last system act restaurants_1 notify_success none none", "system-restaurants_1-offer-city-1": "last system act restaurants_1 offer city 1", "system-restaurants_1-offer-date-1": "last system act restaurants_1 offer date 1", "system-restaurants_1-offer-party_size-1": "last system act restaurants_1 offer party_size 1", "system-restaurants_1-offer-restaurant_name-1": "last system act restaurants_1 offer restaurant_name 1", "system-restaurants_1-offer-time-1": "last system act restaurants_1 offer time 1", "system-restaurants_1-offer_intent-reserverestaurant-1": "last system act restaurants_1 offer_intent ReserveRestaurant 1", "system-restaurants_1-request-city-?": "last system act restaurants_1 request city ?", "system-restaurants_1-request-cuisine-?": "last system act restaurants_1 request cuisine ?", "system-restaurants_1-request-restaurant_name-?": "last system act restaurants_1 request restaurant_name ?", "system-restaurants_1-request-time-?": "last system act restaurants_1 request time ?", "system-restaurants_2-confirm-date-1": "last system act restaurants_2 confirm date 1", "system-restaurants_2-confirm-location-1": "last system act restaurants_2 confirm location 1", "system-restaurants_2-confirm-number_of_seats-1": "last system act restaurants_2 confirm number_of_seats 1", "system-restaurants_2-confirm-restaurant_name-1": "last system act restaurants_2 confirm restaurant_name 1", "system-restaurants_2-confirm-time-1": "last system act restaurants_2 confirm time 1", "system-restaurants_2-inform-address-1": "last system act restaurants_2 inform address 1", "system-restaurants_2-inform-has_seating_outdoors-1": "last system act restaurants_2 inform has_seating_outdoors 1", "system-restaurants_2-inform-has_vegetarian_options-1": "last system act restaurants_2 inform has_vegetarian_options 1", "system-restaurants_2-inform-phone_number-1": "last system act restaurants_2 inform phone_number 1", "system-restaurants_2-inform-price_range-1": "last system act restaurants_2 inform price_range 1", "system-restaurants_2-inform-rating-1": "last system act restaurants_2 inform rating 1", "system-restaurants_2-inform_count-count-1": "last system act restaurants_2 inform_count count 1", "system-restaurants_2-notify_failure-none-none": "last system act restaurants_2 notify_failure none none", "system-restaurants_2-notify_success-none-none": "last system act restaurants_2 notify_success none none", "system-restaurants_2-offer-date-1": "last system act restaurants_2 offer date 1", "system-restaurants_2-offer-location-1": "last system act restaurants_2 offer location 1", "system-restaurants_2-offer-number_of_seats-1": "last system act restaurants_2 offer number_of_seats 1", "system-restaurants_2-offer-restaurant_name-1": "last system act restaurants_2 offer restaurant_name 1", "system-restaurants_2-offer-time-1": "last system act restaurants_2 offer time 1", "system-restaurants_2-offer_intent-reserverestaurant-1": "last system act restaurants_2 offer_intent ReserveRestaurant 1", "system-restaurants_2-request-category-?": "last system act restaurants_2 request category ?", "system-restaurants_2-request-location-?": "last system act restaurants_2 request location ?", "system-restaurants_2-request-restaurant_name-?": "last system act restaurants_2 request restaurant_name ?", "system-restaurants_2-request-time-?": "last system act restaurants_2 request time ?", "system-ridesharing_1-confirm-destination-1": "last system act ridesharing_1 confirm destination 1", "system-ridesharing_1-confirm-number_of_riders-1": "last system act ridesharing_1 confirm number_of_riders 1", "system-ridesharing_1-confirm-shared_ride-1": "last system act ridesharing_1 confirm shared_ride 1", "system-ridesharing_1-inform-approximate_ride_duration-1": "last system act ridesharing_1 inform approximate_ride_duration 1", "system-ridesharing_1-inform-ride_fare-1": "last system act ridesharing_1 inform ride_fare 1", "system-ridesharing_1-notify_success-none-none": "last system act ridesharing_1 notify_success none none", "system-ridesharing_1-request-destination-?": "last system act ridesharing_1 request destination ?", "system-ridesharing_1-request-number_of_riders-?": "last system act ridesharing_1 request number_of_riders ?", "system-ridesharing_1-request-shared_ride-?": "last system act ridesharing_1 request shared_ride ?", "system-ridesharing_2-confirm-destination-1": "last system act ridesharing_2 confirm destination 1", "system-ridesharing_2-confirm-number_of_seats-1": "last system act ridesharing_2 confirm number_of_seats 1", "system-ridesharing_2-confirm-ride_type-1": "last system act ridesharing_2 confirm ride_type 1", "system-ridesharing_2-inform-ride_fare-1": "last system act ridesharing_2 inform ride_fare 1", "system-ridesharing_2-inform-wait_time-1": "last system act ridesharing_2 inform wait_time 1", "system-ridesharing_2-notify_success-none-none": "last system act ridesharing_2 notify_success none none", "system-ridesharing_2-request-destination-?": "last system act ridesharing_2 request destination ?", "system-ridesharing_2-request-number_of_seats-?": "last system act ridesharing_2 request number_of_seats ?", "system-ridesharing_2-request-ride_type-?": "last system act ridesharing_2 request ride_type ?", "system-services_1-confirm-appointment_date-1": "last system act services_1 confirm appointment_date 1", "system-services_1-confirm-appointment_time-1": "last system act services_1 confirm appointment_time 1", "system-services_1-confirm-stylist_name-1": "last system act services_1 confirm stylist_name 1", "system-services_1-inform-average_rating-1": "last system act services_1 inform average_rating 1", "system-services_1-inform-is_unisex-1": "last system act services_1 inform is_unisex 1", "system-services_1-inform-phone_number-1": "last system act services_1 inform phone_number 1", "system-services_1-inform-street_address-1": "last system act services_1 inform street_address 1", "system-services_1-inform_count-count-1": "last system act services_1 inform_count count 1", "system-services_1-notify_failure-none-none": "last system act services_1 notify_failure none none", "system-services_1-notify_success-none-none": "last system act services_1 notify_success none none", "system-services_1-offer-appointment_date-1": "last system act services_1 offer appointment_date 1", "system-services_1-offer-appointment_time-1": "last system act services_1 offer appointment_time 1", "system-services_1-offer-city-1": "last system act services_1 offer city 1", "system-services_1-offer-stylist_name-1": "last system act services_1 offer stylist_name 1", "system-services_1-offer_intent-bookappointment-1": "last system act services_1 offer_intent BookAppointment 1", "system-services_1-request-appointment_date-?": "last system act services_1 request appointment_date ?", "system-services_1-request-appointment_time-?": "last system act services_1 request appointment_time ?", "system-services_1-request-city-?": "last system act services_1 request city ?", "system-services_2-confirm-appointment_date-1": "last system act services_2 confirm appointment_date 1", "system-services_2-confirm-appointment_time-1": "last system act services_2 confirm appointment_time 1", "system-services_2-confirm-dentist_name-1": "last system act services_2 confirm dentist_name 1", "system-services_2-inform-address-1": "last system act services_2 inform address 1", "system-services_2-inform-offers_cosmetic_services-1": "last system act services_2 inform offers_cosmetic_services 1", "system-services_2-inform-phone_number-1": "last system act services_2 inform phone_number 1", "system-services_2-inform_count-count-1": "last system act services_2 inform_count count 1", "system-services_2-notify_failure-none-none": "last system act services_2 notify_failure none none", "system-services_2-notify_success-none-none": "last system act services_2 notify_success none none", "system-services_2-offer-appointment_date-1": "last system act services_2 offer appointment_date 1", "system-services_2-offer-appointment_time-1": "last system act services_2 offer appointment_time 1", "system-services_2-offer-city-1": "last system act services_2 offer city 1", "system-services_2-offer-dentist_name-1": "last system act services_2 offer dentist_name 1", "system-services_2-offer_intent-bookappointment-1": "last system act services_2 offer_intent BookAppointment 1", "system-services_2-request-appointment_date-?": "last system act services_2 request appointment_date ?", "system-services_2-request-appointment_time-?": "last system act services_2 request appointment_time ?", "system-services_2-request-city-?": "last system act services_2 request city ?", "system-services_3-confirm-appointment_date-1": "last system act services_3 confirm appointment_date 1", "system-services_3-confirm-appointment_time-1": "last system act services_3 confirm appointment_time 1", "system-services_3-confirm-doctor_name-1": "last system act services_3 confirm doctor_name 1", "system-services_3-inform-average_rating-1": "last system act services_3 inform average_rating 1", "system-services_3-inform-phone_number-1": "last system act services_3 inform phone_number 1", "system-services_3-inform-street_address-1": "last system act services_3 inform street_address 1", "system-services_3-inform_count-count-1": "last system act services_3 inform_count count 1", "system-services_3-notify_failure-none-none": "last system act services_3 notify_failure none none", "system-services_3-notify_success-none-none": "last system act services_3 notify_success none none", "system-services_3-offer-appointment_date-1": "last system act services_3 offer appointment_date 1", "system-services_3-offer-appointment_time-1": "last system act services_3 offer appointment_time 1", "system-services_3-offer-city-1": "last system act services_3 offer city 1", "system-services_3-offer-doctor_name-1": "last system act services_3 offer doctor_name 1", "system-services_3-offer-type-1": "last system act services_3 offer type 1", "system-services_3-offer_intent-bookappointment-1": "last system act services_3 offer_intent BookAppointment 1", "system-services_3-request-appointment_date-?": "last system act services_3 request appointment_date ?", "system-services_3-request-appointment_time-?": "last system act services_3 request appointment_time ?", "system-services_3-request-city-?": "last system act services_3 request city ?", "system-services_3-request-type-?": "last system act services_3 request type ?", "system-services_4-confirm-appointment_date-1": "last system act services_4 confirm appointment_date 1", "system-services_4-confirm-appointment_time-1": "last system act services_4 confirm appointment_time 1", "system-services_4-confirm-therapist_name-1": "last system act services_4 confirm therapist_name 1", "system-services_4-inform-address-1": "last system act services_4 inform address 1", "system-services_4-inform-phone_number-1": "last system act services_4 inform phone_number 1", "system-services_4-inform_count-count-1": "last system act services_4 inform_count count 1", "system-services_4-notify_failure-none-none": "last system act services_4 notify_failure none none", "system-services_4-notify_success-none-none": "last system act services_4 notify_success none none", "system-services_4-offer-appointment_date-1": "last system act services_4 offer appointment_date 1", "system-services_4-offer-appointment_time-1": "last system act services_4 offer appointment_time 1", "system-services_4-offer-city-1": "last system act services_4 offer city 1", "system-services_4-offer-therapist_name-1": "last system act services_4 offer therapist_name 1", "system-services_4-offer-type-1": "last system act services_4 offer type 1", "system-services_4-offer_intent-bookappointment-1": "last system act services_4 offer_intent BookAppointment 1", "system-services_4-request-appointment_date-?": "last system act services_4 request appointment_date ?", "system-services_4-request-appointment_time-?": "last system act services_4 request appointment_time ?", "system-services_4-request-city-?": "last system act services_4 request city ?", "system-services_4-request-type-?": "last system act services_4 request type ?", "system-trains_1-confirm-class-1": "last system act trains_1 confirm class 1", "system-trains_1-confirm-date_of_journey-1": "last system act trains_1 confirm date_of_journey 1", "system-trains_1-confirm-from-1": "last system act trains_1 confirm from 1", "system-trains_1-confirm-journey_start_time-1": "last system act trains_1 confirm journey_start_time 1", "system-trains_1-confirm-number_of_adults-1": "last system act trains_1 confirm number_of_adults 1", "system-trains_1-confirm-to-1": "last system act trains_1 confirm to 1", "system-trains_1-confirm-trip_protection-1": "last system act trains_1 confirm trip_protection 1", "system-trains_1-inform-from_station-1": "last system act trains_1 inform from_station 1", "system-trains_1-inform-to_station-1": "last system act trains_1 inform to_station 1", "system-trains_1-inform_count-count-1": "last system act trains_1 inform_count count 1", "system-trains_1-notify_success-none-none": "last system act trains_1 notify_success none none", "system-trains_1-offer-journey_start_time-1": "last system act trains_1 offer journey_start_time 1", "system-trains_1-offer-total-1": "last system act trains_1 offer total 1", "system-trains_1-offer_intent-gettraintickets-1": "last system act trains_1 offer_intent GetTrainTickets 1", "system-trains_1-request-date_of_journey-?": "last system act trains_1 request date_of_journey ?", "system-trains_1-request-from-?": "last system act trains_1 request from ?", "system-trains_1-request-number_of_adults-?": "last system act trains_1 request number_of_adults ?", "system-trains_1-request-to-?": "last system act trains_1 request to ?", "system-trains_1-request-trip_protection-?": "last system act trains_1 request trip_protection ?", "system-travel_1-inform-free_entry-1": "last system act travel_1 inform free_entry 1", "system-travel_1-inform-good_for_kids-1": "last system act travel_1 inform good_for_kids 1", "system-travel_1-inform-phone_number-1": "last system act travel_1 inform phone_number 1", "system-travel_1-inform_count-count-1": "last system act travel_1 inform_count count 1", "system-travel_1-offer-attraction_name-1": "last system act travel_1 offer attraction_name 1", "system-travel_1-offer-category-1": "last system act travel_1 offer category 1", "system-travel_1-request-location-?": "last system act travel_1 request location ?", "system-weather_1-inform-humidity-1": "last system act weather_1 inform humidity 1", "system-weather_1-inform-wind-1": "last system act weather_1 inform wind 1", "system-weather_1-offer-precipitation-1": "last system act weather_1 offer precipitation 1", "system-weather_1-offer-temperature-1": "last system act weather_1 offer temperature 1", "system-weather_1-request-city-?": "last system act weather_1 request city ?", "user--affirm-none-none": "user act  affirm none none", "user--goodbye-none-none": "user act  goodbye none none", "user--negate-none-none": "user act  negate none none", "user--thank_you-none-none": "user act  thank_you none none", "user-alarm_1-affirm_intent-none-none": "user act alarm_1 affirm_intent none none", "user-alarm_1-inform-new_alarm_name-1": "user act alarm_1 inform new_alarm_name 1", "user-alarm_1-inform-new_alarm_time-1": "user act alarm_1 inform new_alarm_time 1", "user-alarm_1-inform_intent-addalarm-1": "user act alarm_1 inform_intent addalarm 1", "user-alarm_1-inform_intent-getalarms-1": "user act alarm_1 inform_intent getalarms 1", "user-alarm_1-negate_intent-none-none": "user act alarm_1 negate_intent none none", "user-alarm_1-select-none-none": "user act alarm_1 select none none", "user-banks_1-inform-account_type-1": "user act banks_1 inform account_type 1", "user-banks_1-inform-amount-1": "user act banks_1 inform amount 1", "user-banks_1-inform-recipient_account_name-1": "user act banks_1 inform recipient_account_name 1", "user-banks_1-inform-recipient_account_type-1": "user act banks_1 inform recipient_account_type 1", "user-banks_1-inform_intent-checkbalance-1": "user act banks_1 inform_intent checkbalance 1", "user-banks_1-inform_intent-transfermoney-1": "user act banks_1 inform_intent transfermoney 1", "user-banks_1-negate_intent-none-none": "user act banks_1 negate_intent none none", "user-banks_1-request_alts-none-none": "user act banks_1 request_alts none none", "user-banks_1-select-none-none": "user act banks_1 select none none", "user-banks_2-affirm_intent-none-none": "user act banks_2 affirm_intent none none", "user-banks_2-inform-account_type-1": "user act banks_2 inform account_type 1", "user-banks_2-inform-recipient_account_type-1": "user act banks_2 inform recipient_account_type 1", "user-banks_2-inform-recipient_name-1": "user act banks_2 inform recipient_name 1", "user-banks_2-inform-transfer_amount-1": "user act banks_2 inform transfer_amount 1", "user-banks_2-inform_intent-checkbalance-1": "user act banks_2 inform_intent checkbalance 1", "user-banks_2-inform_intent-transfermoney-1": "user act banks_2 inform_intent transfermoney 1", "user-banks_2-negate_intent-none-none": "user act banks_2 negate_intent none none", "user-banks_2-request-transfer_time-?": "user act banks_2 request transfer_time ?", "user-banks_2-request_alts-none-none": "user act banks_2 request_alts none none", "user-banks_2-select-none-none": "user act banks_2 select none none", "user-buses_1-affirm_intent-none-none": "user act buses_1 affirm_intent none none", "user-buses_1-inform-from_location-1": "user act buses_1 inform from_location 1", "user-buses_1-inform-leaving_date-1": "user act buses_1 inform leaving_date 1", "user-buses_1-inform-leaving_time-1": "user act buses_1 inform leaving_time 1", "user-buses_1-inform-to_location-1": "user act buses_1 inform to_location 1", "user-buses_1-inform-travelers-1": "user act buses_1 inform travelers 1", "user-buses_1-inform_intent-buybusticket-1": "user act buses_1 inform_intent buybusticket 1", "user-buses_1-inform_intent-findbus-1": "user act buses_1 inform_intent findbus 1", "user-buses_1-negate_intent-none-none": "user act buses_1 negate_intent none none", "user-buses_1-request-from_station-?": "user act buses_1 request from_station ?", "user-buses_1-request-to_station-?": "user act buses_1 request to_station ?", "user-buses_1-request-transfers-?": "user act buses_1 request transfers ?", "user-buses_1-request_alts-none-none": "user act buses_1 request_alts none none", "user-buses_1-select-none-none": "user act buses_1 select none none", "user-buses_2-affirm_intent-none-none": "user act buses_2 affirm_intent none none", "user-buses_2-inform-departure_date-1": "user act buses_2 inform departure_date 1", "user-buses_2-inform-departure_time-1": "user act buses_2 inform departure_time 1", "user-buses_2-inform-destination-1": "user act buses_2 inform destination 1", "user-buses_2-inform-fare_type-1": "user act buses_2 inform fare_type 1", "user-buses_2-inform-group_size-1": "user act buses_2 inform group_size 1", "user-buses_2-inform-origin-1": "user act buses_2 inform origin 1", "user-buses_2-inform_intent-buybusticket-1": "user act buses_2 inform_intent buybusticket 1", "user-buses_2-inform_intent-findbus-1": "user act buses_2 inform_intent findbus 1", "user-buses_2-negate_intent-none-none": "user act buses_2 negate_intent none none", "user-buses_2-request-destination_station_name-?": "user act buses_2 request destination_station_name ?", "user-buses_2-request-origin_station_name-?": "user act buses_2 request origin_station_name ?", "user-buses_2-request-price-?": "user act buses_2 request price ?", "user-buses_2-request_alts-none-none": "user act buses_2 request_alts none none", "user-buses_2-select-none-none": "user act buses_2 select none none", "user-buses_3-inform-additional_luggage-1": "user act buses_3 inform additional_luggage 1", "user-buses_3-inform-category-1": "user act buses_3 inform category 1", "user-buses_3-inform-departure_date-1": "user act buses_3 inform departure_date 1", "user-buses_3-inform-departure_time-1": "user act buses_3 inform departure_time 1", "user-buses_3-inform-from_city-1": "user act buses_3 inform from_city 1", "user-buses_3-inform-num_passengers-1": "user act buses_3 inform num_passengers 1", "user-buses_3-inform-to_city-1": "user act buses_3 inform to_city 1", "user-buses_3-inform_intent-buybusticket-1": "user act buses_3 inform_intent buybusticket 1", "user-buses_3-inform_intent-findbus-1": "user act buses_3 inform_intent findbus 1", "user-buses_3-negate_intent-none-none": "user act buses_3 negate_intent none none", "user-buses_3-request-category-?": "user act buses_3 request category ?", "user-buses_3-request-from_station-?": "user act buses_3 request from_station ?", "user-buses_3-request-to_station-?": "user act buses_3 request to_station ?", "user-buses_3-request_alts-none-none": "user act buses_3 request_alts none none", "user-buses_3-select-none-none": "user act buses_3 select none none", "user-calendar_1-inform-event_date-1": "user act calendar_1 inform event_date 1", "user-calendar_1-inform-event_location-1": "user act calendar_1 inform event_location 1", "user-calendar_1-inform-event_name-1": "user act calendar_1 inform event_name 1", "user-calendar_1-inform-event_time-1": "user act calendar_1 inform event_time 1", "user-calendar_1-inform_intent-addevent-1": "user act calendar_1 inform_intent addevent 1", "user-calendar_1-inform_intent-getavailabletime-1": "user act calendar_1 inform_intent getavailabletime 1", "user-calendar_1-inform_intent-getevents-1": "user act calendar_1 inform_intent getevents 1", "user-calendar_1-negate_intent-none-none": "user act calendar_1 negate_intent none none", "user-calendar_1-request_alts-none-none": "user act calendar_1 request_alts none none", "user-calendar_1-select-none-none": "user act calendar_1 select none none", "user-events_1-affirm_intent-none-none": "user act events_1 affirm_intent none none", "user-events_1-inform-category-1": "user act events_1 inform category 1", "user-events_1-inform-city_of_event-1": "user act events_1 inform city_of_event 1", "user-events_1-inform-date-1": "user act events_1 inform date 1", "user-events_1-inform-event_name-1": "user act events_1 inform event_name 1", "user-events_1-inform-number_of_seats-1": "user act events_1 inform number_of_seats 1", "user-events_1-inform-subcategory-1": "user act events_1 inform subcategory 1", "user-events_1-inform_intent-buyeventtickets-1": "user act events_1 inform_intent buyeventtickets 1", "user-events_1-inform_intent-findevents-1": "user act events_1 inform_intent findevents 1", "user-events_1-negate_intent-none-none": "user act events_1 negate_intent none none", "user-events_1-request-address_of_location-?": "user act events_1 request address_of_location ?", "user-events_1-request-event_location-?": "user act events_1 request event_location ?", "user-events_1-request-subcategory-?": "user act events_1 request subcategory ?", "user-events_1-request-time-?": "user act events_1 request time ?", "user-events_1-request_alts-none-none": "user act events_1 request_alts none none", "user-events_1-select-none-none": "user act events_1 select none none", "user-events_2-affirm_intent-none-none": "user act events_2 affirm_intent none none", "user-events_2-inform-category-1": "user act events_2 inform category 1", "user-events_2-inform-city-1": "user act events_2 inform city 1", "user-events_2-inform-date-1": "user act events_2 inform date 1", "user-events_2-inform-event_name-1": "user act events_2 inform event_name 1", "user-events_2-inform-event_type-1": "user act events_2 inform event_type 1", "user-events_2-inform-number_of_tickets-1": "user act events_2 inform number_of_tickets 1", "user-events_2-inform_intent-buyeventtickets-1": "user act events_2 inform_intent buyeventtickets 1", "user-events_2-inform_intent-findevents-1": "user act events_2 inform_intent findevents 1", "user-events_2-inform_intent-geteventdates-1": "user act events_2 inform_intent geteventdates 1", "user-events_2-negate_intent-none-none": "user act events_2 negate_intent none none", "user-events_2-request-category-?": "user act events_2 request category ?", "user-events_2-request-time-?": "user act events_2 request time ?", "user-events_2-request-venue-?": "user act events_2 request venue ?", "user-events_2-request-venue_address-?": "user act events_2 request venue_address ?", "user-events_2-request_alts-none-none": "user act events_2 request_alts none none", "user-events_2-select-none-none": "user act events_2 select none none", "user-events_3-inform-city-1": "user act events_3 inform city 1", "user-events_3-inform-date-1": "user act events_3 inform date 1", "user-events_3-inform-event_name-1": "user act events_3 inform event_name 1", "user-events_3-inform-event_type-1": "user act events_3 inform event_type 1", "user-events_3-inform-number_of_tickets-1": "user act events_3 inform number_of_tickets 1", "user-events_3-inform_intent-buyeventtickets-1": "user act events_3 inform_intent buyeventtickets 1", "user-events_3-inform_intent-findevents-1": "user act events_3 inform_intent findevents 1", "user-events_3-negate_intent-none-none": "user act events_3 negate_intent none none", "user-events_3-request-price_per_ticket-?": "user act events_3 request price_per_ticket ?", "user-events_3-request-venue_address-?": "user act events_3 request venue_address ?", "user-events_3-request_alts-none-none": "user act events_3 request_alts none none", "user-events_3-select-none-none": "user act events_3 select none none", "user-flights_1-affirm_intent-none-none": "user act flights_1 affirm_intent none none", "user-flights_1-inform-airlines-1": "user act flights_1 inform airlines 1", "user-flights_1-inform-departure_date-1": "user act flights_1 inform departure_date 1", "user-flights_1-inform-destination_city-1": "user act flights_1 inform destination_city 1", "user-flights_1-inform-inbound_departure_time-1": "user act flights_1 inform inbound_departure_time 1", "user-flights_1-inform-origin_city-1": "user act flights_1 inform origin_city 1", "user-flights_1-inform-outbound_departure_time-1": "user act flights_1 inform outbound_departure_time 1", "user-flights_1-inform-passengers-1": "user act flights_1 inform passengers 1", "user-flights_1-inform-refundable-1": "user act flights_1 inform refundable 1", "user-flights_1-inform-return_date-1": "user act flights_1 inform return_date 1", "user-flights_1-inform-seating_class-1": "user act flights_1 inform seating_class 1", "user-flights_1-inform_intent-reserveonewayflight-1": "user act flights_1 inform_intent reserveonewayflight 1", "user-flights_1-inform_intent-reserveroundtripflights-1": "user act flights_1 inform_intent reserveroundtripflights 1", "user-flights_1-inform_intent-searchonewayflight-1": "user act flights_1 inform_intent searchonewayflight 1", "user-flights_1-inform_intent-searchroundtripflights-1": "user act flights_1 inform_intent searchroundtripflights 1", "user-flights_1-negate_intent-none-none": "user act flights_1 negate_intent none none", "user-flights_1-request-destination_airport-?": "user act flights_1 request destination_airport ?", "user-flights_1-request-inbound_arrival_time-?": "user act flights_1 request inbound_arrival_time ?", "user-flights_1-request-number_stops-?": "user act flights_1 request number_stops ?", "user-flights_1-request-origin_airport-?": "user act flights_1 request origin_airport ?", "user-flights_1-request-outbound_arrival_time-?": "user act flights_1 request outbound_arrival_time ?", "user-flights_1-request-refundable-?": "user act flights_1 request refundable ?", "user-flights_1-request_alts-none-none": "user act flights_1 request_alts none none", "user-flights_1-select-none-none": "user act flights_1 select none none", "user-flights_2-inform-airlines-1": "user act flights_2 inform airlines 1", "user-flights_2-inform-departure_date-1": "user act flights_2 inform departure_date 1", "user-flights_2-inform-destination-1": "user act flights_2 inform destination 1", "user-flights_2-inform-origin-1": "user act flights_2 inform origin 1", "user-flights_2-inform-passengers-1": "user act flights_2 inform passengers 1", "user-flights_2-inform-return_date-1": "user act flights_2 inform return_date 1", "user-flights_2-inform-seating_class-1": "user act flights_2 inform seating_class 1", "user-flights_2-inform_intent-searchonewayflight-1": "user act flights_2 inform_intent searchonewayflight 1", "user-flights_2-inform_intent-searchroundtripflights-1": "user act flights_2 inform_intent searchroundtripflights 1", "user-flights_2-request-destination_airport-?": "user act flights_2 request destination_airport ?", "user-flights_2-request-is_redeye-?": "user act flights_2 request is_redeye ?", "user-flights_2-request-origin_airport-?": "user act flights_2 request origin_airport ?", "user-flights_2-request-outbound_arrival_time-?": "user act flights_2 request outbound_arrival_time ?", "user-flights_2-request_alts-none-none": "user act flights_2 request_alts none none", "user-flights_2-select-none-none": "user act flights_2 select none none", "user-flights_3-inform-airlines-1": "user act flights_3 inform airlines 1", "user-flights_3-inform-departure_date-1": "user act flights_3 inform departure_date 1", "user-flights_3-inform-destination_city-1": "user act flights_3 inform destination_city 1", "user-flights_3-inform-flight_class-1": "user act flights_3 inform flight_class 1", "user-flights_3-inform-number_checked_bags-1": "user act flights_3 inform number_checked_bags 1", "user-flights_3-inform-origin_city-1": "user act flights_3 inform origin_city 1", "user-flights_3-inform-passengers-1": "user act flights_3 inform passengers 1", "user-flights_3-inform-return_date-1": "user act flights_3 inform return_date 1", "user-flights_3-inform_intent-searchonewayflight-1": "user act flights_3 inform_intent searchonewayflight 1", "user-flights_3-inform_intent-searchroundtripflights-1": "user act flights_3 inform_intent searchroundtripflights 1", "user-flights_3-request-arrives_next_day-?": "user act flights_3 request arrives_next_day ?", "user-flights_3-request-destination_airport_name-?": "user act flights_3 request destination_airport_name ?", "user-flights_3-request-origin_airport_name-?": "user act flights_3 request origin_airport_name ?", "user-flights_3-request-outbound_arrival_time-?": "user act flights_3 request outbound_arrival_time ?", "user-flights_3-request_alts-none-none": "user act flights_3 request_alts none none", "user-flights_3-select-none-none": "user act flights_3 select none none", "user-flights_4-inform-airlines-1": "user act flights_4 inform airlines 1", "user-flights_4-inform-departure_date-1": "user act flights_4 inform departure_date 1", "user-flights_4-inform-destination_airport-1": "user act flights_4 inform destination_airport 1", "user-flights_4-inform-number_of_tickets-1": "user act flights_4 inform number_of_tickets 1", "user-flights_4-inform-origin_airport-1": "user act flights_4 inform origin_airport 1", "user-flights_4-inform-return_date-1": "user act flights_4 inform return_date 1", "user-flights_4-inform-seating_class-1": "user act flights_4 inform seating_class 1", "user-flights_4-inform_intent-searchonewayflight-1": "user act flights_4 inform_intent searchonewayflight 1", "user-flights_4-inform_intent-searchroundtripflights-1": "user act flights_4 inform_intent searchroundtripflights 1", "user-flights_4-request-inbound_arrival_time-?": "user act flights_4 request inbound_arrival_time ?", "user-flights_4-request-number_of_tickets-?": "user act flights_4 request number_of_tickets ?", "user-flights_4-request-outbound_arrival_time-?": "user act flights_4 request outbound_arrival_time ?", "user-flights_4-request-seating_class-?": "user act flights_4 request seating_class ?", "user-flights_4-request_alts-none-none": "user act flights_4 request_alts none none", "user-flights_4-select-none-none": "user act flights_4 select none none", "user-homes_1-affirm_intent-none-none": "user act homes_1 affirm_intent none none", "user-homes_1-inform-area-1": "user act homes_1 inform area 1", "user-homes_1-inform-furnished-1": "user act homes_1 inform furnished 1", "user-homes_1-inform-number_of_baths-1": "user act homes_1 inform number_of_baths 1", "user-homes_1-inform-number_of_beds-1": "user act homes_1 inform number_of_beds 1", "user-homes_1-inform-pets_allowed-1": "user act homes_1 inform pets_allowed 1", "user-homes_1-inform-visit_date-1": "user act homes_1 inform visit_date 1", "user-homes_1-inform_intent-findapartment-1": "user act homes_1 inform_intent findapartment 1", "user-homes_1-inform_intent-schedulevisit-1": "user act homes_1 inform_intent schedulevisit 1", "user-homes_1-negate_intent-none-none": "user act homes_1 negate_intent none none", "user-homes_1-request-furnished-?": "user act homes_1 request furnished ?", "user-homes_1-request-pets_allowed-?": "user act homes_1 request pets_allowed ?", "user-homes_1-request-phone_number-?": "user act homes_1 request phone_number ?", "user-homes_1-request_alts-none-none": "user act homes_1 request_alts none none", "user-homes_1-select-none-none": "user act homes_1 select none none", "user-homes_2-affirm_intent-none-none": "user act homes_2 affirm_intent none none", "user-homes_2-inform-area-1": "user act homes_2 inform area 1", "user-homes_2-inform-intent-1": "user act homes_2 inform intent 1", "user-homes_2-inform-number_of_baths-1": "user act homes_2 inform number_of_baths 1", "user-homes_2-inform-number_of_beds-1": "user act homes_2 inform number_of_beds 1", "user-homes_2-inform-property_name-1": "user act homes_2 inform property_name 1", "user-homes_2-inform-visit_date-1": "user act homes_2 inform visit_date 1", "user-homes_2-inform_intent-findhomebyarea-1": "user act homes_2 inform_intent findhomebyarea 1", "user-homes_2-inform_intent-schedulevisit-1": "user act homes_2 inform_intent schedulevisit 1", "user-homes_2-request-has_garage-?": "user act homes_2 request has_garage ?", "user-homes_2-request-in_unit_laundry-?": "user act homes_2 request in_unit_laundry ?", "user-homes_2-request-phone_number-?": "user act homes_2 request phone_number ?", "user-homes_2-request_alts-none-none": "user act homes_2 request_alts none none", "user-homes_2-select-none-none": "user act homes_2 select none none", "user-hotels_1-affirm_intent-none-none": "user act hotels_1 affirm_intent none none", "user-hotels_1-inform-check_in_date-1": "user act hotels_1 inform check_in_date 1", "user-hotels_1-inform-destination-1": "user act hotels_1 inform destination 1", "user-hotels_1-inform-has_wifi-1": "user act hotels_1 inform has_wifi 1", "user-hotels_1-inform-hotel_name-1": "user act hotels_1 inform hotel_name 1", "user-hotels_1-inform-number_of_days-1": "user act hotels_1 inform number_of_days 1", "user-hotels_1-inform-number_of_rooms-1": "user act hotels_1 inform number_of_rooms 1", "user-hotels_1-inform-star_rating-1": "user act hotels_1 inform star_rating 1", "user-hotels_1-inform_intent-reservehotel-1": "user act hotels_1 inform_intent reservehotel 1", "user-hotels_1-inform_intent-searchhotel-1": "user act hotels_1 inform_intent searchhotel 1", "user-hotels_1-negate_intent-none-none": "user act hotels_1 negate_intent none none", "user-hotels_1-request-has_wifi-?": "user act hotels_1 request has_wifi ?", "user-hotels_1-request-phone_number-?": "user act hotels_1 request phone_number ?", "user-hotels_1-request-price_per_night-?": "user act hotels_1 request price_per_night ?", "user-hotels_1-request-street_address-?": "user act hotels_1 request street_address ?", "user-hotels_1-request_alts-none-none": "user act hotels_1 request_alts none none", "user-hotels_1-select-none-none": "user act hotels_1 select none none", "user-hotels_2-affirm_intent-none-none": "user act hotels_2 affirm_intent none none", "user-hotels_2-inform-check_in_date-1": "user act hotels_2 inform check_in_date 1", "user-hotels_2-inform-check_out_date-1": "user act hotels_2 inform check_out_date 1", "user-hotels_2-inform-has_laundry_service-1": "user act hotels_2 inform has_laundry_service 1", "user-hotels_2-inform-number_of_adults-1": "user act hotels_2 inform number_of_adults 1", "user-hotels_2-inform-rating-1": "user act hotels_2 inform rating 1", "user-hotels_2-inform-where_to-1": "user act hotels_2 inform where_to 1", "user-hotels_2-inform_intent-bookhouse-1": "user act hotels_2 inform_intent bookhouse 1", "user-hotels_2-inform_intent-searchhouse-1": "user act hotels_2 inform_intent searchhouse 1", "user-hotels_2-negate_intent-none-none": "user act hotels_2 negate_intent none none", "user-hotels_2-request-has_laundry_service-?": "user act hotels_2 request has_laundry_service ?", "user-hotels_2-request-phone_number-?": "user act hotels_2 request phone_number ?", "user-hotels_2-request-total_price-?": "user act hotels_2 request total_price ?", "user-hotels_2-request_alts-none-none": "user act hotels_2 request_alts none none", "user-hotels_2-select-none-none": "user act hotels_2 select none none", "user-hotels_3-affirm_intent-none-none": "user act hotels_3 affirm_intent none none", "user-hotels_3-inform-check_in_date-1": "user act hotels_3 inform check_in_date 1", "user-hotels_3-inform-check_out_date-1": "user act hotels_3 inform check_out_date 1", "user-hotels_3-inform-hotel_name-1": "user act hotels_3 inform hotel_name 1", "user-hotels_3-inform-location-1": "user act hotels_3 inform location 1", "user-hotels_3-inform-number_of_rooms-1": "user act hotels_3 inform number_of_rooms 1", "user-hotels_3-inform-pets_welcome-1": "user act hotels_3 inform pets_welcome 1", "user-hotels_3-inform_intent-reservehotel-1": "user act hotels_3 inform_intent reservehotel 1", "user-hotels_3-inform_intent-searchhotel-1": "user act hotels_3 inform_intent searchhotel 1", "user-hotels_3-negate_intent-none-none": "user act hotels_3 negate_intent none none", "user-hotels_3-request-pets_welcome-?": "user act hotels_3 request pets_welcome ?", "user-hotels_3-request-phone_number-?": "user act hotels_3 request phone_number ?", "user-hotels_3-request-price-?": "user act hotels_3 request price ?", "user-hotels_3-request-street_address-?": "user act hotels_3 request street_address ?", "user-hotels_3-request_alts-none-none": "user act hotels_3 request_alts none none", "user-hotels_3-select-none-none": "user act hotels_3 select none none", "user-hotels_4-affirm_intent-none-none": "user act hotels_4 affirm_intent none none", "user-hotels_4-inform-check_in_date-1": "user act hotels_4 inform check_in_date 1", "user-hotels_4-inform-location-1": "user act hotels_4 inform location 1", "user-hotels_4-inform-number_of_rooms-1": "user act hotels_4 inform number_of_rooms 1", "user-hotels_4-inform-smoking_allowed-1": "user act hotels_4 inform smoking_allowed 1", "user-hotels_4-inform-star_rating-1": "user act hotels_4 inform star_rating 1", "user-hotels_4-inform-stay_length-1": "user act hotels_4 inform stay_length 1", "user-hotels_4-inform_intent-reservehotel-1": "user act hotels_4 inform_intent reservehotel 1", "user-hotels_4-inform_intent-searchhotel-1": "user act hotels_4 inform_intent searchhotel 1", "user-hotels_4-negate_intent-none-none": "user act hotels_4 negate_intent none none", "user-hotels_4-request-phone_number-?": "user act hotels_4 request phone_number ?", "user-hotels_4-request-price_per_night-?": "user act hotels_4 request price_per_night ?", "user-hotels_4-request-smoking_allowed-?": "user act hotels_4 request smoking_allowed ?", "user-hotels_4-request-street_address-?": "user act hotels_4 request street_address ?", "user-hotels_4-request_alts-none-none": "user act hotels_4 request_alts none none", "user-hotels_4-select-none-none": "user act hotels_4 select none none", "user-media_1-affirm_intent-none-none": "user act media_1 affirm_intent none none", "user-media_1-inform-directed_by-1": "user act media_1 inform directed_by 1", "user-media_1-inform-genre-1": "user act media_1 inform genre 1", "user-media_1-inform-subtitles-1": "user act media_1 inform subtitles 1", "user-media_1-inform-title-1": "user act media_1 inform title 1", "user-media_1-inform_intent-findmovies-1": "user act media_1 inform_intent findmovies 1", "user-media_1-inform_intent-playmovie-1": "user act media_1 inform_intent playmovie 1", "user-media_1-negate_intent-none-none": "user act media_1 negate_intent none none", "user-media_1-request-directed_by-?": "user act media_1 request directed_by ?", "user-media_1-request-genre-?": "user act media_1 request genre ?", "user-media_1-request_alts-none-none": "user act media_1 request_alts none none", "user-media_1-select-title-1": "user act media_1 select title 1", "user-media_2-affirm_intent-none-none": "user act media_2 affirm_intent none none", "user-media_2-inform-actors-1": "user act media_2 inform actors 1", "user-media_2-inform-director-1": "user act media_2 inform director 1", "user-media_2-inform-genre-1": "user act media_2 inform genre 1", "user-media_2-inform-subtitle_language-1": "user act media_2 inform subtitle_language 1", "user-media_2-inform_intent-findmovies-1": "user act media_2 inform_intent findmovies 1", "user-media_2-inform_intent-rentmovie-1": "user act media_2 inform_intent rentmovie 1", "user-media_2-request-price-?": "user act media_2 request price ?", "user-media_2-select-movie_name-1": "user act media_2 select movie_name 1", "user-media_3-affirm_intent-none-none": "user act media_3 affirm_intent none none", "user-media_3-inform-genre-1": "user act media_3 inform genre 1", "user-media_3-inform-starring-1": "user act media_3 inform starring 1", "user-media_3-inform-subtitle_language-1": "user act media_3 inform subtitle_language 1", "user-media_3-inform-title-1": "user act media_3 inform title 1", "user-media_3-inform_intent-findmovies-1": "user act media_3 inform_intent findmovies 1", "user-media_3-inform_intent-playmovie-1": "user act media_3 inform_intent playmovie 1", "user-media_3-negate_intent-none-none": "user act media_3 negate_intent none none", "user-media_3-request-starring-?": "user act media_3 request starring ?", "user-media_3-request_alts-none-none": "user act media_3 request_alts none none", "user-media_3-select-title-1": "user act media_3 select title 1", "user-messaging_1-inform-contact_name-1": "user act messaging_1 inform contact_name 1", "user-messaging_1-inform-location-1": "user act messaging_1 inform location 1", "user-messaging_1-inform_intent-sharelocation-1": "user act messaging_1 inform_intent sharelocation 1", "user-movies_1-inform-genre-1": "user act movies_1 inform genre 1", "user-movies_1-inform-location-1": "user act movies_1 inform location 1", "user-movies_1-inform-movie_name-1": "user act movies_1 inform movie_name 1", "user-movies_1-inform-number_of_tickets-1": "user act movies_1 inform number_of_tickets 1", "user-movies_1-inform-show_date-1": "user act movies_1 inform show_date 1", "user-movies_1-inform-show_time-1": "user act movies_1 inform show_time 1", "user-movies_1-inform-show_type-1": "user act movies_1 inform show_type 1", "user-movies_1-inform-theater_name-1": "user act movies_1 inform theater_name 1", "user-movies_1-inform_intent-buymovietickets-1": "user act movies_1 inform_intent buymovietickets 1", "user-movies_1-inform_intent-findmovies-1": "user act movies_1 inform_intent findmovies 1", "user-movies_1-inform_intent-gettimesformovie-1": "user act movies_1 inform_intent gettimesformovie 1", "user-movies_1-negate_intent-none-none": "user act movies_1 negate_intent none none", "user-movies_1-request-genre-?": "user act movies_1 request genre ?", "user-movies_1-request-price-?": "user act movies_1 request price ?", "user-movies_1-request-street_address-?": "user act movies_1 request street_address ?", "user-movies_1-request_alts-none-none": "user act movies_1 request_alts none none", "user-movies_1-select-movie_name-1": "user act movies_1 select movie_name 1", "user-movies_1-select-none-none": "user act movies_1 select none none", "user-movies_2-inform-director-1": "user act movies_2 inform director 1", "user-movies_2-inform-genre-1": "user act movies_2 inform genre 1", "user-movies_2-inform-starring-1": "user act movies_2 inform starring 1", "user-movies_2-inform_intent-findmovies-1": "user act movies_2 inform_intent findmovies 1", "user-movies_2-request_alts-none-none": "user act movies_2 request_alts none none", "user-movies_2-select-none-none": "user act movies_2 select none none", "user-movies_3-inform-cast-1": "user act movies_3 inform cast 1", "user-movies_3-inform-directed_by-1": "user act movies_3 inform directed_by 1", "user-movies_3-inform-genre-1": "user act movies_3 inform genre 1", "user-movies_3-inform_intent-findmovies-1": "user act movies_3 inform_intent findmovies 1", "user-movies_3-request-cast-?": "user act movies_3 request cast ?", "user-movies_3-request-directed_by-?": "user act movies_3 request directed_by ?", "user-movies_3-request-genre-?": "user act movies_3 request genre ?", "user-movies_3-select-none-none": "user act movies_3 select none none", "user-music_1-affirm_intent-none-none": "user act music_1 affirm_intent none none", "user-music_1-inform-album-1": "user act music_1 inform album 1", "user-music_1-inform-artist-1": "user act music_1 inform artist 1", "user-music_1-inform-genre-1": "user act music_1 inform genre 1", "user-music_1-inform-playback_device-1": "user act music_1 inform playback_device 1", "user-music_1-inform-song_name-1": "user act music_1 inform song_name 1", "user-music_1-inform-year-1": "user act music_1 inform year 1", "user-music_1-inform_intent-lookupsong-1": "user act music_1 inform_intent lookupsong 1", "user-music_1-inform_intent-playsong-1": "user act music_1 inform_intent playsong 1", "user-music_1-request-album-?": "user act music_1 request album ?", "user-music_1-request-genre-?": "user act music_1 request genre ?", "user-music_1-request-year-?": "user act music_1 request year ?", "user-music_1-request_alts-none-none": "user act music_1 request_alts none none", "user-music_1-select-none-none": "user act music_1 select none none", "user-music_2-affirm_intent-none-none": "user act music_2 affirm_intent none none", "user-music_2-inform-album-1": "user act music_2 inform album 1", "user-music_2-inform-artist-1": "user act music_2 inform artist 1", "user-music_2-inform-genre-1": "user act music_2 inform genre 1", "user-music_2-inform-playback_device-1": "user act music_2 inform playback_device 1", "user-music_2-inform-song_name-1": "user act music_2 inform song_name 1", "user-music_2-inform_intent-lookupmusic-1": "user act music_2 inform_intent lookupmusic 1", "user-music_2-inform_intent-playmedia-1": "user act music_2 inform_intent playmedia 1", "user-music_2-request-genre-?": "user act music_2 request genre ?", "user-music_2-request_alts-none-none": "user act music_2 request_alts none none", "user-music_2-select-none-none": "user act music_2 select none none", "user-music_3-affirm_intent-none-none": "user act music_3 affirm_intent none none", "user-music_3-inform-album-1": "user act music_3 inform album 1", "user-music_3-inform-artist-1": "user act music_3 inform artist 1", "user-music_3-inform-device-1": "user act music_3 inform device 1", "user-music_3-inform-genre-1": "user act music_3 inform genre 1", "user-music_3-inform-year-1": "user act music_3 inform year 1", "user-music_3-inform_intent-lookupmusic-1": "user act music_3 inform_intent lookupmusic 1", "user-music_3-inform_intent-playmedia-1": "user act music_3 inform_intent playmedia 1", "user-music_3-negate_intent-none-none": "user act music_3 negate_intent none none", "user-music_3-request-genre-?": "user act music_3 request genre ?", "user-music_3-request-year-?": "user act music_3 request year ?", "user-music_3-request_alts-none-none": "user act music_3 request_alts none none", "user-music_3-select-none-none": "user act music_3 select none none", "user-payment_1-inform-amount-1": "user act payment_1 inform amount 1", "user-payment_1-inform-payment_method-1": "user act payment_1 inform payment_method 1", "user-payment_1-inform-private_visibility-1": "user act payment_1 inform private_visibility 1", "user-payment_1-inform-receiver-1": "user act payment_1 inform receiver 1", "user-payment_1-inform_intent-makepayment-1": "user act payment_1 inform_intent makepayment 1", "user-payment_1-inform_intent-requestpayment-1": "user act payment_1 inform_intent requestpayment 1", "user-rentalcars_1-affirm_intent-none-none": "user act rentalcars_1 affirm_intent none none", "user-rentalcars_1-inform-dropoff_date-1": "user act rentalcars_1 inform dropoff_date 1", "user-rentalcars_1-inform-pickup_city-1": "user act rentalcars_1 inform pickup_city 1", "user-rentalcars_1-inform-pickup_date-1": "user act rentalcars_1 inform pickup_date 1", "user-rentalcars_1-inform-pickup_location-1": "user act rentalcars_1 inform pickup_location 1", "user-rentalcars_1-inform-pickup_time-1": "user act rentalcars_1 inform pickup_time 1", "user-rentalcars_1-inform-type-1": "user act rentalcars_1 inform type 1", "user-rentalcars_1-inform_intent-getcarsavailable-1": "user act rentalcars_1 inform_intent getcarsavailable 1", "user-rentalcars_1-inform_intent-reservecar-1": "user act rentalcars_1 inform_intent reservecar 1", "user-rentalcars_1-negate_intent-none-none": "user act rentalcars_1 negate_intent none none", "user-rentalcars_1-request-car_name-?": "user act rentalcars_1 request car_name ?", "user-rentalcars_1-request-total_price-?": "user act rentalcars_1 request total_price ?", "user-rentalcars_1-request_alts-none-none": "user act rentalcars_1 request_alts none none", "user-rentalcars_1-select-none-none": "user act rentalcars_1 select none none", "user-rentalcars_2-affirm_intent-none-none": "user act rentalcars_2 affirm_intent none none", "user-rentalcars_2-inform-car_type-1": "user act rentalcars_2 inform car_type 1", "user-rentalcars_2-inform-dropoff_date-1": "user act rentalcars_2 inform dropoff_date 1", "user-rentalcars_2-inform-pickup_city-1": "user act rentalcars_2 inform pickup_city 1", "user-rentalcars_2-inform-pickup_date-1": "user act rentalcars_2 inform pickup_date 1", "user-rentalcars_2-inform-pickup_location-1": "user act rentalcars_2 inform pickup_location 1", "user-rentalcars_2-inform-pickup_time-1": "user act rentalcars_2 inform pickup_time 1", "user-rentalcars_2-inform_intent-getcarsavailable-1": "user act rentalcars_2 inform_intent getcarsavailable 1", "user-rentalcars_2-inform_intent-reservecar-1": "user act rentalcars_2 inform_intent reservecar 1", "user-rentalcars_2-negate_intent-none-none": "user act rentalcars_2 negate_intent none none", "user-rentalcars_2-request-car_name-?": "user act rentalcars_2 request car_name ?", "user-rentalcars_2-request-total_price-?": "user act rentalcars_2 request total_price ?", "user-rentalcars_2-request_alts-none-none": "user act rentalcars_2 request_alts none none", "user-rentalcars_2-select-none-none": "user act rentalcars_2 select none none", "user-rentalcars_3-affirm_intent-none-none": "user act rentalcars_3 affirm_intent none none", "user-rentalcars_3-inform-add_insurance-1": "user act rentalcars_3 inform add_insurance 1", "user-rentalcars_3-inform-car_type-1": "user act rentalcars_3 inform car_type 1", "user-rentalcars_3-inform-city-1": "user act rentalcars_3 inform city 1", "user-rentalcars_3-inform-end_date-1": "user act rentalcars_3 inform end_date 1", "user-rentalcars_3-inform-pickup_location-1": "user act rentalcars_3 inform pickup_location 1", "user-rentalcars_3-inform-pickup_time-1": "user act rentalcars_3 inform pickup_time 1", "user-rentalcars_3-inform-start_date-1": "user act rentalcars_3 inform start_date 1", "user-rentalcars_3-inform_intent-getcarsavailable-1": "user act rentalcars_3 inform_intent getcarsavailable 1", "user-rentalcars_3-inform_intent-reservecar-1": "user act rentalcars_3 inform_intent reservecar 1", "user-rentalcars_3-negate_intent-none-none": "user act rentalcars_3 negate_intent none none", "user-rentalcars_3-request-car_name-?": "user act rentalcars_3 request car_name ?", "user-rentalcars_3-request-price_per_day-?": "user act rentalcars_3 request price_per_day ?", "user-rentalcars_3-request_alts-none-none": "user act rentalcars_3 request_alts none none", "user-rentalcars_3-select-none-none": "user act rentalcars_3 select none none", "user-restaurants_1-affirm_intent-none-none": "user act restaurants_1 affirm_intent none none", "user-restaurants_1-inform-city-1": "user act restaurants_1 inform city 1", "user-restaurants_1-inform-cuisine-1": "user act restaurants_1 inform cuisine 1", "user-restaurants_1-inform-date-1": "user act restaurants_1 inform date 1", "user-restaurants_1-inform-has_live_music-1": "user act restaurants_1 inform has_live_music 1", "user-restaurants_1-inform-party_size-1": "user act restaurants_1 inform party_size 1", "user-restaurants_1-inform-price_range-1": "user act restaurants_1 inform price_range 1", "user-restaurants_1-inform-restaurant_name-1": "user act restaurants_1 inform restaurant_name 1", "user-restaurants_1-inform-serves_alcohol-1": "user act restaurants_1 inform serves_alcohol 1", "user-restaurants_1-inform-time-1": "user act restaurants_1 inform time 1", "user-restaurants_1-inform_intent-findrestaurants-1": "user act restaurants_1 inform_intent findrestaurants 1", "user-restaurants_1-inform_intent-reserverestaurant-1": "user act restaurants_1 inform_intent reserverestaurant 1", "user-restaurants_1-negate_intent-none-none": "user act restaurants_1 negate_intent none none", "user-restaurants_1-request-cuisine-?": "user act restaurants_1 request cuisine ?", "user-restaurants_1-request-has_live_music-?": "user act restaurants_1 request has_live_music ?", "user-restaurants_1-request-phone_number-?": "user act restaurants_1 request phone_number ?", "user-restaurants_1-request-price_range-?": "user act restaurants_1 request price_range ?", "user-restaurants_1-request-serves_alcohol-?": "user act restaurants_1 request serves_alcohol ?", "user-restaurants_1-request-street_address-?": "user act restaurants_1 request street_address ?", "user-restaurants_1-request_alts-none-none": "user act restaurants_1 request_alts none none", "user-restaurants_1-select-none-none": "user act restaurants_1 select none none", "user-restaurants_2-affirm_intent-none-none": "user act restaurants_2 affirm_intent none none", "user-restaurants_2-inform-category-1": "user act restaurants_2 inform category 1", "user-restaurants_2-inform-date-1": "user act restaurants_2 inform date 1", "user-restaurants_2-inform-has_vegetarian_options-1": "user act restaurants_2 inform has_vegetarian_options 1", "user-restaurants_2-inform-location-1": "user act restaurants_2 inform location 1", "user-restaurants_2-inform-number_of_seats-1": "user act restaurants_2 inform number_of_seats 1", "user-restaurants_2-inform-price_range-1": "user act restaurants_2 inform price_range 1", "user-restaurants_2-inform-restaurant_name-1": "user act restaurants_2 inform restaurant_name 1", "user-restaurants_2-inform-time-1": "user act restaurants_2 inform time 1", "user-restaurants_2-inform_intent-findrestaurants-1": "user act restaurants_2 inform_intent findrestaurants 1", "user-restaurants_2-inform_intent-reserverestaurant-1": "user act restaurants_2 inform_intent reserverestaurant 1", "user-restaurants_2-request-address-?": "user act restaurants_2 request address ?", "user-restaurants_2-request-has_seating_outdoors-?": "user act restaurants_2 request has_seating_outdoors ?", "user-restaurants_2-request-has_vegetarian_options-?": "user act restaurants_2 request has_vegetarian_options ?", "user-restaurants_2-request-phone_number-?": "user act restaurants_2 request phone_number ?", "user-restaurants_2-request-price_range-?": "user act restaurants_2 request price_range ?", "user-restaurants_2-request-rating-?": "user act restaurants_2 request rating ?", "user-restaurants_2-request_alts-none-none": "user act restaurants_2 request_alts none none", "user-restaurants_2-select-none-none": "user act restaurants_2 select none none", "user-ridesharing_1-inform-destination-1": "user act ridesharing_1 inform destination 1", "user-ridesharing_1-inform-number_of_riders-1": "user act ridesharing_1 inform number_of_riders 1", "user-ridesharing_1-inform-shared_ride-1": "user act ridesharing_1 inform shared_ride 1", "user-ridesharing_1-inform_intent-getride-1": "user act ridesharing_1 inform_intent getride 1", "user-ridesharing_1-request-approximate_ride_duration-?": "user act ridesharing_1 request approximate_ride_duration ?", "user-ridesharing_1-request-ride_fare-?": "user act ridesharing_1 request ride_fare ?", "user-ridesharing_2-inform-destination-1": "user act ridesharing_2 inform destination 1", "user-ridesharing_2-inform-number_of_seats-1": "user act ridesharing_2 inform number_of_seats 1", "user-ridesharing_2-inform-ride_type-1": "user act ridesharing_2 inform ride_type 1", "user-ridesharing_2-inform_intent-getride-1": "user act ridesharing_2 inform_intent getride 1", "user-ridesharing_2-request-ride_fare-?": "user act ridesharing_2 request ride_fare ?", "user-ridesharing_2-request-wait_time-?": "user act ridesharing_2 request wait_time ?", "user-services_1-affirm_intent-none-none": "user act services_1 affirm_intent none none", "user-services_1-inform-appointment_date-1": "user act services_1 inform appointment_date 1", "user-services_1-inform-appointment_time-1": "user act services_1 inform appointment_time 1", "user-services_1-inform-city-1": "user act services_1 inform city 1", "user-services_1-inform-is_unisex-1": "user act services_1 inform is_unisex 1", "user-services_1-inform-stylist_name-1": "user act services_1 inform stylist_name 1", "user-services_1-inform_intent-bookappointment-1": "user act services_1 inform_intent bookappointment 1", "user-services_1-inform_intent-findprovider-1": "user act services_1 inform_intent findprovider 1", "user-services_1-negate_intent-none-none": "user act services_1 negate_intent none none", "user-services_1-request-average_rating-?": "user act services_1 request average_rating ?", "user-services_1-request-is_unisex-?": "user act services_1 request is_unisex ?", "user-services_1-request-phone_number-?": "user act services_1 request phone_number ?", "user-services_1-request-street_address-?": "user act services_1 request street_address ?", "user-services_1-request_alts-none-none": "user act services_1 request_alts none none", "user-services_1-select-none-none": "user act services_1 select none none", "user-services_2-affirm_intent-none-none": "user act services_2 affirm_intent none none", "user-services_2-inform-appointment_date-1": "user act services_2 inform appointment_date 1", "user-services_2-inform-appointment_time-1": "user act services_2 inform appointment_time 1", "user-services_2-inform-city-1": "user act services_2 inform city 1", "user-services_2-inform-dentist_name-1": "user act services_2 inform dentist_name 1", "user-services_2-inform_intent-bookappointment-1": "user act services_2 inform_intent bookappointment 1", "user-services_2-inform_intent-findprovider-1": "user act services_2 inform_intent findprovider 1", "user-services_2-negate_intent-none-none": "user act services_2 negate_intent none none", "user-services_2-request-address-?": "user act services_2 request address ?", "user-services_2-request-offers_cosmetic_services-?": "user act services_2 request offers_cosmetic_services ?", "user-services_2-request-phone_number-?": "user act services_2 request phone_number ?", "user-services_2-request_alts-none-none": "user act services_2 request_alts none none", "user-services_2-select-none-none": "user act services_2 select none none", "user-services_3-affirm_intent-none-none": "user act services_3 affirm_intent none none", "user-services_3-inform-appointment_date-1": "user act services_3 inform appointment_date 1", "user-services_3-inform-appointment_time-1": "user act services_3 inform appointment_time 1", "user-services_3-inform-city-1": "user act services_3 inform city 1", "user-services_3-inform-doctor_name-1": "user act services_3 inform doctor_name 1", "user-services_3-inform-type-1": "user act services_3 inform type 1", "user-services_3-inform_intent-bookappointment-1": "user act services_3 inform_intent bookappointment 1", "user-services_3-inform_intent-findprovider-1": "user act services_3 inform_intent findprovider 1", "user-services_3-negate_intent-none-none": "user act services_3 negate_intent none none", "user-services_3-request-average_rating-?": "user act services_3 request average_rating ?", "user-services_3-request-phone_number-?": "user act services_3 request phone_number ?", "user-services_3-request-street_address-?": "user act services_3 request street_address ?", "user-services_3-request_alts-none-none": "user act services_3 request_alts none none", "user-services_3-select-none-none": "user act services_3 select none none", "user-services_4-affirm_intent-none-none": "user act services_4 affirm_intent none none", "user-services_4-inform-appointment_date-1": "user act services_4 inform appointment_date 1", "user-services_4-inform-appointment_time-1": "user act services_4 inform appointment_time 1", "user-services_4-inform-city-1": "user act services_4 inform city 1", "user-services_4-inform-type-1": "user act services_4 inform type 1", "user-services_4-inform_intent-bookappointment-1": "user act services_4 inform_intent bookappointment 1", "user-services_4-inform_intent-findprovider-1": "user act services_4 inform_intent findprovider 1", "user-services_4-negate_intent-none-none": "user act services_4 negate_intent none none", "user-services_4-request-address-?": "user act services_4 request address ?", "user-services_4-request-phone_number-?": "user act services_4 request phone_number ?", "user-services_4-request_alts-none-none": "user act services_4 request_alts none none", "user-services_4-select-none-none": "user act services_4 select none none", "user-trains_1-affirm_intent-none-none": "user act trains_1 affirm_intent none none", "user-trains_1-inform-class-1": "user act trains_1 inform class 1", "user-trains_1-inform-date_of_journey-1": "user act trains_1 inform date_of_journey 1", "user-trains_1-inform-from-1": "user act trains_1 inform from 1", "user-trains_1-inform-number_of_adults-1": "user act trains_1 inform number_of_adults 1", "user-trains_1-inform-to-1": "user act trains_1 inform to 1", "user-trains_1-inform-trip_protection-1": "user act trains_1 inform trip_protection 1", "user-trains_1-inform_intent-findtrains-1": "user act trains_1 inform_intent findtrains 1", "user-trains_1-inform_intent-gettraintickets-1": "user act trains_1 inform_intent gettraintickets 1", "user-trains_1-negate_intent-none-none": "user act trains_1 negate_intent none none", "user-trains_1-request-from_station-?": "user act trains_1 request from_station ?", "user-trains_1-request-to_station-?": "user act trains_1 request to_station ?", "user-trains_1-request_alts-none-none": "user act trains_1 request_alts none none", "user-trains_1-select-none-none": "user act trains_1 select none none", "user-travel_1-inform-category-1": "user act travel_1 inform category 1", "user-travel_1-inform-free_entry-1": "user act travel_1 inform free_entry 1", "user-travel_1-inform-good_for_kids-1": "user act travel_1 inform good_for_kids 1", "user-travel_1-inform-location-1": "user act travel_1 inform location 1", "user-travel_1-inform_intent-findattractions-1": "user act travel_1 inform_intent findattractions 1", "user-travel_1-request-free_entry-?": "user act travel_1 request free_entry ?", "user-travel_1-request-good_for_kids-?": "user act travel_1 request good_for_kids ?", "user-travel_1-request-phone_number-?": "user act travel_1 request phone_number ?", "user-travel_1-request_alts-none-none": "user act travel_1 request_alts none none", "user-travel_1-select-none-none": "user act travel_1 select none none", "user-weather_1-inform-city-1": "user act weather_1 inform city 1", "user-weather_1-inform-date-1": "user act weather_1 inform date 1", "user-weather_1-inform_intent-getweather-1": "user act weather_1 inform_intent getweather 1", "user-weather_1-request-humidity-?": "user act weather_1 request humidity ?", "user-weather_1-request-wind-?": "user act weather_1 request wind ?", "user-weather_1-request_alts-none-none": "user act weather_1 request_alts none none", "user-weather_1-select-none-none": "user act weather_1 select none none"}
\ No newline at end of file
diff --git a/convlab/policy/vtrace_DPT/memory.py b/convlab/policy/vtrace_DPT/memory.py
new file mode 100644
index 0000000000000000000000000000000000000000..e9e13eb69bf68fa309ad552d62ff752c90ff230f
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/memory.py
@@ -0,0 +1,192 @@
+# Modified by Microsoft Corporation.
+# Licensed under the MIT license.
+
+import numpy as np
+import os
+import json, random
+import torch
+import pickle
+
+import logging
+from queue import PriorityQueue
+
+from convlab.util.custom_util import set_seed
+
+
+class Memory:
+
+    def __init__(self, seed=0):
+
+        with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.json'), 'r') as f:
+            cfg = json.load(f)
+
+        self.batch_size = cfg.get('batchsz', 32)
+        self.max_size = cfg.get('memory_size', 2000)
+        self.reservoir_sampling = cfg.get("use_reservoir_sampling", False)
+        logging.info(f"We use reservoir sampling: {self.reservoir_sampling}")
+        self.second_r = False
+        self.reward_weight = 1.0
+        self.priority_queue = PriorityQueue()
+
+        self.size = 0  # total experiences stored
+        self.number_episodes = 0
+
+        self.data_keys = ['states', 'actions', 'rewards', 'small_actions', 'mu', 'action_masks', 'critic_value',
+                          'description_idx_list', 'value_list', 'current_domain_mask', 'non_current_domain_mask']
+        self.reset()
+        set_seed(seed)
+
+    def set_seed(self, seed):
+        np.random.seed(seed)
+        torch.random.manual_seed(seed)
+        random.seed(seed)
+        torch.manual_seed(seed)
+        if torch.cuda.is_available():
+            torch.cuda.manual_seed_all(seed)
+
+    def reset(self):
+        for k in self.data_keys:
+            setattr(self, k, [[]])
+
+    def update_episode(self, state_list, action_list, reward_list, small_act_list, mu_list, action_mask_list,
+                       critic_value_list, description_idx_list, value_list, current_domain_mask, non_current_domain_mask):
+
+        if len(self.states) > self.max_size:
+            # delete the oldest episode when max-size is reached
+            #for k in self.data_keys:
+            #    getattr(self, k).pop(0)
+            if not self.reservoir_sampling:
+                # We sample a random experience for deletion
+                remove_index = random.choice(range(len(self.states) - 2))
+            else:
+                item = self.priority_queue.get()
+                remove_index = item[1]
+
+            for k in self.data_keys:
+                getattr(self, k).pop(remove_index)
+
+        self.states[-1] = state_list
+        self.actions[-1] = action_list
+        self.rewards[-1] = [r/40.0 for r in reward_list]
+        self.small_actions[-1] = small_act_list
+        self.mu[-1] = mu_list
+        self.action_masks[-1] = action_mask_list
+        self.critic_value[-1] = critic_value_list
+        self.description_idx_list[-1] = description_idx_list
+        self.value_list[-1] = value_list
+        self.current_domain_mask[-1] = current_domain_mask
+        self.non_current_domain_mask[-1] = non_current_domain_mask
+
+        self.states.append([])
+        self.actions.append([])
+        self.rewards.append([])
+        self.small_actions.append([])
+        self.mu.append([])
+        self.action_masks.append([])
+        self.critic_value.append([])
+        self.description_idx_list.append([])
+        self.value_list.append([])
+        self.current_domain_mask.append([])
+        self.non_current_domain_mask.append([])
+
+        self.number_episodes += 1
+
+        if self.reservoir_sampling:
+            self.priority_queue.put((torch.randn(1), len(self.states) - 2))
+
+    def update(self, state, action, reward, next_state, done):
+
+        self.add_experience(state, action, reward, next_state, done)
+
+    def add_experience(self, state, action, reward, next_state, done, mu=None):
+
+        reward = reward / 40.0
+        if isinstance(action, dict):
+            mu = action.get('mu')
+            action_index = action.get('action_index')
+            mask = action.get('mask')
+        else:
+            action_index = action
+
+        if done:
+            self.states[-1].append(state)
+            self.actions[-1].append(action_index)
+            self.rewards[-1].append(reward)
+            self.next_states[-1].append(next_state)
+            #self.dones[-1].append(done)
+            self.mu[-1].append(mu)
+            self.masks[-1].append(mask)
+
+            self.states.append([])
+            self.actions.append([])
+            self.rewards.append([])
+            self.next_states.append([])
+            #self.dones.append([])
+            self.mu.append([])
+            self.masks.append([])
+
+            if len(self.states) > self.max_size:
+                #self.number_episodes = self.max_size
+                #delete the oldest episode when max-size is reached
+                for k in self.data_keys:
+                    getattr(self, k).pop(0)
+            else:
+                self.number_episodes += 1
+
+        else:
+            self.states[-1].append(state)
+            self.actions[-1].append(action_index)
+            self.rewards[-1].append(reward)
+            self.next_states[-1].append(next_state)
+            #self.dones[-1].append(done)
+            self.mu[-1].append(mu)
+            self.masks[-1].append(mask)
+
+        # Actually occupied size of memory
+        if self.size < self.max_size:
+            self.size += 1
+
+    def sample(self, online_offline_ratio=0.0):
+        '''
+        Returns a batch of batch_size samples. Batch is stored as a dict.
+        Keys are the names of the different elements of an experience. Values are an array of the corresponding sampled elements
+        e.g.
+        batch = {
+            'states'     : states,
+            'actions'    : actions,
+            'rewards'    : rewards,
+            'next_states': next_states,
+            'dones'      : dones}
+        '''
+        number_episodes = len(self.states) - 1
+        num_online = 0
+
+        #Sample batch-size many episodes
+        if number_episodes <= self.batch_size:
+            batch_ids = list(range(number_episodes))
+        elif online_offline_ratio != 0:
+            num_online = int(online_offline_ratio * self.batch_size)
+            batch_ids_online = list(range(number_episodes - num_online, number_episodes - 1))
+            batch_ids_offline = np.random.randint(number_episodes - 1 - num_online, size=self.batch_size - num_online).tolist()
+            batch_ids = batch_ids_online + batch_ids_offline
+        else:
+            batch_ids = np.random.randint(number_episodes - 1, size=self.batch_size).tolist()
+
+        batch = {}
+        for k in self.data_keys:
+            batch[k] = [getattr(self, k)[index] for index in batch_ids]
+
+        return batch, num_online
+
+    def save(self, path):
+
+        # PriorityQueue is not serializable, so only save the list behind it
+        self.priority_queue = self.priority_queue.queue
+        with open(path + f'/vtrace.memory', "wb") as f:
+            pickle.dump(self, f)
+
+    def build_priority_queue(self, queue_list):
+
+        self.priority_queue = PriorityQueue()
+        for element in queue_list:
+            self.priority_queue.put(element)
diff --git a/convlab/policy/vtrace_DPT/multiprocessing_helper.py b/convlab/policy/vtrace_DPT/multiprocessing_helper.py
new file mode 100644
index 0000000000000000000000000000000000000000..945cf4e066265543a0fa19aa8cf2efc84000b93b
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/multiprocessing_helper.py
@@ -0,0 +1,144 @@
+from torch import multiprocessing as mp
+from copy import deepcopy
+
+import logging
+import torch
+import time
+from convlab.util.custom_util import set_seed
+
+torch.multiprocessing.set_sharing_strategy('file_system')
+
+try:
+    mp.set_start_method('spawn', force=True)
+    mp = mp.get_context('spawn')
+except RuntimeError:
+    pass
+
+
+# we use a queue for every process to guarantee reproducibility
+# queues are used for job submission, while episode queues are used for pushing dialogues inside
+def get_queues(train_processes):
+    queues = []
+    episode_queues = []
+    for p in range(train_processes):
+        queues.append(mp.SimpleQueue())
+        episode_queues.append(mp.SimpleQueue())
+
+    return queues, episode_queues
+
+
+# this is our target function for the processes
+def create_episodes_process(do_queue, put_queue, environment, policy, seed, metric_queue):
+    traj_len = 40
+    set_seed(seed)
+
+    while True:
+        if not do_queue.empty():
+            item = do_queue.get()
+            if item == 'stop':
+                print("Got stop signal.")
+                break
+            else:
+                s = environment.reset(item)
+                rl_return = 0
+                user_act_list, sys_act_list, s_vec_list, action_list, reward_list, small_act_list, action_mask_list, mu_list, \
+                trajectory_list, vector_mask_list, critic_value_list, description_idx_list, value_list, current_domain_mask, \
+                non_current_domain_mask = \
+                    [], [], [], [], [], [], [], [], [], [], [], [], [], [], []
+
+                for t in range(traj_len):
+
+                    s_vec, mask = policy.vector.state_vectorize(s)
+                    with torch.no_grad():
+                        a = policy.predict(s)
+
+                    # s_vec_list.append(policy.info_dict['kg'])
+                    action_list.append(policy.info_dict['big_act'].detach())
+                    small_act_list.append(policy.info_dict['small_act'])
+                    action_mask_list.append(policy.info_dict['action_mask'])
+                    mu_list.append(policy.info_dict['a_prob'].detach())
+                    critic_value_list.append(policy.info_dict['critic_value'])
+                    vector_mask_list.append(torch.Tensor(mask))
+                    description_idx_list.append(policy.info_dict["description_idx_list"])
+                    value_list.append(policy.info_dict["value_list"])
+                    current_domain_mask.append(policy.info_dict["current_domain_mask"])
+                    non_current_domain_mask.append(policy.info_dict["non_current_domain_mask"])
+
+                    sys_act_list.append(policy.vector.action_vectorize(a))
+                    trajectory_list.extend([s['user_action'], a])
+
+                    # interact with env
+                    next_s, r, done = environment.step(a)
+                    rl_return += r
+                    reward_list.append(torch.Tensor([r]))
+
+                    next_s_vec, next_mask = policy.vector.state_vectorize(next_s)
+
+                    # update per step
+                    s = next_s
+
+                    if done:
+                        metric_queue.put({"success": environment.evaluator.success_strict, "return": rl_return,
+                                          "avg_actions": torch.stack(action_list).sum(dim=-1).mean().item(),
+                                          "turns": t, "goal": item.domain_goals})
+                        put_queue.put((description_idx_list, action_list, reward_list, small_act_list, mu_list,
+                                       action_mask_list, critic_value_list, description_idx_list, value_list,
+                                       current_domain_mask, non_current_domain_mask))
+                        break
+
+
+def start_processes(train_processes, queues, episode_queues, env, policy_sys, seed, metric_queue):
+    logging.info("Spawning processes..")
+    processes = []
+    for i in range(train_processes):
+        process_args = (queues[i], episode_queues[i], env, policy_sys, seed, metric_queue)
+        p = mp.Process(target=create_episodes_process, args=process_args)
+        processes.append(p)
+    for b, p in enumerate(processes):
+        p.daemon = True
+        p.start()
+        logging.info(f"Started process {b}")
+    return processes
+
+
+def terminate_processes(processes, queues):
+    # kill processes properly
+    logging.info("Terminating processes..")
+    for b, p in enumerate(processes):
+        queues[b].put('stop')
+    time.sleep(2)
+    for b, p in enumerate(processes):
+        p.terminate()
+        logging.info(f"Terminated process {b}")
+
+
+def submit_jobs(num_jobs, queues, episode_queues, train_processes, memory, goals, metric_queue):
+    # first create goals with global environment and put them into queue.
+    # If every environment process would do that itself, it could happen that environment 1 creates 24 dialogues in
+    # one run and 25 in another run (for two processes and 50 jobs for instance)
+    metrics = []
+    for job in range(num_jobs):
+        if goals:
+            goal = goals.pop()
+            queues[job % train_processes].put(goal)
+    time_now = time.time()
+    collected_dialogues = 0
+    episode_list = []
+    for b in range(train_processes):
+        episode_list.append([])
+    # we need to have a dialogue list for every process, otherwise it could happen that the order in which dialogues
+    # are pushed into the list is different for different runs
+    # in the end the dialogue lists are just appended basically instead of being possibly mixed
+    while collected_dialogues != num_jobs:
+        for b in range(train_processes):
+            if not episode_queues[b].empty():
+                metrics.append(metric_queue.get())
+                dialogue = episode_queues[b].get()
+                dialogue_ = deepcopy(dialogue)
+                episode_list[b].append(dialogue_)
+                del dialogue
+                collected_dialogues += 1
+    for b in range(train_processes):
+        for dialogue in episode_list[b]:
+            memory.update_episode(*dialogue)
+    return time_now, metrics
diff --git a/convlab/policy/vtrace_DPT/semantic_level_config.json b/convlab/policy/vtrace_DPT/semantic_level_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..7f6d266751a32642c9842af3bca0d80a33a97beb
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/semantic_level_config.json
@@ -0,0 +1,48 @@
+{"goals": {"single_domains": false, "allowed_domains": null},
+
+	"model": {
+		"load_path": "",
+		"use_pretrained_initialisation": false,
+		"pretrained_load_path": "",
+		"seed": 0,
+        "process_num": 4,
+		"eval_frequency": 1000,
+        "num_eval_dialogues": 500,
+        "process_num_train": 1,
+        "total_dialogues": 10000,
+        "update_rounds": 1,
+		"new_dialogues": 2,
+		"sys_semantic_to_usr": false
+	},
+	"vectorizer_sys": {
+		"uncertainty_vector_mul": {
+			"class_path": "convlab.policy.vector.vector_nodes.VectorNodes",
+			"ini_params": {
+				"use_masking": true,
+				"manually_add_entity_names": true,
+				"seed": 0,
+				"dataset_name": "multiwoz21",
+				"filter_state": true
+			}
+		}
+	},
+	"nlu_sys": {},
+	"dst_sys": {
+		"RuleDST": {
+			"class_path": "convlab.dst.rule.multiwoz.dst.RuleDST",
+			"ini_params": {}
+		}
+	},
+	"sys_nlg": {},
+	"nlu_usr": {},
+	"dst_usr": {},
+	"policy_usr": {
+		"RulePolicy": {
+			"class_path": "convlab.policy.rule.multiwoz.RulePolicy",
+			"ini_params": {
+				"character": "usr"
+			}
+		}
+	},
+	"usr_nlg": {}
+}
\ No newline at end of file
diff --git a/convlab/policy/vtrace_DPT/supervised/loader.py b/convlab/policy/vtrace_DPT/supervised/loader.py
new file mode 100644
index 0000000000000000000000000000000000000000..8dbbcaafb95672711d17715294c6dcd9ded9326b
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/supervised/loader.py
@@ -0,0 +1,169 @@
+import os
+import pickle
+import torch
+import time
+import torch.utils.data as data
+
+from convlab.policy.vector.vector_binary import VectorBinary
+from convlab.util import load_policy_data, load_dataset
+from convlab.util.custom_util import flatten_acts
+from convlab.util.multiwoz.state import default_state
+from convlab.policy.vector.dataset import ActDatasetKG
+from tqdm import tqdm
+
+mwoz_domains = ['restaurant', 'hotel', 'train', 'taxi', 'attraction']
+
+
+class PolicyDataVectorizer:
+
+    def __init__(self, dataset_name='multiwoz21', vector=None, percentage=1.0, dialogue_order=0):
+        self.dataset_name = dataset_name
+        self.percentage = percentage
+        self.dialogue_order = dialogue_order
+        if vector is None:
+            self.vector = VectorBinary(dataset_name)
+        else:
+            self.vector = vector
+        self.process_data()
+
+    def process_data(self):
+
+        processed_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                     f'processed_data/{self.dataset_name}_{type(self.vector).__name__}')
+        if self.percentage != 1.0:
+            processed_dir += f"_{self.percentage}_{self.dialogue_order}"
+        if os.path.exists(processed_dir):
+            print('Load processed data file')
+            self._load_data(processed_dir)
+        else:
+            print('Start preprocessing the dataset, this can take a while..')
+            self._build_data(processed_dir)
+
+    def _build_data(self, processed_dir):
+        self.data = {}
+
+        os.makedirs(processed_dir, exist_ok=True)
+        dataset = load_dataset(self.dataset_name, dial_ids_order=self.dialogue_order,
+                               split2ratio={'train': self.percentage, 'validation': self.percentage,
+                                            'test': self.percentage})
+        data_split = load_policy_data(dataset, context_window_size=2)
+
+        for split in data_split:
+            self.data[split] = []
+            raw_data = data_split[split]
+
+            for data_point in tqdm(raw_data):
+                state = default_state()
+
+                state['belief_state'] = data_point['context'][-1]['state']
+                state['user_action'] = flatten_acts(data_point['context'][-1]['dialogue_acts'])
+                last_system_act = data_point['context'][-2]['dialogue_acts'] \
+                    if len(data_point['context']) > 1 else {}
+                state['system_action'] = flatten_acts(last_system_act)
+                state['terminated'] = data_point['terminated']
+                if 'booked' in data_point:
+                    state['booked'] = data_point['booked']
+                dialogue_act = flatten_acts(data_point['dialogue_acts'])
+
+                vectorized_state, mask = self.vector.state_vectorize(state)
+                vectorized_action = self.vector.action_vectorize(dialogue_act)
+                self.data[split].append({"state": self.vector.kg_info, "action": vectorized_action, "mask": mask,
+                                         "terminated": state['terminated']})
+
+            with open(os.path.join(processed_dir, '{}.pkl'.format(split)), 'wb') as f:
+                pickle.dump(self.data[split], f)
+
+        print("Data processing done.")
+
+    def _load_data(self, processed_dir):
+        self.data = {}
+        for part in ['train', 'validation', 'test']:
+            with open(os.path.join(processed_dir, '{}.pkl'.format(part)), 'rb') as f:
+                self.data[part] = pickle.load(f)
+
+    def is_multiwoz_like(self, item):
+
+        state = item['state']
+        is_like = False
+        for node in state:
+            domain = node['domain'].lower()
+            for mw_domain in mwoz_domains:
+                # we check if the mw_domain as a string is contained in the domain of the node
+                if mw_domain in domain:
+                    is_like = True
+                    break
+            if is_like:
+                break
+        return is_like
+
+    def create_dataset(self, part, batchsz, policy, multiwoz_like=False):
+        print('Start creating {} dataset'.format(part))
+        time_now = time.time()
+
+        root_dir = os.path.dirname(os.path.abspath(__file__))
+        data_dir = os.path.join(root_dir, "data", self.dataset_name)
+        os.makedirs(data_dir, exist_ok=True)
+        file_path = os.path.join(data_dir, part)
+        if multiwoz_like:
+            file_path += "mw"
+
+        if self.percentage != 1.0:
+            file_path += f"_{self.percentage}_{self.dialogue_order}"
+
+        if os.path.exists(file_path):
+            action_batch, a_masks, max_length, small_act_batch, \
+            current_domain_mask_batch, non_current_domain_mask_batch, \
+            description_batch, value_batch, kg_list = torch.load(file_path)
+            print(f"Loaded data from {file_path}")
+        else:
+            print("Creating data from scratch.")
+
+            action_batch, small_act_batch, \
+            current_domain_mask_batch, non_current_domain_mask_batch, \
+            description_batch, value_batch = [], [], [], [], [], []
+            kg_list = []
+
+            for num, item in tqdm(enumerate(self.data[part])):
+
+                if item['action'].sum() == 0 or len(item['state']) == 0:
+                    continue
+                if multiwoz_like:
+                    if not self.is_multiwoz_like(item):
+                        continue
+                action_batch.append(torch.Tensor(item['action']))
+
+                kg = [item['state']]
+                kg_list.append(item['state'])
+
+                description_idx_list, value_list = policy.get_descriptions_and_values(kg)
+                description_batch.append(description_idx_list)
+                value_batch.append(value_list)
+
+                current_domains = policy.get_current_domains(kg)
+                current_domain_mask = policy.action_embedder.get_current_domain_mask(current_domains[0], current=True)
+                non_current_domain_mask = policy.action_embedder.get_current_domain_mask(current_domains[0], current=False)
+                current_domain_mask_batch.append(current_domain_mask)
+                non_current_domain_mask_batch.append(non_current_domain_mask)
+
+                small_act_batch.append(torch.Tensor(policy.action_embedder.real_action_to_small_action_list(torch.Tensor(item['action']))))
+
+            print("Creating action masks..")
+            a_masks, max_length = policy.get_action_masks(action_batch)
+            action_batch = torch.stack(action_batch)
+            current_domain_mask_batch = torch.stack(current_domain_mask_batch)
+            non_current_domain_mask_batch = torch.stack(non_current_domain_mask_batch)
+
+            print(f"Finished data set, time spent: {time.time() - time_now}")
+
+            torch.save([action_batch, a_masks, max_length, small_act_batch,
+                        current_domain_mask_batch, non_current_domain_mask_batch,
+                        description_batch, value_batch, kg_list], file_path)
+
+        dataset = ActDatasetKG(action_batch, a_masks, current_domain_mask_batch, non_current_domain_mask_batch)
+        dataloader = data.DataLoader(dataset, batchsz, True)
+        print("NUMBER OF EXAMPLES:", len(current_domain_mask_batch))
+        return dataloader, max_length, small_act_batch, description_batch, value_batch, kg_list
+
+
+if __name__ == '__main__':
+    data_loader = PolicyDataVectorizer()
diff --git a/convlab/policy/vtrace_DPT/supervised/train_supervised.py b/convlab/policy/vtrace_DPT/supervised/train_supervised.py
new file mode 100644
index 0000000000000000000000000000000000000000..1807a671da7e2938173a18277cd21980ee577a11
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/supervised/train_supervised.py
@@ -0,0 +1,245 @@
+import argparse
+import os
+import torch
+import logging
+import json
+import sys
+
+from torch import optim
+from copy import deepcopy
+from convlab.policy.vtrace_DPT.supervised.loader import PolicyDataVectorizer
+from convlab.util.custom_util import set_seed, init_logging, save_config
+from convlab.util.train_util import to_device
+from convlab.policy.vtrace_DPT.transformer_model.EncoderDecoder import EncoderDecoder
+from convlab.policy.vector.vector_nodes import VectorNodes
+
+root_dir = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+sys.path.append(root_dir)
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+class MLE_Trainer:
+    def __init__(self, manager, cfg, policy):
+        self.start_policy = deepcopy(policy)
+        self.policy = policy
+        self.policy_optim = optim.Adam(list(self.policy.parameters()), lr=cfg['supervised_lr'])
+        self.entropy_weight = cfg['entropy_weight']
+        self.regularization_weight = cfg['regularization_weight']
+        self._init_data(manager, cfg)
+
+    def _init_data(self, manager, cfg):
+        multiwoz_like = cfg['multiwoz_like']
+        self.data_train, self.max_length_train, self.small_act_train, self.descriptions_train, self.values_train, \
+            self.kg_train = manager.create_dataset('train', cfg['batchsz'], self.policy, multiwoz_like)
+        self.data_valid, self.max_length_valid, self.small_act_valid, self.descriptions_valid, self.values_valid, \
+            self.kg_valid = manager.create_dataset('validation', cfg['batchsz'], self.policy, multiwoz_like)
+        self.data_test, self.max_length_test, self.small_act_test, self.descriptions_test, self.values_test, \
+            self.kg_test = manager.create_dataset('test', cfg['batchsz'], self.policy, multiwoz_like)
+        self.save_dir = cfg['save_dir']
+
+    def policy_loop(self, data):
+
+        actions, action_masks, current_domain_mask, non_current_domain_mask, indices = to_device(data)
+
+        small_act_batch = [self.small_act_train[i].to(DEVICE) for i in indices]
+        description_batch = [self.descriptions_train[i].to(DEVICE) for i in indices]
+        value_batch = [self.values_train[i].to(DEVICE) for i in indices]
+
+        log_prob, entropy = self.policy.get_log_prob(actions, action_masks, self.max_length_train, small_act_batch,
+                                 current_domain_mask, non_current_domain_mask,
+                                 description_batch, value_batch)
+        loss_a = -1 * log_prob.mean()
+
+        weight_loss = self.weight_loss()
+
+        return loss_a, -entropy, weight_loss
+
+    def weight_loss(self):
+
+        loss = 0
+        num_params = sum(p.numel() for p in self.policy.parameters() if p.requires_grad)
+        for paramA, paramB in zip(self.policy.parameters(), self.start_policy.parameters()):
+            loss += torch.sum(torch.abs(paramA - paramB.detach()))
+        return loss / num_params
+
+    def imitating(self):
+        """
+        pretrain the policy by simple imitation learning (behavioral cloning)
+        """
+        self.policy.train()
+        a_loss = 0.
+        for i, data in enumerate(self.data_train):
+            self.policy_optim.zero_grad()
+            loss_a, entropy_loss, weight_loss = self.policy_loop(data)
+            a_loss += loss_a.item()
+            loss_a = loss_a + self.entropy_weight * entropy_loss + self.regularization_weight * weight_loss
+
+            if i % 20 == 0 and i != 0:
+                print("LOSS:", a_loss / 20.0)
+                a_loss = 0
+            loss_a.backward()
+            for p in self.policy.parameters():
+                if p.grad is not None:
+                    p.grad[p.grad != p.grad] = 0.0
+            self.policy_optim.step()
+
+        self.policy.eval()
+
+    def validate(self):
+        def f1(a, target):
+            TP, FP, FN = 0, 0, 0
+            real = target.nonzero().tolist()
+            predict = a.nonzero().tolist()
+            for item in real:
+                if item in predict:
+                    TP += 1
+                else:
+                    FN += 1
+            for item in predict:
+                if item not in real:
+                    FP += 1
+            return TP, FP, FN
+
+        average_actions, average_target_actions, counter = 0, 0, 0
+        a_TP, a_FP, a_FN = 0, 0, 0
+        for i, data in enumerate(self.data_valid):
+            counter += 1
+            target_a, action_masks, current_domain_mask, non_current_domain_mask, indices = to_device(data)
+
+            kg_batch = [self.kg_valid[i] for i in indices]
+            a = torch.stack([self.policy.select_action([kg]) for kg in kg_batch])
+
+            TP, FP, FN = f1(a, target_a)
+            a_TP += TP
+            a_FP += FP
+            a_FN += FN
+
+            average_actions += a.float().sum(dim=-1).mean()
+            average_target_actions += target_a.float().sum(dim=-1).mean()
+
+        logging.info(f"Average actions: {average_actions / counter}")
+        logging.info(f"Average target actions: {average_target_actions / counter}")
+        prec = a_TP / (a_TP + a_FP)
+        rec = a_TP / (a_TP + a_FN)
+        F1 = 2 * prec * rec / (prec + rec)
+        return prec, rec, F1
+
+    def test(self):
+        def f1(a, target):
+            TP, FP, FN = 0, 0, 0
+            real = target.nonzero().tolist()
+            predict = a.nonzero().tolist()
+            for item in real:
+                if item in predict:
+                    TP += 1
+                else:
+                    FN += 1
+            for item in predict:
+                if item not in real:
+                    FP += 1
+            return TP, FP, FN
+
+        a_TP, a_FP, a_FN = 0, 0, 0
+        for i, data in enumerate(self.data_test):
+            s, target_a = to_device(data)
+            a_weights = self.policy(s)
+            a = a_weights.ge(0)
+            TP, FP, FN = f1(a, target_a)
+            a_TP += TP
+            a_FP += FP
+            a_FN += FN
+
+        prec = a_TP / (a_TP + a_FP)
+        rec = a_TP / (a_TP + a_FN)
+        F1 = 2 * prec * rec / (prec + rec)
+        print(a_TP, a_FP, a_FN, F1)
+
+    def save(self, directory, epoch):
+        if not os.path.exists(directory):
+            os.makedirs(directory)
+
+        torch.save(self.policy.state_dict(), directory + '/supervised.pol.mdl')
+
+        logging.info('<<dialog policy>> epoch {}: saved network to mdl'.format(epoch))
+
+
+def arg_parser():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--seed", type=int, default=0)
+    parser.add_argument("--eval_freq", type=int, default=1)
+    parser.add_argument("--dataset_name", type=str, default="multiwoz21")
+    parser.add_argument("--model_path", type=str, default="")
+
+    args = parser.parse_args()
+    return args
+
+
+if __name__ == '__main__':
+
+    args = arg_parser()
+
+    root_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    with open(os.path.join(root_directory, 'config.json'), 'r') as f:
+        cfg = json.load(f)
+
+    cfg['dataset_name'] = args.dataset_name
+
+    logger, tb_writer, current_time, save_path, config_save_path, dir_path, log_save_path = \
+        init_logging(os.path.dirname(os.path.abspath(__file__)), "info")
+    save_config(vars(args), cfg, config_save_path)
+
+    set_seed(args.seed)
+    logging.info(f"Seed used: {args.seed}")
+    logging.info(f"Batch size: {cfg['batchsz']}")
+    logging.info(f"Epochs: {cfg['epoch']}")
+    logging.info(f"Learning rate: {cfg['supervised_lr']}")
+    logging.info(f"Entropy weight: {cfg['entropy_weight']}")
+    logging.info(f"Regularization weight: {cfg['regularization_weight']}")
+    logging.info(f"Only use multiwoz like domains: {cfg['multiwoz_like']}")
+    logging.info(f"We use: {cfg['data_percentage']*100}% of the data")
+    logging.info(f"Dialogue order used: {cfg['dialogue_order']}")
+
+    vector = VectorNodes(dataset_name=args.dataset_name, use_masking=False, filter_state=True)
+    manager = PolicyDataVectorizer(dataset_name=args.dataset_name, vector=vector,
+                                   percentage=cfg['data_percentage'], dialogue_order=cfg["dialogue_order"])
+    policy = EncoderDecoder(**cfg, action_dict=vector.act2vec).to(device=DEVICE)
+    try:
+        policy.load_state_dict(torch.load(args.model_path, map_location=DEVICE))
+        logging.info(f"Loaded model from {args.model_path}")
+    except:
+        logging.info("Didnt load a model")
+    agent = MLE_Trainer(manager, cfg, policy)
+
+    logging.info('Start training')
+
+    best_recall = 0.0
+    best_precision = 0.0
+    best_f1 = 0.0
+    precision = 0
+    recall = 0
+    f1 = 0
+
+    for e in range(cfg['epoch']):
+        agent.imitating()
+        logging.info(f"Epoch: {e}")
+
+        if e % args.eval_freq == 0:
+            precision, recall, f1 = agent.validate()
+
+        logging.info(f"Precision: {precision}")
+        logging.info(f"Recall: {recall}")
+        logging.info(f"F1: {f1}")
+
+        if precision > best_precision:
+            best_precision = precision
+        if recall > best_recall:
+            best_recall = recall
+        if f1 > best_f1:
+            best_f1 = f1
+            agent.save(save_path, e)
+        logging.info(f"Best Precision: {best_precision}")
+        logging.info(f"Best Recall: {best_recall}")
+        logging.info(f"Best F1: {best_f1}")
diff --git a/convlab/policy/vtrace_DPT/train.py b/convlab/policy/vtrace_DPT/train.py
new file mode 100644
index 0000000000000000000000000000000000000000..f441da29cf44fb366daf9f5afd3f2f21d1bd420d
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/train.py
@@ -0,0 +1,224 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Sun Jul 14 16:14:07 2019
+@author: Chris Geishauser
+"""
+
+import sys
+import os
+import logging
+import time
+import torch
+
+from torch import multiprocessing as mp
+from argparse import ArgumentParser
+from convlab.policy.vtrace_DPT import VTRACE
+from convlab.policy.vtrace_DPT.memory import Memory
+from convlab.policy.vtrace_DPT.multiprocessing_helper import get_queues, start_processes, submit_jobs, \
+    terminate_processes
+from convlab.task.multiwoz.goal_generator import GoalGenerator
+from convlab.util.custom_util import set_seed, init_logging, save_config, move_finished_training, env_config, \
+    eval_policy, log_start_args, save_best, load_config_file, create_goals, get_config
+from datetime import datetime
+
+sys.path.append(os.path.dirname(os.path.dirname(
+    os.path.dirname(os.path.abspath(__file__)))))
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+device = DEVICE
+
+try:
+    mp.set_start_method('spawn', force=True)
+    mp = mp.get_context('spawn')
+except RuntimeError:
+    pass
+
+
+def create_episodes(environment, policy, num_episodes, memory, goals):
+    sampled_num = 0
+    traj_len = 40
+
+    while sampled_num < num_episodes:
+        goal = goals.pop()
+        s = environment.reset(goal)
+
+        user_act_list, sys_act_list, s_vec_list, action_list, reward_list, small_act_list, action_mask_list, mu_list, \
+        trajectory_list, vector_mask_list, critic_value_list, description_idx_list, value_list, current_domain_mask, \
+        non_current_domain_mask = \
+            [], [], [], [], [], [], [], [], [], [], [], [], [], [], []
+
+        for t in range(traj_len):
+
+            s_vec, mask = policy.vector.state_vectorize(s)
+            with torch.no_grad():
+                a = policy.predict(s)
+
+            # s_vec_list.append(policy.info_dict['kg'])
+            action_list.append(policy.info_dict['big_act'].detach())
+            small_act_list.append(policy.info_dict['small_act'])
+            action_mask_list.append(policy.info_dict['action_mask'])
+            mu_list.append(policy.info_dict['a_prob'].detach())
+            critic_value_list.append(policy.info_dict['critic_value'])
+            vector_mask_list.append(torch.Tensor(mask))
+            description_idx_list.append(policy.info_dict["description_idx_list"])
+            value_list.append(policy.info_dict["value_list"])
+            current_domain_mask.append(policy.info_dict["current_domain_mask"])
+            non_current_domain_mask.append(policy.info_dict["non_current_domain_mask"])
+
+            sys_act_list.append(policy.vector.action_vectorize(a))
+            trajectory_list.extend([s['user_action'], a])
+
+            # interact with env
+            next_s, r, done = environment.step(a)
+            reward_list.append(torch.Tensor([r]))
+
+            next_s_vec, next_mask = policy.vector.state_vectorize(next_s)
+
+            # update per step
+            s = next_s
+
+            if done:
+                memory.update_episode(description_idx_list, action_list, reward_list, small_act_list, mu_list,
+                                      action_mask_list, critic_value_list, description_idx_list, value_list,
+                                      current_domain_mask, non_current_domain_mask)
+                break
+
+        sampled_num += 1
+
+
+def log_train_configs():
+    logging.info('Train seed is ' + str(seed))
+    logging.info("Start of Training: " +
+                 time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()))
+    logging.info(f"Number of processes for training: {train_processes}")
+    logging.info(f"Number of new dialogues per update: {new_dialogues}")
+    logging.info(f"Number of total dialogues: {total_dialogues}")
+
+
+if __name__ == '__main__':
+
+    time_now = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
+
+    begin_time = datetime.now()
+    parser = ArgumentParser()
+    parser.add_argument("--path", type=str, default='convlab/policy/vtrace_DPT/semantic_level_config.json',
+                        help="Load path for config file")
+    parser.add_argument("--seed", type=int, default=None,
+                        help="Seed for the policy parameter initialization")
+    parser.add_argument("--mode", type=str, default='info',
+                        help="Set level for logger")
+    parser.add_argument("--save_eval_dials", type=bool, default=False,
+                        help="Flag for saving dialogue_info during evaluation")
+
+    path = parser.parse_args().path
+    seed = parser.parse_args().seed
+    mode = parser.parse_args().mode
+    save_eval = parser.parse_args().save_eval_dials
+
+    logger, tb_writer, current_time, save_path, config_save_path, dir_path, log_save_path = \
+        init_logging(os.path.dirname(os.path.abspath(__file__)), mode)
+
+    args = [('model', 'seed', seed)] if seed is not None else list()
+
+    environment_config = load_config_file(path)
+
+    conf = get_config(path, args)
+    seed = conf['model']['seed']
+    set_seed(seed)
+
+    policy_sys = VTRACE(is_train=True, seed=seed, vectorizer=conf['vectorizer_sys_activated'],
+                        load_path=conf['model']['load_path'])
+    policy_sys.share_memory()
+    memory = Memory(seed=seed)
+    policy_sys.current_time = current_time
+    policy_sys.log_dir = config_save_path.replace('configs', 'logs')
+    policy_sys.save_dir = save_path
+
+    save_config(vars(parser.parse_args()), environment_config, config_save_path, policy_config=policy_sys.cfg)
+
+    env, sess = env_config(conf, policy_sys)
+
+    # Setup uncertainty thresholding
+    if env.sys_dst:
+        try:
+            if env.sys_dst.use_confidence_scores:
+                policy_sys.vector.setup_uncertain_query(env.sys_dst.thresholds)
+        except:
+            logging.info('Uncertainty threshold not set.')
+
+    single_domains = conf['goals']['single_domains']
+    allowed_domains = conf['goals']['allowed_domains']
+    logging.info(f"Single domains only: {single_domains}")
+    logging.info(f"Allowed domains {allowed_domains}")
+
+    logging.info(f"Evaluating at start - {time_now}" + '-'*60)
+    time_now = time.time()
+    eval_dict = eval_policy(conf, policy_sys, env, sess, save_eval, log_save_path,
+                            single_domain_goals=single_domains, allowed_domains=allowed_domains)
+    logging.info(f"Finished evaluating, time spent: {time.time() - time_now}")
+
+    for key in eval_dict:
+        tb_writer.add_scalar(key, eval_dict[key], 0)
+    best_complete_rate = eval_dict['complete_rate']
+    best_success_rate = eval_dict['success_rate_strict']
+    best_return = eval_dict['avg_return']
+
+    train_processes = conf['model']["process_num_train"]
+
+    if train_processes > 1:
+        # We use multiprocessing
+        queues, episode_queues = get_queues(train_processes)
+        online_metric_queue = mp.SimpleQueue()
+        processes = start_processes(train_processes, queues, episode_queues, env, policy_sys, seed,
+                                    online_metric_queue)
+    goal_generator = GoalGenerator()
+
+    num_dialogues = 0
+    new_dialogues = conf['model']["new_dialogues"]
+    total_dialogues = conf['model']["total_dialogues"]
+
+    log_train_configs()
+
+    while num_dialogues < total_dialogues:
+
+        goals = create_goals(goal_generator, new_dialogues, single_domains=single_domains,
+                             allowed_domains=allowed_domains)
+        if train_processes > 1:
+            time_now, metrics = submit_jobs(new_dialogues, queues, episode_queues, train_processes, memory, goals,
+                                            online_metric_queue)
+        else:
+            create_episodes(env, policy_sys, new_dialogues, memory, goals)
+        num_dialogues += new_dialogues
+
+        for r in range(conf['model']['update_rounds']):
+            if num_dialogues > 50:
+                policy_sys.update(memory)
+                torch.cuda.empty_cache()
+
+        if num_dialogues % conf['model']['eval_frequency'] == 0:
+            time_now = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
+            logging.info(f"Evaluating after Dialogues: {num_dialogues} - {time_now}" + '-' * 60)
+
+            eval_dict = eval_policy(conf, policy_sys, env, sess, save_eval, log_save_path,
+                                    single_domain_goals=single_domains, allowed_domains=allowed_domains)
+
+            best_complete_rate, best_success_rate, best_return = \
+                save_best(policy_sys, best_complete_rate, best_success_rate, best_return,
+                          eval_dict["complete_rate"], eval_dict["success_rate_strict"],
+                          eval_dict["avg_return"], save_path)
+            policy_sys.save(save_path, "last")
+            for key in eval_dict:
+                tb_writer.add_scalar(key, eval_dict[key], num_dialogues)
+
+    logging.info("End of Training: " +
+                 time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()))
+
+    if train_processes > 1:
+        terminate_processes(processes, queues)
+
+    f = open(os.path.join(dir_path, "time.txt"), "a")
+    f.write(str(datetime.now() - begin_time))
+    f.close()
+
+    move_finished_training(dir_path, os.path.join(
+        os.path.dirname(os.path.abspath(__file__)), "finished_experiments"))
diff --git a/convlab/policy/vtrace_DPT/transformer_model/EncoderCritic.py b/convlab/policy/vtrace_DPT/transformer_model/EncoderCritic.py
new file mode 100644
index 0000000000000000000000000000000000000000..9d64deffe2a8b4c74072a3f6f625bfb47f359685
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/transformer_model/EncoderCritic.py
@@ -0,0 +1,109 @@
+import torch
+import torch.nn as nn
+import logging
+
+from torch.nn.utils.rnn import pad_sequence
+from .noisy_linear import NoisyLinear
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+class EncoderCritic(nn.Module):
+
+    def __init__(self, node_embedder, encoder, cls_dim=128, independent=True, enc_nhead=2, noisy_linear=False,
+                 **kwargs):
+        super(EncoderCritic, self).__init__()
+
+        self.node_embedder = node_embedder
+        self.encoder = encoder
+        #self.cls = torch.nn.Parameter(torch.randn(cls_dim), requires_grad=True).to(DEVICE)
+        self.cls = torch.randn(cls_dim)
+        self.cls.requires_grad = True
+        self.cls = torch.nn.Parameter(self.cls)
+        if noisy_linear:
+            logging.info("EncoderCritic: We use noisy linear layers.")
+            self.linear = NoisyLinear(cls_dim, 1).to(DEVICE)
+        else:
+            self.linear = torch.nn.Linear(cls_dim, 1).to(DEVICE)
+        self.num_heads = enc_nhead
+
+        logging.info(f"Initialised critic. Independent: {independent}")
+
+    def forward(self, descriptions_list, value_list):
+        # return output of cls token
+        return self.linear(self.encode_kg(descriptions_list, value_list)[:, 0, :])
+
+    def encode_kg(self, descriptions_list, value_list):
+        #encoder_mask = self.compute_mask_extended(kg_list)
+        encoder_mask = self.compute_mask(descriptions_list)
+        embedded_nodes = self.embedd_nodes(descriptions_list, value_list)
+        padded_graphs = pad_sequence(embedded_nodes, batch_first=False).to(DEVICE)
+        encoded_nodes, att_weights = self.encoder(padded_graphs, src_key_padding_mask=encoder_mask)
+        # size [num_graphs, max_num_nodes, enc_input_dim]
+        return encoded_nodes.permute(1, 0, 2)
+
+    def embedd_nodes(self, descriptions_list, value_list):
+        kg_sizes = [len(kg) for kg in descriptions_list]
+
+        # we view the kg_list as one huge knowledge graph to embed all nodes simultaneously
+        flattened_descriptions = torch.stack(
+            [descr_idx for descr_list in descriptions_list for descr_idx in descr_list]).to(DEVICE)
+        flattened_values = torch.stack(
+            [value for values in value_list for value in values])
+        flat_embedded_nodes = self.node_embedder(flattened_descriptions, flattened_values).to(DEVICE)
+
+        #now get back the individual knowledge graphs
+        embedded_nodes = []
+        counter = 0
+        for size in kg_sizes:
+            embedded_nodes.append(
+                torch.cat([self.cls.unsqueeze(0), flat_embedded_nodes[counter:counter + size, :]], dim=0))
+            counter += size
+        return embedded_nodes
+
+    def compute_mask(self, kg_list, all=True):
+        # we add 1 for the cls_node in every graph
+        kg_sizes = [len(kg) + 1 for kg in kg_list]
+        max_size = max(kg_sizes)
+
+        attention_mask = torch.ones((len(kg_list), max_size))
+
+        for idx, size in enumerate(kg_sizes):
+            if not all:
+                attention_mask[idx, idx] = 0
+            else:
+                attention_mask[idx, :size] = torch.zeros(size)
+
+        return attention_mask.bool().to(DEVICE)
+
+    def compute_mask_extended(self, kg_list):
+
+        kg_sizes = [len(kg) + 1 for kg in kg_list]
+        max_size = max(kg_sizes)
+        attention_mask = torch.ones((len(kg_list), max_size, max_size))
+
+        domain_list = []
+        for kg in kg_list:
+            node_dict = {}
+            for idx, node in enumerate(kg):
+                domain = node['domain']
+                if domain not in node_dict:
+                    node_dict[domain] = torch.ones(max_size)
+                    node_dict[domain][idx + 1] = 0
+                else:
+                    node_dict[domain][idx + 1] = 0
+
+            domain_list.append(node_dict)
+
+        for idx, kg in enumerate(kg_list):
+            for idx_n, node in enumerate(kg):
+                domain = node['domain']
+                attention_mask[idx, idx_n + 1] = domain_list[idx][domain]
+
+            attention_mask[idx, 0, :len(kg) + 1] = torch.zeros(len(kg) + 1)
+            pad_size = max_size - (len(kg) + 1)
+            attention_mask[idx, len(kg) + 1:, :] = torch.zeros(pad_size, max_size)
+
+        attention_mask = attention_mask.repeat(self.num_heads, 1, 1)
+
+        return attention_mask.bool().to(DEVICE)
diff --git a/convlab/policy/vtrace_DPT/transformer_model/EncoderDecoder.py b/convlab/policy/vtrace_DPT/transformer_model/EncoderDecoder.py
new file mode 100644
index 0000000000000000000000000000000000000000..cc2065f57dac4f17108f4db219368b4bc592dbee
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/transformer_model/EncoderDecoder.py
@@ -0,0 +1,487 @@
+from torch.nn.utils.rnn import pad_sequence
+from .node_embedder import NodeEmbedderRoberta
+from .transformer import TransformerModelEncoder, TransformerModelDecoder
+from .action_embedder import ActionEmbedder
+from torch.distributions.categorical import Categorical
+from .noisy_linear import NoisyLinear
+from tqdm import tqdm
+
+import torch
+import torch.nn as nn
+import sys
+import logging
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+class EncoderDecoder(nn.Module):
+    '''
+    Documentation
+    '''
+    def __init__(self, enc_input_dim, enc_nhead, enc_d_hid, enc_nlayers, enc_dropout,
+                 dec_input_dim, dec_nhead, dec_d_hid, dec_nlayers, dec_dropout,
+                 action_embedding_dim, action_dict, domain_embedding_dim, value_embedding_dim,
+                 node_embedding_dim, roberta_path="", node_attention=True, max_length=25, semantic_descriptions=True,
+                 freeze_roberta=True, use_pooled=False, verbose=False, mean=False, ignore_features=None,
+                 only_active_values=False, roberta_actions=False, independent_descriptions=False, need_weights=True,
+                 random_matrix=False, distance_metric=False, noisy_linear=False, dataset_name='multiwoz21', **kwargs):
+        super(EncoderDecoder, self).__init__()
+        self.node_embedder = NodeEmbedderRoberta(node_embedding_dim, freeze_roberta=freeze_roberta,
+                                                 use_pooled=use_pooled, roberta_path=roberta_path,
+                                                 semantic_descriptions=semantic_descriptions, mean=mean,
+                                                 dataset_name=dataset_name).to(DEVICE)
+        #TODO: Encoder input dim should be same as projection dim or use another linear layer?
+        self.encoder = TransformerModelEncoder(enc_input_dim, enc_nhead, enc_d_hid, enc_nlayers, enc_dropout, need_weights).to(DEVICE)
+        self.decoder = TransformerModelDecoder(action_embedding_dim, dec_nhead, dec_d_hid, dec_nlayers, dec_dropout, need_weights).to(DEVICE)
+        if not roberta_actions:
+            self.action_embedder = ActionEmbedder(action_dict, domain_embedding_dim, value_embedding_dim,
+                                                  action_embedding_dim,
+                                                  random_matrix=random_matrix,
+                                                  distance_metric=distance_metric).to(DEVICE)
+        else:
+            self.action_embedder = ActionEmbedder(action_dict, domain_embedding_dim, value_embedding_dim,
+                                                  action_embedding_dim, node_embedder=self.node_embedder,
+                                                  random_matrix=random_matrix,
+                                                  distance_metric=distance_metric).to(DEVICE)
+        #TODO: Ignore features for better robustness and simulating absence of certain information
+        self.ignore_features = ignore_features
+        self.node_attention = node_attention
+        self.freeze_roberta = freeze_roberta
+        self.max_length = max_length
+        self.verbose = verbose
+        self.only_active_values = only_active_values
+        self.num_heads = enc_nhead
+        self.action_embedding_dim = action_embedding_dim
+        # embeddings for "domain", intent", "slot" and "start"
+        self.embedding = nn.Embedding(4, action_embedding_dim).to(DEVICE)
+        self.info_dict = {}
+
+        if noisy_linear:
+            logging.info("EncoderDecoder: We use noisy linear layers.")
+            self.action_projector = NoisyLinear(dec_input_dim, action_embedding_dim).to(DEVICE)
+            self.current_domain_predictor = NoisyLinear(dec_input_dim, 1).to(DEVICE)
+        else:
+            self.action_projector = torch.nn.Linear(dec_input_dim, action_embedding_dim).to(DEVICE)
+            self.current_domain_predictor = torch.nn.Linear(dec_input_dim, 1).to(DEVICE)
+        self.softmax = torch.nn.Softmax(dim=-1)
+        self.sigmoid = torch.nn.Sigmoid()
+
+        self.num_book = 0
+        self.num_nobook = 0
+        self.num_selected = 0
+
+    def get_current_domain_mask(self, kg_list, current=True):
+
+        current_domains = self.get_current_domains(kg_list)
+        current_domain_mask = self.action_embedder.get_current_domain_mask(current_domains[0], current=current).to(DEVICE)
+        return current_domain_mask
+
+    def get_descriptions_and_values(self, kg_list):
+
+        description_idx_list = self.node_embedder.description_2_idx(kg_list[0]).to(DEVICE)
+        value_list = torch.Tensor([node['value'] for node in kg_list[0]]).unsqueeze(1).to(DEVICE)
+        return description_idx_list, value_list
+
+    def select_action(self, kg_list, mask=None, eval=False):
+        '''
+        :param kg_list: A single knowledge graph consisting of a list of nodes
+        :return: multi-action
+        Will also return tensors that are used for calculating log-probs, i.e. for doing RL training
+        '''
+
+        kg_list = [[node for node in kg if node['node_type'] not in self.ignore_features] for kg in kg_list]
+        # this is a bug during supervised training that they use ticket instead of people in book information
+        kg_list = [[node for node in kg if node['description'] != "user goal-train-ticket"] for kg in kg_list]
+
+        current_domains = self.get_current_domains(kg_list)
+        legal_mask = self.action_embedder.get_legal_mask(mask)
+
+        if self.only_active_values:
+            kg_list = [[node for node in kg if node['value'] != 0.0] for kg in kg_list]
+
+        description_idx_list, value_list = self.get_descriptions_and_values(kg_list)
+        encoded_nodes, att_weights_encoder = self.encode_kg([description_idx_list], [value_list])
+        encoded_nodes = encoded_nodes.to(DEVICE)
+
+        active_domains = set([node['domain'].lower() for node in kg_list[0]] + ['general', 'booking'])
+
+        decoder_input = self.embedding(torch.Tensor([3]).long().to(DEVICE)) + self.embedding(torch.Tensor([0]).to(DEVICE).long())
+        decoder_input = decoder_input.view(1, 1, -1).to(DEVICE)
+        start = True
+        action_mask = self.action_embedder.get_action_mask(start=start)
+        action_mask = action_mask + legal_mask
+        action_mask = action_mask.bool().float()
+        action_mask_list = [action_mask]
+        action_list = []
+        action_list_num = []
+        distribution_list = []
+        attention_weights_list = []
+
+        current_domain_mask = self.action_embedder.get_current_domain_mask(current_domains[0], current=True).to(DEVICE)
+        non_current_domain_mask = self.action_embedder.get_current_domain_mask(current_domains[0], current=False).to(DEVICE)
+
+        domains = [d for d, i in sorted(self.action_embedder.domain_dict.items(), key=lambda item: item[1])]
+        domain_indices = [self.action_embedder.small_action_dict[d] for d in domains]
+
+        intents = [d for d, i in sorted(self.action_embedder.intent_dict.items(), key=lambda item: item[1])]
+        intent_indices = [self.action_embedder.small_action_dict[d] for d in intents]
+
+        slot_values = [d for d, i in sorted(self.action_embedder.slot_value_dict.items(), key=lambda item: item[1])]
+        s_v_indices = [self.action_embedder.small_action_dict[d] for d in slot_values]
+
+        for t in range(self.max_length):
+            decoder_output, att_weights_decoder = self.decoder(decoder_input, encoded_nodes.permute(1, 0, 2))
+            attention_weights_list.append(att_weights_decoder)
+            action_logits = self.action_embedder(self.action_projector(decoder_output))
+
+            if t % 3 == 0:
+                # We need to choose a domain
+                current_domain_empty = float((len(current_domains[0]) == 0))
+                # we mask taking a current domain if there is none
+                pick_current_domain_prob = self.sigmoid(
+                    self.current_domain_predictor(decoder_output) - current_domain_empty * sys.maxsize)
+
+                # only pick from current domains
+                action_logits_current_domain = action_logits - (
+                            action_mask + current_domain_mask).bool().float() * sys.maxsize
+                action_distribution_current_domain = self.softmax(action_logits_current_domain)
+                action_distribution_current_domain = (action_distribution_current_domain * pick_current_domain_prob)
+
+                # only pick from non-current domains
+                action_logits_non_current_domain = action_logits - (
+                            action_mask + non_current_domain_mask).bool().float() * sys.maxsize
+                action_distribution_non_current_domain = self.softmax(action_logits_non_current_domain)
+                action_distribution_non_current_domain = (
+                            action_distribution_non_current_domain * (1.0 - pick_current_domain_prob))
+
+                action_distribution = action_distribution_non_current_domain + action_distribution_current_domain
+                action_distribution = (action_distribution / action_distribution.sum(dim=-1, keepdim=True)).squeeze(-1)
+
+            else:
+                action_logits = action_logits - action_mask * sys.maxsize
+                action_distribution = self.softmax(action_logits).squeeze(-1)
+
+            if not eval or t % 3 != 0:
+                dist = Categorical(action_distribution)
+                rand_state = torch.random.get_rng_state()
+                action = dist.sample().tolist()[-1]
+                torch.random.set_rng_state(rand_state)
+                semantic_action = self.action_embedder.small_action_dict_reversed[action[-1]]
+                action_list.append(semantic_action)
+                action_list_num.append(action[-1])
+            else:
+                action = action_distribution[-1, -1, :]
+                action = torch.argmax(action).item()
+                semantic_action = self.action_embedder.small_action_dict_reversed[action]
+                action_list.append(semantic_action)
+                action_list_num.append(action)
+
+            #prepare for next step
+            next_input = self.action_embedder.action_projector(self.action_embedder.action_embeddings[action]).view(1, 1, -1) + \
+                         self.embedding(torch.Tensor([(t + 1) % 3]).to(DEVICE).long())
+            decoder_input = torch.cat([decoder_input, next_input], dim=0)
+
+            if t % 3 == 0:
+                # We chose a domain
+                action_mask_restricted = action_mask[domain_indices]
+                domain_dist = action_distribution[0, -1, :][domain_indices]
+                distribution_list.append(
+                    [semantic_action, dict((domain, (distri, m)) for domain, distri, m in
+                                           zip(domains, domain_dist, action_mask_restricted))])
+
+                if semantic_action == 'eos':
+                    break
+                chosen_domain = semantic_action
+                # focus only on the chosen domain information
+
+                action_mask = self.action_embedder.get_action_mask(domain=semantic_action, start=False)
+                action_mask = action_mask + self.action_embedder.get_legal_mask(mask, domain=semantic_action)
+                action_mask = action_mask.bool().float()
+            elif t % 3 == 1:
+                # We chose an intent
+                if semantic_action == "book":
+                    self.num_book += 1
+                if semantic_action == "nobook":
+                    self.num_nobook += 1
+
+                action_mask = self.action_embedder.get_action_mask(domain=chosen_domain,
+                                                                   intent=semantic_action, start=False)
+                action_mask = action_mask + self.action_embedder.get_legal_mask(mask, domain=chosen_domain,
+                                                                                intent=semantic_action)
+                action_mask = action_mask.bool().float()
+                #intent_dist = action_distribution[0, -1, :][intent_indices]
+                #distribution_list.append(
+                #    [semantic_action, dict((intent, (distri, m)) for intent, distri, m in
+                 #                          zip(intents, intent_dist, action_mask_restricted))])
+            else:
+                # We chose a slot-value pair
+                action_mask = self.action_embedder.get_action_mask(start=False)
+                action_mask = action_mask + self.action_embedder.get_legal_mask(mask)
+                action_mask = action_mask.bool().float()
+
+            action_mask_list.append(action_mask)
+
+        self.num_selected += 1
+
+        if action_list[-1] != 'eos':
+            action_mask_list = action_mask_list[:-1]
+
+        self.info_dict["kg"] = kg_list[0]
+        self.info_dict["small_act"] = torch.Tensor(action_list_num)
+        self.info_dict["action_mask"] = torch.stack(action_mask_list)
+        self.info_dict["description_idx_list"] = description_idx_list
+        self.info_dict["value_list"] = value_list
+        self.info_dict["semantic_action"] = action_list
+        self.info_dict["current_domain_mask"] = current_domain_mask
+        self.info_dict["non_current_domain_mask"] = non_current_domain_mask
+        self.info_dict["active_domains"] = active_domains
+        self.info_dict["attention_weights"] = attention_weights_list
+
+        if self.verbose:
+            print("NEW SELECTION **************************")
+            print(f"KG: {kg_list}")
+            print(f"Active Domains: {active_domains}")
+            print(f"Semantic Act: {action_list}")
+            #print("DISTRIBUTION LIST", distribution_list)
+            print("Attention:", attention_weights_list[1][1][1])
+
+        return self.action_embedder.small_action_list_to_real_actions(action_list)
+
+    def get_log_prob(self, actions, action_mask_list, max_length, action_targets,
+                 current_domain_mask, non_current_domain_mask, descriptions_list, value_list, no_slots=False):
+
+        action_probs, entropy_probs = self.get_prob(actions, action_mask_list, max_length, action_targets,
+                 current_domain_mask, non_current_domain_mask, descriptions_list, value_list)
+        log_probs = torch.log(action_probs)
+
+        entropy_probs = torch.where(entropy_probs < 0.00001, torch.ones(entropy_probs.size()).to(DEVICE), entropy_probs)
+        entropy_probs = torch.where(entropy_probs > 1.0, torch.ones(entropy_probs.size()).to(DEVICE), entropy_probs)
+        entropy = -(entropy_probs * torch.log(entropy_probs)).sum(-1).sum(-1).mean()
+
+        # sometimes a domain will be masked because it is inactive due to labelling error. Will ignore these cases.
+        log_probs[log_probs == -float("Inf")] = 0
+
+        if no_slots:
+            time_steps = torch.arange(0, max_length)
+            slot_steps = torch.where(time_steps % 3 == 2, torch.zeros(max_length), torch.ones(max_length))\
+                .view(1, -1).to(DEVICE)
+            log_probs *= slot_steps
+
+        return log_probs.sum(-1), entropy
+
+    def get_prob(self, actions, action_mask_list, max_length, action_targets,
+                 current_domain_mask, non_current_domain_mask, descriptions_list, value_list):
+        if not self.freeze_roberta:
+            self.node_embedder.form_embedded_descriptions()
+
+        current_domain_mask = current_domain_mask.unsqueeze(1).to(DEVICE)
+        non_current_domain_mask = non_current_domain_mask.unsqueeze(1).to(DEVICE)
+
+        encoded_nodes, att_weights_encoder = self.encode_kg(descriptions_list, value_list)
+        encoder_mask = self.compute_mask(descriptions_list)
+        padded_decoder_input, padded_action_targets = self.get_decoder_tensors(actions, max_length, action_targets)
+        # produde decoder mask to not attend to future time-steps
+        decoder_mask = torch.triu(torch.ones(max_length, max_length) * float('-inf'), diagonal=1)
+
+        decoder_output, att_weights_decoder = self.decoder(padded_decoder_input.permute(1, 0, 2).to(DEVICE),
+                                                           encoded_nodes.permute(1, 0, 2).to(DEVICE),
+                                                           decoder_mask.to(DEVICE), encoder_mask.to(DEVICE))
+
+        pick_current_domain_prob = self.sigmoid(self.current_domain_predictor(decoder_output.permute(1, 0, 2)).clone())
+
+        action_logits = self.action_embedder(self.action_projector(decoder_output.permute(1, 0, 2)))
+
+        # do the general mask for intent and slots, domain must be treated separately
+        action_logits_general = action_logits - action_mask_list * sys.maxsize
+        action_distribution_general = self.softmax(action_logits_general)
+
+        # only pick from current domains
+        action_logits_current_domain = action_logits - (action_mask_list + current_domain_mask).bool().float() * sys.maxsize
+        action_distribution_current_domain = self.softmax(action_logits_current_domain)
+        action_distribution_current_domain = (action_distribution_current_domain * pick_current_domain_prob)
+
+        # only pick from non-current domains
+        action_logits_non_current_domain = action_logits - (action_mask_list + non_current_domain_mask).bool().float() * sys.maxsize
+        action_distribution_non_current_domain = self.softmax(action_logits_non_current_domain)
+        action_distribution_non_current_domain = (action_distribution_non_current_domain * (1.0 - pick_current_domain_prob))
+
+        action_distribution_domain = action_distribution_non_current_domain + action_distribution_current_domain
+
+        time_steps = torch.arange(0, max_length)
+        non_domain_steps = torch.where(time_steps % 3 == 0, torch.zeros(max_length), torch.ones(max_length))\
+            .view(1, -1, 1).to(DEVICE)
+        domain_steps = torch.where(time_steps % 3 == 0, torch.ones(max_length), torch.zeros(max_length))\
+            .view(1, -1, 1).to(DEVICE)
+
+        action_distribution_domain = (action_distribution_domain * domain_steps)
+        action_distribution_general = (action_distribution_general * non_domain_steps)
+        action_distribution = action_distribution_domain + action_distribution_general
+        # make sure it sums up to 1 in every time-step
+        action_distribution = (action_distribution / action_distribution.sum(dim=-1, keepdim=True))
+
+        action_probs = action_distribution.gather(-1, padded_action_targets.long().unsqueeze(-1)).squeeze()
+        # padded time-steps can have very low probability, taking log can be unstable. This prevents it.
+        action_prob_helper = torch.Tensor(
+            [[1] * len(actions) + [0] * (max_length - len(actions)) for actions in action_targets]).to(DEVICE)
+        action_prob_helper_rev = torch.Tensor(
+            [[0] * len(actions) + [1] * (max_length - len(actions)) for actions in action_targets]).to(DEVICE)
+        # set all padded time-steps to 0 probability
+        action_probs = action_probs * action_prob_helper
+        # set padded time-steps to probability 1, so that log will be 0
+        action_probs = action_probs + action_prob_helper_rev
+
+        entropy_probs = action_distribution_general * action_prob_helper.unsqueeze(-1) + action_prob_helper_rev.unsqueeze(-1)
+        #entropy_probs = entropy_probs + domain_steps
+        return action_probs, entropy_probs
+
+    def get_current_domains(self, kg_list):
+        current_domains = []
+        for kg in kg_list:
+            curr_list = []
+            for node in kg:
+                if node['node_type'] == 'user act':
+                    if node['domain'].lower() not in current_domains:
+                        curr_list.append(node['domain'].lower())
+            current_domains.append(curr_list)
+        return current_domains
+
+    def get_decoder_tensors(self, actions, max_length, action_targets):
+
+        # Map the actions to action embeddings that are fed as input to decoder model
+        # pad input and remove "eos" token
+        padded_decoder_input = torch.stack(
+            [torch.cat([act[:-1].to(DEVICE), torch.zeros(max_length - len(act)).to(DEVICE)], dim=-1) for act in action_targets], dim=0) \
+            .to(DEVICE).long()
+
+        padded_action_targets = torch.stack(
+            [torch.cat([act.to(DEVICE), torch.zeros(max_length - len(act)).to(DEVICE)], dim=-1) for act in action_targets], dim=0) \
+            .to(DEVICE)
+
+        decoder_input = self.action_embedder.action_embeddings[padded_decoder_input]
+        decoder_input = self.action_embedder.action_projector(decoder_input)
+        # Add "start" token
+        start_input = self.embedding(torch.Tensor([3]).to(DEVICE).long()).to(DEVICE).repeat(len(actions), 1, 1)
+        decoder_input = torch.cat([start_input, decoder_input], dim=1)
+        # Add "domain", "intent" or "slot" token to input so model knows what to predict
+        type_tokens = self.embedding(torch.remainder(torch.Tensor(range(max_length)).to(DEVICE), 3).long())
+        decoder_input += type_tokens
+
+        return decoder_input, padded_action_targets
+
+    def encode_kg(self, descriptions_list, value_list):
+        #encoder_mask = self.compute_mask_extended(kg_list)
+        encoder_mask = self.compute_mask(descriptions_list)
+        embedded_nodes = self.embedd_nodes(descriptions_list, value_list)
+        padded_graphs = pad_sequence(embedded_nodes, batch_first=False).to(DEVICE)
+        encoded_nodes, att_weights = self.encoder(padded_graphs, src_key_padding_mask=encoder_mask)
+        # size [num_graphs, max_num_nodes, enc_input_dim]
+
+        return encoded_nodes.permute(1, 0, 2), att_weights
+
+    def embedd_nodes(self, descriptions_list, value_list):
+        kg_sizes = [len(descr_list) for descr_list in descriptions_list]
+
+        # we view the kg_list as one huge knowledge graph to embed all nodes simultaneously
+        flattened_descriptions = torch.stack(
+            [descr_idx for descr_list in descriptions_list for descr_idx in descr_list]).to(DEVICE)
+        flattened_values = torch.stack(
+            [value for values in value_list for value in values])
+        flat_embedded_nodes = self.node_embedder(flattened_descriptions, flattened_values).to(DEVICE)
+
+        #now get back the individual knowledge graphs
+        embedded_nodes = []
+        counter = 0
+        for size in kg_sizes:
+            embedded_nodes.append(flat_embedded_nodes[counter:counter + size, :])
+            counter += size
+        return embedded_nodes
+
+    def compute_mask(self, descriptions_list):
+        kg_sizes = [len(descr_list) for descr_list in descriptions_list]
+        max_size = max(kg_sizes)
+        attention_mask = torch.ones((len(descriptions_list), max_size))
+
+        for idx, size in enumerate(kg_sizes):
+            attention_mask[idx, :size] = torch.zeros(size)
+
+        return attention_mask.bool().to(DEVICE)
+
+    def compute_mask_extended(self, kg_list):
+
+        kg_sizes = [len(kg) for kg in kg_list]
+        max_size = max(kg_sizes)
+        attention_mask = torch.ones((len(kg_list), max_size, max_size))
+
+        domain_list = []
+        for kg in kg_list:
+            node_dict = {}
+            for idx, node in enumerate(kg):
+                domain = node['domain']
+                if domain not in node_dict:
+                    node_dict[domain] = torch.ones(max_size)
+                    node_dict[domain][idx] = 0
+                else:
+                    node_dict[domain][idx] = 0
+
+            domain_list.append(node_dict)
+
+        for idx, kg in enumerate(kg_list):
+            for idx_n, node in enumerate(kg):
+                domain = node['domain']
+                attention_mask[idx, idx_n] = domain_list[idx][domain]
+            pad_size = max_size - len(kg)
+            attention_mask[idx, len(kg):, :] = torch.zeros(pad_size, max_size)
+
+        attention_mask = attention_mask.repeat(self.num_heads, 1, 1)
+
+        return attention_mask.bool().to(DEVICE)
+
+    def get_action_masks(self, actions):
+        # active domains
+        # active_domain_list = [set([node['domain'].lower() for node in kg] + ['general', 'booking']) for kg in kg_list]
+        # print("active domain list", active_domain_list)
+
+        action_targets = [self.action_embedder.real_action_to_small_action_list(act) for act in actions]
+        action_lengths = [len(actions) for actions in action_targets]
+        max_length = max(action_lengths)
+
+        semantic_acts = [self.action_embedder.real_action_to_small_action_list(act, semantic=True) for act in actions]
+        action_mask_list = []
+        decoder_encoder_mask_list = []
+        for i, act_sequence in tqdm(enumerate(semantic_acts)):
+            action_mask = [self.action_embedder.get_action_mask(start=True)]
+
+            for t, act in enumerate(act_sequence):
+
+                if t % 3 == 0:
+                    # We chose a domain
+                    if act == 'eos':
+                        break
+                    chosen_domain = act
+                    # focus only on the chosen domain information
+                    # TODO: Decoder encoder mask is unfinished, but I need to modify the self-attention mask for that
+                    # decoder_encoder_mask = [0 if node['domain'] in ['booking', 'general', chosen_domain] else 1
+                    #                        for node in kg_list[i]]
+                    # decoder_encoder_mask.append(decoder_encoder_mask)
+
+                    action_mask.append(self.action_embedder.get_action_mask(domain=act, start=False))
+                elif t % 3 == 1:
+                    # We chose an intent
+                    action_mask.append(self.action_embedder.get_action_mask(domain=chosen_domain,
+                                                                            intent=act, start=(t == 0)))
+                else:
+                    # We chose a slot-value pair
+                    action_mask.append(self.action_embedder.get_action_mask(start=False))
+
+            # pad action mask to get list of max_length
+            action_mask = torch.cat([
+                torch.stack(action_mask).to(DEVICE),
+                torch.zeros(max_length - len(action_mask), len(self.action_embedder.small_action_dict)).to(DEVICE)],
+                dim=0)
+            action_mask_list.append(action_mask)
+
+        action_mask_list = torch.stack(action_mask_list).to(DEVICE)
+        # print("semantic acts:", semantic_acts)
+        return action_mask_list, max_length
+
+
diff --git a/convlab/policy/vtrace_DPT/transformer_model/TransformerLayerCustom.py b/convlab/policy/vtrace_DPT/transformer_model/TransformerLayerCustom.py
new file mode 100644
index 0000000000000000000000000000000000000000..0414114acc58860d1391572f3d587247b4675c64
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/transformer_model/TransformerLayerCustom.py
@@ -0,0 +1,160 @@
+from torch.nn import TransformerEncoder, TransformerEncoderLayer, TransformerDecoderLayer, TransformerDecoder
+from torch import Tensor
+from typing import Optional
+from torch.nn import ModuleList
+from torch.nn import functional as F
+
+import copy
+
+
+class TransformerDecoderLayerCustom(TransformerDecoderLayer):
+
+    def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation="relu",
+                 layer_norm_eps=1e-5, batch_first=False, device=None, dtype=None, need_weights=False):
+        super().__init__(d_model, nhead, dim_feedforward, dropout, activation, layer_norm_eps, batch_first, device, dtype)
+        self.need_weights = need_weights
+
+    def forward(self, tgt: Tensor, memory: Tensor, tgt_mask: Optional[Tensor] = None, memory_mask: Optional[Tensor] = None,
+                tgt_key_padding_mask: Optional[Tensor] = None, memory_key_padding_mask: Optional[Tensor] = None):
+        r"""Pass the inputs (and mask) through the decoder layer.
+
+        Args:
+            tgt: the sequence to the decoder layer (required).
+            memory: the sequence from the last layer of the encoder (required).
+            tgt_mask: the mask for the tgt sequence (optional).
+            memory_mask: the mask for the memory sequence (optional).
+            tgt_key_padding_mask: the mask for the tgt keys per batch (optional).
+            memory_key_padding_mask: the mask for the memory keys per batch (optional).
+
+        Shape:
+            see the docs in Transformer class.
+        """
+        tgt2, self_att_weights = self.self_attn(tgt, tgt, tgt, attn_mask=tgt_mask,
+                              key_padding_mask=tgt_key_padding_mask, need_weights=self.need_weights)
+        tgt = tgt + self.dropout1(tgt2)
+        tgt = self.norm1(tgt)
+        tgt2, att_weights = self.multihead_attn(tgt, memory, memory, attn_mask=memory_mask,
+                                   key_padding_mask=memory_key_padding_mask, need_weights=self.need_weights)
+        tgt = tgt + self.dropout2(tgt2)
+        tgt = self.norm2(tgt)
+        tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt))))
+        tgt = tgt + self.dropout3(tgt2)
+        tgt = self.norm3(tgt)
+        return tgt, (self_att_weights, att_weights)
+
+
+class TransformerEncoderLayerCustom(TransformerEncoderLayer):
+
+    def __init__(self, d_model, nhead, dim_feedforward=2048, dropout=0.1, activation="relu",
+                 layer_norm_eps=1e-5, batch_first=False, device=None, dtype=None, need_weights=False):
+        super().__init__(d_model, nhead, dim_feedforward, dropout, activation, layer_norm_eps, batch_first, device, dtype)
+
+        self.need_weights = need_weights
+
+    def forward(self, src: Tensor, src_mask: Optional[Tensor] = None,
+                src_key_padding_mask: Optional[Tensor] = None):
+        r"""Pass the input through the encoder layer.
+
+        Args:
+            src: the sequence to the encoder layer (required).
+            src_mask: the mask for the src sequence (optional).
+            src_key_padding_mask: the mask for the src keys per batch (optional).
+
+        Shape:
+            see the docs in Transformer class.
+        """
+        src2, attention_weights = self.self_attn(src, src, src, attn_mask=src_mask,
+                              key_padding_mask=src_key_padding_mask, need_weights=self.need_weights)
+        src = src + self.dropout1(src2)
+        src = self.norm1(src)
+        src2 = self.linear2(self.dropout(self.activation(self.linear1(src))))
+        src = src + self.dropout2(src2)
+        src = self.norm2(src)
+        return src, attention_weights
+
+
+class TransformerEncoderCustom(TransformerEncoder):
+
+    def __init__(self, encoder_layer, num_layers, norm=None, need_weights=False):
+        super(TransformerEncoder, self).__init__()
+        self.layers = _get_clones(encoder_layer, num_layers)
+        self.num_layers = num_layers
+        self.norm = norm
+        self.need_weights = need_weights
+
+    def forward(self, src: Tensor, mask: Optional[Tensor] = None, src_key_padding_mask: Optional[Tensor] = None):
+        r"""Pass the input through the encoder layers in turn.
+
+        Args:
+            src: the sequence to the encoder (required).
+            mask: the mask for the src sequence (optional).
+            src_key_padding_mask: the mask for the src keys per batch (optional).
+
+        Shape:
+            see the docs in Transformer class.
+        """
+        output = src
+        attention_weights_list = []
+
+        for mod in self.layers:
+            output, attention_weights = mod(output, src_mask=mask, src_key_padding_mask=src_key_padding_mask)
+            attention_weights_list.append(attention_weights)
+
+        if self.norm is not None:
+            output = self.norm(output)
+
+        return output, attention_weights_list
+
+
+class TransformerDecoderCustom(TransformerDecoder):
+
+    def __init__(self, decoder_layer, num_layers, norm=None, need_weights=False):
+        super(TransformerDecoder, self).__init__()
+        self.layers = _get_clones(decoder_layer, num_layers)
+        self.num_layers = num_layers
+        self.norm = norm
+        self.need_weights = need_weights
+
+    def forward(self, tgt: Tensor, memory: Tensor, tgt_mask: Optional[Tensor] = None,
+                memory_mask: Optional[Tensor] = None, tgt_key_padding_mask: Optional[Tensor] = None,
+                memory_key_padding_mask: Optional[Tensor] = None):
+        r"""Pass the inputs (and mask) through the decoder layer in turn.
+
+        Args:
+            tgt: the sequence to the decoder (required).
+            memory: the sequence from the last layer of the encoder (required).
+            tgt_mask: the mask for the tgt sequence (optional).
+            memory_mask: the mask for the memory sequence (optional).
+            tgt_key_padding_mask: the mask for the tgt keys per batch (optional).
+            memory_key_padding_mask: the mask for the memory keys per batch (optional).
+
+        Shape:
+            see the docs in Transformer class.
+        """
+        output = tgt
+        att_weights_list = []
+
+        for mod in self.layers:
+            output, att_weights_tuple = mod(output, memory, tgt_mask=tgt_mask,
+                         memory_mask=memory_mask,
+                         tgt_key_padding_mask=tgt_key_padding_mask,
+                         memory_key_padding_mask=memory_key_padding_mask)
+            att_weights_list.append(att_weights_tuple)
+
+        if self.norm is not None:
+            output = self.norm(output)
+
+        return output, att_weights_list
+
+
+def _get_clones(module, N):
+    return ModuleList([copy.deepcopy(module) for i in range(N)])
+
+
+def _get_activation_fn(activation):
+    if activation == "relu":
+        return F.relu
+    elif activation == "gelu":
+        return F.gelu
+
+    raise RuntimeError("activation should be relu/gelu, not {}".format(activation))
\ No newline at end of file
diff --git a/convlab/policy/vtrace_DPT/transformer_model/__init__.py b/convlab/policy/vtrace_DPT/transformer_model/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/convlab/policy/vtrace_DPT/transformer_model/action_embedder.py b/convlab/policy/vtrace_DPT/transformer_model/action_embedder.py
new file mode 100644
index 0000000000000000000000000000000000000000..283443eb2daf55c56427e6de01f9cf6a2b4049cf
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/transformer_model/action_embedder.py
@@ -0,0 +1,292 @@
+import os
+import torch
+import torch.nn as nn
+import logging
+import json
+
+from copy import deepcopy
+from convlab.policy.vtrace_DPT.transformer_model.noisy_linear import NoisyLinear
+
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+class ActionEmbedder(nn.Module):
+    '''
+    Obtains the action-dictionary with all actions and creates embeddings for domain, intent and slot-value pairs
+    The embeddings are used for creating the domain, intent and slot-value actions in the EncoderDecoder
+    '''
+    def __init__(self, action_dict, embedding_dim, value_embedding_dim, action_embedding_dim, node_embedder=None,
+                 random_matrix=False, distance_metric=False):
+        super(ActionEmbedder, self).__init__()
+
+        self.domain_dict, self.intent_dict, self.slot_dict, self.value_dict, self.slot_value_dict \
+            = self.create_dicts(action_dict)
+
+        #EOS token is considered a "domain"
+        self.action_dict = dict((key.lower(), value) for key, value in action_dict.items())
+        self.action_dict_reversed = dict((value, key) for key, value in self.action_dict.items())
+        self.embed_domain = torch.randn(len(self.domain_dict), embedding_dim)
+        self.embed_intent = torch.randn(len(self.intent_dict), embedding_dim)
+        self.embed_slot = torch.randn(len(self.slot_dict), embedding_dim - value_embedding_dim)
+        self.embed_value = torch.randn(len(self.value_dict), value_embedding_dim)
+        self.embed_rest = torch.randn(1, embedding_dim)     #Pad token
+        self.use_random_matrix = random_matrix
+        self.distance_metric = distance_metric
+        self.forbidden_domains = []
+
+        if not node_embedder:
+            logging.info("We train action embeddings from scratch.")
+            self.action_embeddings, self.small_action_dict = self.create_action_embeddings(embedding_dim)
+            self.action_embeddings.requires_grad = True
+            self.action_embeddings = nn.Parameter(self.action_embeddings)
+        else:
+            logging.info("We use Roberta to embed actions.")
+            self.dataset_name = node_embedder.dataset_name
+            self.create_action_embeddings_roberta(node_embedder)
+            self.action_embeddings.requires_grad = False
+            embedding_dim = 768
+
+        #logging.info(f"Small Action Dict: {self.small_action_dict}")
+
+        self.small_action_dict_reversed = dict((value, key) for key, value in self.small_action_dict.items())
+
+        self.linear = torch.nn.Linear(embedding_dim, action_embedding_dim).to(DEVICE)
+        #self.linear = NoisyLinear(embedding_dim, action_embedding_dim).to(DEVICE)
+        self.random_matrix = torch.randn(embedding_dim, action_embedding_dim).to(DEVICE) / \
+                             torch.sqrt(torch.Tensor([768])).to(DEVICE)
+
+    def action_projector(self, actions):
+        if self.use_random_matrix:
+            return torch.matmul(actions, self.random_matrix).to(DEVICE)
+        else:
+            return self.linear(actions)
+
+    def forward(self, state):
+        # state [batch-size, action_dim], self.action_embeddings [num_actions, embedding_dim]
+        action_embeddings = self.action_projector(self.action_embeddings)
+
+        if not self.distance_metric:
+            # We use scalar product for similarity
+            output = torch.matmul(state, action_embeddings.permute(1, 0))
+        else:
+            # We use distance metric for similarity as in SUMBT
+            output = -torch.cdist(state, action_embeddings, p=2)
+
+        return output
+
+    def get_legal_mask(self, legal_mask, domain="", intent=""):
+
+        if legal_mask is None:
+            return torch.zeros(len(self.small_action_dict)).to(DEVICE)
+
+        action_mask = torch.ones(len(self.small_action_dict))
+        if not domain:
+            for domain in self.domain_dict:
+                # check whether we can use that domain, at the moment we want to allow all domains
+                action_mask[self.small_action_dict[domain]] = 0
+        elif not intent:
+            # Domain was selected, check intents that are allowed
+            for intent in self.intent_dict:
+                domain_intent = f"{domain}-{intent}"
+                for idx, not_allow in enumerate(legal_mask):
+                    semantic_act = self.action_dict_reversed[idx]
+                    if domain_intent in semantic_act and not_allow == 0:
+                        action_mask[self.small_action_dict[intent]] = 0
+                        break
+        else:
+            # Selected domain and intent, need slot-value
+            for slot_value in self.slot_value_dict:
+                domain_intent_slot = f"{domain}-{intent}-{slot_value}"
+                for idx, not_allow in enumerate(legal_mask):
+                    semantic_act = self.action_dict_reversed[idx]
+                    if domain_intent_slot in semantic_act and not_allow == 0:
+                        action_mask[self.small_action_dict[slot_value]] = 0
+                        break
+
+        return action_mask.to(DEVICE)
+
+    def get_action_mask(self, domain=None, intent="", start=False):
+
+        action_mask = torch.ones(len(self.small_action_dict))
+
+        # This is for predicting end of sequence token <eos>
+        if not start and domain is None:
+            action_mask[self.small_action_dict['eos']] = 0
+
+        if domain is None:
+            #TODO: I allow all domains now for checking supervised training
+            for domain in self.domain_dict:
+                if domain not in self.forbidden_domains:
+                    action_mask[self.small_action_dict[domain]] = 0
+            if start:
+                action_mask[self.small_action_dict['eos']] = 1
+            # Only active domains can be selected
+            #for domain in active_domains:
+            #    action_mask[self.small_action_dict[domain]] = 0
+
+        elif not intent:
+            # Domain was selected, need intent now
+            for intent in self.intent_dict:
+                domain_intent = f"{domain}-{intent}"
+                valid = self.is_valid(domain_intent + "-")
+                if valid:
+                    action_mask[self.small_action_dict[intent]] = 0
+        else:
+            # Selected domain and intent, need slot-value
+            for slot_value in self.slot_value_dict:
+                domain_intent_slot = f"{domain}-{intent}-{slot_value}"
+                valid = self.is_valid(domain_intent_slot)
+                if valid:
+                    action_mask[self.small_action_dict[slot_value]] = 0
+
+        assert not torch.equal(action_mask, torch.ones(len(self.small_action_dict)))
+
+        return action_mask.to(DEVICE)
+
+    def get_current_domain_mask(self, current_domains, current=True):
+
+        action_mask = torch.ones(len(self.small_action_dict))
+        if current:
+            for domain in current_domains:
+                action_mask[self.small_action_dict[domain]] = 0
+        else:
+            for domain in self.domain_dict:
+                if domain not in current_domains:
+                    action_mask[self.small_action_dict[domain]] = 0
+
+        return action_mask.to(DEVICE)
+
+    def is_valid(self, part_action):
+
+        for act in self.action_dict:
+            if act.startswith(part_action):
+                return True
+
+        return False
+
+    def create_action_embeddings(self, embedding_dim):
+
+        action_embeddings = torch.zeros((len(self.domain_dict) + len(self.intent_dict) + len(self.slot_value_dict) + 1,
+                                         embedding_dim))
+
+        small_action_dict = {}
+        for domain, idx in self.domain_dict.items():
+            action_embeddings[len(small_action_dict)] = self.embed_domain[idx]
+            small_action_dict[domain] = len(small_action_dict)
+        for intent, idx in self.intent_dict.items():
+            action_embeddings[len(small_action_dict)] = self.embed_intent[idx]
+            small_action_dict[intent] = len(small_action_dict)
+        for slot_value in self.slot_value_dict:
+            slot, value = slot_value.split("-")
+            slot_idx = self.slot_dict[slot]
+            value_idx = self.value_dict[value]
+            action_embeddings[len(small_action_dict)] = torch.cat(
+                (self.embed_slot[slot_idx], self.embed_value[value_idx]))
+            small_action_dict[slot_value] = len(small_action_dict)
+
+        action_embeddings[len(small_action_dict)] = self.embed_rest[0]      #add the PAD token
+        small_action_dict['pad'] = len(small_action_dict)
+        return action_embeddings.to(DEVICE), small_action_dict
+
+    def create_action_embeddings_roberta(self, node_embedder):
+
+        action_embeddings = []
+
+        small_action_dict = {}
+        for domain, idx in self.domain_dict.items():
+            action_embeddings.append(domain)
+            small_action_dict[domain] = len(small_action_dict)
+        for intent, idx in self.intent_dict.items():
+            action_embeddings.append(intent)
+            small_action_dict[intent] = len(small_action_dict)
+        for slot_value in self.slot_value_dict:
+            slot, value = slot_value.split("-")
+            action_embeddings.append(f"{slot} {value}")
+            small_action_dict[slot_value] = len(small_action_dict)
+
+        action_embeddings.append("pad")     #add the PAD token
+        small_action_dict['pad'] = len(small_action_dict)
+
+        action_embeddings_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                              f'action_embeddings_{self.dataset_name}.pt')
+        small_action_dict_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                              f'small_action_dict_{self.dataset_name}.json')
+
+        if os.path.exists(action_embeddings_path):
+            self.action_embeddings = torch.load(action_embeddings_path).to(DEVICE)
+        else:
+            self.action_embeddings = node_embedder.embed_sentences(action_embeddings).to(DEVICE)
+            torch.save(self.action_embeddings, action_embeddings_path)
+
+        if os.path.exists(small_action_dict_path):
+            self.small_action_dict = json.load(open(small_action_dict_path, 'r'))
+        else:
+            self.small_action_dict = small_action_dict
+            with open(small_action_dict_path, 'w') as f:
+                json.dump(self.small_action_dict, f)
+
+        self.small_action_dict = small_action_dict
+
+    def create_dicts(self, action_dict):
+        domain_dict = {}
+        intent_dict = {}
+        slot_dict = {}
+        value_dict = {}
+        slot_value_dict = {}
+        for action in action_dict:
+            domain, intent, slot, value = [act.lower() for act in action.split('-')]
+            if domain not in domain_dict:
+                domain_dict[domain] = len(domain_dict)
+            if intent not in intent_dict:
+                intent_dict[intent] = len(intent_dict)
+            if slot not in slot_dict:
+                slot_dict[slot] = len(slot_dict)
+            if value not in value_dict:
+                value_dict[value] = len(value_dict)
+            if slot + "-" + value not in slot_value_dict:
+                slot_value_dict[slot + "-" + value] = len(slot_value_dict)
+
+        domain_dict['eos'] = len(domain_dict)
+
+        return domain_dict, intent_dict, slot_dict, value_dict, slot_value_dict
+
+    def small_action_list_to_real_actions(self, small_action_list):
+
+        #print("SMALL ACTION LIST:", small_action_list)
+        action_vector = torch.zeros(len(self.action_dict))
+        act_string = ""
+        for idx, act in enumerate(small_action_list):
+            if act == 'eos':
+                break
+
+            if idx % 3 != 2:
+                act_string += f"{act}-"
+            else:
+                act_string += act
+                action_vector[self.action_dict[act_string]] = 1
+                act_string = ""
+
+        return action_vector
+
+    def real_action_to_small_action_list(self, action, semantic=False, permute=False):
+        '''
+        :param action: [hotel-req-address, taxi-inform-phone]
+        :return: [hotel, req, address, taxi, inform, phone, eos]
+        '''
+
+        action_list = []
+        for idx, i in enumerate(action):
+            if i == 1:
+                action_list += self.action_dict_reversed[idx].split("-", 2)
+
+        if permute and len(action_list) > 3:
+            action_list_new = deepcopy(action_list[-3:]) + deepcopy(action_list[:-3])
+            action_list = action_list_new
+        action_list.append("eos")
+
+        if semantic:
+            return action_list
+
+        action_list = [self.small_action_dict[act] for act in action_list]
+        return action_list
\ No newline at end of file
diff --git a/convlab/policy/vtrace_DPT/transformer_model/action_embeddings_multiwoz21.pt b/convlab/policy/vtrace_DPT/transformer_model/action_embeddings_multiwoz21.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2f7e5e9cdf949b082b9ae752c09d2eafeb322bfb
Binary files /dev/null and b/convlab/policy/vtrace_DPT/transformer_model/action_embeddings_multiwoz21.pt differ
diff --git a/convlab/policy/vtrace_DPT/transformer_model/action_embeddings_sgd.pt b/convlab/policy/vtrace_DPT/transformer_model/action_embeddings_sgd.pt
new file mode 100644
index 0000000000000000000000000000000000000000..67e557ce5ef7a0bfd40c60ae5a03937b47b92de2
Binary files /dev/null and b/convlab/policy/vtrace_DPT/transformer_model/action_embeddings_sgd.pt differ
diff --git a/convlab/policy/vtrace_DPT/transformer_model/embedded_descriptions_base_multiwoz21.pt b/convlab/policy/vtrace_DPT/transformer_model/embedded_descriptions_base_multiwoz21.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5c6193163b6a1fc788b959eff2d0153b3a4bf7cb
Binary files /dev/null and b/convlab/policy/vtrace_DPT/transformer_model/embedded_descriptions_base_multiwoz21.pt differ
diff --git a/convlab/policy/vtrace_DPT/transformer_model/embedded_descriptions_base_sgd.pt b/convlab/policy/vtrace_DPT/transformer_model/embedded_descriptions_base_sgd.pt
new file mode 100644
index 0000000000000000000000000000000000000000..619824588654a36cab3bf795e6fe94527b04ba68
Binary files /dev/null and b/convlab/policy/vtrace_DPT/transformer_model/embedded_descriptions_base_sgd.pt differ
diff --git a/convlab/policy/vtrace_DPT/transformer_model/node_embedder.py b/convlab/policy/vtrace_DPT/transformer_model/node_embedder.py
new file mode 100644
index 0000000000000000000000000000000000000000..ab57df122369e9457c78e04fe2e64eab389d5abd
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/transformer_model/node_embedder.py
@@ -0,0 +1,123 @@
+import os, json, logging
+import torch
+import torch.nn as nn
+
+from transformers import RobertaTokenizer, RobertaModel
+from convlab.policy.vtrace_DPT.transformer_model.noisy_linear import NoisyLinear
+from convlab.policy.vtrace_DPT.create_descriptions import create_description_dicts
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+class NodeEmbedderRoberta(nn.Module):
+    '''
+    Class to build node embeddings
+    Nodes have attributes: Description, value and node type that are used for building embedding
+    '''
+
+    def __init__(self, projection_dim, freeze_roberta=True, use_pooled=False, max_length=25, roberta_path="",
+                 description_dict=None, semantic_descriptions=True, mean=False, dataset_name="multiwoz21"):
+        super(NodeEmbedderRoberta, self).__init__()
+
+        self.dataset_name = dataset_name
+        self.max_length = max_length
+        self.description_size = 768
+        self.projection_dim = projection_dim
+        self.feature_projection = torch.nn.Linear(2 * self.description_size, projection_dim).to(DEVICE)
+        #self.feature_projection = NoisyLinear(2 * self.description_size, projection_dim).to(DEVICE)
+        self.value_embedding = torch.nn.Linear(1, self.description_size).to(DEVICE)
+
+        self.semantic_descriptions = semantic_descriptions
+        self.init_description_dict()
+
+        self.description2idx = dict((descr, i) for i, descr in enumerate(self.description_dict))
+        self.idx2description = dict((i, descr) for descr, i in self.description2idx.items())
+        self.use_pooled = use_pooled
+        self.mean = mean
+        self.embedded_descriptions = None
+
+        if roberta_path:
+            embedded_descriptions_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                                      f'embedded_descriptions_{self.dataset_name}.pt')
+            if os.path.exists(embedded_descriptions_path):
+                self.embedded_descriptions = torch.load(embedded_descriptions_path).to(DEVICE)
+            else:
+                logging.info(f"Loading Roberta from path {roberta_path}")
+                self.tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
+                self.roberta_model = RobertaModel.from_pretrained(roberta_path).to(DEVICE)
+
+        else:
+            embedded_descriptions_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                                                      f'embedded_descriptions_base_{self.dataset_name}.pt')
+            if os.path.exists(embedded_descriptions_path):
+                self.embedded_descriptions = torch.load(embedded_descriptions_path).to(DEVICE)
+            else:
+                self.tokenizer = RobertaTokenizer.from_pretrained("roberta-base")
+                self.roberta_model = RobertaModel.from_pretrained("roberta-base").to(DEVICE)
+
+        if self.embedded_descriptions is None:
+            if freeze_roberta:
+                for param in self.roberta_model.parameters():
+                    param.requires_grad = False
+            #We embed descriptions beforehand and only make a lookup for better efficiency
+            self.form_embedded_descriptions()
+            torch.save(self.embedded_descriptions, embedded_descriptions_path)
+
+        logging.info(f"Embedding semantic descriptions: {semantic_descriptions}")
+        logging.info(f"Embedded descriptions successfully. Size: {self.embedded_descriptions.size()}")
+        logging.info(f"Data set used for descriptions: {dataset_name}")
+
+    def form_embedded_descriptions(self):
+
+        self.embedded_descriptions = self.embed_sentences(
+            [self.description_dict[self.idx2description[i]] for i in range(len(self.description_dict))])
+
+    def description_2_idx(self, kg_info):
+        embedded_descriptions_idx = torch.Tensor([self.description2idx[node["description"]] for node in kg_info])\
+            .long()
+        return embedded_descriptions_idx
+
+    def forward(self, description_idx, values):
+
+        #embedded_descriptions = torch.stack(
+        #    [self.embedded_descriptions[idx] for idx in description_idx]).to(DEVICE)
+        embedded_descriptions = self.embedded_descriptions[description_idx]
+        description_value_tensor = torch.cat((self.value_embedding(values),
+                                                  embedded_descriptions), dim=-1).to(DEVICE)
+
+        node_embedding = self.feature_projection(description_value_tensor).to(DEVICE)
+
+        return node_embedding
+
+    def embed_sentences(self, sentences):
+
+        tokenized = [self.tokenizer.encode_plus(sen, add_special_tokens=True, max_length=self.max_length,
+                                                padding='max_length') for sen in sentences]
+
+        input_ids = torch.Tensor([feat['input_ids'] for feat in tokenized]).long().to(DEVICE)
+        attention_mask = torch.Tensor([feat['attention_mask'] for feat in tokenized]).long().to(DEVICE)
+
+        roberta_output = self.roberta_model(input_ids, attention_mask)
+        output_states = roberta_output.last_hidden_state
+        pooled = roberta_output.pooler_output
+
+        if self.mean:
+            length_mask = torch.Tensor([[1 if id_ != 1 else 0 for id_ in ids] for ids in input_ids]).unsqueeze(-1)\
+                .to(DEVICE)
+            output = (output_states * length_mask).sum(dim=1) / length_mask.sum(dim=1)
+        else:
+            output = pooled if self.use_pooled else output_states[:, 0, :]
+
+        return output
+
+    def init_description_dict(self):
+
+        create_description_dicts(self.dataset_name)
+        root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        if self.semantic_descriptions:
+            path = os.path.join(root_dir, f'descriptions/semantic_information_descriptions_{self.dataset_name}.json')
+        else:
+            path = os.path.join(root_dir, 'information_descriptions.json')
+        with open(path, "r") as f:
+            self.description_dict = json.load(f)
+
diff --git a/convlab/policy/vtrace_DPT/transformer_model/noisy_linear.py b/convlab/policy/vtrace_DPT/transformer_model/noisy_linear.py
new file mode 100644
index 0000000000000000000000000000000000000000..90fcb9fd38b07f4dc2a0ee60833c4d1803812e54
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/transformer_model/noisy_linear.py
@@ -0,0 +1,34 @@
+import math
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class NoisyLinear(nn.Linear):
+
+    def __init__(self, in_features, out_features, sigma_init=0.017, bias=True):
+        super(NoisyLinear, self).__init__(in_features, out_features, bias=bias)
+        w = torch.full((out_features, in_features), sigma_init)
+        self.sigma_weight = nn.Parameter(w)
+        z = torch.zeros(out_features, in_features)
+        self.register_buffer("epsilon_weight", z)
+        if bias:
+            w = torch.full((out_features,), sigma_init)
+            self.sigma_bias = nn.Parameter(w)
+            z = torch.zeros(out_features)
+            self.register_buffer("epsilon_bias", z)
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        std = math.sqrt(3 / self.in_features)
+        self.weight.data.uniform_(-std, std)
+        self.bias.data.uniform_(-std, std)
+
+    def forward(self, input):
+        self.epsilon_weight.normal_()
+        bias = self.bias
+        if bias is not None:
+            self.epsilon_bias.normal_()
+            bias = bias + self.sigma_bias * self.epsilon_bias.data
+        v = self.sigma_weight * self.epsilon_weight.data + self.weight
+        return F.linear(input, v, bias)
\ No newline at end of file
diff --git a/convlab/policy/vtrace_DPT/transformer_model/small_action_dict.json b/convlab/policy/vtrace_DPT/transformer_model/small_action_dict.json
new file mode 100644
index 0000000000000000000000000000000000000000..0d5bd2002fa7d0082e7589b80ae3664781732ece
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/transformer_model/small_action_dict.json
@@ -0,0 +1 @@
+{"attraction": 0, "general": 1, "hospital": 2, "hotel": 3, "police": 4, "restaurant": 5, "taxi": 6, "train": 7, "eos": 8, "inform": 9, "nooffer": 10, "recommend": 11, "request": 12, "select": 13, "bye": 14, "greet": 15, "reqmore": 16, "welcome": 17, "book": 18, "offerbook": 19, "nobook": 20, "address-1": 21, "address-2": 22, "address-3": 23, "area-1": 24, "area-2": 25, "area-3": 26, "choice-1": 27, "choice-2": 28, "choice-3": 29, "entrance fee-1": 30, "entrance fee-2": 31, "name-1": 32, "name-2": 33, "name-3": 34, "name-4": 35, "phone-1": 36, "postcode-1": 37, "type-1": 38, "type-2": 39, "type-3": 40, "type-4": 41, "type-5": 42, "none-none": 43, "area-?": 44, "entrance fee-?": 45, "name-?": 46, "type-?": 47, "department-1": 48, "department-?": 49, "book day-1": 50, "book people-1": 51, "book stay-1": 52, "internet-1": 53, "parking-1": 54, "price range-1": 55, "price range-2": 56, "ref-1": 57, "stars-1": 58, "stars-2": 59, "book day-?": 60, "book people-?": 61, "book stay-?": 62, "internet-?": 63, "parking-?": 64, "price range-?": 65, "stars-?": 66, "book time-1": 67, "food-1": 68, "food-2": 69, "food-3": 70, "food-4": 71, "postcode-2": 72, "book time-?": 73, "food-?": 74, "arrive by-1": 75, "departure-1": 76, "destination-1": 77, "leave at-1": 78, "arrive by-?": 79, "departure-?": 80, "destination-?": 81, "leave at-?": 82, "arrive by-2": 83, "day-1": 84, "duration-1": 85, "leave at-2": 86, "leave at-3": 87, "price-1": 88, "train id-1": 89, "day-?": 90, "pad": 91}
\ No newline at end of file
diff --git a/convlab/policy/vtrace_DPT/transformer_model/small_action_dict_multiwoz21.json b/convlab/policy/vtrace_DPT/transformer_model/small_action_dict_multiwoz21.json
new file mode 100644
index 0000000000000000000000000000000000000000..0d5bd2002fa7d0082e7589b80ae3664781732ece
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/transformer_model/small_action_dict_multiwoz21.json
@@ -0,0 +1 @@
+{"attraction": 0, "general": 1, "hospital": 2, "hotel": 3, "police": 4, "restaurant": 5, "taxi": 6, "train": 7, "eos": 8, "inform": 9, "nooffer": 10, "recommend": 11, "request": 12, "select": 13, "bye": 14, "greet": 15, "reqmore": 16, "welcome": 17, "book": 18, "offerbook": 19, "nobook": 20, "address-1": 21, "address-2": 22, "address-3": 23, "area-1": 24, "area-2": 25, "area-3": 26, "choice-1": 27, "choice-2": 28, "choice-3": 29, "entrance fee-1": 30, "entrance fee-2": 31, "name-1": 32, "name-2": 33, "name-3": 34, "name-4": 35, "phone-1": 36, "postcode-1": 37, "type-1": 38, "type-2": 39, "type-3": 40, "type-4": 41, "type-5": 42, "none-none": 43, "area-?": 44, "entrance fee-?": 45, "name-?": 46, "type-?": 47, "department-1": 48, "department-?": 49, "book day-1": 50, "book people-1": 51, "book stay-1": 52, "internet-1": 53, "parking-1": 54, "price range-1": 55, "price range-2": 56, "ref-1": 57, "stars-1": 58, "stars-2": 59, "book day-?": 60, "book people-?": 61, "book stay-?": 62, "internet-?": 63, "parking-?": 64, "price range-?": 65, "stars-?": 66, "book time-1": 67, "food-1": 68, "food-2": 69, "food-3": 70, "food-4": 71, "postcode-2": 72, "book time-?": 73, "food-?": 74, "arrive by-1": 75, "departure-1": 76, "destination-1": 77, "leave at-1": 78, "arrive by-?": 79, "departure-?": 80, "destination-?": 81, "leave at-?": 82, "arrive by-2": 83, "day-1": 84, "duration-1": 85, "leave at-2": 86, "leave at-3": 87, "price-1": 88, "train id-1": 89, "day-?": 90, "pad": 91}
\ No newline at end of file
diff --git a/convlab/policy/vtrace_DPT/transformer_model/small_action_dict_sgd.json b/convlab/policy/vtrace_DPT/transformer_model/small_action_dict_sgd.json
new file mode 100644
index 0000000000000000000000000000000000000000..c48573262f50cabe1d4cd3811638ebf4c046a7e0
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/transformer_model/small_action_dict_sgd.json
@@ -0,0 +1 @@
+{"": 0, "alarm_1": 1, "banks_1": 2, "banks_2": 3, "buses_1": 4, "buses_2": 5, "buses_3": 6, "calendar_1": 7, "events_1": 8, "events_2": 9, "events_3": 10, "flights_1": 11, "flights_2": 12, "flights_3": 13, "flights_4": 14, "homes_1": 15, "homes_2": 16, "hotels_1": 17, "hotels_2": 18, "hotels_3": 19, "hotels_4": 20, "media_1": 21, "media_2": 22, "media_3": 23, "messaging_1": 24, "movies_1": 25, "movies_2": 26, "movies_3": 27, "music_1": 28, "music_2": 29, "music_3": 30, "payment_1": 31, "rentalcars_1": 32, "rentalcars_2": 33, "rentalcars_3": 34, "restaurants_1": 35, "restaurants_2": 36, "ridesharing_1": 37, "ridesharing_2": 38, "services_1": 39, "services_2": 40, "services_3": 41, "services_4": 42, "trains_1": 43, "travel_1": 44, "weather_1": 45, "eos": 46, "goodbye": 47, "req_more": 48, "confirm": 49, "inform_count": 50, "notify_success": 51, "offer": 52, "offer_intent": 53, "request": 54, "inform": 55, "notify_failure": 56, "none-none": 57, "new_alarm_name-1": 58, "new_alarm_time-1": 59, "count-1": 60, "alarm_name-1": 61, "alarm_time-1": 62, "addalarm-1": 63, "new_alarm_time-?": 64, "account_type-1": 65, "amount-1": 66, "recipient_account_name-1": 67, "recipient_account_type-1": 68, "balance-1": 69, "transfermoney-1": 70, "account_type-?": 71, "amount-?": 72, "recipient_account_name-?": 73, "recipient_name-1": 74, "transfer_amount-1": 75, "transfer_time-1": 76, "account_balance-1": 77, "recipient_name-?": 78, "transfer_amount-?": 79, "from_location-1": 80, "leaving_date-1": 81, "leaving_time-1": 82, "to_location-1": 83, "travelers-1": 84, "from_station-1": 85, "to_station-1": 86, "transfers-1": 87, "fare-1": 88, "buybusticket-1": 89, "from_location-?": 90, "leaving_date-?": 91, "leaving_time-?": 92, "to_location-?": 93, "travelers-?": 94, "departure_date-1": 95, "departure_time-1": 96, "destination-1": 97, "fare_type-1": 98, "group_size-1": 99, "origin-1": 100, "destination_station_name-1": 101, "origin_station_name-1": 102, "price-1": 103, "departure_date-?": 104, "departure_time-?": 105, "destination-?": 106, "group_size-?": 107, "origin-?": 108, "additional_luggage-1": 109, "from_city-1": 110, "num_passengers-1": 111, "to_city-1": 112, "category-1": 113, "from_city-?": 114, "num_passengers-?": 115, "to_city-?": 116, "event_date-1": 117, "event_location-1": 118, "event_name-1": 119, "event_time-1": 120, "available_end_time-1": 121, "available_start_time-1": 122, "addevent-1": 123, "event_date-?": 124, "event_location-?": 125, "event_name-?": 126, "event_time-?": 127, "city_of_event-1": 128, "date-1": 129, "number_of_seats-1": 130, "address_of_location-1": 131, "subcategory-1": 132, "time-1": 133, "buyeventtickets-1": 134, "category-?": 135, "city_of_event-?": 136, "date-?": 137, "number_of_seats-?": 138, "city-1": 139, "number_of_tickets-1": 140, "venue-1": 141, "venue_address-1": 142, "city-?": 143, "event_type-?": 144, "number_of_tickets-?": 145, "price_per_ticket-1": 146, "airlines-1": 147, "destination_city-1": 148, "inbound_departure_time-1": 149, "origin_city-1": 150, "outbound_departure_time-1": 151, "passengers-1": 152, "return_date-1": 153, "seating_class-1": 154, "destination_airport-1": 155, "inbound_arrival_time-1": 156, "number_stops-1": 157, "origin_airport-1": 158, "outbound_arrival_time-1": 159, "refundable-1": 160, "reserveonewayflight-1": 161, "reserveroundtripflights-1": 162, "airlines-?": 163, "destination_city-?": 164, "inbound_departure_time-?": 165, "origin_city-?": 166, "outbound_departure_time-?": 167, "return_date-?": 168, "is_redeye-1": 169, "arrives_next_day-1": 170, "destination_airport_name-1": 171, "origin_airport_name-1": 172, "is_nonstop-1": 173, "destination_airport-?": 174, "origin_airport-?": 175, "property_name-1": 176, "visit_date-1": 177, "furnished-1": 178, "pets_allowed-1": 179, "phone_number-1": 180, "address-1": 181, "number_of_baths-1": 182, "number_of_beds-1": 183, "rent-1": 184, "schedulevisit-1": 185, "area-?": 186, "number_of_beds-?": 187, "visit_date-?": 188, "has_garage-1": 189, "in_unit_laundry-1": 190, "intent-?": 191, "number_of_baths-?": 192, "check_in_date-1": 193, "hotel_name-1": 194, "number_of_days-1": 195, "number_of_rooms-1": 196, "has_wifi-1": 197, "price_per_night-1": 198, "street_address-1": 199, "star_rating-1": 200, "reservehotel-1": 201, "check_in_date-?": 202, "hotel_name-?": 203, "number_of_days-?": 204, "check_out_date-1": 205, "number_of_adults-1": 206, "where_to-1": 207, "has_laundry_service-1": 208, "total_price-1": 209, "rating-1": 210, "bookhouse-1": 211, "check_out_date-?": 212, "number_of_adults-?": 213, "where_to-?": 214, "location-1": 215, "pets_welcome-1": 216, "average_rating-1": 217, "location-?": 218, "place_name-1": 219, "stay_length-1": 220, "smoking_allowed-1": 221, "stay_length-?": 222, "subtitles-1": 223, "title-1": 224, "directed_by-1": 225, "genre-1": 226, "title-2": 227, "title-3": 228, "playmovie-1": 229, "genre-?": 230, "title-?": 231, "movie_name-1": 232, "subtitle_language-1": 233, "movie_name-2": 234, "movie_name-3": 235, "rentmovie-1": 236, "starring-1": 237, "contact_name-1": 238, "contact_name-?": 239, "show_date-1": 240, "show_time-1": 241, "show_type-1": 242, "theater_name-1": 243, "buymovietickets-1": 244, "movie_name-?": 245, "show_date-?": 246, "show_time-?": 247, "show_type-?": 248, "aggregate_rating-1": 249, "cast-1": 250, "movie_title-1": 251, "percent_rating-1": 252, "playback_device-1": 253, "song_name-1": 254, "album-1": 255, "year-1": 256, "artist-1": 257, "playsong-1": 258, "song_name-?": 259, "playmedia-1": 260, "device-1": 261, "track-1": 262, "payment_method-1": 263, "private_visibility-1": 264, "receiver-1": 265, "payment_method-?": 266, "receiver-?": 267, "dropoff_date-1": 268, "pickup_date-1": 269, "pickup_location-1": 270, "pickup_time-1": 271, "type-1": 272, "car_name-1": 273, "reservecar-1": 274, "dropoff_date-?": 275, "pickup_city-?": 276, "pickup_date-?": 277, "pickup_location-?": 278, "pickup_time-?": 279, "type-?": 280, "car_type-1": 281, "car_type-?": 282, "add_insurance-1": 283, "end_date-1": 284, "start_date-1": 285, "price_per_day-1": 286, "add_insurance-?": 287, "end_date-?": 288, "start_date-?": 289, "party_size-1": 290, "restaurant_name-1": 291, "cuisine-1": 292, "has_live_music-1": 293, "price_range-1": 294, "serves_alcohol-1": 295, "reserverestaurant-1": 296, "cuisine-?": 297, "restaurant_name-?": 298, "time-?": 299, "has_seating_outdoors-1": 300, "has_vegetarian_options-1": 301, "number_of_riders-1": 302, "shared_ride-1": 303, "approximate_ride_duration-1": 304, "ride_fare-1": 305, "number_of_riders-?": 306, "shared_ride-?": 307, "ride_type-1": 308, "wait_time-1": 309, "ride_type-?": 310, "appointment_date-1": 311, "appointment_time-1": 312, "stylist_name-1": 313, "is_unisex-1": 314, "bookappointment-1": 315, "appointment_date-?": 316, "appointment_time-?": 317, "dentist_name-1": 318, "offers_cosmetic_services-1": 319, "doctor_name-1": 320, "therapist_name-1": 321, "class-1": 322, "date_of_journey-1": 323, "from-1": 324, "journey_start_time-1": 325, "to-1": 326, "trip_protection-1": 327, "total-1": 328, "gettraintickets-1": 329, "date_of_journey-?": 330, "from-?": 331, "to-?": 332, "trip_protection-?": 333, "free_entry-1": 334, "good_for_kids-1": 335, "attraction_name-1": 336, "humidity-1": 337, "wind-1": 338, "precipitation-1": 339, "temperature-1": 340, "pad": 341}
\ No newline at end of file
diff --git a/convlab/policy/vtrace_DPT/transformer_model/transformer.py b/convlab/policy/vtrace_DPT/transformer_model/transformer.py
new file mode 100644
index 0000000000000000000000000000000000000000..bb26a5319f1ddc6feeccf35b4096ec6f934efde5
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/transformer_model/transformer.py
@@ -0,0 +1,99 @@
+import math
+import torch
+
+from torch import nn, Tensor
+from .TransformerLayerCustom import TransformerEncoderCustom, TransformerEncoderLayerCustom, \
+    TransformerDecoderLayerCustom, TransformerDecoderCustom
+
+
+class TransformerModelEncoder(nn.Module):
+
+    def __init__(self, d_model: int, nhead: int, d_hid: int, nlayers: int, dropout: float = 0.5, need_weights=False):
+        super().__init__()
+        self.model_type = 'TransformerEncoder'
+        self.pos_encoder = PositionalEncoding(d_model, dropout)
+        encoder_layers = TransformerEncoderLayerCustom(d_model, nhead, d_hid, dropout, need_weights=need_weights)
+        self.encoder = TransformerEncoderCustom(encoder_layers, nlayers, need_weights=need_weights)
+        self.d_model = d_model
+
+        #self.init_weights()
+
+    def init_weights(self) -> None:
+        initrange = 0.1
+        self.encoder.weight.data.uniform_(-initrange, initrange)
+        self.decoder.bias.data.zero_()
+        self.decoder.weight.data.uniform_(-initrange, initrange)
+
+    def forward(self, src, mask=None, src_key_padding_mask=None):
+        """
+        Args:
+            src: Tensor, shape [seq_len, batch_size]
+            src_mask: Tensor, shape [seq_len, seq_len]
+
+        Returns:
+            output Tensor of shape [seq_len, batch_size, ntoken]
+        """
+        src = self.pos_encoder(src)
+        output, attention_weights = self.encoder(src, mask=mask, src_key_padding_mask=src_key_padding_mask)
+        return output, attention_weights
+
+
+class TransformerModelDecoder(nn.Module):
+
+    def __init__(self, d_model: int, nhead: int, d_hid: int, nlayers: int, dropout: float = 0.5, need_weights=False):
+        super().__init__()
+        self.model_type = 'TransformerDecoder'
+        self.pos_encoder = PositionalEncoding(d_model, dropout)
+        decoder_layers = TransformerDecoderLayerCustom(d_model, nhead, d_hid, dropout, need_weights=need_weights)
+        self.decoder = TransformerDecoderCustom(decoder_layers, nlayers, need_weights=need_weights)
+        self.d_model = d_model
+
+        #self.init_weights()
+
+    def init_weights(self) -> None:
+        initrange = 0.1
+        self.encoder.weight.data.uniform_(-initrange, initrange)
+        #self.decoder.bias.data.zero_()
+        #self.decoder.weight.data.uniform_(-initrange, initrange)
+
+    def forward(self, decoder_input, encoder_output, tgt_mask=None, memory_key_padding_mask=None):
+        """
+        Args:
+            src: Tensor, shape [seq_len, batch_size]
+            src_mask: Tensor, shape [seq_len, seq_len]
+
+        Returns:
+            output Tensor of shape [seq_len, batch_size, ntoken]
+        """
+        decoder_input = self.pos_encoder(decoder_input)
+        output, att_weights = self.decoder(tgt=decoder_input, memory=encoder_output, tgt_mask=tgt_mask,
+                                          tgt_key_padding_mask=None,
+                                          memory_key_padding_mask=memory_key_padding_mask)
+        return output, att_weights
+
+
+def generate_square_subsequent_mask(sz: int) -> Tensor:
+    """Generates an upper-triangular matrix of -inf, with zeros on diag."""
+    return torch.triu(torch.ones(sz, sz) * float('-inf'), diagonal=1)
+
+
+class PositionalEncoding(nn.Module):
+
+    def __init__(self, d_model: int, dropout: float = 0.1, max_len: int = 5000):
+        super().__init__()
+        self.dropout = nn.Dropout(p=dropout)
+
+        position = torch.arange(max_len).unsqueeze(1)
+        div_term = torch.exp(torch.arange(0, d_model, 2) * (-math.log(10000.0) / d_model))
+        pe = torch.zeros(max_len, 1, d_model)
+        pe[:, 0, 0::2] = torch.sin(position * div_term)
+        pe[:, 0, 1::2] = torch.cos(position * div_term)
+        self.register_buffer('pe', pe)
+
+    def forward(self, x: Tensor) -> Tensor:
+        """
+        Args:
+            x: Tensor, shape [seq_len, batch_size, embedding_dim]
+        """
+        x = x + self.pe[:x.size(0)]
+        return self.dropout(x)
diff --git a/convlab/policy/vtrace_DPT/vtrace.py b/convlab/policy/vtrace_DPT/vtrace.py
new file mode 100644
index 0000000000000000000000000000000000000000..b03662c60539a2e3aa80a5132618a3f7563a0f09
--- /dev/null
+++ b/convlab/policy/vtrace_DPT/vtrace.py
@@ -0,0 +1,368 @@
+import numpy as np
+import logging
+import json
+import os
+import sys
+import torch
+import torch.nn as nn
+
+from torch import optim
+from convlab.policy.vtrace_DPT.transformer_model.EncoderDecoder import EncoderDecoder
+from convlab.policy.vtrace_DPT.transformer_model.EncoderCritic import EncoderCritic
+from ... import Policy
+from ...util.custom_util import set_seed
+
+root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+sys.path.append(root_dir)
+
+DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+class VTRACE(nn.Module, Policy):
+
+    def __init__(self, is_train=True, seed=0, vectorizer=None, load_path=""):
+
+        super(VTRACE, self).__init__()
+
+        dir_name = os.path.dirname(os.path.abspath(__file__))
+        self.config_path = os.path.join(dir_name, 'config.json')
+
+        with open(self.config_path, 'r') as f:
+            cfg = json.load(f)
+
+        self.cfg = cfg
+        self.save_dir = os.path.join(dir_name, cfg['save_dir'])
+        self.save_per_epoch = cfg['save_per_epoch']
+        self.gamma = cfg['gamma']
+        self.tau = cfg['tau']
+        self.is_train = is_train
+        self.entropy_weight = cfg.get('entropy_weight', 0.0)
+        self.behaviour_cloning_weight = cfg.get('behaviour_cloning_weight', 0.0)
+        self.online_offline_ratio = cfg.get('online_offline_ratio', 0.0)
+        self.hidden_size = cfg['hidden_size']
+        self.policy_freq = cfg['policy_freq']
+        self.seed = seed
+        self.total_it = 0
+        self.rho_bar = cfg.get('rho_bar', 10)
+        self.c = cfg['c']
+        self.info_dict = {}
+        self.use_regularization = False
+        self.supervised_weight = cfg.get('supervised_weight', 0.0)
+
+        logging.info(f"Entropy weight: {self.entropy_weight}")
+        logging.info(f"Online-Offline-ratio: {self.online_offline_ratio}")
+        logging.info(f"Behaviour cloning weight: {self.behaviour_cloning_weight}")
+        logging.info(f"Supervised weight: {self.supervised_weight}")
+
+        set_seed(seed)
+
+        self.last_action = None
+
+        self.vector = vectorizer
+        self.policy = EncoderDecoder(**self.cfg, action_dict=self.vector.act2vec).to(device=DEVICE)
+        self.value_helper = EncoderDecoder(**self.cfg, action_dict=self.vector.act2vec).to(device=DEVICE)
+
+        try:
+            self.load_policy(load_path)
+        except Exception as e:
+            print(f"Could not load the policy, Exception: {e}")
+
+        if self.cfg['independent']:
+            self.value = EncoderCritic(self.value_helper.node_embedder, self.value_helper.encoder, **self.cfg).to(
+                device=DEVICE)
+        else:
+            self.value = EncoderCritic(self.policy.node_embedder, self.policy.encoder, **self.cfg).to(device=DEVICE)
+
+        try:
+            self.load_value(load_path)
+        except Exception as e:
+            print(f"Could not load the critic, Exception: {e}")
+
+        self.optimizer = optim.Adam([
+            {'params': self.policy.parameters(), 'lr': cfg['policy_lr'], 'betas': (0.0, 0.999)},
+            {'params': self.value.parameters(), 'lr': cfg['value_lr']}
+        ])
+
+        try:
+            self.load_optimizer_dicts(load_path)
+        except Exception as e:
+            print(f"Could not load optimiser dicts, Exception: {e}")
+
+    def predict(self, state):
+        """
+        Predict an system action given state.
+        Args:
+            state (dict): Dialog state. Please refer to util/state.py
+        Returns:
+            action : System act, with the form of (act_type, {slot_name_1: value_1, slot_name_2, value_2, ...})
+        """
+
+        if not self.is_train:
+            for param in self.policy.parameters():
+                param.requires_grad = False
+            for param in self.value.parameters():
+                param.requires_grad = False
+
+        s, action_mask = self.vector.state_vectorize(state)
+
+        kg_states = [self.vector.kg_info]
+        a = self.policy.select_action(kg_states, mask=action_mask, eval=not self.is_train).detach().cpu()
+        self.info_dict = self.policy.info_dict
+
+        descr_list = self.info_dict["description_idx_list"]
+        value_list = self.info_dict["value_list"]
+        current_domain_mask = self.info_dict["current_domain_mask"].unsqueeze(0)
+        non_current_domain_mask = self.info_dict["non_current_domain_mask"].unsqueeze(0)
+
+        a_prob, _ = self.policy.get_prob(a.unsqueeze(0), self.info_dict['action_mask'].unsqueeze(0),
+                                         len(self.info_dict['small_act']), [self.info_dict['small_act']],
+                                         current_domain_mask, non_current_domain_mask, [descr_list], [value_list])
+
+        self.info_dict['big_act'] = a
+        self.info_dict['a_prob'] = a_prob.prod()
+        self.info_dict['critic_value'] = self.value([descr_list], [value_list]).squeeze()
+
+        action = self.vector.action_devectorize(a.detach().numpy())
+
+        return action
+
+    def update(self, memory):
+        p_loss, v_loss = self.get_loss(memory)
+        loss = v_loss
+        if p_loss is not None:
+            loss += p_loss
+
+        self.optimizer.zero_grad()
+        loss.backward()
+
+        torch.nn.utils.clip_grad_norm_(self.value.parameters(), 40)
+        for p in self.policy.parameters():
+            if p.grad is not None:
+                p.grad[p.grad != p.grad] = 0.0
+        torch.nn.utils.clip_grad_norm_(self.policy.parameters(), 10)
+
+        self.optimizer.step()
+
+    def get_loss(self, memory):
+
+        self.is_train = True
+
+        if self.is_train:
+            self.total_it += 1
+
+            for param in self.policy.parameters():
+                param.requires_grad = True
+            for param in self.value.parameters():
+                param.requires_grad = True
+
+            batch, num_online = self.get_batch(memory)
+
+            action_masks, actions, critic_v, current_domain_mask, description_batch, max_length, mu, \
+            non_current_domain_mask, rewards, small_actions, unflattened_states, value_batch \
+                = self.prepare_batch(batch)
+
+            with torch.no_grad():
+                values = self.value(description_batch, value_batch).squeeze(-1)
+
+                pi_prob, _ = self.policy.get_prob(actions, action_masks, max_length, small_actions,
+                                                  current_domain_mask, non_current_domain_mask,
+                                                  description_batch, value_batch)
+                pi_prob = pi_prob.prod(dim=-1)
+
+                rho = torch.min(torch.Tensor([self.rho_bar]).to(DEVICE), pi_prob / mu)
+                cs = torch.min(torch.Tensor([self.c]).to(DEVICE), pi_prob / mu)
+
+                vtrace_target, advantages = self.compute_vtrace_advantage(unflattened_states, rewards, rho, cs, values)
+
+            # Compute critic loss
+            current_v = self.value(description_batch, value_batch).to(DEVICE)
+            critic_loss = torch.square(vtrace_target.unsqueeze(-1).to(DEVICE) - current_v).mean()
+
+            if self.use_regularization:
+                # do behaviour cloning on the buffer data
+                num_online = sum([len(reward_list) for reward_list in batch['rewards'][:num_online]])
+
+                behaviour_loss_critic = torch.square(
+                    critic_v[num_online:].unsqueeze(-1).to(DEVICE) - current_v[num_online:]).mean()
+                critic_loss += self.behaviour_cloning_weight * behaviour_loss_critic
+
+            actor_loss = None
+
+            # Delayed policy updates
+            if self.total_it % self.policy_freq == 0:
+
+                actor_loss, entropy = self.policy.get_log_prob(actions, action_masks, max_length, small_actions,
+                                                               current_domain_mask, non_current_domain_mask,
+                                                               description_batch, value_batch)
+                actor_loss = -1 * actor_loss
+                actor_loss = actor_loss * (advantages.to(DEVICE) * rho)
+                actor_loss = actor_loss.mean() - entropy * self.entropy_weight
+
+                if self.use_regularization:
+                    log_prob, entropy = self.policy.get_log_prob(actions[num_online:], action_masks[num_online:],
+                                                                 max_length, small_actions[num_online:],
+                                                                 current_domain_mask[num_online:],
+                                                                 non_current_domain_mask[num_online:],
+                                                                 description_batch[num_online:],
+                                                                 value_batch[num_online:])
+                    actor_loss = actor_loss - log_prob.mean() * self.behaviour_cloning_weight
+
+            return actor_loss, critic_loss
+
+        else:
+            return np.nan
+
+    def get_batch(self, memory):
+
+        if self.use_regularization or self.online_offline_ratio == 1.0:
+            batch, num_online = memory.sample(self.online_offline_ratio)
+        else:
+            batch, num_online = memory.sample(0.0)
+        return batch, num_online
+
+    def prepare_batch(self, batch):
+        unflattened_states = batch['states']
+        states = [kg for kg_list in unflattened_states for kg in kg_list]
+        description_batch = batch['description_idx_list']
+        description_batch = [descr_ for descr_episode in description_batch for descr_ in descr_episode]
+        value_batch = batch['value_list']
+        value_batch = [value_ for value_episode in value_batch for value_ in value_episode]
+
+        current_domain_mask = batch['current_domain_mask']
+        current_domain_mask = torch.stack([curr_mask for curr_mask_episode in current_domain_mask
+                                           for curr_mask in curr_mask_episode]).to(DEVICE)
+        non_current_domain_mask = batch['non_current_domain_mask']
+        non_current_domain_mask = torch.stack([non_curr_mask for non_curr_mask_episode in non_current_domain_mask
+                                               for non_curr_mask in non_curr_mask_episode]).to(DEVICE)
+        actions = batch['actions']
+        actions = torch.stack([act for act_list in actions for act in act_list], dim=0).to(DEVICE)
+        small_actions = batch['small_actions']
+        small_actions = [act for act_list in small_actions for act in act_list]
+        rewards = batch['rewards']
+        rewards = torch.stack([r for r_episode in rewards for r in r_episode]).to(DEVICE)
+        # rewards = torch.from_numpy(np.concatenate(np.array(rewards), axis=0)).to(DEVICE)
+        mu = batch['mu']
+        mu = torch.stack([mu_ for mu_list in mu for mu_ in mu_list], dim=0).to(DEVICE)
+        critic_v = batch['critic_value']
+        critic_v = torch.stack([v for v_list in critic_v for v in v_list]).to(DEVICE)
+        max_length = max(len(act) for act in small_actions)
+        action_masks = batch['action_masks']
+        action_mask_list = [mask for mask_list in action_masks for mask in mask_list]
+        action_masks = torch.stack([torch.cat([
+            action_mask.to(DEVICE),
+            torch.zeros(max_length - len(action_mask), len(self.policy.action_embedder.small_action_dict)).to(
+                DEVICE)],
+            dim=0) for action_mask in action_mask_list]).to(DEVICE)
+        return action_masks, actions, critic_v, current_domain_mask, description_batch, max_length, mu, \
+               non_current_domain_mask, rewards, small_actions, unflattened_states, value_batch
+
+    def compute_vtrace_advantage(self, states, rewards, rho, cs, values):
+
+        vtraces, advantages, offset = [], [], 0
+        #len(states) is number of episodes sampled, so we iterate over episodes
+        for j in range(0, len(states)):
+            vtrace_list, advantage_list, new_vtrace, v_next = [], [], 0, 0
+            for i in range(len(states[j]) - 1, -1, -1):
+                v_now = values[offset + i]
+                delta = rewards[offset + i] + self.gamma * v_next - v_now
+                delta = rho[offset + i] * delta
+                advantage = rewards[offset + i] + self.gamma * new_vtrace - v_now
+                new_vtrace = v_now + delta + self.gamma * cs[offset + i] * (new_vtrace - v_next)
+                v_next = v_now
+                vtrace_list.append(new_vtrace)
+                advantage_list.append(advantage)
+            vtrace_list = list(reversed(vtrace_list))
+            advantange_list = list(reversed(advantage_list))
+            vtraces.append(vtrace_list)
+            advantages.append(advantange_list)
+            offset += len(states[j])
+
+        vtraces_flat = torch.Tensor([v for v_episode in vtraces for v in v_episode])
+        advantages_flat = torch.Tensor([a for a_episode in advantages for a in a_episode])
+        return vtraces_flat, advantages_flat
+
+    def save(self, directory, addition=""):
+        if not os.path.exists(directory):
+            os.makedirs(directory)
+
+        torch.save(self.value.state_dict(), directory + f'/{addition}_vtrace.val.mdl')
+        torch.save(self.policy.state_dict(), directory + f'/{addition}_vtrace.pol.mdl')
+        torch.save(self.optimizer.state_dict(), directory + f'/{addition}_vtrace.optimizer')
+
+        logging.info(f"Saved policy, critic and optimizer.")
+
+    def load(self, filename):
+
+        value_mdl_candidates = [
+            filename + '.val.mdl',
+            filename + '_vtrace.val.mdl',
+            os.path.join(os.path.dirname(
+                os.path.abspath(__file__)), filename + '.val.mdl'),
+            os.path.join(os.path.dirname(os.path.abspath(
+                __file__)), filename + '_vtrace.val.mdl')
+        ]
+        for value_mdl in value_mdl_candidates:
+            if os.path.exists(value_mdl):
+                self.value.load_state_dict(torch.load(value_mdl, map_location=DEVICE))
+                print('<<dialog policy>> loaded checkpoint from file: {}'.format(value_mdl))
+                break
+
+        policy_mdl_candidates = [
+            filename + '.pol.mdl',
+            filename + '_vtrace.pol.mdl',
+            os.path.join(os.path.dirname(
+                os.path.abspath(__file__)), filename + '.pol.mdl'),
+            os.path.join(os.path.dirname(os.path.abspath(
+                __file__)), filename + '_vtrace.pol.mdl')
+        ]
+
+        for policy_mdl in policy_mdl_candidates:
+            if os.path.exists(policy_mdl):
+                self.policy.load_state_dict(torch.load(policy_mdl, map_location=DEVICE))
+                self.value_helper.load_state_dict(torch.load(policy_mdl, map_location=DEVICE))
+                print('<<dialog policy>> loaded checkpoint from file: {}'.format(policy_mdl))
+                break
+
+    def load_policy(self, filename):
+
+        policy_mdl_candidates = [
+            filename + '.pol.mdl',
+            filename + '_vtrace.pol.mdl',
+            os.path.join(os.path.dirname(
+                os.path.abspath(__file__)), filename + '.pol.mdl'),
+            os.path.join(os.path.dirname(os.path.abspath(
+                __file__)), filename + '_vtrace.pol.mdl')
+        ]
+
+        for policy_mdl in policy_mdl_candidates:
+            if os.path.exists(policy_mdl):
+                self.policy.load_state_dict(torch.load(policy_mdl, map_location=DEVICE))
+                self.value_helper.load_state_dict(torch.load(policy_mdl, map_location=DEVICE))
+                logging.info('<<dialog policy>> loaded checkpoint from file: {}'.format(policy_mdl))
+                break
+
+    def load_value(self, filename):
+
+        value_mdl_candidates = [
+            filename + '.val.mdl',
+            filename + '_vtrace.val.mdl',
+            os.path.join(os.path.dirname(
+                os.path.abspath(__file__)), filename + '.val.mdl'),
+            os.path.join(os.path.dirname(os.path.abspath(
+                __file__)), filename + '_vtrace.val.mdl')
+        ]
+        for value_mdl in value_mdl_candidates:
+            if os.path.exists(value_mdl):
+                self.value.load_state_dict(torch.load(value_mdl, map_location=DEVICE))
+                logging.info('<<dialog policy>> loaded checkpoint from file: {}'.format(value_mdl))
+                break
+
+    def load_optimizer_dicts(self, filename):
+        self.optimizer.load_state_dict(torch.load(filename + f".optimizer", map_location=DEVICE))
+        logging.info('<<dialog policy>> loaded optimisers from file: {}'.format(filename))
+
+    def from_pretrained(self):
+        raise NotImplementedError
+
+    def init_session(self):
+        pass
diff --git a/convlab/task/multiwoz/goal_generator.py b/convlab/task/multiwoz/goal_generator.py
index 3a709c08e061a57d0885063bda5a5c97a501097d..b1f8443b720eb803cdb9852d64f2e5c34bc798aa 100755
--- a/convlab/task/multiwoz/goal_generator.py
+++ b/convlab/task/multiwoz/goal_generator.py
@@ -151,7 +151,7 @@ class GoalGenerator:
                  boldify=False,
                  sample_info_from_trainset=True,
                  sample_reqt_from_trainset=False,
-                 seed = 0):
+                 seed = 0, domain_ordering_dist=None):
         """
         Args:
             goal_model_path: path to a goal model
@@ -174,6 +174,8 @@ class GoalGenerator:
         else:
             self._build_goal_model()
             print('Building goal model is done')
+        if domain_ordering_dist is not None:
+            self.domain_ordering_dist = domain_ordering_dist
         np.random.seed(seed)
         random.seed(seed)
         # remove some slot
diff --git a/convlab/util/custom_util.py b/convlab/util/custom_util.py
index f3abff7009a21adacd33aef50bd497634d391ae1..38d8b92a36efd67bdf9166c1c5f9f20734d1ecb5 100644
--- a/convlab/util/custom_util.py
+++ b/convlab/util/custom_util.py
@@ -8,6 +8,7 @@ import zipfile
 import numpy as np
 import torch
 from tensorboardX import SummaryWriter
+from convlab.task.multiwoz.goal_generator import GoalGenerator
 from convlab.util.file_util import cached_path
 from convlab.policy.evaluate_distributed import evaluate_distributed
 from convlab.util.train_util_neo import init_logging_nunu
@@ -18,16 +19,15 @@ from convlab.dst.rule.multiwoz import RuleDST
 from convlab.policy.rule.multiwoz import RulePolicy
 from convlab.evaluator.multiwoz_eval import MultiWozEvaluator
 from convlab.util import load_dataset
-from convlab.policy.rule.multiwoz.policy_agenda_multiwoz import Goal
 
 import shutil
+import signal
 
 
 slot_mapping = {"pricerange": "price range", "post": "postcode", "arriveBy": "arrive by", "leaveAt": "leave at",
                 "Id": "trainid", "ref": "reference"}
 
 
-
 sys.path.append(os.path.dirname(os.path.dirname(
     os.path.dirname(os.path.abspath(__file__)))))
 
@@ -35,6 +35,22 @@ DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 device = DEVICE
 
 
+class timeout:
+    def __init__(self, seconds=10, error_message='Timeout'):
+        self.seconds = seconds
+        self.error_message = error_message
+
+    def handle_timeout(self, signum, frame):
+        raise TimeoutError(self.error_message)
+
+    def __enter__(self):
+        signal.signal(signal.SIGALRM, self.handle_timeout)
+        signal.alarm(self.seconds)
+
+    def __exit__(self, type, value, traceback):
+        signal.alarm(0)
+
+
 class NumpyEncoder(json.JSONEncoder):
     """ Special json encoder for numpy types """
 
@@ -58,7 +74,8 @@ def flatten_acts(dialogue_acts):
     act_list = []
     for act_type in dialogue_acts:
         for act in dialogue_acts[act_type]:
-            act_list.append([act['intent'], act['domain'], act['slot'], act.get('value', "")])
+            act_list.append([act['intent'], act['domain'],
+                            act['slot'], act.get('value', "")])
     return act_list
 
 
@@ -84,17 +101,20 @@ def load_config_file(filepath: str = None) -> dict:
     return conf
 
 
-def save_config(terminal_args, config_file_args, config_save_path):
+def save_config(terminal_args, config_file_args, config_save_path, policy_config=None):
     config_save_path = os.path.join(config_save_path, f'config_saved.json')
-    args_dict = {"args": terminal_args, "config": config_file_args}
+    args_dict = {"args": terminal_args, "config": config_file_args, "policy_config": policy_config}
     json.dump(args_dict, open(config_save_path, 'w'))
 
 
 def set_seed(seed):
     random.seed(seed)
     np.random.seed(seed)
+    torch.manual_seed(seed)
     if torch.cuda.is_available():
         torch.cuda.manual_seed_all(seed)
+        torch.backends.cudnn.deterministic = True
+        torch.backends.cudnn.benchmark = False
 
 
 def init_logging(root_dir, mode):
@@ -138,24 +158,33 @@ def save_best(policy_sys, best_complete_rate, best_success_rate, best_return, co
     return best_complete_rate, best_success_rate, best_return
 
 
-def eval_policy(conf, policy_sys, env, sess, save_eval, log_save_path):
+def eval_policy(conf, policy_sys, env, sess, save_eval, log_save_path, single_domain_goals=False, allowed_domains=None):
     policy_sys.is_train = False
+
+    goal_generator = GoalGenerator()
+    goals = []
+    for seed in range(1000, 1000 + conf['model']['num_eval_dialogues']):
+        set_seed(seed)
+        goal = create_goals(goal_generator, 1, single_domain_goals, allowed_domains)
+        goals.append(goal[0])
+
     if conf['model']['process_num'] == 1:
         complete_rate, success_rate, success_rate_strict, avg_return, turns, \
             avg_actions, task_success, book_acts, inform_acts, request_acts, \
-                select_acts, offer_acts = evaluate(sess,
+                select_acts, offer_acts, recommend_acts = evaluate(sess,
                                                 num_dialogues=conf['model']['num_eval_dialogues'],
                                                 sys_semantic_to_usr=conf['model'][
                                                     'sys_semantic_to_usr'],
-                                                save_flag=save_eval, save_path=log_save_path)
-        total_acts = book_acts + inform_acts + request_acts + select_acts + offer_acts
+                                                save_flag=save_eval, save_path=log_save_path, goals=goals)
+
+        total_acts = book_acts + inform_acts + request_acts + select_acts + offer_acts + recommend_acts
     else:
         complete_rate, success_rate, success_rate_strict, avg_return, turns, \
-        avg_actions, task_success, book_acts, inform_acts, request_acts, \
-        select_acts, offer_acts = \
+            avg_actions, task_success, book_acts, inform_acts, request_acts, \
+            select_acts, offer_acts, recommend_acts = \
             evaluate_distributed(sess, list(range(1000, 1000 + conf['model']['num_eval_dialogues'])),
-                                 conf['model']['process_num'])
-        total_acts = book_acts + inform_acts + request_acts + select_acts + offer_acts
+                                 conf['model']['process_num'], goals)
+        total_acts = book_acts + inform_acts + request_acts + select_acts + offer_acts + recommend_acts
 
         task_success_gathered = {}
         for task_dict in task_success:
@@ -164,24 +193,42 @@ def eval_policy(conf, policy_sys, env, sess, save_eval, log_save_path):
                     task_success_gathered[key] = []
                 task_success_gathered[key].append(value)
         task_success = task_success_gathered
-    
+
     policy_sys.is_train = True
-    logging.info(f"Complete: {complete_rate}, Success: {success_rate}, Success strict: {success_rate_strict}, "
-                 f"Average Return: {avg_return}, Turns: {turns}, Average Actions: {avg_actions}, "
+
+    mean_complete, err_complete = np.average(complete_rate), np.std(complete_rate) / np.sqrt(len(complete_rate))
+    mean_success, err_success = np.average(success_rate), np.std(success_rate) / np.sqrt(len(success_rate))
+    mean_success_strict, err_success_strict = np.average(success_rate_strict), np.std(success_rate_strict) / np.sqrt(len(success_rate_strict))
+    mean_return, err_return = np.average(avg_return), np.std(avg_return) / np.sqrt(len(avg_return))
+    mean_turns, err_turns = np.average(turns), np.std(turns) / np.sqrt(len(turns))
+    mean_actions, err_actions = np.average(avg_actions), np.std(avg_actions) / np.sqrt(len(avg_actions))
+
+    logging.info(f"Complete: {mean_complete}+-{round(err_complete, 2)}, "
+                 f"Success: {mean_success}+-{round(err_success, 2)}, "
+                 f"Success strict: {mean_success_strict}+-{round(err_success_strict, 2)}, "
+                 f"Average Return: {mean_return}+-{round(err_return, 2)}, "
+                 f"Turns: {mean_turns}+-{round(err_turns, 2)}, "
+                 f"Average Actions: {mean_actions}+-{round(err_actions, 2)}, "
                  f"Book Actions: {book_acts/total_acts}, Inform Actions: {inform_acts/total_acts}, "
                  f"Request Actions: {request_acts/total_acts}, Select Actions: {select_acts/total_acts}, "
-                 f"Offer Actions: {offer_acts/total_acts}")
+                 f"Offer Actions: {offer_acts/total_acts}, Recommend Actions: {recommend_acts/total_acts}")
 
     for key in task_success:
         logging.info(
             f"{key}: Num: {len(task_success[key])} Success: {np.average(task_success[key]) if len(task_success[key]) > 0 else 0}")
 
-    return {"complete_rate": complete_rate,
-            "success_rate": success_rate,
-            "success_rate_strict": success_rate_strict,
-            "avg_return": avg_return,
-            "turns": turns,
-            "avg_actions": avg_actions}
+    return {"complete_rate": mean_complete,
+            "success_rate": mean_success,
+            "success_rate_strict": mean_success_strict,
+            "avg_return": mean_return,
+            "turns": mean_turns,
+            "avg_actions": mean_actions,
+            "book_acts": book_acts/total_acts,
+            "inform_acts": inform_acts/total_acts,
+            "request_acts": request_acts/total_acts,
+            "select_acts": select_acts/total_acts,
+            "offer_acts": offer_acts/total_acts,
+            "recommend_acts": recommend_acts/total_acts}
 
 
 def env_config(conf, policy_sys, check_book_constraints=True):
@@ -193,12 +240,21 @@ def env_config(conf, policy_sys, check_book_constraints=True):
     policy_usr = conf['policy_usr_activated']
     usr_nlg = conf['usr_nlg_activated']
 
+    # Setup uncertainty thresholding
+    if dst_sys:
+        try:
+            if dst_sys.return_confidence_scores:
+                policy_sys.vector.setup_uncertain_query(dst_sys.confidence_thresholds)
+        except:
+            logging.info('Uncertainty threshold not set.')
+
     simulator = PipelineAgent(nlu_usr, dst_usr, policy_usr, usr_nlg, 'user')
     system_pipeline = PipelineAgent(nlu_sys, dst_sys, policy_sys, sys_nlg,
                                     'sys', return_semantic_acts=conf['model']['sys_semantic_to_usr'])
 
     # assemble
-    evaluator = MultiWozEvaluator(check_book_constraints=check_book_constraints)
+    evaluator = MultiWozEvaluator(
+        check_book_constraints=check_book_constraints)
     env = Environment(sys_nlg, simulator, nlu_sys, dst_sys, evaluator=evaluator,
                       use_semantic_acts=conf['model']['sys_semantic_to_usr'])
     sess = BiSession(system_pipeline, simulator, None, evaluator)
@@ -264,12 +320,7 @@ def create_env(args, policy_sys):
     return env, sess
 
 
-def evaluate(sess, num_dialogues=400, sys_semantic_to_usr=False, save_flag=False, save_path=None):
-    seed = 0
-    random.seed(seed)
-    np.random.seed(seed)
-
-    # sess = BiSession(agent_sys, simulator, None, evaluator)
+def evaluate(sess, num_dialogues=400, sys_semantic_to_usr=False, save_flag=False, save_path=None, goals=None):
 
     eval_save = {}
     turn_counter_dict = {}
@@ -278,11 +329,12 @@ def evaluate(sess, num_dialogues=400, sys_semantic_to_usr=False, save_flag=False
     task_success = {'All_user_sim': [], 'All_evaluator': [], "All_evaluator_strict": [],
                     'total_return': [], 'turns': [], 'avg_actions': [],
                     'total_booking_acts': [], 'total_inform_acts': [], 'total_request_acts': [],
-                    'total_select_acts': [], 'total_offer_acts': []}
+                    'total_select_acts': [], 'total_offer_acts': [], 'total_recommend_acts': []}
     dial_count = 0
     for seed in range(1000, 1000 + num_dialogues):
         set_seed(seed)
-        sess.init_session()
+        goal = goals.pop()
+        sess.init_session(goal=goal)
         sys_response = [] if sess.sys_agent.nlg is None else ''
         sys_response = [] if sys_semantic_to_usr else sys_response
         avg_actions = 0
@@ -293,6 +345,7 @@ def evaluate(sess, num_dialogues=400, sys_semantic_to_usr=False, save_flag=False
         request = 0
         select = 0
         offer = 0
+        recommend = 0
         # this 40 represents the max turn of dialogue
         for i in range(40):
             sys_response, user_response, session_over, reward = sess.next_turn(
@@ -315,6 +368,8 @@ def evaluate(sess, num_dialogues=400, sys_semantic_to_usr=False, save_flag=False
                     select += 1
                 if intent.lower() == 'offerbook':
                     offer += 1
+                if intent.lower() == 'recommend':
+                    recommend += 1
             avg_actions += len(acts)
             turn_counter += 1
             turns += 1
@@ -345,12 +400,14 @@ def evaluate(sess, num_dialogues=400, sys_semantic_to_usr=False, save_flag=False
         task_success['total_return'].append(total_return)
         task_success['turns'].append(turns)
         task_success['avg_actions'].append(avg_actions / turns)
-        
+
         task_success['total_booking_acts'].append(book)
         task_success['total_inform_acts'].append(inform)
         task_success['total_request_acts'].append(request)
         task_success['total_select_acts'].append(select)
         task_success['total_offer_acts'].append(offer)
+        task_success['total_offer_acts'].append(offer)
+        task_success['total_recommend_acts'].append(recommend)
 
         # print(agent_sys.agent_saves)
         eval_save['Conversation {}'.format(str(dial_count))] = [
@@ -367,11 +424,11 @@ def evaluate(sess, num_dialogues=400, sys_semantic_to_usr=False, save_flag=False
     # save dialogue_info and clear mem
 
     return np.average(task_success['All_user_sim']), np.average(task_success['All_evaluator']), \
-           np.average(task_success['All_evaluator_strict']), np.average(task_success['total_return']), \
-           np.average(task_success['turns']), np.average(task_success['avg_actions']), task_success, \
-           np.average(task_success['total_booking_acts']), np.average(task_success['total_inform_acts']), \
-           np.average(task_success['total_request_acts']), np.average(task_success['total_select_acts']), \
-           np.average(task_success['total_offer_acts'])
+        np.average(task_success['All_evaluator_strict']), np.average(task_success['total_return']), \
+        np.average(task_success['turns']), np.average(task_success['avg_actions']), task_success, \
+        np.average(task_success['total_booking_acts']), np.average(task_success['total_inform_acts']), \
+        np.average(task_success['total_request_acts']), np.average(task_success['total_select_acts']), \
+        np.average(task_success['total_offer_acts']), np.average(task_success['total_recommend_acts'])
 
 
 def model_downloader(download_dir, model_path):
@@ -395,7 +452,8 @@ def get_goal_distribution(dataset_name='multiwoz21'):
         data = data_split[key]
         for dialogue in data:
             goal = dialogue['goal']
-            domains = list(set(goal['inform'].keys()) | set(goal['request'].keys()))
+            domains = list(set(goal['inform'].keys()) |
+                           set(goal['request'].keys()))
             domains.sort()
             domains = "-".join(domains)
 
@@ -417,14 +475,16 @@ def get_goal_distribution(dataset_name='multiwoz21'):
     domain_combinations.sort(key=lambda x: x[1], reverse=True)
     print(domain_combinations)
     print(single_domain_counter)
-    print("Number of combinations:", sum([value for _, value in domain_combinations]))
+    print("Number of combinations:", sum(
+        [value for _, value in domain_combinations]))
 
 
 def unified_format(acts):
     new_acts = {'categorical': []}
     for act in acts:
         intent, domain, slot, value = act
-        new_acts['categorical'].append({"intent": intent, "domain": domain, "slot": slot, "value": value})
+        new_acts['categorical'].append(
+            {"intent": intent, "domain": domain, "slot": slot, "value": value})
 
     return new_acts
 
@@ -438,6 +498,7 @@ def act_dict_to_flat_tuple(acts):
 
 
 def create_goals(goal_generator, num_goals, single_domains=False, allowed_domains=None):
+    from convlab.policy.rule.multiwoz.policy_agenda_multiwoz import Goal
 
     collected_goals = []
     while len(collected_goals) != num_goals:
@@ -450,6 +511,32 @@ def create_goals(goal_generator, num_goals, single_domains=False, allowed_domain
     return collected_goals
 
 
+def build_domains_goal(goal_generator, domains=None):
+    from convlab.policy.rule.multiwoz.policy_agenda_multiwoz import Goal
+    found = False
+    while not found:
+        goal = Goal(goal_generator)
+        if domains is None:
+            found = True
+        if set(goal.domain_goals) == domains:
+            found = True
+    return goal
+
+
+def data_goals(num_goals, dataset="multiwoz21", dial_ids_order=0):
+    from convlab.policy.tus.unify.Goal import Goal
+    from convlab.policy.tus.unify.util import create_goal
+    data = load_dataset(dataset, dial_ids_order)
+    collected_goals = []
+    for dialog in data["test"]:
+        goal = Goal(create_goal(dialog))
+        collected_goals.append(goal)
+    if len(collected_goals) < num_goals:
+        print(f"# of data goals ({data['test']}) < num_goals {num_goals}")
+    # reorder goals?
+    return collected_goals
+
+
 def map_class(cls_path: str):
     """
     Map to class via package text path
@@ -492,18 +579,21 @@ def get_config(filepath, args) -> dict:
     vec_name = [model for model in conf['vectorizer_sys']]
     vec_name = vec_name[0] if vec_name else None
     if dst_name and 'setsumbt' in dst_name.lower():
-        if 'get_confidence_scores' in conf['dst_sys'][dst_name]['ini_params']:
-            conf['vectorizer_sys'][vec_name]['ini_params']['use_confidence_scores'] = conf['dst_sys'][dst_name]['ini_params']['get_confidence_scores']
+        if 'return_confidence_scores' in conf['dst_sys'][dst_name]['ini_params']:
+            param = conf['dst_sys'][dst_name]['ini_params']['return_confidence_scores']
+            conf['vectorizer_sys'][vec_name]['ini_params']['use_confidence_scores'] = param
         else:
             conf['vectorizer_sys'][vec_name]['ini_params']['use_confidence_scores'] = False
-        if 'return_mutual_info' in conf['dst_sys'][dst_name]['ini_params']:
-            conf['vectorizer_sys'][vec_name]['ini_params']['use_mutual_info'] = conf['dst_sys'][dst_name]['ini_params']['return_mutual_info']
+        if 'return_belief_state_mutual_info' in conf['dst_sys'][dst_name]['ini_params']:
+            param = conf['dst_sys'][dst_name]['ini_params']['return_belief_state_mutual_info']
+            conf['vectorizer_sys'][vec_name]['ini_params']['use_state_knowledge_uncertainty'] = param
         else:
-            conf['vectorizer_sys'][vec_name]['ini_params']['use_mutual_info'] = False
-        if 'return_entropy' in conf['dst_sys'][dst_name]['ini_params']:
-            conf['vectorizer_sys'][vec_name]['ini_params']['use_entropy'] = conf['dst_sys'][dst_name]['ini_params']['return_entropy']
+            conf['vectorizer_sys'][vec_name]['ini_params']['use_state_knowledge_uncertainty'] = False
+        if 'return_belief_state_entropy' in conf['dst_sys'][dst_name]['ini_params']:
+            param = conf['dst_sys'][dst_name]['ini_params']['return_belief_state_entropy']
+            conf['vectorizer_sys'][vec_name]['ini_params']['use_state_total_uncertainty'] = param
         else:
-            conf['vectorizer_sys'][vec_name]['ini_params']['use_entropy'] = False
+            conf['vectorizer_sys'][vec_name]['ini_params']['use_state_total_uncertainty'] = False
 
     from convlab.nlu import NLU
     from convlab.dst import DST
@@ -532,13 +622,10 @@ def get_config(filepath, args) -> dict:
                 cls_path = infos.get('class_path', '')
                 cls = map_class(cls_path)
                 conf[unit + '_class'] = cls
-                conf[unit + '_activated'] = conf[unit +
-                                                 '_class'](**conf[unit][model]['ini_params'])
+                conf[unit + '_activated'] = conf[unit + '_class'](**conf[unit][model]['ini_params'])
                 print("Loaded " + model + " for " + unit)
     return conf
 
 
 if __name__ == '__main__':
     get_goal_distribution()
-
-
diff --git a/convlab/util/multiwoz/lexicalize.py b/convlab/util/multiwoz/lexicalize.py
index 9eab25e26d82d7f20d92343a2ac64bb807239074..4fd9f262ece3fe38935c26bc449b1e93121db498 100755
--- a/convlab/util/multiwoz/lexicalize.py
+++ b/convlab/util/multiwoz/lexicalize.py
@@ -86,7 +86,8 @@ def lexicalize_da(meta, entities, state, requestable):
                             pair[1] = entities[domain][n][slot_old]
                         elif slot in state[domain]:
                             pair[1] = state[domain][slot]
-                        pair[1] = pair[1] if pair[1] else 'not available'
+                        else:
+                            pair[1] = 'not available'
                     elif slot in state[domain]:
                         pair[1] = state[domain][slot] if state[domain][slot] else 'none'
                     else:
diff --git a/convlab/util/multiwoz/state.py b/convlab/util/multiwoz/state.py
index 5b65ba066b33cf9cd1e2fb49515406d962f03588..116403ba98ea9604f011aa5c9eb1bf03cda2d708 100755
--- a/convlab/util/multiwoz/state.py
+++ b/convlab/util/multiwoz/state.py
@@ -1,7 +1,14 @@
 def default_state():
     state = dict(user_action=[],
                  system_action=[],
-                 belief_state={},
+                 belief_state={
+                     'attraction': {'type': '', 'name': '', 'area': ''}, 
+                     'hotel': {'name': '', 'area': '', 'parking': '', 'price range': '', 'stars': '4', 'internet': 'yes', 'type': 'hotel', 'book stay': '', 'book day': '', 'book people': ''}, 
+                     'restaurant': {'food': '', 'price range': '', 'name': '', 'area': '', 'book time': '', 'book day': '', 'book people': ''}, 
+                     'taxi': {'leave at': '', 'destination': '', 'departure': '', 'arrive by': ''}, 
+                     'train': {'leave at': '', 'destination': '', 'day': '', 'arrive by': '', 'departure': '', 'book people': ''}, 
+                     'hospital': {'department': ''}
+                     },
                  booked={},
                  request_state={},
                  terminated=False,
diff --git a/convlab/util/unified_datasets_util.py b/convlab/util/unified_datasets_util.py
index aff31be6742be3908d5ff4ab65e141b3427471d9..2c16bc5cc6d99bc6a00005fc3652ee556321c8df 100644
--- a/convlab/util/unified_datasets_util.py
+++ b/convlab/util/unified_datasets_util.py
@@ -16,17 +16,19 @@ from tqdm import tqdm
 
 class BaseDatabase(ABC):
     """Base class of unified database. Should override the query function."""
+
     def __init__(self):
         """extract data.zip and load the database."""
 
     @abstractmethod
-    def query(self, domain:str, state:dict, topk:int, **kwargs)->list:
+    def query(self, domain: str, state: dict, topk: int, **kwargs) -> list:
         """return a list of topk entities (dict containing slot-value pairs) for a given domain based on the dialogue state."""
 
+
 def download_unified_datasets(dataset_name, filename, data_dir):
     """
     It downloads the file of unified datasets from HuggingFace's datasets if it doesn't exist in the data directory
-    
+
     :param dataset_name: The name of the dataset
     :param filename: the name of the file you want to download
     :param data_dir: the directory where the file will be downloaded to
@@ -41,22 +43,26 @@ def download_unified_datasets(dataset_name, filename, data_dir):
         shutil.move(cache_path, data_path)
     return data_path
 
+
 def relative_import_module_from_unified_datasets(dataset_name, filename, names2import):
     """
     It downloads a file from the unified datasets repository, imports it as a module, and returns the
     variable(s) you want from that module
-    
+
     :param dataset_name: the name of the dataset, e.g. 'multiwoz21'
     :param filename: the name of the file to download, e.g. 'preprocess.py'
     :param names2import: a string or a list of strings. If it's a string, it's the name of the variable
     to import. If it's a list of strings, it's the names of the variables to import
     :return: the variable(s) that are being imported from the module.
     """
-    data_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), f'../../../data/unified_datasets/{dataset_name}'))
+    data_dir = os.path.abspath(os.path.join(os.path.abspath(
+        __file__), f'../../../data/unified_datasets/{dataset_name}'))
     assert filename.endswith('.py')
-    assert isinstance(names2import, str) or (isinstance(names2import, list) and len(names2import) > 0)
+    assert isinstance(names2import, str) or (
+        isinstance(names2import, list) and len(names2import) > 0)
     data_path = download_unified_datasets(dataset_name, filename, data_dir)
-    module_spec = importlib.util.spec_from_file_location(filename[:-3], data_path)
+    module_spec = importlib.util.spec_from_file_location(
+        filename[:-3], data_path)
     module = importlib.util.module_from_spec(module_spec)
     module_spec.loader.exec_module(module)
     if isinstance(names2import, str):
@@ -67,7 +73,8 @@ def relative_import_module_from_unified_datasets(dataset_name, filename, names2i
             variables.append(eval(f'module.{name}'))
         return variables
 
-def load_dataset(dataset_name:str, dial_ids_order=None, split2ratio={}) -> Dict:
+
+def load_dataset(dataset_name: str, dial_ids_order=None, split2ratio={}) -> Dict:
     """load unified dataset from `data/unified_datasets/$dataset_name`
 
     Args:
@@ -79,7 +86,8 @@ def load_dataset(dataset_name:str, dial_ids_order=None, split2ratio={}) -> Dict:
     Returns:
         dataset (dict): keys are data splits and the values are lists of dialogues
     """
-    data_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), f'../../../data/unified_datasets/{dataset_name}'))
+    data_dir = os.path.abspath(os.path.join(os.path.abspath(
+        __file__), f'../../../data/unified_datasets/{dataset_name}'))
     data_path = download_unified_datasets(dataset_name, 'data.zip', data_dir)
 
     archive = ZipFile(data_path)
@@ -87,11 +95,13 @@ def load_dataset(dataset_name:str, dial_ids_order=None, split2ratio={}) -> Dict:
         dialogues = json.loads(f.read())
     dataset = {}
     if dial_ids_order is not None:
-        data_path = download_unified_datasets(dataset_name, 'shuffled_dial_ids.json', data_dir)
+        data_path = download_unified_datasets(
+            dataset_name, 'shuffled_dial_ids.json', data_dir)
         dial_ids = json.load(open(data_path))[dial_ids_order]
         for data_split in dial_ids:
             ratio = split2ratio.get(data_split, 1)
-            dataset[data_split] = [dialogues[i] for i in dial_ids[data_split][:round(len(dial_ids[data_split])*ratio)]]
+            dataset[data_split] = [dialogues[i]
+                                   for i in dial_ids[data_split][:round(len(dial_ids[data_split])*ratio)]]
     else:
         for dialogue in dialogues:
             if dialogue['data_split'] not in dataset:
@@ -100,10 +110,12 @@ def load_dataset(dataset_name:str, dial_ids_order=None, split2ratio={}) -> Dict:
                 dataset[dialogue['data_split']].append(dialogue)
         for data_split in dataset:
             if data_split in split2ratio:
-                dataset[data_split] = dataset[data_split][:round(len(dataset[data_split])*split2ratio[data_split])]
+                dataset[data_split] = dataset[data_split][:round(
+                    len(dataset[data_split])*split2ratio[data_split])]
     return dataset
 
-def load_ontology(dataset_name:str) -> Dict:
+
+def load_ontology(dataset_name: str) -> Dict:
     """load unified ontology from `data/unified_datasets/$dataset_name`
 
     Args:
@@ -112,7 +124,8 @@ def load_ontology(dataset_name:str) -> Dict:
     Returns:
         ontology (dict): dataset ontology
     """
-    data_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), f'../../../data/unified_datasets/{dataset_name}'))
+    data_dir = os.path.abspath(os.path.join(os.path.abspath(
+        __file__), f'../../../data/unified_datasets/{dataset_name}'))
     data_path = download_unified_datasets(dataset_name, 'data.zip', data_dir)
 
     archive = ZipFile(data_path)
@@ -120,7 +133,8 @@ def load_ontology(dataset_name:str) -> Dict:
         ontology = json.loads(f.read())
     return ontology
 
-def load_database(dataset_name:str):
+
+def load_database(dataset_name: str):
     """load database from `data/unified_datasets/$dataset_name`
 
     Args:
@@ -129,17 +143,21 @@ def load_database(dataset_name:str):
     Returns:
         database: an instance of BaseDatabase
     """
-    data_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), f'../../../data/unified_datasets/{dataset_name}'))
-    data_path = download_unified_datasets(dataset_name, 'database.py', data_dir)
+    data_dir = os.path.abspath(os.path.join(os.path.abspath(
+        __file__), f'../../../data/unified_datasets/{dataset_name}'))
+    data_path = download_unified_datasets(
+        dataset_name, 'database.py', data_dir)
     module_spec = importlib.util.spec_from_file_location('database', data_path)
     module = importlib.util.module_from_spec(module_spec)
     module_spec.loader.exec_module(module)
-    Database = relative_import_module_from_unified_datasets(dataset_name, 'database.py', 'Database')
+    Database = relative_import_module_from_unified_datasets(
+        dataset_name, 'database.py', 'Database')
     assert issubclass(Database, BaseDatabase)
     database = Database()
     assert isinstance(database, BaseDatabase)
     return database
 
+
 def load_unified_data(
         dataset, 
         data_split='all', 
@@ -148,6 +166,7 @@ def load_unified_data(
         dialogue_acts=False, 
         state=False, 
         db_results=False,
+        delex_utterance=False,
         use_context=False, 
         context_window_size=0, 
         terminated=False, 
@@ -158,7 +177,7 @@ def load_unified_data(
     """
     > This function takes in a dataset, and returns a dictionary of data splits, where each data split
     is a list of samples
-    
+
     :param dataset: dataset object from `load_dataset`
     :param data_split: which split of the data to load. Can be 'train', 'validation', 'test', or 'all',
     defaults to all (optional)
@@ -182,7 +201,7 @@ def load_unified_data(
     data_splits = dataset.keys() if data_split == 'all' else [data_split]
     assert speaker in ['user', 'system', 'all']
     assert not use_context or context_window_size > 0
-    info_list = list(filter(eval, ['utterance', 'dialogue_acts', 'state', 'db_results']))
+    info_list = list(filter(eval, ['utterance', 'dialogue_acts', 'state', 'db_results', 'delex_utterance']))
     info_list += ['utt_idx']
     data_by_split = {}
     for data_split in data_splits:
@@ -194,7 +213,7 @@ def load_unified_data(
                 for ele in info_list:
                     if ele in turn:
                         sample[ele] = turn[ele]
-                
+
                 if use_context or not split_to_turn:
                     sample_copy = deepcopy(sample)
                     context.append(sample_copy)
@@ -207,7 +226,8 @@ def load_unified_data(
                     if active_domains:
                         sample['domains'] = dialogue['domains']
                     if terminated:
-                        sample['terminated'] = turn['utt_idx'] == len(dialogue['turns']) - 1
+                        sample['terminated'] = turn['utt_idx'] == len(
+                            dialogue['turns']) - 1
                     if speaker == 'system' and 'booked' in turn:
                         sample['booked'] = turn['booked']
                     data_by_split[data_split].append(sample)
@@ -221,7 +241,7 @@ def load_nlu_data(dataset, data_split='all', speaker='user', use_context=False,
     """
     It loads the data from the specified dataset, and returns it in a format that is suitable for
     training a NLU model
-    
+
     :param dataset: dataset object from `load_dataset`
     :param data_split: 'train', 'validation', 'test', or 'all', defaults to all (optional)
     :param speaker: 'user' or 'system', defaults to user (optional)
@@ -243,7 +263,7 @@ def load_dst_data(dataset, data_split='all', speaker='user', context_window_size
     """
     It loads the data from the specified dataset, with the specified data split, speaker, context window
     size, suitable for training a DST model
-    
+
     :param dataset: dataset object from `load_dataset`
     :param data_split: 'train', 'validation', 'test', or 'all', defaults to all (optional)
     :param speaker: 'user' or 'system', defaults to user (optional)
@@ -259,11 +279,12 @@ def load_dst_data(dataset, data_split='all', speaker='user', context_window_size
     kwargs.setdefault('state', True)
     return load_unified_data(dataset, **kwargs)
 
+
 def load_policy_data(dataset, data_split='all', speaker='system', context_window_size=1, **kwargs):
     """
     It loads the data from the specified dataset, and returns it in a format that is suitable for
     training a policy
-    
+
     :param dataset: dataset object from `load_dataset`
     :param data_split: 'train', 'validation', 'test', or 'all', defaults to all (optional)
     :param speaker: 'system' or 'user', defaults to system (optional)
@@ -283,11 +304,12 @@ def load_policy_data(dataset, data_split='all', speaker='system', context_window
     kwargs.setdefault('terminated', True)
     return load_unified_data(dataset, **kwargs)
 
+
 def load_nlg_data(dataset, data_split='all', speaker='system', use_context=False, context_window_size=0, **kwargs):
     """
     It loads the data from the specified dataset, and returns it in a format that is suitable for
     training a NLG model
-    
+
     :param dataset: dataset object from `load_dataset`
     :param data_split: 'train', 'validation', 'test', or 'all', defaults to all (optional)
     :param speaker: 'system' or 'user', defaults to system (optional)
@@ -304,11 +326,12 @@ def load_nlg_data(dataset, data_split='all', speaker='system', use_context=False
     kwargs.setdefault('dialogue_acts', True)
     return load_unified_data(dataset, **kwargs)
 
+
 def load_e2e_data(dataset, data_split='all', speaker='system', context_window_size=100, **kwargs):
     """
     It loads the data from the specified dataset, and returns it in a format that is suitable for
     training an End2End model
-    
+
     :param dataset: dataset object from `load_dataset`
     :param data_split: 'train', 'validation', 'test', or 'all', defaults to all (optional)
     :param speaker: 'system' or 'user', defaults to system (optional)
@@ -327,11 +350,12 @@ def load_e2e_data(dataset, data_split='all', speaker='system', context_window_si
     kwargs.setdefault('dialogue_acts', True)
     return load_unified_data(dataset, **kwargs)
 
+
 def load_rg_data(dataset, data_split='all', speaker='system', context_window_size=100, **kwargs):
     """
     It loads the data from the dataset, and returns it in a format that is suitable for training a 
     response generation model
-    
+
     :param dataset: dataset object from `load_dataset`
     :param data_split: 'train', 'validation', 'test', or 'all', defaults to all (optional)
     :param speaker: 'system' or 'user', defaults to system (optional)
@@ -347,7 +371,7 @@ def load_rg_data(dataset, data_split='all', speaker='system', context_window_siz
     return load_unified_data(dataset, **kwargs)
 
 
-def create_delex_data(dataset, delex_func=lambda d,s,v: f'[({d})-({s})]', ignore_values=['yes', 'no']):
+def create_delex_data(dataset, delex_func=lambda d, s, v: f'[({d})-({s})]', ignore_values=['yes', 'no']):
     """add delex_utterance to the dataset according to dialogue acts and belief_state
     delex_func: function that return the placeholder (e.g. "[(domain_name)-(slot_name)]") given (domain, slot, value)
     ignore_values: ignored values when delexicalizing using the categorical acts and states
@@ -357,7 +381,7 @@ def create_delex_data(dataset, delex_func=lambda d,s,v: f'[({d})-({s})]', ignore
         It takes a list of strings and placeholders, and a regex pattern. If the pattern matches exactly
         one string, it replaces that string with a placeholder and returns True. Otherwise, it returns
         False
-        
+
         :param texts_placeholders: a list of tuples, each tuple is a string and a boolean. The boolean
         indicates whether the string is a placeholder or not
         :param value_pattern: a regular expression that matches the value to be delexicalized
@@ -379,7 +403,8 @@ def create_delex_data(dataset, delex_func=lambda d,s,v: f'[({d})-({s})]', ignore
             searchObj = re.search(value_pattern, substring)
             assert searchObj
             start, end = searchObj.span(1)
-            texts_placeholders[idx:idx+1] = [(substring[0:start], False), (placeholder, True), (substring[end:], False)]
+            texts_placeholders[idx:idx+1] = [
+                (substring[0:start], False), (placeholder, True), (substring[end:], False)]
             return True
         return False
 
@@ -392,7 +417,8 @@ def create_delex_data(dataset, delex_func=lambda d,s,v: f'[({d})-({s})]', ignore
                 delex_utt = []
                 last_end = 0
                 # ignore the non-categorical das that do not have span annotation
-                spans = [x for x in turn['dialogue_acts']['non-categorical'] if 'start' in x]
+                spans = [x for x in turn['dialogue_acts']
+                         ['non-categorical'] if 'start' in x]
                 for da in sorted(spans, key=lambda x: x['start']):
                     # from left to right
                     start, end = da['start'], da['end']
@@ -412,7 +438,8 @@ def create_delex_data(dataset, delex_func=lambda d,s,v: f'[({d})-({s})]', ignore
                     domain, slot, value = da['domain'], da['slot'], da['value']
                     if value.lower() not in ignore_values:
                         placeholder = delex_func(domain, slot, value)
-                        pattern = re.compile(r'\b({})\b'.format(value), flags=re.I)
+                        pattern = re.compile(
+                            r'\b({})\b'.format(value), flags=re.I)
                         if delex_inplace(delex_utt, pattern):
                             delex_vocab.add(placeholder)
 
@@ -426,12 +453,17 @@ def create_delex_data(dataset, delex_func=lambda d,s,v: f'[({d})-({s})]', ignore
                             for value in values.split('|'):
                                 if value.lower() not in ignore_values:
                                     placeholder = delex_func(domain, slot, value)
-                                    pattern = re.compile(r'\b({})\b'.format(value), flags=re.I)
+                                    #TODO: value = ?
+                                    value = '\?' if value == '?' else value
+                                    try:
+                                        pattern = re.compile(r'\b({})\b'.format(value), flags=re.I)
+                                    except Exception:
+                                        print(value)
                                     if delex_inplace(delex_utt, pattern):
                                         delex_vocab.add(placeholder)
 
                 turn['delex_utterance'] = ''.join([x[0] for x in delex_utt])
-    
+
     return dataset, sorted(list(delex_vocab))
 
 
@@ -468,7 +500,8 @@ def retrieve_utterances(query_turns, turn_pool, top_k, model_name):
 if __name__ == "__main__":
     dataset = load_dataset('multiwoz21', dial_ids_order=0)
     train_ratio = 0.1
-    dataset['train'] = dataset['train'][:round(len(dataset['train'])*train_ratio)]
+    dataset['train'] = dataset['train'][:round(
+        len(dataset['train'])*train_ratio)]
     print(len(dataset['train']))
     print(dataset.keys())
     print(len(dataset['test']))
@@ -477,7 +510,7 @@ if __name__ == "__main__":
     database = load_database('multiwoz21')
     res = database.query("train", {'train':{'departure':'cambridge', 'destination':'peterborough', 'day':'tuesday', 'arrive by':'11:15'}}, topk=3)
     print(res[0], len(res))
-    
+
     data_by_split = load_nlu_data(dataset, data_split='test', speaker='user')
     query_turns = data_by_split['test'][:10]
     pool_dataset = load_dataset('camrest')
@@ -490,8 +523,10 @@ if __name__ == "__main__":
         return f'[{slot}]'
 
     dataset, delex_vocab = create_delex_data(dataset, delex_slot)
-    json.dump(dataset['test'], open('new_delex_multiwoz21_test.json', 'w', encoding='utf-8'), indent=2, ensure_ascii=False)
-    json.dump(delex_vocab, open('new_delex_vocab.json', 'w', encoding='utf-8'), indent=2, ensure_ascii=False)
+    json.dump(dataset['test'], open('new_delex_multiwoz21_test.json',
+              'w', encoding='utf-8'), indent=2, ensure_ascii=False)
+    json.dump(delex_vocab, open('new_delex_vocab.json', 'w',
+              encoding='utf-8'), indent=2, ensure_ascii=False)
     with open('new_delex_cmp.txt', 'w') as f:
         for dialog in dataset['test']:
             for turn in dialog['turns']:
diff --git a/data/unified_datasets/crosswoz/README.md b/data/unified_datasets/crosswoz/README.md
index 2b8b037c1b6bbf0e98989d5959291529346da44b..07156f0e869180308d824d0a5a35abac12659004 100644
--- a/data/unified_datasets/crosswoz/README.md
+++ b/data/unified_datasets/crosswoz/README.md
@@ -13,13 +13,13 @@ CrossWOZ is the first large-scale Chinese Cross-Domain Wizard-of-Oz task-oriente
   - Run `python preprocess.py` in the current directory. Need `../../crosswoz/` as the original data.
 - **Main changes of the transformation:**
   - Add simple description for domains, slots, and intents.
-  - switch intent&domain of `General` dialog acts => domain == 'General' and intent in ['thank','bye','greet','welcome']
+  - Switch intent&domain of `General` dialog acts => domain == 'General' and intent in ['thank','bye','greet','welcome']
   - Binary dialog acts include: 1) domain == 'General'; 2) intent in ['NoOffer', 'Request', 'Select']; 3) slot in ['酒店设施']
   - Categorical dialog acts include: slot in ['酒店类型', '车型', '车牌']
   - Non-categorical dialogue acts: others. assert intent in ['Inform', 'Recommend'] and slot != 'none' and value != 'none'
   - Transform original user goal to list of `{domain: {'inform': {slot: [value, mentioned/not mentioned]}, 'request': {slot: [value, mentioned/not mentioned]}}}`, stored as `user_state` of user turns.
   - Transform `sys_state_init` (first API call of system turns) without `selectedResults` as belief state in user turns.
-  - Transform `sys_state` (last API call of system turns) to `db_query` with domain states that contain non-empty `selectedResults`. The `selectedResults` are saved as `db_results`. Both stored in system turns.
+  - Transform `sys_state` (last API call of system turns) to `db_query` with domain states that contain non-empty `selectedResults`. The `selectedResults` are saved as `db_results` (only contain entity name). Both stored in system turns.
 - **Annotations:**
   - user goal, user state, dialogue acts, state, db query, db results.
   - Multiple values in state are separated by spaces, meaning all constraints should be satisfied.
@@ -36,10 +36,10 @@ Chinese
 
 | split      |   dialogues |   utterances |   avg_utt |   avg_tokens |   avg_domains |   cat slot match(state) | cat slot match(goal)   |   cat slot match(dialogue act) |   non-cat slot span(dialogue act) |
 |------------|-------------|--------------|-----------|--------------|---------------|-------------------------|------------------------|--------------------------------|-----------------------------------|
-| train      |        5010 |        84660 |     16.9  |        20.55 |          3.02 |                   99.67 | -                      |                            100 |                             94.39 |
+| train      |        5012 |        84674 |     16.89 |        20.55 |          3.02 |                   99.67 | -                      |                            100 |                             94.39 |
 | validation |         500 |         8458 |     16.92 |        20.53 |          3.04 |                   99.62 | -                      |                            100 |                             94.36 |
 | test       |         500 |         8476 |     16.95 |        20.51 |          3.08 |                   99.61 | -                      |                            100 |                             94.85 |
-| all        |        6010 |       101594 |     16.9  |        20.54 |          3.03 |                   99.66 | -                      |                            100 |                             94.43 |
+| all        |        6012 |       101608 |     16.9  |        20.54 |          3.03 |                   99.66 | -                      |                            100 |                             94.43 |
 
 6 domains: ['景点', '餐馆', '酒店', '地铁', '出租', 'General']
 - **cat slot match**: how many values of categorical slots are in the possible values of ontology in percentage.
diff --git a/data/unified_datasets/crosswoz/data.zip b/data/unified_datasets/crosswoz/data.zip
index 067bb6ab80b28af8cfc5a3038c0e7bc91e6f0de4..c686a08351c47571cd5539273497f8680ea882fc 100644
Binary files a/data/unified_datasets/crosswoz/data.zip and b/data/unified_datasets/crosswoz/data.zip differ
diff --git a/data/unified_datasets/crosswoz/preprocess.py b/data/unified_datasets/crosswoz/preprocess.py
index 0eb97dc7b5b795f0fa7a3e2cd5d0f7e8b9dc42d3..b36f9e1aa688c006d0a506f1f639708cd916e243 100644
--- a/data/unified_datasets/crosswoz/preprocess.py
+++ b/data/unified_datasets/crosswoz/preprocess.py
@@ -432,9 +432,6 @@ def preprocess():
             split = 'validation'
     
         for ori_dialog_id, ori_dialog in data.items():
-            if ori_dialog_id in ['10550', '11724']:
-                # skip error dialog
-                continue
             dialogue_id = f'{dataset}-{split}-{len(dialogues_by_split[split])}'
 
             # get user goal and involved domains
@@ -456,15 +453,20 @@ def preprocess():
             }
 
             for turn_id, turn in enumerate(ori_dialog['messages']):
+                # skip error turns
                 if ori_dialog_id == '2660' and turn_id in [8,9]:
-                    # skip error turns
                     continue
                 elif ori_dialog_id == '7467' and turn_id in [14,15]:
-                    # skip error turns
                     continue
                 elif ori_dialog_id == '11726' and turn_id in [4,5]:
-                    # skip error turns
                     continue
+                elif ori_dialog_id == '10550' and turn_id == 6:
+                    dialogue['user_state_final'] = dialogue['turns'][-2]['user_state']
+                    break
+                elif ori_dialog_id == '11724' and turn_id == 8:
+                    dialogue['user_state_final'] = dialogue['turns'][-2]['user_state']
+                    break
+
                 speaker = 'user' if turn['role'] == 'usr' else 'system'
                 utt = turn['content']
 
diff --git a/data/unified_datasets/crosswoz/shuffled_dial_ids.json b/data/unified_datasets/crosswoz/shuffled_dial_ids.json
index 149bf1d24637290b5d3ea70d91b17cdd83c7517a..85cd4dc52ebfca561863cf259f33c42a91571d73 100644
--- a/data/unified_datasets/crosswoz/shuffled_dial_ids.json
+++ b/data/unified_datasets/crosswoz/shuffled_dial_ids.json
@@ -1 +1 @@
-[{"train": [4134, 296, 4325, 1662, 4156, 1105, 1065, 376, 1729, 4471, 393, 847, 3320, 2830, 3927, 3418, 1430, 2485, 3276, 1089, 180, 2064, 2104, 4440, 3661, 3351, 857, 1718, 2912, 4535, 2229, 893, 3336, 4111, 824, 4822, 4106, 2164, 930, 1491, 595, 2114, 1459, 1371, 4791, 15, 2262, 3007, 4911, 2137, 2908, 1753, 1368, 1859, 2884, 3196, 4155, 2853, 1700, 472, 593, 3017, 2328, 4765, 1375, 1243, 4040, 2094, 238, 1580, 4433, 3624, 1615, 18, 4772, 854, 3937, 1919, 4335, 4482, 872, 4109, 473, 73, 3875, 251, 3842, 1011, 2719, 41, 294, 88, 2204, 4337, 2994, 838, 2517, 2268, 3422, 4577, 2826, 3251, 1335, 963, 1578, 3391, 3228, 3030, 2843, 2227, 1776, 1868, 2477, 4035, 3601, 4159, 1852, 826, 3764, 4236, 3833, 4958, 435, 4858, 1452, 3705, 741, 900, 3707, 4769, 431, 3803, 2440, 3277, 331, 1551, 2432, 1137, 4606, 3169, 2613, 1802, 773, 4648, 2244, 2031, 2142, 348, 3091, 3830, 830, 155, 2481, 1128, 3841, 799, 558, 4013, 1074, 3665, 165, 3164, 833, 914, 2010, 4107, 630, 4928, 589, 1239, 2018, 3046, 649, 42, 427, 2333, 2374, 4771, 4991, 3226, 2400, 956, 2196, 1999, 4210, 3510, 1091, 2463, 3756, 3885, 2384, 3048, 695, 4286, 2772, 2901, 4044, 3203, 1274, 2717, 1238, 4584, 2596, 758, 1594, 2846, 2202, 2199, 1446, 3919, 163, 4411, 2609, 202, 1302, 4194, 0, 4104, 2933, 3237, 722, 3361, 4703, 4613, 3788, 3086, 3036, 3070, 3980, 4652, 723, 1916, 22, 3161, 4891, 933, 4492, 117, 1487, 371, 4123, 4659, 2159, 2738, 3859, 4946, 1898, 4864, 71, 1144, 256, 61, 3655, 3517, 417, 337, 853, 3437, 1773, 3231, 1727, 1291, 4913, 2015, 798, 977, 3653, 2880, 221, 4448, 362, 4124, 4181, 432, 4808, 4570, 1070, 133, 32, 1600, 225, 2217, 297, 2293, 711, 3119, 3310, 4587, 1661, 2993, 2292, 2761, 401, 2130, 1096, 30, 2915, 675, 1173, 3649, 489, 2818, 139, 788, 1214, 4823, 1982, 3704, 3663, 2765, 47, 2637, 3463, 4953, 4962, 1196, 4502, 388, 1869, 4674, 1197, 3131, 4497, 1121, 2845, 1155, 2703, 2634, 4902, 1339, 4074, 3662, 1675, 836, 4193, 869, 1904, 716, 1198, 4026, 4091, 4192, 1681, 293, 56, 1057, 3657, 3068, 1743, 3744, 4030, 2686, 3498, 2862, 2856, 3990, 4724, 3928, 2847, 1466, 1166, 1161, 2722, 2982, 3526, 2097, 101, 2918, 1994, 1259, 1075, 1078, 3214, 2575, 3355, 4235, 3677, 1107, 742, 1830, 2487, 4368, 3479, 543, 2540, 2739, 2929, 5002, 1149, 2509, 3014, 3897, 3710, 78, 2527, 4316, 2061, 2799, 2564, 1782, 3245, 4579, 3153, 1115, 4400, 10, 3144, 3471, 1502, 708, 2814, 4912, 4320, 1025, 3563, 4567, 729, 915, 4319, 1850, 2668, 364, 551, 1017, 2013, 2423, 3864, 4554, 4264, 730, 672, 2512, 1899, 4979, 3709, 846, 3262, 84, 96, 4118, 183, 2740, 2678, 2820, 3239, 3098, 1709, 268, 2474, 3887, 2392, 692, 4843, 1190, 1329, 1778, 1277, 1649, 4031, 2387, 3082, 1344, 52, 3192, 1272, 715, 4716, 3802, 4180, 2970, 4105, 2841, 1223, 891, 3219, 97, 3547, 2526, 4357, 896, 1181, 1199, 1151, 4245, 2998, 4876, 1232, 3955, 196, 1269, 492, 4249, 2797, 1012, 4431, 2409, 1636, 3901, 2215, 3721, 1389, 2523, 4681, 391, 3435, 86, 3279, 3614, 3583, 4223, 813, 4859, 998, 320, 3118, 2331, 1808, 1611, 1049, 4727, 146, 3762, 4401, 3148, 4650, 1622, 809, 1722, 2260, 1419, 4491, 3426, 4564, 2413, 483, 1499, 2510, 481, 4764, 1398, 1685, 266, 3791, 1532, 430, 995, 1001, 3794, 639, 4786, 4166, 2214, 775, 2490, 4894, 939, 4172, 3331, 955, 3997, 3697, 2849, 1433, 4212, 3263, 2867, 875, 36, 4870, 2747, 1597, 2877, 4281, 2991, 4904, 143, 2458, 917, 4816, 1475, 4130, 4462, 2958, 1966, 2705, 587, 4879, 283, 1122, 65, 1069, 4133, 305, 4670, 4529, 1653, 4234, 7, 104, 4900, 4756, 533, 309, 1981, 1187, 4077, 3557, 3968, 4331, 1029, 4701, 4761, 3016, 2298, 8, 4096, 1404, 1347, 3107, 4950, 3278, 2832, 874, 4430, 3891, 1860, 3184, 4523, 2138, 3814, 3933, 1664, 4379, 4079, 92, 757, 167, 1576, 1525, 4372, 4698, 1932, 2742, 3366, 3421, 831, 700, 2614, 886, 1865, 3886, 4530, 3804, 1795, 1461, 3283, 2245, 51, 1293, 3175, 3234, 2989, 4818, 2451, 3308, 4658, 582, 1742, 1381, 3524, 518, 4004, 3077, 162, 4560, 1058, 1767, 3776, 1247, 2129, 769, 3486, 2287, 4168, 1539, 619, 4572, 2133, 2352, 453, 1283, 2896, 1774, 1665, 1106, 4412, 2771, 3627, 4208, 2428, 2934, 4247, 1102, 2021, 4559, 4705, 1139, 3097, 2768, 1039, 3205, 1334, 4205, 1041, 1643, 2436, 2134, 1688, 1168, 3610, 3995, 1787, 392, 3931, 4023, 1409, 2663, 3045, 2894, 3862, 108, 4436, 2275, 602, 1740, 4973, 1134, 3034, 3101, 3645, 793, 601, 2957, 2291, 2375, 469, 1156, 4633, 3257, 2022, 3637, 1563, 4054, 1275, 2857, 4898, 960, 264, 1769, 2480, 642, 1027, 445, 699, 4685, 4916, 43, 1547, 197, 2495, 968, 3381, 1278, 4952, 150, 4457, 2255, 860, 1870, 2937, 514, 35, 2347, 1131, 80, 4072, 1915, 1318, 3894, 2673, 3460, 1655, 428, 2748, 2356, 2242, 3210, 232, 1929, 774, 4871, 4152, 352, 684, 4590, 2321, 683, 4888, 1229, 4299, 2414, 3438, 2320, 1658, 3009, 1357, 2604, 4183, 4512, 456, 2574, 3767, 2128, 379, 189, 2671, 1314, 657, 4480, 3751, 1369, 2117, 2899, 407, 4468, 1922, 2264, 1061, 3938, 2026, 3069, 4896, 1465, 2184, 2528, 5008, 3452, 4611, 640, 4085, 1835, 4721, 1985, 2942, 4548, 3322, 2589, 3676, 4315, 3977, 585, 402, 3623, 98, 4138, 1108, 288, 676, 1147, 4201, 1031, 1394, 1427, 1798, 419, 3924, 1969, 3127, 1635, 2330, 3293, 3505, 621, 1367, 1353, 1930, 3451, 4170, 494, 2357, 608, 346, 95, 3910, 3430, 4565, 3812, 922, 2380, 3449, 1683, 3174, 3988, 1458, 20, 1088, 1383, 3502, 1878, 3022, 3656, 4402, 4009, 2012, 776, 666, 2909, 3732, 1206, 1405, 4713, 1123, 384, 559, 4314, 4498, 902, 504, 2670, 3019, 170, 4700, 750, 2283, 3598, 49, 4255, 2645, 110, 2767, 4926, 358, 69, 3358, 4714, 3135, 3225, 3722, 3224, 4517, 3394, 1832, 2322, 2272, 3363, 2395, 1290, 4575, 3084, 2980, 3168, 4432, 243, 4649, 4348, 4747, 144, 2150, 2304, 1501, 3254, 3971, 4256, 2821, 1453, 943, 1512, 4924, 3606, 1524, 4640, 4273, 4355, 3911, 1987, 2155, 3752, 1019, 2095, 3018, 2314, 1312, 2080, 705, 1652, 3647, 4831, 1713, 4380, 3139, 4025, 228, 365, 805, 2922, 2505, 910, 3702, 1077, 2019, 4059, 1632, 3103, 3197, 498, 983, 4251, 3629, 4063, 3956, 1766, 2107, 1856, 3484, 4474, 1513, 3078, 206, 4394, 2984, 2499, 2425, 4311, 3141, 2891, 1970, 4518, 1604, 4131, 4573, 1140, 412, 2294, 2257, 3396, 40, 1079, 3838, 4792, 184, 3350, 1004, 1986, 3787, 1008, 4897, 4220, 3612, 2471, 2118, 1194, 1200, 3410, 2461, 726, 4970, 449, 114, 545, 4938, 1650, 3528, 4202, 2396, 732, 4918, 119, 1608, 4120, 3750, 4887, 4982, 1951, 553, 4838, 4203, 3823, 704, 1697, 4722, 3132, 1952, 39, 3147, 1076, 4084, 4047, 1349, 2600, 4167, 3010, 4442, 2855, 1780, 1021, 3376, 503, 3402, 2121, 4878, 2566, 2182, 1704, 1779, 724, 4162, 4636, 1460, 4006, 2743, 3371, 887, 3636, 4369, 4591, 3747, 250, 3672, 4324, 263, 908, 3431, 2924, 44, 1854, 1407, 4746, 4641, 2720, 3120, 2205, 3093, 126, 2096, 3761, 1429, 1770, 3633, 4604, 3281, 2514, 1138, 1825, 4877, 1678, 356, 3593, 2996, 4278, 4842, 4708, 728, 3506, 3939, 2838, 4718, 2697, 82, 2774, 2562, 2371, 3856, 2306, 3837, 2075, 1570, 425, 1548, 3053, 3386, 2218, 1976, 3936, 4396, 2054, 1873, 1355, 1813, 1984, 415, 2503, 478, 4725, 3217, 651, 668, 1579, 2872, 534, 1250, 2438, 1534, 2650, 2938, 1162, 3540, 807, 273, 2597, 1596, 2549, 4992, 1552, 2472, 845, 1064, 3289, 2796, 1554, 2105, 2103, 122, 1988, 859, 487, 4068, 1937, 212, 629, 3648, 1946, 4723, 1479, 2581, 1109, 3810, 4225, 5003, 4715, 2885, 2888, 2092, 2007, 1010, 3213, 754, 3480, 2561, 4382, 2456, 4309, 2524, 3347, 2895, 4190, 2890, 2727, 1676, 636, 578, 952, 4762, 821, 2936, 1807, 457, 2594, 3129, 1028, 4944, 4001, 4269, 3631, 2887, 2960, 852, 2072, 2986, 2408, 858, 1132, 72, 2507, 2946, 4367, 2828, 2930, 2921, 1493, 3590, 3429, 1648, 2376, 3387, 2858, 2783, 2616, 522, 3947, 2190, 1322, 3475, 1400, 113, 4230, 1820, 3689, 3490, 4506, 4661, 3481, 4610, 130, 3291, 557, 3736, 686, 3708, 3579, 3064, 1589, 1971, 1708, 530, 1059, 2226, 3370, 4191, 2415, 23, 4340, 3527, 4997, 3443, 4076, 4371, 4347, 4362, 928, 3195, 3033, 176, 3585, 3535, 1747, 2177, 4277, 135, 2233, 3415, 1299, 3012, 800, 669, 3258, 3428, 4663, 1047, 2250, 3215, 3008, 2863, 3390, 2194, 3487, 1098, 3290, 4749, 3950, 2241, 850, 2811, 4942, 4907, 2741, 1974, 123, 81, 441, 4070, 28, 3176, 2464, 34, 3000, 2450, 1744, 3666, 4197, 861, 1164, 4971, 2313, 648, 4800, 24, 1081, 3953, 3315, 3317, 1023, 4248, 3335, 790, 1877, 490, 111, 3348, 1846, 4767, 386, 1043, 4262, 1060, 2968, 3973, 2875, 54, 4504, 4227, 1244, 2143, 2559, 4473, 1890, 4389, 1320, 3798, 507, 3356, 2421, 3898, 1447, 2296, 3642, 2829, 286, 205, 4985, 4161, 2213, 3299, 1617, 843, 310, 4242, 4219, 3447, 1992, 4312, 1514, 3316, 4752, 2580, 3699, 140, 292, 1812, 3827, 272, 783, 4043, 3099, 752, 3916, 5007, 1642, 182, 784, 3801, 145, 978, 3507, 1421, 4366, 2726, 3128, 2349, 16, 1698, 911, 1392, 1063, 463, 1215, 1654, 2657, 3087, 437, 2, 194, 1603, 2835, 1342, 2583, 4740, 3253, 4188, 1730, 4538, 811, 2905, 2132, 2052, 1550, 761, 3912, 2806, 3880, 1340, 191, 4929, 4719, 2179, 3340, 270, 4910, 3212, 4975, 1, 2089, 1995, 2149, 2439, 4452, 3380, 3427, 3494, 2716, 99, 1397, 4427, 2624, 2232, 4332, 2565, 3193, 691, 4809, 661, 2028, 2579, 3405, 4741, 3684, 814, 1911, 4697, 733, 2805, 1360, 1300, 4391, 1496, 1567, 2928, 5, 1191, 4413, 999, 4037, 755, 1422, 4955, 3124, 1055, 4943, 2822, 4177, 3206, 4036, 768, 2115, 2582, 2381, 4224, 1771, 291, 1707, 4041, 1384, 4672, 2544, 659, 1456, 2372, 511, 271, 3359, 259, 2399, 567, 1612, 4631, 4957, 3781, 2995, 1855, 2473, 4292, 1657, 4228, 2550, 1775, 662, 1668, 1103, 4804, 4706, 1090, 4775, 3146, 2093, 4571, 1834, 1607, 2419, 4137, 339, 2541, 4875, 1249, 3183, 2466, 2948, 1204, 3602, 3605, 4386, 2398, 2189, 899, 3047, 2954, 3896, 4052, 4712, 3660, 3844, 2353, 4960, 2990, 2688, 870, 2534, 3483, 4836, 3723, 3319, 3416, 1549, 4489, 751, 2536, 1587, 2695, 4539, 3929, 3353, 3695, 2165, 3292, 2926, 2680, 4890, 3439, 936, 1598, 344, 2267, 1101, 460, 2535, 4932, 4254, 451, 4779, 2014, 63, 1157, 4195, 2024, 4116, 3136, 1454, 2043, 1007, 2578, 1944, 2332, 1337, 1035, 1150, 3055, 1669, 399, 3343, 4275, 19, 735, 2623, 3314, 2273, 3497, 383, 287, 3166, 1759, 341, 1136, 354, 1881, 1726, 4678, 1301, 4655, 380, 4143, 785, 4038, 2618, 4920, 4846, 1754, 3455, 289, 1420, 4763, 3790, 239, 3607, 4216, 241, 1756, 3604, 1376, 2418, 4742, 1258, 3207, 1639, 2708, 3204, 2515, 3111, 1264, 3095, 1731, 3182, 5001, 3638, 1202, 2539, 4645, 795, 2339, 641, 1308, 1080, 2315, 3117, 537, 1957, 3240, 2221, 4241, 749, 2140, 1897, 1637, 208, 4163, 128, 4196, 3423, 4796, 4250, 570, 3817, 4260, 3028, 4164, 796, 879, 1953, 920, 404, 1257, 4550, 59, 3388, 4388, 1889, 4308, 3304, 377, 4731, 829, 1980, 4851, 946, 4511, 1146, 677, 4434, 3759, 2062, 1056, 2556, 4300, 3934, 3615, 3706, 1892, 1327, 4385, 1739, 2443, 4860, 1455, 3417, 91, 4828, 3942, 4799, 3553, 1426, 4692, 1941, 4229, 975, 1839, 989, 2223, 4667, 1606, 3917, 702, 638, 2869, 290, 1495, 1938, 4739, 175, 851, 2608, 3275, 4304, 2050, 3866, 3364, 3156, 2750, 1205, 3529, 4889, 948, 801, 766, 4092, 452, 2547, 1408, 2733, 3906, 1062, 3044, 4745, 3180, 3446, 201, 2069, 216, 4798, 4778, 679, 4585, 818, 1324, 3873, 4969, 1153, 90, 1254, 2455, 340, 727, 3031, 1346, 2823, 3318, 223, 2653, 4384, 2303, 2391, 4419, 4872, 1833, 1436, 736, 1619, 75, 1113, 4179, 3256, 2950, 3932, 3167, 3001, 2151, 820, 1192, 4329, 1224, 3556, 3013, 2327, 2675, 866, 1464, 2112, 4754, 3015, 495, 2598, 2073, 4544, 2907, 1378, 4689, 4064, 4865, 4061, 4376, 5000, 4145, 2759, 1972, 2945, 2427, 429, 4987, 2592, 4676, 4341, 55, 4493, 2106, 4374, 2715, 3639, 1212, 409, 3362, 2222, 3327, 4185, 3664, 2532, 4773, 574, 2248, 667, 4785, 1601, 1179, 1005, 1296, 3514, 3472, 4012, 4951, 2664, 4576, 643, 981, 3985, 3236, 2654, 3673, 4893, 3259, 4108, 4115, 1333, 3477, 1686, 1670, 4988, 685, 901, 2819, 2060, 1646, 2240, 3006, 1723, 680, 810, 2621, 4574, 623, 3515, 2941, 4397, 2554, 4486, 1216, 3060, 3581, 1476, 505, 4213, 3742, 2972, 157, 1602, 4378, 3733, 862, 4704, 890, 4638, 3287, 2420, 2081, 4296, 4849, 550, 284, 2035, 779, 1403, 3244, 1067, 2997, 2865, 2003, 4258, 1853, 2676, 2091, 4103, 3392, 2074, 1588, 3826, 1195, 50, 4303, 3504, 3600, 1449, 367, 3250, 3300, 3779, 3178, 3946, 3062, 4102, 4922, 418, 1104, 1474, 4039, 2379, 2951, 374, 3957, 3869, 4053, 1595, 2638, 2084, 4370, 1097, 4377, 3632, 4566, 3793, 351, 4055, 760, 3599, 1537, 1457, 4268, 4276, 3301, 2992, 121, 1336, 4444, 1395, 311, 950, 1790, 400, 219, 1416, 4199, 321, 2192, 1572, 1358, 1593, 984, 2475, 4965, 315, 3004, 2665, 3734, 2683, 1288, 4098, 3777, 4776, 2186, 190, 2610, 1947, 3459, 941, 174, 4186, 3815, 4034, 3080, 2977, 2794, 3984, 4017, 4807, 2251, 4688, 2434, 2801, 2363, 4114, 4416, 925, 3404, 1880, 2766, 3882, 2854, 1625, 1094, 2944, 1033, 3218, 2274, 609, 2754, 394, 438, 734, 2974, 4093, 70, 1174, 3337, 2839, 4453, 1684, 2453, 4784, 4048, 3958, 247, 3309, 1867, 2071, 4782, 3096, 4182, 278, 4580, 3321, 1689, 4873, 198, 304, 138, 3288, 240, 4947, 2000, 2307, 693, 4360, 689, 1887, 2009, 4690, 1054, 2790, 2045, 1006, 4832, 3442, 1351, 803, 624, 3949, 3420, 4680, 3042, 597, 3821, 187, 1695, 1281, 1518, 2261, 1311, 4071, 4032, 2836, 4781, 2100, 1473, 3501, 3255, 1471, 2802, 1909, 4795, 1159, 4178, 2935, 4383, 3550, 3959, 3668, 3323, 3920, 867, 4080, 1241, 1363, 855, 1963, 4629, 770, 2763, 3904, 1544, 1099, 4198, 780, 4923, 4524, 2919, 2920, 3546, 2522, 832, 3609, 4149, 4293, 2482, 2170, 4121, 1690, 2382, 2338, 591, 1231, 4313, 3696, 3037, 2203, 753, 1541, 2873, 611, 3811, 1705, 3121, 2148, 4126, 1410, 192, 4825, 4978, 1749, 2674, 3542, 3669, 1127, 1230, 1693, 905, 3674, 3525, 3200, 4257, 1313, 1306, 2311, 4128, 3682, 4568, 231, 3774, 3209, 959, 4463, 4833, 3029, 1002, 4903, 5005, 137, 413, 4634, 844, 3065, 3701, 1717, 131, 462, 4794, 1129, 635, 2815, 1886, 2416, 4533, 1553, 2178, 4113, 3765, 4231, 2492, 93, 14, 5009, 521, 4801, 3378, 2200, 731, 485, 1424, 544, 2208, 4062, 1538, 1546, 3106, 4446, 4359, 4545, 554, 4005, 4596, 4789, 4101, 1933, 282, 246, 4046, 261, 2833, 4835, 2658, 299, 2368, 892, 12, 4556, 3737, 3948, 1262, 1893, 3440, 4148, 3357, 3820, 3890, 3140, 540, 3544, 227, 1824, 4557, 4284, 2563, 3993, 841, 476, 3974, 4475, 2462, 3877, 613, 2699, 2605, 828, 2713, 2243, 252, 459, 2310, 319, 1481, 2163, 2449, 4515, 335, 1013, 894, 3800, 1928, 4483, 3616, 1530, 4291, 627, 1958, 547, 3922, 904, 3558, 2469, 4755, 4673, 2800, 4551, 2728, 1256, 1906, 484, 4405, 3755, 1472, 4656, 46, 2346, 4233, 1236, 3269, 1093, 1829, 964, 4555, 1068, 27, 2729, 1325, 4934, 575, 4662, 390, 3522, 4966, 4734, 3075, 4757, 1660, 718, 791, 2325, 3870, 1380, 3847, 2904, 1053, 1879, 2644, 1354, 461, 789, 4127, 1917, 2702, 549, 2861, 2870, 4478, 1861, 4218, 1372, 4263, 2961, 2326, 2773, 37, 153, 3419, 4142, 2029, 4717, 148, 720, 1711, 4643, 3807, 2191, 3991, 3554, 366, 242, 2892, 2969, 1176, 816, 4710, 3305, 1666, 646, 2254, 4346, 2914, 4027, 1934, 4437, 1692, 4735, 3499, 953, 3860, 4010, 4810, 993, 980, 2403, 4993, 447, 3457, 1112, 2302, 1110, 2206, 4949, 4675, 1962, 1050, 3341, 1418, 2812, 4033, 4821, 3635, 3373, 3478, 3848, 4743, 3003, 3424, 3861, 3575, 1120, 2295, 1428, 1965, 3537, 2002, 3076, 1183, 1219, 921, 4261, 3267, 1142, 1009, 787, 2758, 3822, 1082, 1451, 2237, 1402, 2183, 3987, 4499, 1285, 3199, 4908, 4484, 3646, 3549, 849, 2124, 565, 255, 3829, 2504, 3881, 1680, 2756, 1691, 620, 1715, 1284, 4447, 3102, 781, 3713, 2345, 129, 992, 605, 3445, 3998, 4169, 2147, 698, 1967, 3154, 3576, 932, 2365, 2369, 1087, 1943, 3079, 4614, 4919, 1260, 4496, 3170, 448, 1356, 1508, 68, 1945, 1227, 2305, 5006, 2180, 2087, 2077, 1377, 3685, 4150, 4637, 1609, 4507, 4616, 4307, 3857, 4395, 3005, 3935, 610, 3964, 2127, 1940, 3393, 4726, 4856, 2690, 164, 2048, 1644, 2085, 4854, 3961, 4078, 1497, 4732, 2985, 1528, 1613, 2879, 527, 486, 2502, 4066, 3778, 4990, 1844, 2065, 4603, 3659, 1182, 4660, 1484, 797, 2301, 2335, 209, 3878, 4356, 2693, 1651, 3851, 3846, 548, 1765, 440, 4758, 4354, 4914, 1927, 2679, 2447, 2027, 1152, 2734, 199, 3383, 2389, 3513, 3996, 3083, 3640, 1610, 368, 4883, 2981, 3468, 2042, 4226, 2108, 2040, 652, 4915, 2752, 2366, 4390, 654, 1016, 1712, 4803, 1573, 1188, 3746, 396, 577, 1015, 4221, 3536, 501, 1910, 4087, 4477, 434, 2900, 2109, 3775, 4267, 3617, 1414, 4981, 3611, 3071, 2484, 1393, 835, 1902, 1983, 4246, 1800, 4805, 3591, 298, 3618, 1218, 3072, 1003, 1978, 3813, 2629, 1186, 2971, 2568, 2039, 124, 3085, 3202, 4863, 2343, 863, 2667, 1084, 2362, 2627, 3467, 332, 116, 4318, 4671, 1382, 2923, 3651, 4777, 4330, 4751, 2378, 2424, 690, 2625, 2058, 4595, 4056, 2516, 4465, 2560, 512, 2949, 2256, 2601, 804, 4874, 2975, 4602, 2125, 1417, 923, 1321, 3160, 674, 1245, 3835, 2122, 508, 4592, 3163, 4298, 3247, 745, 2407, 2620, 3628, 525, 1411, 2119, 2785, 1908, 1843, 2886, 2631, 1298, 513, 3975, 4132, 1677, 3188, 2153, 3020, 2911, 4627, 3715, 353, 2893, 531, 2259, 1914, 1511, 2344, 3051, 1478, 3541, 3360, 969, 4343, 4294, 497, 3564, 2083, 1480, 4454, 3989, 2868, 4848, 1228, 1189, 2220, 2252, 265, 4240, 2735, 1309, 2793, 359, 1823, 707, 4328, 3332, 3799, 502, 4699, 4352, 4024, 1169, 2506, 878, 1827, 4029, 1066, 4707, 4814, 4829, 1585, 2120, 4632, 4321, 106, 464, 4963, 3582, 2660, 1287, 3760, 4774, 2168, 3115, 4546, 3324, 4290, 1523, 3796, 4165, 2781, 628, 3552, 132, 2323, 1568, 1483, 1500, 2642, 966, 275, 4094, 142, 1794, 3474, 2488, 4217, 3670, 655, 2782, 2101, 4426, 1663, 2611, 2749, 307, 234, 3749, 3944, 1781, 3840, 1444, 4449, 1831, 1439, 1818, 2825, 682, 1745, 2939, 3260, 4605, 1255, 1210, 3712, 4909, 327, 3150, 1315, 1203, 2234, 739, 3369, 3448, 4327, 1738, 338, 4501, 2126, 4844, 3450, 2834, 4618, 4531, 3620, 3265, 3344, 1638, 1789, 1507, 509, 4569, 4090, 2685, 3088, 3041, 1171, 782, 2649, 149, 1037, 4626, 2224, 4527, 2111, 3252, 4687, 171, 3159, 179, 825, 4016, 151, 1874, 2932, 3819, 4812, 746, 2587, 4147, 2848, 1903, 2804, 4003, 3941, 4067, 2236, 3923, 181, 546, 1361, 3011, 2383, 3571, 3909, 2691, 3002, 987, 1412, 276, 4589, 66, 1038, 1529, 1584, 2603, 3189, 1925, 1599, 4683, 2269, 3211, 3333, 2228, 3694, 277, 1477, 2144, 350, 2737, 4620, 4996, 1706, 2337, 985, 1445, 2518, 6, 3425, 1193, 588, 4051, 990, 2355, 2110, 2588, 67, 2195, 4867, 2047, 3476, 4811, 1872, 2551, 4069, 4429, 976, 195, 4738, 1233, 973, 1180, 455, 4421, 2161, 2493, 3678, 4285, 612, 883, 4176, 77, 324, 4733, 2520, 1837, 3173, 2889, 4933, 4599, 2851, 1935, 3594, 156, 3216, 598, 1373, 4342, 2927, 3783, 1305, 4287, 919, 4014, 3492, 1581, 965, 1531, 2056, 1623, 218, 3785, 109, 3382, 1847, 909, 3720, 4995, 603, 4490, 4141, 1117, 4728, 3125, 343, 4609, 4460, 2113, 2913, 4271, 4487, 1520, 4326, 2626, 213, 3965, 4845, 4288, 817, 663, 1714, 2160, 4306, 1564, 1154, 3151, 3731, 4817, 4748, 3266, 1506, 3384, 1921, 1267, 4301, 4563, 958, 3379, 3066, 3081, 1167, 4607, 1819, 972, 2947, 3858, 1492, 3584, 178, 4509, 3577, 678, 3349, 3158, 4065, 740, 664, 631, 1543, 3644, 536, 3412, 3220, 1379, 3489, 3328, 3233, 3025, 2210, 3157, 2079, 4766, 3951, 316, 4481, 4691, 927, 786, 1991, 3138, 1399, 877, 3534, 3530, 3769, 74, 3408, 2082, 1734, 3795, 3853, 382, 2156, 2807, 3313, 3532, 4204, 523, 3728, 4323, 1542, 2289, 4744, 3500, 2903, 2437, 528, 1605, 4593, 2860, 2687, 3538, 1432, 579, 2718, 2386, 333, 1365, 615, 2468, 4000, 2964, 1032, 1163, 1583, 2778, 2285, 387, 3587, 3654, 1370, 2651, 4302, 3232, 3930, 2448, 2780, 1385, 3172, 1674, 1586, 1158, 1527, 1490, 650, 2497, 2141, 3400, 4930, 1793, 2898, 1014, 4456, 706, 2646, 4414, 345, 2925, 1406, 423, 1845, 253, 4770, 2053, 1948, 4684, 1316, 771, 2669, 2341, 1842, 168, 3596, 3243, 3735, 2377, 1895, 1574, 2076, 2433, 3680, 4184, 4200, 314, 1175, 4272, 233, 154, 3683, 1891, 1271, 4095, 3569, 1441, 230, 4558, 1125, 4018, 1240, 2586, 1811, 4479, 279, 1801, 618, 1294, 4583, 4654, 2558, 3818, 2633, 3740, 2789, 1838, 3899, 4972, 2309, 3496, 1900, 2023, 2572, 3165, 725, 3863, 2684, 3109, 4948, 626, 2467, 710, 808, 1048, 599, 421, 3771, 3726, 3338, 2643, 3354, 2979, 1438, 1431, 3179, 4793, 2088, 3643, 467, 3518, 2090, 312, 688, 3191, 1364, 3983, 4536, 4349, 1270, 3945, 322, 2146, 1640, 2412, 1784, 996, 433, 1788, 360, 3687, 2824, 3407, 1884, 2531, 4266, 703, 1330, 85, 1659, 3865, 3454, 1710, 2622, 4338, 1656, 2348, 1758, 2358, 4238, 318, 1883, 4702, 1786, 173, 3039, 1626, 4644, 4624, 3302, 2736, 3413, 2288, 2049, 3926, 4310, 2181, 4578, 529, 3889, 4976, 4830, 2037, 3560, 491, 3824, 1185, 1046, 3130, 4407, 3711, 865, 1234, 4974, 31, 224, 2454, 1618, 903, 186, 1263, 4657, 2158, 4317, 2429, 4783, 2777, 1160, 454, 1760, 1413, 466, 2098, 3092, 882, 4125, 1696, 1810, 3925, 4381, 1736, 2711, 1814, 1114, 334, 2760, 4622, 2530, 4110, 3508, 637, 215, 347, 2442, 4534, 3181, 4019, 2955, 1276, 1515, 644, 1396, 3641, 1719, 1540, 881, 4694, 3056, 3753, 125, 3650, 4373, 4815, 285, 1641, 2692, 2319, 3992, 743, 25, 1437, 1391, 3903, 3122, 2878, 1119, 738, 4280, 1095, 2840, 3397, 2694, 2494, 107, 2280, 2632, 3831, 2513, 2546, 3352, 3914, 2297, 4086, 1757, 3306, 926, 2351, 1141, 4206, 470, 1505, 3592, 3963, 1434, 3832, 3667, 918, 1975, 4601, 633, 4868, 2078, 2367, 424, 4768, 1279, 2459, 3979, 4387, 2953, 3555, 1732, 3073, 2725, 3578, 1448, 1682, 3757, 2744, 1350, 3884, 2300, 2402, 245, 4581, 3718, 3325, 1526, 3026, 3049, 395, 1768, 381, 349, 4461, 373, 274, 1942, 2813, 625, 1533, 4639, 2724, 3613, 1748, 1280, 389, 4968, 1703, 4403, 1857, 3389, 1388, 4459, 935, 1849, 1761, 880, 3545, 3915, 2258, 3149, 3595, 2666, 4075, 3493, 4736, 2874, 3248, 3768, 1762, 2479, 2070, 3797, 4642, 2173, 4451, 3836, 4625, 2916, 2246, 2397, 2593, 2557, 1126, 516, 269, 607, 4209, 1319, 330, 301, 3773, 2917, 2008, 442, 2810, 4696, 1645, 3470, 4852, 249, 127, 2635, 4927, 4472, 2135, 2698, 4365, 3208, 2978, 177, 4214, 907, 2063, 1851, 4283, 2431, 2102, 1482, 3186, 4408, 1950, 4998, 3054, 4730, 4866, 1724, 3038, 519, 214, 1086, 713, 4682, 1326, 2411, 974, 3229, 3235, 1467, 60, 539, 105, 3548, 2707, 450, 714, 3270, 3918, 3792, 532, 592, 3516, 1135, 11, 2441, 3688, 3902, 604, 3171, 4060, 954, 4917, 3330, 884, 4959, 4679, 2270, 3134, 4466, 281, 4279, 2312, 3223, 515, 997, 1366, 3222, 207, 87, 4117, 4476, 2628, 868, 2714, 4954, 3843, 3375, 3473, 500, 777, 1130, 1630, 248, 4289, 2641, 3693, 4207, 1504, 4669, 406, 1485, 4651, 967, 1821, 1816, 889, 1517, 4297, 134, 1145, 1671, 2521, 2004, 3414, 405, 2682, 1699, 4252, 4073, 2976, 3040, 2405, 4941, 4358, 2852, 3043, 2746, 1519, 1858, 1591, 555, 2201, 4050, 4899, 398, 475, 2017, 4097, 152, 185, 572, 477, 1672, 2659, 3962, 2871, 4420, 1901, 3374, 2602, 258, 2652, 906, 2498, 3748, 1343, 2299, 4157, 2537, 102, 1931, 957, 4861, 4787, 1521, 2615, 3368, 1560, 2394, 4153, 4439, 3067, 4345, 1509, 1862, 506, 2197, 4802, 57, 1616, 3238, 3895, 3883, 4600, 3982, 3464, 1627, 4175, 2966, 2770, 3523, 924, 3089, 576, 308, 1918, 471, 3294, 1387, 3565, 3057, 4841, 3303, 4850, 3867, 4623, 3743, 1052, 673, 3401, 1116, 4129, 2055, 4588, 1498, 4151, 2209, 3221, 541, 822, 719, 1701, 3061, 4935, 4513, 4171, 369, 538, 2999, 1494, 2373, 136, 4986, 3230, 1265, 161, 2647, 2570, 4022, 397, 4711, 560, 3433, 4336, 2700, 3143, 2952, 3094, 2776, 1026, 2543, 778, 2607, 897, 1443, 3284, 2486, 942, 3272, 3177, 1172, 4984, 3703, 3411, 2034, 3052, 2483, 4136, 4525, 3050, 3187, 2809, 3543, 4008, 33, 1124, 1621, 2696, 4122, 89, 1209, 4608, 1211, 2354, 1470, 2342, 1562, 971, 436, 1979, 4495, 4956, 815, 1237, 4980, 4081, 1024, 4112, 4339, 3940, 2617, 1225, 2406, 1415, 3871, 1809, 302, 647, 2238, 2775, 1178, 3691, 4994, 3295, 3681, 38, 2731, 4435, 1882, 1040, 2612, 4305, 951, 586, 2388, 3738, 2038, 986, 2798, 2538, 1566, 3566, 2496, 323, 717, 696, 737, 3246, 2792, 4295, 1687, 4709, 1000, 4521, 1764, 1535, 1468, 1201, 229, 3261, 1328, 188, 118, 3700, 1226, 2842, 681, 1817, 4088, 4582, 2883, 2595, 3114, 403, 4007, 1777, 370, 3058, 931, 1177, 3772, 2803, 2585, 961, 280, 2529, 147, 2216, 3573, 4514, 3110, 3634, 2943, 2567, 4146, 3908, 1556, 4617, 3519, 2762, 3491, 1345, 3488, 328, 2661, 4989, 2837, 2552, 1386, 4693, 4485, 3201, 1923, 306, 632, 3972, 4422, 4361, 1679, 590, 2959, 62, 1871, 3729, 2154, 408, 1803, 3461, 3580, 4425, 3562, 4215, 2417, 3954, 4664, 517, 303, 4057, 1246, 3162, 3123, 4797, 2606, 357, 3365, 1913, 4470, 1956, 1545, 2636, 4520, 3090, 2723, 2827, 3967, 3658, 1217, 4011, 2445, 2317, 4237, 2188, 2784, 3399, 4886, 571, 1118, 1561, 3142, 1184, 3741, 1996, 3868, 3952, 3403, 2239, 2390, 4082, 1261, 962, 3780, 929, 4528, 763, 3342, 112, 2545, 1297, 1960, 236, 4840, 3970, 4686, 2359, 1073, 4940, 4668, 422, 3621, 4083, 2963, 1536, 444, 3385, 3784, 9, 3311, 949, 2648, 2067, 2576, 721, 3725, 300, 3280, 2361, 4058, 4619, 2701, 1317, 856, 2630, 2730, 4561, 2850, 3850, 747, 4265, 4443, 1486, 4983, 4540, 3854, 1435, 4406, 2207, 169, 4813, 748, 211, 2051, 4353, 3539, 4635, 658, 3586, 4045, 4099, 2230, 3297, 2491, 3839, 1018, 3597, 2364, 4853, 1359, 488, 4939, 1165, 2176, 1220, 1841, 940, 4510, 1620, 4244, 1912, 583, 3719, 2316, 2360, 1896, 267, 2385, 3849, 4253, 3346, 21, 1282, 614, 1423, 1964, 1286, 665, 2167, 4921, 1702, 812, 493, 2501, 3512, 2435, 3626, 3690, 4455, 1848, 3409, 970, 2033, 697, 4547, 1222, 1242, 439, 606, 4028, 3282, 535, 934, 3913, 2068, 420, 1510, 2791, 3608, 3921, 3845, 2329, 100, 840, 4021, 4399, 2393, 446, 372, 2247, 1036, 115, 2476, 4621, 2956, 1863, 2426, 1590, 823, 2318, 2265, 3966, 1772, 4905, 1516, 979, 4925, 1577, 988, 1888, 4892, 2511, 2219, 3717, 2882, 2709, 3485, 2816, 4820, 3023, 2508, 1993, 4438, 1815, 2410, 762, 141, 3671, 4020, 3900, 1968, 4665, 4628, 1977, 4847, 1565, 4232, 83, 2263, 2290, 2788, 2116, 2906, 1737, 3994, 1489, 2662, 4516, 235, 2755, 4753, 4500, 1997, 3692, 2284, 709, 2011, 2640, 2198, 3568, 3808, 3960, 2212, 4274, 1442, 3567, 1253, 3766, 2967, 3551, 864, 4015, 4458, 103, 2795, 4695, 564, 3035, 4906, 4364, 2844, 336, 4826, 2470, 834, 1989, 622, 1624, 3249, 3074, 2555, 210, 4615, 2548, 1341, 4882, 1374, 3021, 2099, 4542, 1785, 937, 3879, 1034, 1954, 1248, 166, 1633, 4494, 254, 329, 562, 1836, 2866, 426, 158, 3852, 670, 2066, 3453, 2279, 4423, 2764, 2677, 3724, 1733, 2681, 4553, 2131, 4505, 1488, 3782, 1111, 3511, 687, 222, 3227, 220, 4937, 1045, 3828, 4839, 3312, 2753, 3185, 237, 4759, 3570, 2235, 1797, 3999, 1806, 4409, 3432, 3194, 2446, 2881, 48, 3907, 3116, 160, 3190, 3441, 1694, 510, 3406, 3816, 4445, 2710, 3339, 3398, 526, 1752, 1251, 3137, 1894, 1235, 4901, 660, 4424, 2619, 2751, 2225, 1920, 2340, 634, 3603, 2249, 1959, 3969, 4270, 2271, 2025, 2123, 3686, 3888, 1303, 3652, 4549, 3495, 4135, 3625, 2599, 4351, 3806, 1092, 3739, 994, 2041, 411, 3805, 4243, 1440, 1792, 3482, 873, 45, 1208, 1998, 3273, 4049, 3145, 1592, 3395, 3730, 596, 3892, 4936, 378, 200, 1207, 1631, 4824, 2542, 1924, 765, 2745, 4222, 2324, 694, 1390, 3876, 4187, 1750, 3622, 4653, 3809, 2639, 3521, 1721, 2350, 2489, 3, 4488, 3326, 2465, 1425, 3105, 3444, 1571, 1042, 4541, 2422, 3296, 2672, 3345, 2533, 4154, 2460, 1020, 2571, 2704, 573, 3533, 2983, 1614, 4410, 4259, 13, 3572, 363, 2086, 1557, 3155, 947, 4189, 3574, 4737, 2831, 3559, 888, 313, 2174, 1462, 2139, 944, 226, 2712, 4964, 3274, 4173, 2876, 2286, 568, 2282, 2689, 3264, 1582, 193, 1826, 1864, 2500, 2864, 3770, 1634, 1822, 3855, 317, 3133, 885, 1304, 4880, 2032, 3372, 2721, 1469, 203, 3377, 1450, 1331, 1939, 1266, 4760, 4586, 2897, 842, 2044, 1720, 2931, 326, 2987, 2525, 342, 1044, 3520, 1213, 4526, 3024, 295, 4469, 1221, 616, 4855, 79, 4594, 3503, 4519, 3619, 4862, 2145, 1071, 2732, 29, 361, 2266, 4790, 756, 416, 2057, 4857, 4140, 2162, 1667, 2808, 1725, 2308, 2231, 4977, 4884, 876, 945, 4967, 1100, 848, 2452, 4543, 64, 4333, 2478, 4450, 3589, 1292, 2334, 3027, 2001, 563, 2859, 1085, 4750, 4144, 1746, 4404, 600, 819, 4612, 3754, 76, 594, 2590, 3745, 2457, 1072, 3242, 2175, 1955, 2573, 1673, 2136, 2591, 4282, 4630, 3434, 671, 325, 2016, 4666, 982, 3307, 1926, 556, 561, 1522, 656, 4160, 465, 1289, 4350, 4119, 3905, 410, 468, 2569, 499, 4441, 479, 1783, 2430, 1555, 1268, 3986, 4552, 3465, 2336, 3727, 2172, 3198, 17, 3104, 1348, 480, 3285, 1751, 3943, 3978, 3329, 1362, 1936, 764, 120, 4428, 414, 802, 4508, 3630, 617, 2046, 3789, 4806, 1503, 4999, 1148, 3675, 4393, 1558, 1569, 2036, 895, 2779, 3298, 496, 806, 443, 3825, 3367, 3469, 2902, 3531, 794, 772, 1559, 3334, 3874, 1990, 3872, 4720, 4677, 1083, 4834, 1755, 3976, 2281, 1949, 4211, 580, 2706, 257, 552, 1875, 482, 581, 58, 1804, 1885, 4537, 4819, 172, 2769, 524, 1840, 2030, 991, 3698, 4239, 3588, 3059, 3268, 2553, 1647, 4417, 1735, 3466, 4322, 2171, 1352, 4503, 3893, 1051, 1030, 4363, 566, 3981, 701, 4961, 4647, 1973, 474, 1961, 2519, 2973, 916, 159, 4002, 2655, 4, 4344, 4418, 1323, 3063, 4895, 1629, 4869, 2444, 871, 4158, 1463, 4100, 2157, 26, 3716, 1295, 3561, 2404, 913, 2786, 2185, 4398, 938, 94, 4532, 2059, 4334, 3834, 4881, 3126, 3152, 520, 4885, 3458, 1310, 1252, 898, 385, 744, 4042, 4174, 1133, 1796, 2965, 3271, 4780, 3509, 4788, 2152, 4415, 4598, 2020, 5004, 2169, 1170, 3758, 3241, 4089, 1741, 2577, 4646, 1728, 542, 2193, 3286, 2584, 262, 1876, 458, 2656, 1799, 4562, 2211, 3108, 3786, 1338, 2005, 4375, 1401, 584, 2187, 1716, 2910, 3032, 1332, 2988, 3714, 2277, 3113, 827, 1907, 653, 2370, 1866, 375, 569, 1575, 4729, 2962, 2401, 4522, 645, 3100, 1022, 4392, 3763, 355, 2166, 4945, 2817, 2940, 792, 3112, 759, 837, 2757, 1763, 1273, 2276, 2787, 3462, 1307, 53, 2278, 4827, 3679, 1805, 3436, 4464, 1628, 4597, 217, 4931, 4139, 1905, 1791, 767, 244, 260, 3456, 4837, 712, 4467, 839, 1143, 1828, 2006, 2253, 204, 912], "validation": [5260, 5105, 5215, 5150, 5387, 5504, 5072, 5132, 5413, 5241, 5093, 5468, 5355, 5135, 5271, 5447, 5191, 5237, 5474, 5362, 5127, 5351, 5058, 5183, 5152, 5394, 5035, 5051, 5023, 5436, 5420, 5159, 5339, 5137, 5065, 5131, 5010, 5125, 5219, 5469, 5347, 5301, 5226, 5366, 5156, 5388, 5049, 5040, 5169, 5153, 5370, 5418, 5116, 5365, 5439, 5163, 5452, 5144, 5016, 5327, 5391, 5034, 5459, 5367, 5235, 5319, 5249, 5402, 5195, 5476, 5099, 5112, 5038, 5097, 5317, 5364, 5224, 5313, 5168, 5378, 5204, 5312, 5086, 5472, 5223, 5146, 5497, 5401, 5423, 5028, 5107, 5471, 5350, 5451, 5027, 5129, 5398, 5335, 5180, 5331, 5041, 5371, 5265, 5198, 5145, 5085, 5280, 5323, 5120, 5275, 5199, 5024, 5173, 5228, 5242, 5399, 5039, 5094, 5344, 5114, 5322, 5087, 5056, 5455, 5346, 5254, 5257, 5251, 5345, 5306, 5239, 5340, 5475, 5202, 5011, 5316, 5182, 5098, 5100, 5250, 5189, 5214, 5157, 5074, 5416, 5247, 5203, 5333, 5106, 5256, 5287, 5473, 5230, 5200, 5080, 5143, 5498, 5311, 5176, 5021, 5376, 5220, 5321, 5305, 5375, 5062, 5184, 5181, 5310, 5244, 5026, 5069, 5407, 5066, 5020, 5262, 5445, 5299, 5456, 5190, 5492, 5055, 5050, 5263, 5507, 5433, 5505, 5185, 5167, 5071, 5266, 5428, 5213, 5470, 5329, 5357, 5140, 5296, 5264, 5454, 5084, 5243, 5460, 5434, 5067, 5110, 5103, 5048, 5490, 5177, 5337, 5148, 5149, 5076, 5393, 5019, 5427, 5336, 5206, 5113, 5396, 5298, 5374, 5017, 5277, 5308, 5400, 5450, 5282, 5278, 5385, 5415, 5236, 5197, 5077, 5379, 5147, 5422, 5432, 5314, 5478, 5081, 5225, 5440, 5165, 5500, 5464, 5443, 5269, 5493, 5481, 5506, 5381, 5124, 5128, 5255, 5292, 5078, 5461, 5151, 5233, 5358, 5136, 5170, 5494, 5096, 5229, 5368, 5196, 5390, 5318, 5178, 5483, 5258, 5482, 5307, 5045, 5155, 5061, 5130, 5075, 5192, 5386, 5424, 5111, 5060, 5453, 5477, 5448, 5356, 5302, 5246, 5496, 5382, 5300, 5102, 5435, 5109, 5139, 5467, 5479, 5092, 5442, 5272, 5221, 5118, 5186, 5484, 5166, 5053, 5290, 5397, 5211, 5499, 5488, 5509, 5031, 5073, 5082, 5294, 5293, 5015, 5426, 5162, 5083, 5334, 5419, 5289, 5210, 5079, 5480, 5363, 5090, 5252, 5025, 5212, 5437, 5234, 5430, 5384, 5320, 5491, 5158, 5064, 5267, 5486, 5383, 5033, 5309, 5042, 5457, 5444, 5408, 5409, 5070, 5354, 5209, 5018, 5171, 5231, 5217, 5403, 5240, 5208, 5218, 5142, 5126, 5304, 5392, 5295, 5138, 5101, 5037, 5501, 5054, 5029, 5466, 5377, 5245, 5429, 5502, 5361, 5046, 5341, 5232, 5389, 5369, 5410, 5115, 5063, 5227, 5360, 5465, 5332, 5353, 5117, 5043, 5495, 5013, 5283, 5285, 5405, 5095, 5059, 5052, 5425, 5438, 5108, 5343, 5161, 5032, 5259, 5446, 5270, 5421, 5273, 5201, 5404, 5274, 5022, 5154, 5380, 5286, 5141, 5207, 5449, 5417, 5205, 5174, 5359, 5503, 5134, 5328, 5324, 5338, 5133, 5238, 5187, 5352, 5068, 5014, 5342, 5458, 5431, 5297, 5253, 5281, 5326, 5395, 5121, 5160, 5194, 5463, 5508, 5349, 5119, 5462, 5123, 5188, 5057, 5216, 5284, 5288, 5222, 5091, 5268, 5412, 5044, 5276, 5104, 5122, 5279, 5172, 5372, 5411, 5047, 5193, 5291, 5303, 5414, 5261, 5164, 5348, 5441, 5325, 5330, 5487, 5406, 5036, 5315, 5175, 5485, 5088, 5030, 5089, 5489, 5373, 5248, 5179, 5012], "test": [5629, 5876, 5782, 5511, 5747, 5635, 5849, 5811, 5553, 5733, 5604, 5790, 5731, 5962, 5973, 5840, 5648, 5850, 5663, 5819, 5869, 5588, 5573, 5720, 5649, 5905, 5675, 5727, 5680, 5954, 5915, 5984, 5888, 5693, 5519, 5998, 5740, 5807, 5948, 5538, 5997, 5833, 5945, 5616, 5532, 5752, 5679, 5558, 5890, 5678, 5773, 5636, 5872, 5842, 5789, 5669, 5707, 5802, 5521, 5672, 5661, 5673, 5572, 5901, 5990, 5991, 5831, 5662, 5987, 5704, 5864, 5659, 5608, 5681, 5626, 5908, 5755, 5546, 5668, 5913, 5549, 5559, 5708, 5923, 5982, 5722, 5606, 5610, 5743, 5674, 5889, 5516, 5965, 5914, 5518, 5989, 5523, 5723, 5827, 5942, 5520, 5698, 5972, 5760, 5929, 5594, 5909, 5999, 5963, 5828, 5953, 5533, 5786, 5562, 5567, 5677, 5585, 5729, 5724, 5881, 5863, 5566, 5877, 5701, 5739, 5941, 5587, 5741, 5934, 5591, 5696, 6003, 5958, 5768, 5654, 5960, 5980, 5841, 5894, 5552, 5940, 5985, 5583, 5893, 5530, 5624, 6002, 5640, 5750, 5600, 5582, 5966, 5971, 5664, 5895, 5710, 5665, 5766, 5541, 5797, 5907, 5598, 5957, 5917, 5812, 5676, 5749, 5746, 5595, 5725, 5534, 5924, 5937, 5822, 5565, 5525, 5551, 5758, 5922, 5611, 5522, 5943, 5777, 5574, 5586, 5838, 5764, 5861, 5646, 5780, 5783, 5829, 5735, 5935, 5930, 5839, 5631, 5813, 5926, 5882, 5560, 5834, 5620, 5652, 6006, 5579, 5891, 5933, 5856, 5568, 5814, 5932, 5543, 5903, 5873, 5865, 5581, 5801, 5737, 5605, 6009, 5689, 5617, 5778, 5618, 5975, 5939, 5779, 5597, 5748, 5800, 5578, 5961, 5577, 5781, 5900, 5576, 5730, 5666, 5602, 5593, 5535, 5658, 5806, 5563, 5682, 5787, 5859, 5970, 5887, 5853, 5830, 5956, 5540, 5759, 5836, 5857, 5711, 5897, 5851, 5526, 5754, 5992, 5537, 5575, 5927, 5967, 5653, 5899, 5844, 5761, 5650, 5788, 5825, 5700, 5832, 5531, 5513, 5762, 5925, 5571, 5854, 6000, 5692, 5818, 5950, 5826, 6007, 5645, 5633, 5630, 5878, 5765, 5607, 5603, 5647, 5803, 5621, 5691, 5670, 5753, 5978, 5793, 5548, 5685, 5884, 5557, 5657, 5637, 5928, 5514, 5993, 5898, 5883, 5983, 6004, 5959, 5622, 5705, 5544, 5642, 5848, 5756, 5524, 5879, 5745, 5744, 5949, 5634, 5974, 5798, 5703, 5736, 5638, 5944, 5785, 5641, 5772, 5569, 5684, 5660, 5767, 5547, 5875, 6001, 5769, 5718, 5981, 5995, 5938, 5804, 5716, 5742, 5902, 5820, 5625, 5885, 5687, 5688, 5808, 5976, 5655, 5964, 5936, 5545, 5870, 5517, 5904, 5612, 5810, 5717, 5712, 5852, 5690, 5714, 5988, 5554, 5590, 5510, 5918, 5632, 5694, 5528, 5855, 5596, 5919, 5628, 5867, 5774, 5771, 5816, 5527, 5609, 5916, 5775, 5846, 5613, 5515, 5734, 5763, 5837, 5599, 5969, 5539, 5815, 5874, 5656, 5776, 6008, 5946, 5951, 5706, 5880, 5529, 5821, 5550, 5977, 5623, 5644, 5542, 5555, 5732, 5817, 5751, 5584, 5931, 5556, 6005, 5512, 5835, 5709, 5955, 5536, 5968, 5860, 5866, 5824, 5791, 5920, 5683, 5912, 5715, 5770, 5619, 5561, 5592, 5868, 5910, 5738, 5627, 5892, 5570, 5847, 5952, 5994, 5580, 5886, 5986, 5871, 5906, 5686, 5805, 5667, 5996, 5979, 5794, 5796, 5699, 5601, 5757, 5795, 5911, 5792, 5721, 5614, 5896, 5713, 5858, 5651, 5823, 5697, 5799, 5726, 5784, 5947, 5643, 5639, 5809, 5843, 5728, 5589, 5564, 5695, 5719, 5671, 5921, 5862, 5615, 5845, 5702]}, {"train": [2462, 1803, 2727, 4949, 2114, 3193, 973, 4552, 2787, 2548, 2610, 725, 4418, 3264, 4561, 1719, 2492, 1012, 2915, 2393, 1032, 4822, 769, 2023, 1620, 281, 3882, 4017, 3844, 2066, 2794, 3987, 1708, 3603, 2708, 4404, 1669, 4582, 4617, 2736, 686, 1677, 1072, 234, 2566, 3346, 1377, 3580, 741, 1786, 205, 3555, 3730, 3609, 3552, 3447, 2432, 4417, 1167, 1534, 915, 1630, 3537, 3819, 1285, 2340, 2663, 4749, 3839, 3638, 2192, 4145, 2944, 1279, 2486, 2906, 1348, 2278, 2274, 4157, 3787, 819, 733, 860, 4933, 2671, 3972, 1161, 893, 1697, 2421, 384, 4394, 4519, 1732, 2509, 486, 3982, 2611, 2331, 3163, 2703, 3996, 2452, 2504, 897, 1327, 485, 520, 3315, 4677, 3914, 4520, 2326, 3501, 4162, 4758, 3487, 2816, 1563, 2099, 1597, 54, 1504, 236, 4321, 3308, 979, 3191, 1806, 4688, 1345, 4523, 828, 3354, 914, 4419, 4064, 1574, 2412, 4205, 3454, 3467, 197, 2605, 4019, 4356, 3001, 4533, 3184, 1849, 4596, 3327, 1177, 455, 4310, 1239, 2487, 1935, 438, 3973, 3223, 2589, 1988, 229, 4228, 1066, 4306, 2186, 2139, 2537, 3167, 522, 3601, 2070, 1737, 1120, 1734, 1699, 4065, 565, 219, 72, 3986, 4482, 4345, 385, 2621, 2523, 759, 2702, 45, 3323, 3822, 4871, 4779, 4792, 3532, 4045, 851, 1226, 3699, 4728, 4873, 463, 4805, 3704, 3220, 3876, 1814, 86, 4497, 2033, 2437, 3209, 1655, 2167, 3684, 652, 4581, 3488, 3736, 23, 1132, 2091, 3877, 3039, 2715, 4251, 3442, 1576, 831, 3781, 2133, 3561, 931, 4767, 2620, 2751, 4050, 417, 3682, 775, 1927, 4979, 2733, 1789, 118, 46, 1497, 3510, 3126, 4975, 2430, 4632, 3675, 2414, 1553, 285, 988, 4855, 616, 4909, 1247, 1770, 350, 1246, 4254, 3560, 465, 59, 2728, 1880, 1236, 740, 3446, 210, 3240, 895, 3356, 3847, 570, 958, 1151, 4692, 2086, 3732, 3696, 399, 627, 3125, 4959, 3585, 4002, 1891, 3414, 2147, 4998, 2642, 2478, 4714, 2707, 1687, 4049, 4001, 3540, 2789, 2410, 1363, 426, 2719, 1779, 1318, 1304, 839, 3985, 1379, 3237, 232, 133, 3639, 4684, 2624, 1342, 4280, 4107, 998, 3678, 4547, 2057, 2909, 1675, 2154, 3531, 738, 3832, 1290, 2648, 1808, 1254, 4626, 429, 4531, 1500, 4588, 3608, 2358, 4216, 1496, 1139, 1391, 2840, 1606, 119, 917, 3768, 2101, 1896, 2841, 1473, 1015, 2939, 1841, 3900, 888, 1965, 546, 4181, 4768, 4442, 3194, 689, 660, 3359, 3108, 4624, 2557, 925, 4723, 368, 776, 4221, 3459, 204, 1471, 4664, 2687, 149, 3565, 1397, 474, 3956, 3066, 1449, 4362, 4605, 2934, 1838, 3520, 3186, 225, 3776, 4, 938, 755, 1073, 551, 4852, 999, 912, 2571, 4598, 976, 2704, 3785, 2666, 2701, 1758, 2554, 3628, 4734, 2854, 3857, 3969, 2660, 1521, 2172, 1911, 263, 3020, 2228, 2752, 2112, 3623, 3405, 4125, 576, 3491, 3041, 1812, 936, 191, 962, 4183, 3752, 4989, 4295, 2897, 3692, 2110, 1010, 4339, 4388, 2449, 334, 4170, 4083, 3012, 975, 2937, 2011, 3634, 1980, 4343, 4072, 4971, 549, 2810, 4277, 1546, 4604, 2382, 4330, 4868, 2543, 2141, 4235, 1522, 684, 557, 4819, 3525, 1701, 3598, 2456, 2348, 2056, 2145, 1637, 3450, 1212, 4047, 880, 4869, 778, 4105, 3713, 3633, 4824, 2262, 4615, 3771, 1946, 4492, 2176, 1938, 3204, 3737, 3695, 3032, 4448, 3282, 3296, 1399, 4073, 4695, 376, 2935, 3427, 2970, 2584, 4358, 271, 4812, 2673, 400, 2803, 2209, 2480, 3114, 3954, 4007, 1186, 2435, 2440, 1696, 1283, 2020, 3003, 4193, 2405, 1048, 3380, 3293, 3485, 2026, 2490, 2510, 3024, 2322, 3635, 2868, 3396, 253, 777, 3048, 3925, 2351, 4611, 4268, 3621, 1547, 4957, 2637, 1341, 1468, 4097, 1232, 1187, 3106, 4101, 3920, 2895, 3919, 2210, 2163, 1198, 706, 182, 4369, 4440, 903, 58, 4549, 1710, 2253, 730, 723, 3092, 1077, 989, 4471, 1536, 4787, 4133, 1882, 453, 3416, 703, 3504, 173, 1646, 2457, 4616, 695, 3484, 3593, 3602, 1223, 3993, 2867, 1575, 735, 3175, 3181, 3940, 2916, 2006, 1827, 601, 169, 4504, 2570, 4941, 4668, 4179, 870, 2078, 1811, 3152, 2880, 3102, 1802, 3107, 2901, 2217, 1191, 63, 398, 4584, 805, 1346, 2022, 1491, 2283, 2152, 1417, 2967, 1668, 744, 4673, 1888, 301, 4540, 1878, 3383, 3142, 3963, 850, 1157, 4256, 4305, 3351, 168, 3860, 2093, 3906, 3544, 4319, 2591, 471, 345, 3262, 2562, 206, 4650, 3843, 1711, 4846, 996, 314, 1745, 919, 139, 2270, 618, 780, 1182, 3301, 3774, 2062, 587, 3911, 1618, 3991, 2951, 586, 3777, 286, 4396, 682, 2609, 389, 2654, 608, 1795, 2814, 287, 4867, 1925, 116, 3753, 4397, 4764, 3479, 3641, 3858, 1322, 1549, 813, 1275, 4538, 4414, 2799, 1358, 2087, 691, 3814, 2129, 3243, 916, 2824, 4126, 2608, 146, 666, 1160, 354, 1080, 4542, 869, 1519, 3168, 3136, 3676, 171, 416, 1200, 4304, 957, 3724, 3394, 1571, 4300, 2024, 4406, 781, 646, 2563, 1992, 37, 993, 216, 3325, 3716, 336, 1398, 2494, 2266, 4679, 4024, 4693, 4089, 4149, 1023, 3280, 663, 2265, 2798, 1542, 4807, 315, 3008, 2076, 1457, 2104, 1944, 396, 3093, 965, 3051, 4456, 4927, 2536, 1922, 4991, 3619, 1064, 3330, 3423, 1258, 3726, 1131, 4900, 3743, 2220, 2392, 795, 2031, 3063, 1954, 3230, 3541, 3368, 3225, 1382, 1065, 4197, 3556, 333, 2540, 4014, 3306, 3221, 2327, 4501, 4230, 2933, 4167, 2679, 4763, 1835, 3341, 1793, 1505, 2493, 3180, 249, 2294, 906, 1949, 3409, 3604, 105, 488, 2166, 2311, 1601, 3018, 2426, 1039, 1141, 3617, 4322, 583, 4095, 3745, 276, 2674, 4433, 1213, 1540, 4599, 2846, 3581, 2825, 3300, 1741, 243, 1943, 2084, 4479, 3121, 318, 3091, 1425, 1666, 751, 4491, 3820, 556, 2786, 1889, 3851, 4502, 1249, 2489, 3830, 808, 4246, 1009, 4066, 101, 3689, 2770, 575, 2764, 2125, 2359, 1917, 3017, 15, 4063, 2613, 71, 4752, 4886, 2149, 2957, 4737, 3727, 2982, 1028, 1918, 3085, 4643, 2586, 1513, 311, 4793, 564, 1580, 1350, 4618, 624, 3391, 2396, 3203, 533, 457, 654, 1626, 2553, 3388, 3490, 122, 484, 526, 3527, 4817, 3950, 96, 2386, 4980, 3200, 3870, 1140, 2902, 4370, 683, 2298, 3372, 3644, 4366, 3712, 1751, 3422, 366, 3095, 4571, 3109, 3757, 1721, 3382, 1665, 2850, 2454, 1054, 3984, 4027, 4438, 4155, 4353, 2729, 3747, 1483, 3884, 2793, 4740, 3441, 1147, 2866, 941, 2929, 3470, 110, 2328, 3731, 4528, 269, 3778, 3412, 3798, 4715, 4015, 3481, 1129, 4459, 4772, 3811, 4078, 2373, 3889, 4192, 4811, 3265, 2506, 1142, 1544, 569, 75, 863, 4920, 3514, 1545, 3404, 604, 1932, 693, 4303, 3661, 406, 2397, 560, 2886, 1331, 3707, 167, 2108, 4644, 3511, 4859, 2481, 1062, 421, 1753, 513, 272, 2943, 2235, 2143, 1071, 3670, 585, 548, 339, 2823, 475, 4476, 2560, 2395, 3060, 3072, 841, 4689, 1276, 3274, 2123, 3512, 2034, 1611, 4827, 2974, 2849, 1577, 1447, 4712, 4938, 4913, 2851, 1624, 2061, 3744, 4203, 2017, 844, 4041, 3438, 4288, 3927, 4875, 4000, 4904, 4690, 2291, 2758, 896, 4843, 1158, 195, 4021, 181, 2195, 239, 2651, 4821, 2301, 1130, 2369, 4667, 3117, 2047, 4472, 428, 4735, 2342, 2898, 3655, 502, 4765, 1731, 3679, 680, 4171, 2681, 1240, 3615, 3990, 1089, 3806, 3000, 3064, 2667, 351, 1372, 3268, 4951, 4741, 1210, 391, 1295, 3673, 4011, 3233, 1851, 1800, 2512, 3294, 3417, 3120, 361, 1360, 2232, 1000, 327, 4648, 1013, 1101, 2711, 794, 1205, 3428, 2542, 1870, 658, 3567, 2334, 3758, 479, 3298, 305, 4267, 4325, 2446, 4328, 1717, 1674, 4694, 2694, 2261, 410, 3983, 1281, 3076, 1763, 3873, 1977, 2089, 3939, 812, 2706, 509, 2927, 2199, 1854, 4589, 2158, 1043, 4110, 3395, 1312, 2773, 4231, 423, 1467, 1920, 2316, 3845, 933, 3588, 1456, 3172, 3433, 1046, 1874, 492, 3563, 3228, 1747, 1401, 335, 625, 3156, 4647, 1755, 2596, 2812, 4473, 4833, 3196, 4395, 2482, 2940, 1172, 4272, 1439, 3902, 1999, 1537, 577, 1096, 375, 2676, 4455, 5, 4891, 1118, 809, 3508, 1865, 1337, 1511, 1853, 791, 4830, 620, 3050, 910, 1941, 1469, 1092, 9, 2119, 1242, 4929, 3783, 1355, 4427, 364, 4212, 662, 1994, 4722, 89, 3765, 207, 57, 4995, 1311, 1490, 2843, 1912, 4018, 4666, 1081, 1116, 1429, 343, 3951, 594, 3856, 4968, 4026, 3266, 4870, 3502, 3589, 4022, 4227, 4299, 2219, 4477, 1143, 4385, 2870, 444, 582, 4823, 4020, 461, 3358, 3141, 4441, 1810, 2193, 2821, 2016, 4804, 902, 3703, 92, 111, 3775, 3322, 3328, 1628, 4425, 1438, 256, 4156, 228, 3355, 3740, 3559, 697, 378, 2629, 273, 2709, 401, 843, 3006, 1287, 1221, 43, 3891, 642, 1523, 2254, 584, 3786, 3622, 2371, 3415, 4086, 538, 746, 848, 2738, 28, 718, 1055, 4040, 1887, 2411, 4266, 1233, 553, 2643, 2187, 721, 2080, 4257, 1970, 355, 3503, 2329, 1224, 3174, 1211, 4112, 2107, 2962, 1228, 4163, 4905, 4841, 517, 3115, 4849, 4290, 3665, 3921, 2109, 2151, 3653, 2337, 3871, 3756, 1495, 2310, 1394, 414, 1238, 170, 4789, 4346, 4351, 1995, 4004, 2300, 4726, 1791, 3957, 3310, 81, 4635, 1641, 4318, 3016, 1245, 3738, 2105, 4813, 3961, 4832, 3793, 478, 4245, 3411, 4660, 1627, 2306, 1235, 707, 4461, 3443, 4092, 3517, 1409, 1886, 4367, 677, 699, 4445, 4128, 1901, 3057, 874, 3542, 1419, 2472, 3173, 4093, 2809, 3401, 949, 3627, 2197, 628, 4003, 3838, 328, 753, 1568, 4866, 418, 3406, 3208, 2529, 3010, 981, 2652, 1347, 1260, 1836, 2339, 3097, 2363, 1979, 1638, 764, 3755, 724, 4906, 1179, 2012, 5005, 3642, 2662, 2861, 3632, 4281, 4143, 842, 4828, 802, 4430, 2043, 2829, 4800, 379, 3123, 1110, 1738, 3399, 4423, 4597, 3764, 425, 2399, 4344, 800, 2675, 2321, 3705, 4851, 3456, 2128, 3421, 1261, 42, 4198, 2118, 3801, 3384, 2367, 1383, 593, 4943, 2966, 3049, 2767, 2588, 2766, 1635, 3549, 3027, 87, 1526, 2640, 2682, 2863, 90, 1268, 4085, 1412, 1968, 430, 2879, 4030, 2891, 4999, 127, 945, 3424, 2878, 2948, 3449, 4686, 4966, 297, 3566, 2375, 2131, 5000, 3620, 4934, 3733, 3904, 2778, 3059, 160, 3591, 650, 4777, 4592, 2844, 2599, 300, 3967, 3134, 3352, 251, 1138, 1446, 152, 4309, 4010, 3070, 2617, 3334, 3812, 2098, 1315, 2712, 4850, 3907, 4897, 121, 325, 3794, 2541, 2630, 838, 2763, 4558, 1174, 2260, 2590, 1127, 760, 2445, 1272, 1807, 702, 3505, 532, 3238, 4490, 2869, 1189, 2845, 1599, 4436, 1883, 4732, 2164, 2517, 3478, 1308, 70, 2832, 1632, 3717, 2202, 3702, 2914, 1387, 1502, 1437, 4129, 2095, 4508, 1222, 4121, 4796, 2979, 2013, 2887, 4088, 3660, 712, 2189, 154, 1550, 2441, 441, 2942, 3978, 292, 2795, 1817, 3616, 2980, 2177, 3916, 2632, 1106, 2603, 2938, 2238, 4573, 3946, 922, 4559, 2683, 78, 653, 1448, 147, 4893, 3910, 4642, 1206, 855, 530, 852, 3197, 4279, 3808, 4548, 4191, 639, 431, 4282, 1100, 1715, 4743, 930, 4982, 2576, 3099, 3219, 313, 1231, 950, 3347, 2465, 1255, 3821, 3647, 4512, 4380, 854, 1324, 2792, 2174, 4608, 4602, 1498, 3081, 1095, 1353, 3250, 4176, 163, 1430, 4168, 1578, 352, 2005, 2247, 2697, 2206, 4320, 743, 534, 3855, 595, 1866, 705, 1826, 4233, 2271, 2447, 2724, 632, 1862, 1503, 3254, 1942, 674, 3570, 4801, 3400, 1861, 913, 4287, 4377, 4834, 2873, 2409, 4340, 675, 1270, 2053, 4264, 2884, 2739, 4389, 3725, 460, 2338, 4776, 994, 1070, 524, 3226, 1595, 2813, 353, 4551, 1093, 4511, 3211, 1651, 2936, 2431, 2731, 926, 4831, 4629, 1543, 4452, 3065, 4945, 443, 4612, 1723, 21, 3868, 3080, 4298, 606, 1885, 369, 4106, 1076, 68, 447, 3782, 613, 1815, 5003, 1797, 165, 1830, 3198, 3879, 3353, 3796, 573, 4935, 107, 3534, 1720, 1724, 1300, 4700, 2341, 3513, 536, 1434, 2744, 377, 3287, 2018, 566, 214, 1420, 2575, 4775, 3564, 3722, 3258, 4910, 3161, 4469, 4978, 3122, 3779, 252, 765, 3045, 3769, 590, 3143, 1209, 669, 2649, 3813, 5008, 1031, 4435, 722, 3440, 1581, 4720, 4324, 1725, 3150, 3962, 3924, 4961, 4223, 1695, 2069, 2256, 2693, 85, 1214, 4096, 3606, 1585, 643, 833, 1875, 3084, 2323, 3046, 732, 506, 3590, 4082, 3026, 388, 4043, 490, 3370, 4729, 2450, 4517, 200, 1004, 473, 2384, 4359, 4042, 2357, 4368, 1676, 3052, 4144, 4495, 2805, 4662, 288, 3043, 1014, 4786, 1410, 2356, 1892, 4009, 2144, 319, 4915, 3465, 2263, 2476, 4781, 4973, 3407, 76, 3335, 3866, 757, 1413, 4489, 3337, 2156, 1219, 131, 4453, 3116, 2236, 3687, 382, 2226, 1484, 189, 2995, 62, 772, 1728, 1784, 4034, 2196, 3362, 128, 1707, 1762, 1945, 951, 923, 3885, 600, 1748, 3852, 3371, 102, 2628, 547, 3896, 2989, 3413, 3579, 803, 2044, 2140, 1899, 3339, 3075, 3047, 2532, 4403, 719, 1730, 2945, 3789, 158, 55, 2960, 3741, 41, 3941, 847, 135, 3289, 873, 4363, 3, 3019, 4355, 1757, 4510, 3314, 1783, 1691, 3165, 2518, 909, 1301, 3869, 1418, 2249, 3770, 4916, 4888, 1144, 2042, 940, 4215, 2958, 2655, 162, 1289, 2234, 3377, 1766, 215, 3435, 4013, 2455, 942, 2700, 3260, 1947, 4458, 1729, 1670, 1026, 26, 1250, 2250, 265, 4877, 3029, 442, 2442, 3849, 4788, 1388, 4165, 673, 3379, 4468, 543, 3898, 4214, 3058, 67, 1084, 865, 3477, 1559, 3979, 4429, 2081, 4237, 1257, 1192, 3524, 1684, 2598, 882, 1910, 358, 1099, 4331, 3943, 1406, 1926, 2004, 2519, 2376, 599, 4646, 2507, 4887, 2325, 4278, 537, 3128, 2852, 4545, 3148, 2400, 2883, 4814, 1416, 4211, 4139, 4795, 2032, 2126, 3013, 31, 1602, 4922, 4625, 307, 3374, 3840, 3190, 701, 2419, 2885, 4760, 1474, 4131, 2064, 2286, 3316, 4544, 2661, 3594, 2658, 4123, 3014, 211, 184, 1756, 4036, 4201, 969, 4639, 363, 1509, 1193, 2686, 132, 1385, 3629, 2324, 3550, 247, 1984, 4075, 7, 4932, 1034, 3025, 4457, 3784, 621, 3452, 527, 2065, 4439, 394, 3386, 3132, 514, 2899, 278, 2335, 4274, 801, 815, 2389, 1227, 1181, 4140, 987, 974, 433, 4940, 641, 2730, 758, 4100, 4202, 2320, 856, 3526, 4255, 2994, 3875, 2516, 814, 3146, 469, 1402, 540, 3693, 1658, 1356, 2817, 2088, 114, 226, 4983, 2050, 4506, 14, 982, 3734, 1856, 1656, 1190, 2756, 4426, 4936, 2833, 2990, 2956, 3331, 1739, 3242, 634, 2500, 1933, 928, 1754, 3299, 2188, 3551, 4594, 2361, 4912, 737, 1768, 3473, 3728, 4218, 1361, 3636, 4606, 4890, 4562, 829, 2245, 402, 1554, 4348, 3613, 1479, 1079, 1090, 3336, 1488, 3419, 4621, 4947, 3721, 2920, 2594, 397, 2085, 4273, 3694, 2496, 3842, 3297, 3912, 1038, 2657, 2807, 2276, 2019, 944, 871, 1364, 248, 605, 4332, 275, 2913, 497, 412, 2535, 2365, 1924, 295, 2889, 1297, 1069, 4883, 1477, 561, 4622, 2233, 2685, 1112, 943, 3674, 779, 516, 525, 678, 260, 1195, 1990, 4964, 143, 1204, 1259, 2699, 1598, 1804, 2103, 2828, 293, 3650, 4109, 4988, 804, 1370, 1323, 4641, 2471, 1998, 3408, 1460, 3804, 4710, 2049, 3790, 2135, 3691, 1686, 1305, 4269, 2505, 4035, 3735, 8, 588, 1760, 4146, 1821, 3350, 1647, 1570, 254, 1592, 407, 2015, 1820, 2051, 4392, 690, 3671, 1987, 3360, 4384, 2835, 2402, 2434, 4946, 3652, 4837, 1796, 1349, 4329, 1427, 3094, 4685, 2665, 3667, 3645, 1303, 2780, 4187, 1586, 3723, 464, 4840, 2635, 142, 4196, 629, 2892, 2601, 3626, 2559, 4408, 5001, 4244, 2160, 3385, 4570, 4669, 4070, 1149, 2305, 2246, 3646, 3945, 3458, 2796, 2772, 3823, 1561, 4919, 4360, 3663, 3244, 1314, 3313, 437, 1937, 2287, 1164, 3178, 1403, 123, 4653, 2762, 810, 2008, 3378, 1589, 2473, 4450, 1660, 4628, 88, 459, 4754, 73, 39, 289, 2161, 3160, 3607, 727, 1702, 4593, 1445, 872, 4412, 2698, 2837, 3720, 1396, 1736, 4505, 4630, 4432, 498, 1876, 1916, 439, 4226, 1983, 997, 1975, 2622, 2230, 3953, 1776, 4557, 507, 1332, 1622, 747, 4770, 4159, 2602, 885, 2146, 4620, 4697, 186, 3444, 4120, 1775, 3980, 262, 4698, 1313, 579, 3015, 521, 3420, 4484, 4874, 499, 2474, 2491, 4631, 3895, 3543, 2221, 1082, 3077, 2092, 611, 4023, 2985, 348, 644, 3271, 2579, 1493, 4711, 768, 4854, 1957, 4769, 1615, 3668, 2429, 4184, 141, 3528, 4039, 3690, 2988, 2650, 657, 4464, 2972, 4365, 4953, 2527, 4815, 1860, 3195, 2364, 349, 2931, 4536, 136, 2180, 835, 3471, 4958, 2214, 3135, 1884, 2284, 4601, 1389, 2678, 2547, 1981, 126, 1336, 4204, 4539, 1480, 4160, 4981, 2353, 4006, 2633, 710, 2205, 4103, 2225, 2272, 3189, 2317, 1256, 2574, 480, 1020, 2, 3994, 2896, 729, 3800, 1338, 4137, 283, 2627, 1743, 2801, 3304, 3780, 672, 505, 655, 4671, 761, 2212, 2567, 4952, 2336, 1489, 344, 1864, 1119, 3252, 3133, 1510, 201, 728, 3216, 891, 4704, 1785, 1904, 2467, 1705, 739, 2484, 550, 4984, 782, 4451, 2573, 3279, 2921, 52, 4173, 687, 1572, 3127, 4336, 4158, 4847, 4651, 4118, 946, 4853, 2157, 2905, 1271, 1154, 1913, 4400, 2781, 2790, 2498, 1678, 80, 2173, 2839, 2853, 4420, 10, 1600, 4709, 2379, 2063, 2969, 50, 2401, 1617, 2433, 615, 4746, 4234, 4141, 387, 2216, 3915, 4744, 2243, 4896, 4757, 1750, 4316, 1459, 5004, 4124, 2448, 4518, 3955, 4153, 2159, 3162, 4199, 331, 2740, 3539, 4398, 3073, 3448, 1919, 1869, 1649, 1639, 3227, 2413, 4939, 320, 578, 4327, 2963, 1722, 4563, 4876, 826, 3246, 1386, 2959, 1125, 2299, 3582, 1562, 1654, 2072, 1320, 667, 1003, 481, 3874, 3463, 340, 2281, 4716, 2919, 4182, 1033, 816, 4060, 3571, 1088, 3062, 4079, 4460, 1850, 4283, 648, 230, 2285, 2344, 1825, 190, 1321, 1368, 1908, 2213, 12, 3288, 3710, 1215, 2182, 3680, 1936, 4232, 4467, 1539, 493, 5002, 3637, 4154, 2925, 321, 2201, 932, 2453, 2224, 763, 2717, 3827, 129, 785, 1698, 3038, 1616, 1381, 1169, 762, 1582, 2555, 104, 371, 4148, 1742, 2030, 3624, 324, 4424, 2444, 1310, 1801, 787, 4220, 2280, 4903, 1005, 1893, 1392, 27, 432, 4378, 954, 196, 1852, 365, 3054, 2757, 3899, 4122, 3053, 3344, 4364, 3583, 3089, 992, 4614, 834, 3516, 3933, 393, 2227, 1165, 1184, 1749, 890, 3888, 3974, 4225, 649, 3831, 1690, 1873, 4955, 2647, 2604, 767, 3574, 806, 2203, 2134, 4275, 1905, 1857, 2014, 1718, 696, 3493, 1928, 2417, 4907, 2917, 2592, 4222, 4721, 1122, 3498, 1829, 2713, 2120, 3343, 2418, 518, 714, 3706, 2273, 2162, 1148, 991, 468, 3553, 2546, 1319, 4262, 2279, 2137, 3474, 4672, 2749, 1202, 2488, 4386, 16, 3521, 4974, 4444, 789, 1380, 3475, 4486, 3403, 1714, 552, 1485, 1414, 2568, 4799, 1273, 4409, 4607, 1201, 3893, 3022, 4623, 227, 774, 1805, 2961, 3086, 972, 742, 1840, 496, 3988, 4810, 3483, 3251, 1556, 542, 4524, 1366, 2877, 4748, 4773, 3082, 1939, 3666, 3934, 1002, 309, 4803, 222, 2955, 3159, 2549, 4699, 4895, 4663, 2710, 3061, 2485, 4372, 4610, 679, 1653, 3392, 1208, 1335, 1083, 3810, 198, 4658, 3042, 3917, 3333, 4077, 1846, 3947, 2366, 424, 665, 1306, 3069, 4898, 2723, 258, 3759, 1051, 2297, 4521, 1555, 4357, 4954, 3234, 2138, 1016, 1843, 2479, 4797, 1156, 2774, 2319, 1024, 1293, 4993, 3486, 1443, 2838, 4219, 3968, 4286, 2218, 3462, 3259, 2670, 726, 2021, 2890, 179, 3992, 3748, 2383, 3519, 1963, 572, 2259, 4390, 2469, 4683, 4347, 2211, 797, 4207, 4835, 3349, 4285, 717, 4186, 1832, 294, 3437, 150, 2721, 1011, 3685, 4860, 2618, 1244, 2427, 1685, 1518, 4751, 4921, 2438, 100, 2669, 2977, 2264, 1404, 2035, 3557, 2636, 1045, 908, 4560, 268, 4759, 1621, 2275, 1657, 4902, 3928, 4778, 3381, 221, 4271, 1902, 4901, 1050, 3364, 2872, 3321, 322, 3096, 4963, 495, 4681, 2973, 2404, 1634, 1809, 2968, 2672, 2360, 4613, 274, 241, 4099, 4102, 4084, 310, 130, 559, 2269, 3977, 4240, 3137, 115, 3664, 3241, 715, 134, 4924, 1170, 875, 2067, 106, 1693, 1716, 4151, 4825, 937, 1688, 811, 4738, 4928, 1842, 3100, 4188, 4177, 3545, 3201, 194, 298, 1593, 3788, 151, 1991, 4675, 3662, 4465, 640, 2864, 4032, 3863, 4516, 3376, 3766, 3111, 2755, 2569, 3309, 1733, 2407, 1771, 2993, 3267, 1973, 4334, 1823, 3824, 921, 3212, 264, 3130, 1847, 36, 1833, 2827, 4577, 1824, 2718, 4113, 4969, 4994, 4818, 4382, 501, 4785, 1243, 519, 3964, 4514, 4992, 4488, 1532, 2290, 3439, 1171, 2631, 4375, 2551, 192, 4091, 2155, 4296, 3398, 4783, 4586, 2248, 1607, 1867, 2776, 822, 1104, 2725, 4527, 2073, 2010, 1307, 1881, 180, 1424, 529, 178, 3656, 2181, 3817, 3033, 983, 4059, 2932, 4645, 1052, 472, 2096, 2303, 1972, 405, 3451, 454, 2251, 1007, 2856, 4926, 4293, 4986, 4862, 2038, 3797, 4750, 3905, 2597, 1683, 4802, 3248, 4247, 4678, 818, 2171, 1778, 3997, 3281, 3859, 1565, 279, 3803, 3762, 1974, 4480, 79, 784, 849, 3825, 466, 4798, 955, 99, 511, 3862, 1207, 4333, 1930, 4670, 920, 3312, 4108, 2983, 2907, 2818, 3872, 3436, 456, 2332, 2848, 3971, 3105, 934, 1365, 235, 3303, 69, 4076, 898, 2737, 3009, 380, 4371, 2370, 3922, 91, 2483, 1415, 2292, 4466, 984, 671, 4236, 927, 2614, 3995, 2045, 4270, 3011, 4554, 656, 4889, 1837, 953, 2241, 2984, 166, 4315, 82, 1603, 2585, 3361, 1042, 33, 1461, 392, 1569, 1135, 512, 2842, 2964, 1344, 2748, 3681, 1703, 3562, 4914, 1183, 2978, 736, 4209, 1097, 3816, 3249, 1063, 1362, 3897, 60, 1199, 1915, 4747, 2148, 3276, 4655, 1890, 2581, 889, 4261, 1435, 3554, 1588, 4217, 1667, 3573, 2001, 1411, 403, 1643, 1524, 2578, 175, 2037, 4878, 1087, 342, 4443, 4899, 4619, 2106, 1564, 2903, 1316, 2350, 383, 3506, 4755, 3365, 878, 2538, 1740, 4535, 857, 4074, 2060, 4525, 700, 2947, 244, 242, 125, 220, 708, 3698, 630, 3357, 1713, 4731, 1058, 1317, 270, 2893, 2352, 1997, 3074, 4816, 887, 372, 1645, 2741, 436, 3558, 2689, 1623, 1798, 1605, 3708, 3177, 4383, 554, 4056, 4431, 2470, 4836, 1282, 1903, 4470, 3176, 1264, 1369, 2646, 2777, 1458, 1328, 4302, 1906, 1067, 3802, 4387, 4654, 2558, 1291, 3461, 2428, 3807, 4745, 3229, 2860, 1178, 636, 1018, 555, 1960, 3040, 483, 4856, 4848, 1652, 2463, 2403, 1378, 4335, 491, 4574, 1475, 4016, 224, 30, 3363, 4857, 3909, 4701, 2009, 3202, 4585, 4114, 1188, 1844, 1948, 420, 2998, 2165, 2333, 4449, 74, 34, 4652, 861, 155, 837, 290, 1175, 1659, 138, 1673, 4885, 2953, 3578, 528, 1773, 1612, 623, 1512, 904, 3749, 2277, 3342, 2142, 2991, 2255, 825, 4308, 255, 4839, 901, 2775, 3835, 924, 84, 11, 563, 487, 2347, 2714, 2582, 3164, 591, 3931, 2117, 3595, 1813, 603, 156, 1153, 4578, 1339, 2313, 2530, 240, 3253, 1078, 3935, 2124, 3036, 4960, 2888, 3651, 1098, 145, 2307, 1871, 1636, 2664, 3158, 3818, 508, 1907, 2830, 2577, 597, 4587, 4691, 233, 631, 633, 4407, 1248, 404, 4185, 1216, 4845, 1516, 4965, 3418, 1538, 1482, 1299, 2083, 3278, 1952, 4292, 929, 3841, 3295, 3079, 2502, 1029, 2865, 3937, 330, 373, 3078, 4844, 93, 1661, 1173, 773, 1017, 1194, 966, 4806, 846, 5006, 3305, 3430, 749, 2113, 187, 4136, 3239, 3742, 1609, 108, 4706, 193, 3767, 4213, 711, 4434, 2894, 4838, 3683, 3612, 1898, 3390, 567, 38, 1040, 3572, 3273, 788, 664, 2451, 1008, 1059, 2626, 4326, 4399, 2785, 3182, 4487, 2924, 1292, 3576, 47, 4522, 4880, 4861, 1604, 2857, 4627, 1329, 960, 4428, 137, 2068, 1531, 1672, 2732, 4564, 1265, 1515, 2862, 1252, 4829, 2215, 698, 1325, 494, 4338, 3878, 2800, 22, 3677, 2302, 2268, 3169, 3460, 2425, 4708, 1203, 2380, 4161, 1426, 3110, 2423, 1094, 4500, 1454, 3270, 32, 1788, 824, 1393, 1371, 1241, 2946, 3908, 3247, 124, 2464, 3815, 1929, 3375, 2677, 3611, 3222, 2314, 6, 1218, 3533, 3031, 2550, 4284, 4603, 40, 257, 2497, 3147, 1792, 4942, 4636, 3056, 1613, 720, 2406, 1828, 4312, 3625, 3700, 2564, 589, 2424, 4048, 3387, 2343, 3124, 638, 1583, 3936, 4911, 323, 918, 3958, 748, 4918, 4987, 188, 4352, 2745, 1111, 2385, 2525, 3476, 277, 3952, 792, 4374, 2954, 1436, 1931, 4194, 97, 4972, 2185, 756, 3272, 4703, 1060, 4976, 2136, 246, 98, 619, 4169, 4882, 1146, 1894, 1453, 4537, 213, 1035, 1872, 284, 4702, 2696, 2684, 4405, 4529, 1799, 1877, 1551, 3903, 734, 3232, 670, 1333, 1253, 3989, 3489, 1650, 4892, 771, 4753, 4229, 4354, 4950, 4067, 4291, 3171, 2436, 1053, 1277, 2691, 1700, 1986, 881, 3672, 2514, 4437, 1286, 4311, 2720, 3597, 4381, 4962, 374, 4727, 4543, 1044, 2075, 1868, 3850, 4147, 3596, 668, 209, 4411, 2200, 1774, 261, 2759, 2237, 3772, 449, 231, 4481, 571, 754, 3425, 3286, 2521, 2153, 2244, 4572, 4826, 807, 4600, 4242, 3455, 1047, 2111, 544, 1464, 3548, 1955, 766, 866, 4791, 3584, 2000, 3348, 4682, 0, 1176, 858, 892, 3587, 3949, 1680, 4595, 306, 2859, 2282, 3002, 3199, 4376, 2922, 2722, 4717, 3836, 4416, 1197, 1288, 1113, 3761, 4509, 2355, 4373, 609, 3088, 1507, 357, 2634, 745, 4012, 4556, 3192, 1340, 3799, 3236, 4794, 799, 1137, 2052, 4005, 3959, 2746, 4116, 2508, 367, 3324, 2656, 4090, 1263, 4258, 3261, 2229, 1220, 840, 2390, 3658, 140, 3754, 4739, 2533, 2169, 4742, 3599, 452, 1134, 3763, 2515, 477, 2644, 1629, 3853, 3773, 3044, 1440, 3497, 1845, 1508, 1433, 3657, 4172, 4550, 3846, 1772, 1761, 1831, 65, 1375, 3393, 4665, 1442, 4166, 3118, 434, 2459, 1976, 1664, 341, 2288, 2726, 4252, 1185, 4224, 3037, 968, 370, 4676, 3205, 3640, 1108, 1159, 796, 1422, 2680, 2822, 4485, 113, 1428, 1834, 894, 2252, 3469, 4058, 900, 2513, 356, 3104, 3887, 978, 2572, 3669, 1309, 1662, 4104, 4081, 4820, 1950, 4719, 614, 1162, 1114, 4401, 4317, 1359, 4276, 2191, 2207, 3140, 2097, 386, 2881, 2782, 3028, 1558, 308, 4289, 2561, 2362, 2312, 1432, 2178, 212, 2616, 94, 2923, 1465, 4152, 1839, 4046, 2791, 1855, 3217, 2071, 3529, 4761, 2784, 4756, 883, 4134, 1895, 2345, 1180, 2753, 3948, 422, 2747, 1294, 346, 1075, 61, 2027, 4150, 2910, 1752, 2779, 3496, 817, 4656, 1441, 2949, 3098, 964, 3119, 2926, 2952, 4762, 3345, 510, 4948, 1782, 4313, 4997, 2391, 1472, 1958, 4180, 2819, 4238, 2208, 4546, 1996, 3923, 4674, 4087, 1610, 2768, 2150, 4132, 1909, 4248, 3530, 3701, 879, 1463, 4766, 4241, 4483, 3975, 3569, 3630, 2831, 362, 2115, 4576, 2612, 1400, 2240, 409, 3643, 1527, 4297, 1879, 4462, 1528, 172, 4200, 2184, 2257, 1914, 4463, 53, 1145, 2607, 2735, 4314, 4780, 1499, 4864, 2388, 450, 2048, 2734, 2132, 2079, 338, 3004, 1548, 2223, 4872, 3464, 2659, 3586, 884, 3210, 1560, 1027, 3466, 2871, 3366, 2466, 2040, 4265, 3434, 1126, 1642, 3157, 3892, 4705, 183, 1117, 676, 2231, 1993, 117, 4970, 4567, 4736, 2039, 3373, 1966, 790, 2941, 4634, 4842, 4071, 3445, 4454, 1150, 3938, 1452, 2645, 867, 3429, 1964, 4178, 103, 1280, 1967, 2170, 731, 291, 2688, 448, 2036, 3575, 4210, 596, 2528, 1107, 1343, 458, 1041, 1166, 4033, 4111, 3224, 4301, 2641, 3649, 4195, 704, 4590, 3837, 3659, 1962, 4956, 332, 4413, 1735, 4057, 1470, 4494, 2545, 2874, 1769, 2769, 1746, 985, 1405, 2296, 2639, 208, 4127, 1326, 1529, 4341, 1923, 1334, 312, 2100, 645, 1451, 3499, 4575, 2565, 3577, 3215, 990, 4863, 337, 905, 859, 3791, 4029, 2981, 3166, 3981, 2460, 1068, 1085, 446, 2976, 1989, 1816, 3631, 1124, 1704, 2761, 541, 3284, 1978, 199, 259, 4985, 1487, 3500, 2378, 1712, 750, 2908, 2127, 3970, 390, 1934, 2041, 4055, 4568, 3319, 2204, 3207, 203, 3746, 3214, 4135, 4349, 19, 144, 1557, 4044, 2304, 3320, 2372, 1262, 3834, 1421, 1777, 1679, 970, 2552, 3410, 4707, 174, 3829, 617, 4062, 1237, 2116, 3185, 3547, 568, 4038, 3966, 2090, 1765, 770, 3714, 2855, 4175, 4475, 3426, 347, 4894, 4733, 2587, 4507, 4555, 445, 2130, 4259, 217, 2580, 635, 1971, 783, 1330, 95, 4579, 1168, 3083, 2346, 977, 2503, 489, 1478, 692, 845, 3340, 3245, 2623, 1541, 3515, 4119, 3886, 3389, 1900, 3792, 2999, 4250, 4493, 961, 3007, 408, 4361, 2046, 2826, 4053, 2912, 1694, 2025, 4649, 911, 282, 607, 3457, 1692, 3302, 2668, 1633, 2054, 2415, 3648, 2900, 3317, 1466, 1373, 77, 4661, 3854, 4609, 3482, 4208, 3369, 159, 49, 4633, 3103, 3535, 1940, 827, 3112, 3688, 1587, 3918, 4393, 1594, 4730, 326, 467, 3138, 2121, 3507, 504, 2422, 793, 713, 1858, 4718, 4253, 1822, 1481, 1234, 2194, 1671, 5009, 153, 2468, 381, 3890, 1352, 29, 3719, 1921, 3131, 4323, 329, 2501, 1476, 3068, 515, 1086, 4565, 1631, 1196, 4925, 1374, 2082, 2002, 598, 1681, 2377, 4534, 862, 4117, 1644, 3326, 395, 1514, 266, 451, 3292, 3942, 3179, 4680, 435, 4583, 798, 2750, 659, 176, 250, 2055, 3976, 4503, 2806, 1706, 3067, 876, 2600, 161, 4530, 1663, 202, 1959, 1619, 3318, 304, 1302, 2179, 1608, 419, 661, 3213, 4080, 3509, 4478, 626, 688, 3998, 1390, 4725, 4447, 3880, 1501, 864, 939, 1284, 2858, 415, 2987, 3402, 1863, 980, 3431, 3929, 4937, 1407, 3055, 3750, 237, 177, 440, 2295, 2760, 2524, 1296, 3865, 1408, 4028, 218, 3610, 4591, 2308, 2625, 2007, 3275, 1230, 4659, 1506, 2475, 3614, 3546, 1956, 4061, 1354, 3760, 359, 1298, 1462, 2930, 877, 482, 1573, 1450, 2788, 2420, 1155, 2374, 868, 2606, 2398, 3494, 1787, 2198, 1225, 4908, 1357, 3311, 4342, 2653, 3151, 476, 558, 1367, 1781, 947, 581, 995, 238, 185, 3709, 2836, 2534, 462, 2797, 24, 2992, 4724, 3023, 4069, 830, 2996, 971, 3495, 4098, 4239, 2619, 2267, 1376, 562, 1726, 3468, 1091, 3523, 2102, 148, 4402, 2975, 2477, 4446, 1596, 4415, 1709, 4713, 1102, 2986, 4996, 4052, 820, 2918, 1533, 651, 935, 4294, 1744, 2029, 1274, 2122, 1030, 4037, 531, 4569, 4657, 3518, 1123, 1133, 17, 2692, 4580, 4879, 4115, 157, 823, 2876, 1103, 523, 4881, 3183, 4189, 3187, 1021, 545, 3592, 3090, 1163, 2771, 1625, 3654, 3686, 1115, 1494, 1767, 3071, 1951, 1552, 3795, 2368, 3397, 1455, 3034, 120, 2593, 786, 1025, 4190, 580, 1217, 4990, 4068, 1266, 4553, 3154, 716, 2318, 4142, 2820, 109, 4337, 948, 1019, 2531, 470, 3715, 3536, 4031, 2742, 3113, 1982, 2175, 35, 1566, 2875, 2354, 3285, 4379, 4164, 4790, 3930, 4977, 4054, 4350, 1790, 3932, 2074, 1961, 4784, 44, 2408, 2168, 967, 2765, 1278, 3129, 64, 4884, 4541, 1486, 694, 2522, 4771, 2183, 2315, 2443, 3522, 1859, 2904, 503, 1423, 3277, 2815, 2293, 296, 3538, 4696, 3235, 3944, 4566, 4243, 4051, 2539, 1251, 317, 3472, 3005, 25, 3338, 1584, 164, 3805, 4499, 3332, 3149, 3035, 2783, 3618, 2239, 4930, 411, 2349, 3290, 1985, 637, 500, 4094, 3307, 4138, 1640, 4410, 2059, 5007, 2808, 4174, 2882, 2544, 4391, 3965, 3568, 223, 3751, 2242, 1848, 1897, 1525, 4263, 2458, 836, 83, 4782, 2526, 2804, 360, 1614, 1818, 3901, 2615, 3283, 681, 1136, 3030, 4130, 2556, 1037, 3153, 1395, 2511, 1764, 1819, 4967, 1727, 1229, 3206, 647, 1969, 2695, 853, 752, 2461, 2847, 1001, 1036, 3894, 4931, 2077, 2309, 4637, 413, 821, 3145, 4422, 1590, 2928, 1267, 3453, 1049, 3170, 2222, 2834, 1128, 1, 2520, 112, 1105, 592, 4687, 3828, 2754, 1530, 907, 2965, 709, 3263, 3291, 3809, 4923, 2394, 610, 2911, 48, 2716, 4526, 1567, 1535, 3999, 4515, 3155, 1682, 302, 1121, 3600, 1780, 1794, 4421, 427, 1384, 3718, 3188, 13, 3881, 3256, 986, 2499, 2330, 3021, 952, 4774, 1269, 4249, 2971, 3257, 303, 1648, 2997, 602, 1022, 4944, 4532, 4513, 3329, 56, 18, 685, 2289, 4640, 3367, 832, 2811, 4858, 2690, 4206, 4865, 4498, 1953, 3864, 1444, 3605, 3269, 4008, 2638, 4025, 535, 3739, 1057, 2190, 4307, 3861, 1492, 3867, 1759, 3913, 2495, 1517, 899, 4808, 4638, 2258, 1520, 2003, 3139, 963, 3492, 3826, 2381, 3087, 3883, 3432, 2705, 316, 1689, 2387, 1579, 51, 1152, 3231, 1591, 66, 3729, 3848, 3926, 2058, 4260, 280, 3833, 1109, 3101, 2950, 2094, 245, 4474, 1351, 959, 1061, 2439, 956, 622, 3144, 3480, 1074, 4496, 4809, 2595, 2802, 539, 4917, 20, 299, 2743, 2583, 3697, 2028, 1431, 1006, 3960, 2416, 3255, 267, 574, 3711, 886, 3218, 1056, 612], "validation": [5082, 5023, 5306, 5145, 5268, 5116, 5146, 5072, 5421, 5060, 5432, 5469, 5431, 5402, 5226, 5443, 5257, 5481, 5285, 5409, 5453, 5129, 5369, 5130, 5360, 5122, 5278, 5458, 5132, 5362, 5196, 5286, 5076, 5408, 5169, 5390, 5389, 5208, 5010, 5416, 5358, 5016, 5361, 5202, 5107, 5292, 5377, 5065, 5061, 5294, 5204, 5454, 5391, 5334, 5328, 5081, 5085, 5315, 5308, 5123, 5309, 5378, 5448, 5241, 5111, 5172, 5480, 5422, 5077, 5240, 5051, 5321, 5491, 5048, 5280, 5345, 5451, 5355, 5192, 5013, 5028, 5445, 5397, 5177, 5318, 5297, 5270, 5386, 5452, 5195, 5151, 5434, 5249, 5039, 5289, 5143, 5279, 5223, 5435, 5501, 5054, 5449, 5494, 5175, 5193, 5093, 5403, 5502, 5089, 5363, 5233, 5127, 5250, 5236, 5133, 5393, 5011, 5485, 5012, 5265, 5227, 5338, 5216, 5343, 5423, 5157, 5509, 5252, 5490, 5385, 5311, 5100, 5498, 5478, 5136, 5263, 5040, 5098, 5429, 5472, 5394, 5030, 5224, 5185, 5396, 5180, 5486, 5412, 5062, 5194, 5149, 5203, 5115, 5504, 5015, 5239, 5299, 5437, 5047, 5253, 5492, 5337, 5027, 5352, 5246, 5099, 5221, 5266, 5344, 5058, 5310, 5069, 5046, 5400, 5426, 5230, 5138, 5254, 5371, 5034, 5500, 5259, 5134, 5198, 5161, 5281, 5219, 5024, 5356, 5238, 5417, 5457, 5503, 5141, 5293, 5087, 5096, 5037, 5209, 5248, 5019, 5275, 5073, 5206, 5243, 5066, 5291, 5277, 5433, 5350, 5215, 5438, 5488, 5370, 5428, 5187, 5163, 5365, 5125, 5424, 5142, 5392, 5109, 5131, 5333, 5495, 5070, 5094, 5313, 5055, 5336, 5324, 5103, 5298, 5162, 5288, 5483, 5354, 5212, 5152, 5381, 5155, 5117, 5190, 5411, 5147, 5025, 5405, 5038, 5479, 5316, 5474, 5118, 5041, 5090, 5043, 5222, 5186, 5188, 5463, 5468, 5312, 5200, 5084, 5493, 5471, 5035, 5399, 5323, 5168, 5304, 5420, 5104, 5256, 5124, 5199, 5036, 5189, 5383, 5078, 5176, 5092, 5237, 5302, 5247, 5364, 5068, 5255, 5380, 5473, 5317, 5508, 5379, 5307, 5441, 5460, 5217, 5228, 5341, 5235, 5505, 5305, 5427, 5264, 5335, 5482, 5442, 5348, 5021, 5260, 5102, 5329, 5376, 5283, 5207, 5382, 5158, 5273, 5272, 5020, 5231, 5144, 5108, 5106, 5295, 5475, 5201, 5059, 5017, 5477, 5210, 5357, 5326, 5349, 5079, 5137, 5484, 5300, 5242, 5026, 5049, 5506, 5032, 5258, 5476, 5462, 5489, 5064, 5353, 5121, 5340, 5173, 5359, 5140, 5181, 5086, 5271, 5374, 5439, 5388, 5465, 5332, 5401, 5372, 5346, 5455, 5045, 5384, 5464, 5327, 5282, 5497, 5314, 5075, 5487, 5080, 5156, 5128, 5319, 5191, 5461, 5126, 5496, 5044, 5097, 5218, 5331, 5095, 5179, 5296, 5029, 5342, 5174, 5154, 5022, 5057, 5456, 5301, 5499, 5245, 5395, 5091, 5290, 5114, 5450, 5033, 5184, 5466, 5183, 5160, 5083, 5413, 5375, 5419, 5014, 5398, 5430, 5148, 5170, 5440, 5234, 5444, 5031, 5415, 5167, 5251, 5404, 5261, 5171, 5074, 5178, 5459, 5276, 5135, 5112, 5205, 5101, 5110, 5018, 5197, 5105, 5325, 5446, 5244, 5050, 5436, 5211, 5373, 5067, 5303, 5507, 5056, 5274, 5220, 5088, 5119, 5042, 5071, 5284, 5113, 5053, 5407, 5410, 5225, 5406, 5164, 5330, 5229, 5166, 5213, 5414, 5425, 5368, 5120, 5153, 5351, 5052, 5139, 5320, 5150, 5232, 5182, 5159, 5387, 5418, 5063, 5267, 5269, 5165, 5322, 5467, 5447, 5262, 5339, 5214, 5367, 5470, 5366, 5287, 5347], "test": [5945, 5926, 5786, 5564, 5831, 5591, 5752, 5638, 5545, 5895, 5984, 5568, 5719, 5963, 5594, 5872, 5919, 5896, 5652, 5552, 5627, 5849, 5925, 5813, 5547, 5699, 5567, 5893, 5955, 5910, 5762, 5519, 6005, 5796, 5927, 5833, 5724, 5658, 5572, 5917, 5944, 5973, 5520, 5826, 5805, 5640, 5950, 5728, 5621, 5518, 5903, 5542, 5740, 5535, 5705, 5790, 5710, 5850, 5852, 5760, 5650, 5995, 5744, 5735, 5758, 5924, 5860, 5575, 5839, 5684, 5715, 5556, 5774, 5576, 5942, 5691, 5937, 5998, 5755, 5683, 5753, 5656, 5983, 5523, 5673, 5553, 5968, 5701, 5587, 5881, 5800, 5751, 5810, 5888, 5516, 5996, 6003, 5702, 5776, 5565, 5819, 5909, 5928, 5600, 5586, 5768, 6001, 5837, 5763, 5794, 5592, 5577, 5797, 5820, 5889, 5988, 5940, 5874, 5923, 5562, 5585, 5908, 5721, 5787, 5583, 5986, 5549, 5809, 5892, 5573, 5827, 5842, 5994, 5970, 5544, 5844, 5651, 5706, 5778, 5883, 5784, 5746, 5584, 5722, 5676, 5510, 5836, 5939, 5953, 5905, 5528, 5863, 5580, 5756, 5609, 5607, 5629, 5754, 5686, 5731, 5559, 5737, 5999, 5517, 5633, 5951, 5513, 5680, 5546, 5838, 5812, 5884, 5779, 5840, 5666, 5783, 5717, 5897, 5672, 5932, 5732, 5637, 5654, 5929, 5571, 5916, 5720, 5738, 5766, 5736, 5653, 5807, 5678, 5864, 5733, 5972, 5693, 5915, 5521, 5855, 5799, 5598, 5966, 5921, 5644, 5772, 5815, 5597, 5647, 5976, 5771, 5560, 5912, 5566, 5643, 5961, 5657, 5534, 5975, 5642, 5851, 5689, 5798, 5835, 6007, 5578, 5713, 5696, 5803, 5613, 5541, 5557, 5806, 5527, 5821, 5739, 5608, 5626, 5615, 5590, 5818, 5646, 5623, 5875, 5885, 5543, 5902, 5962, 5612, 5525, 5631, 5795, 5793, 5743, 5550, 5886, 5604, 5660, 5854, 5662, 5823, 5857, 6006, 5749, 5979, 5695, 5862, 5832, 5938, 5757, 5515, 5911, 5645, 5648, 5725, 5865, 5670, 5936, 5869, 5624, 5628, 5707, 5898, 5574, 5848, 5617, 5622, 5659, 5708, 5952, 5791, 5669, 5841, 5700, 5920, 5709, 5764, 5742, 5625, 5620, 5593, 5512, 5934, 5871, 5634, 5703, 5816, 5814, 5668, 5785, 5847, 5601, 5698, 5830, 6009, 5636, 5667, 5767, 5579, 5989, 5611, 5782, 5789, 5824, 5971, 5595, 5900, 5867, 5697, 5777, 5946, 5887, 5959, 5730, 5663, 5868, 5974, 5817, 5533, 5834, 5878, 5524, 5876, 5828, 5960, 5582, 5859, 5664, 5750, 5661, 5531, 5561, 5589, 5822, 5802, 5829, 5726, 5930, 5540, 5581, 5675, 5522, 5846, 5987, 5861, 5948, 5674, 5677, 5899, 5769, 5606, 5748, 5514, 5588, 5770, 5845, 5671, 5569, 5941, 5858, 5825, 5808, 5969, 5947, 5511, 5694, 5723, 6004, 5811, 5933, 5901, 5687, 5792, 5599, 5804, 5537, 5759, 5682, 5704, 5714, 5526, 5718, 5801, 5635, 5956, 5692, 5536, 5632, 5616, 5954, 5603, 5982, 5558, 5877, 5690, 5538, 5997, 5879, 5679, 5907, 5775, 5856, 5931, 5532, 5745, 5665, 5729, 5977, 5904, 5967, 5605, 5985, 6002, 5711, 5602, 6008, 5943, 5618, 5914, 5781, 5655, 5873, 5639, 5614, 5741, 5734, 5964, 5780, 5949, 5712, 5727, 5980, 5554, 5630, 5935, 5681, 5990, 5641, 5913, 5870, 5992, 5716, 5747, 5958, 5991, 5548, 5563, 5891, 5866, 5570, 5685, 5843, 5788, 5539, 5981, 5918, 5880, 5551, 5965, 5882, 5596, 5957, 5610, 5649, 5906, 5773, 5993, 5894, 6000, 5530, 5619, 5765, 5688, 5922, 5761, 5890, 5853, 5529, 5555, 5978]}, {"train": [3981, 99, 1371, 4704, 296, 196, 2638, 3467, 3550, 4876, 1174, 3271, 3552, 4089, 4044, 1277, 1220, 3292, 4694, 418, 2350, 3177, 4863, 1457, 1463, 136, 4966, 2198, 2569, 1085, 489, 1845, 2572, 4923, 4051, 3903, 787, 152, 2476, 4765, 563, 3968, 1175, 2994, 1244, 272, 417, 1209, 4465, 2063, 2464, 290, 416, 2699, 2351, 4767, 1566, 2624, 414, 2432, 2352, 820, 3693, 4456, 2915, 295, 1387, 3076, 1068, 400, 2129, 4647, 4190, 2048, 3224, 950, 68, 3669, 3249, 4171, 2413, 2190, 1961, 181, 1515, 1974, 1112, 573, 2500, 3844, 2688, 4864, 2252, 3063, 3193, 2004, 4560, 874, 4551, 3024, 2294, 4076, 1782, 1035, 2453, 1973, 785, 940, 2233, 4731, 3227, 45, 1838, 3668, 3140, 3852, 2700, 3560, 130, 1930, 3945, 470, 2199, 3815, 2674, 3862, 4973, 3712, 1622, 1690, 1819, 201, 1317, 957, 1530, 469, 3910, 268, 3774, 3221, 4508, 247, 244, 120, 2450, 3695, 3315, 4386, 4748, 4422, 1904, 1062, 4774, 4810, 4503, 3462, 4624, 1531, 4780, 1899, 1937, 3414, 4293, 3138, 567, 4510, 1560, 1773, 4099, 838, 1729, 1481, 4073, 399, 315, 1422, 818, 222, 2639, 1335, 763, 3766, 666, 142, 4070, 231, 24, 4604, 4446, 863, 2771, 832, 900, 3526, 2866, 2670, 1434, 2648, 4899, 1119, 2823, 1073, 1590, 59, 1332, 291, 2467, 3859, 4988, 2532, 1814, 653, 2574, 4430, 3428, 623, 4628, 258, 850, 1258, 1122, 4302, 4825, 2949, 4432, 1883, 2083, 4336, 1310, 4477, 4002, 2625, 4268, 1671, 680, 3911, 4784, 4486, 2451, 2092, 2616, 3565, 2460, 1651, 223, 926, 3244, 2469, 2283, 1789, 3821, 2363, 2647, 3641, 3068, 1150, 4826, 792, 4799, 2618, 1886, 2311, 154, 684, 534, 2709, 4805, 2681, 3784, 566, 3090, 4118, 4833, 862, 4212, 4272, 3786, 2005, 1737, 1390, 1237, 3119, 734, 1828, 3661, 4919, 318, 1685, 2707, 2891, 3143, 4413, 4170, 98, 590, 902, 1616, 2710, 1659, 2977, 454, 3893, 2695, 578, 4189, 4093, 1436, 813, 2292, 2074, 176, 932, 1563, 2397, 3435, 3567, 3794, 193, 4889, 4509, 2248, 368, 2817, 2173, 934, 3146, 1781, 3869, 55, 3491, 1416, 2906, 3635, 1734, 4048, 2210, 4261, 3285, 9, 1401, 1573, 3772, 3304, 1218, 2511, 357, 2465, 4986, 3783, 2692, 4841, 3837, 4494, 3009, 1383, 213, 2881, 426, 4595, 1443, 4102, 1403, 995, 3717, 4000, 3295, 2204, 1873, 2643, 2134, 2276, 917, 4127, 2516, 4016, 2901, 3653, 4698, 1885, 3729, 1739, 1238, 4278, 309, 3643, 4031, 446, 2980, 690, 4381, 3112, 2056, 4497, 3629, 4678, 1585, 3037, 1464, 3592, 163, 3288, 1718, 4353, 28, 3662, 935, 2843, 4557, 3877, 3705, 2963, 3650, 3833, 325, 4387, 1512, 2014, 2954, 4157, 3847, 4321, 1253, 4848, 4133, 1751, 2364, 4959, 3094, 184, 1634, 1890, 2884, 2164, 4318, 4522, 4304, 4130, 1028, 2542, 2117, 4811, 265, 2153, 4238, 1948, 1213, 22, 4921, 2607, 1920, 4544, 2254, 4262, 1338, 3545, 384, 4892, 2008, 4467, 3183, 1136, 3581, 4590, 1452, 3426, 4998, 248, 3871, 2787, 132, 1180, 1805, 302, 849, 2293, 2668, 2200, 2981, 4740, 2347, 481, 4448, 914, 977, 4813, 2297, 4603, 398, 3270, 1575, 2177, 3268, 1021, 1215, 4722, 3721, 3555, 2731, 2365, 3093, 4429, 696, 3977, 976, 1670, 545, 4013, 3806, 4149, 1914, 1265, 3880, 4890, 2101, 4054, 2883, 1409, 2513, 550, 866, 1844, 113, 3670, 2126, 1856, 1015, 4981, 853, 1480, 3733, 4433, 404, 2604, 3953, 4120, 3620, 3820, 3964, 4242, 1842, 2187, 2758, 1195, 2196, 669, 1049, 3600, 650, 1712, 4165, 1744, 1412, 1088, 4084, 1689, 4258, 280, 4414, 2606, 1610, 1688, 1880, 661, 2889, 2821, 1641, 2194, 3098, 1102, 1442, 532, 4932, 3711, 2587, 3769, 4907, 1716, 1913, 1648, 3320, 2082, 3584, 2370, 764, 266, 4308, 3534, 3932, 1373, 4939, 966, 613, 3369, 1386, 2271, 1488, 38, 610, 1063, 2371, 4410, 1461, 212, 4730, 1147, 3540, 74, 1726, 1645, 816, 1283, 4230, 1103, 4449, 891, 663, 1755, 986, 479, 1516, 3217, 981, 2111, 3448, 3789, 3362, 3927, 4379, 2000, 1960, 1411, 4416, 2612, 3182, 4594, 821, 338, 127, 2143, 3129, 4025, 4734, 2013, 3734, 1223, 3486, 3798, 3148, 440, 825, 4808, 4141, 2818, 1857, 4662, 557, 2603, 2683, 2515, 2343, 1753, 982, 3778, 2086, 2161, 1932, 2640, 253, 1182, 4721, 1055, 1126, 1743, 4915, 480, 3355, 466, 3856, 3522, 931, 300, 3393, 554, 519, 2505, 4570, 3045, 3015, 4384, 579, 4423, 985, 4476, 391, 4199, 851, 2938, 3763, 4204, 4775, 1459, 1366, 1591, 2138, 4527, 1307, 2211, 3029, 4895, 4233, 4457, 423, 2096, 2387, 1806, 3925, 4894, 901, 4273, 3422, 1285, 4478, 4637, 4963, 3075, 3225, 3723, 4844, 2792, 2870, 562, 2377, 2995, 2395, 3608, 3518, 346, 2804, 4470, 4428, 2842, 909, 3725, 4072, 2380, 1980, 2666, 3917, 1224, 3323, 2672, 2496, 2721, 4798, 4454, 1432, 4057, 4228, 2984, 4543, 744, 3743, 842, 1945, 401, 1786, 4391, 3698, 2797, 1598, 2428, 4787, 4533, 5002, 332, 1189, 3832, 1194, 873, 4471, 4312, 4092, 3915, 4619, 3277, 4665, 1290, 3828, 4123, 2832, 1541, 1450, 831, 4553, 4108, 2812, 1415, 1164, 4713, 3982, 1552, 1197, 3850, 1038, 3498, 2411, 2209, 4307, 3840, 1226, 4882, 905, 929, 4285, 1075, 1556, 4675, 1822, 1588, 3617, 1796, 3144, 4737, 4535, 4055, 1074, 342, 3767, 301, 4613, 1284, 2830, 2546, 2322, 51, 2141, 1184, 4162, 3278, 2893, 324, 4636, 1896, 1719, 2988, 4246, 4953, 3312, 1029, 4400, 2613, 3473, 3396, 542, 1758, 2931, 3149, 1991, 71, 3360, 3331, 4082, 1230, 1592, 2711, 1191, 639, 761, 3459, 3350, 4275, 4769, 3876, 620, 3509, 4155, 4232, 4283, 930, 4614, 117, 2396, 2669, 3395, 4310, 3879, 1699, 1120, 3020, 1410, 2776, 5001, 138, 4320, 4567, 1611, 1970, 4834, 833, 4023, 2867, 1367, 2760, 4481, 3896, 3432, 806, 1135, 4290, 4703, 4640, 4341, 3792, 1909, 1741, 581, 3027, 1675, 311, 3351, 1214, 879, 1368, 2730, 2159, 2583, 137, 3957, 4138, 710, 1549, 737, 3002, 2972, 2568, 3752, 942, 2131, 1660, 2090, 569, 2289, 4491, 3000, 18, 600, 2929, 4552, 2492, 111, 1656, 688, 616, 2990, 3272, 472, 3406, 3937, 326, 4779, 4850, 3799, 1185, 4854, 601, 4255, 4615, 4259, 3210, 4188, 2106, 1066, 4038, 4579, 2947, 363, 3523, 4226, 2826, 1264, 1486, 1505, 4323, 4958, 1270, 2498, 4390, 822, 2110, 1818, 2275, 1315, 3622, 1296, 339, 993, 64, 2044, 1898, 2400, 214, 3939, 2282, 1010, 4993, 2989, 2726, 3206, 3571, 3703, 1658, 124, 2137, 3095, 3667, 3240, 1545, 4480, 4950, 694, 4859, 3674, 4649, 633, 3759, 471, 3454, 4764, 4855, 3787, 811, 3704, 3483, 2882, 1351, 1099, 3566, 2725, 691, 4877, 593, 617, 4083, 3252, 2459, 3091, 4142, 3535, 1567, 3081, 3863, 4124, 4347, 1004, 4621, 1249, 457, 789, 660, 3706, 3843, 1779, 3379, 1328, 4733, 606, 2418, 3088, 3817, 4158, 3060, 3310, 1576, 4062, 455, 2629, 4437, 4493, 185, 847, 289, 2619, 575, 3603, 4269, 3226, 1504, 1823, 3574, 224, 1211, 1478, 3445, 4781, 2744, 3011, 4225, 2737, 3637, 1524, 2408, 39, 2242, 3546, 3963, 925, 2307, 1918, 4645, 4365, 2203, 3696, 2566, 4250, 2926, 2435, 3979, 2646, 717, 565, 42, 1746, 2641, 2743, 2344, 313, 2579, 4061, 1311, 3959, 2029, 2125, 2441, 1267, 2162, 1172, 4795, 1673, 1597, 4115, 528, 4085, 2645, 2238, 911, 622, 3466, 2551, 4547, 3542, 3327, 2503, 3916, 1787, 1455, 275, 4874, 3253, 3287, 3544, 2076, 397, 1157, 1638, 3457, 3407, 699, 4514, 4521, 4550, 2740, 3599, 4832, 4691, 923, 1929, 1807, 725, 1846, 619, 3071, 1975, 2880, 1580, 712, 3970, 2894, 4079, 3021, 3159, 4296, 3321, 3003, 572, 3054, 4322, 4357, 3255, 1381, 3134, 826, 2840, 1797, 1825, 4332, 4857, 1093, 3100, 3057, 4837, 496, 2942, 4316, 3383, 2130, 2697, 26, 576, 4984, 698, 664, 1110, 2169, 3746, 4705, 1018, 4856, 4502, 1196, 3232, 3948, 4052, 3382, 3790, 4169, 3345, 4772, 3359, 1225, 3577, 2904, 945, 3590, 4245, 1756, 1178, 195, 758, 3175, 2874, 841, 269, 3052, 2156, 987, 3864, 2268, 2800, 134, 3460, 2442, 92, 3813, 3563, 1672, 2896, 510, 2955, 3133, 2393, 1369, 3236, 1406, 1439, 3530, 3171, 4196, 4215, 4762, 4896, 1340, 4752, 4377, 3441, 4, 88, 1529, 4372, 2801, 3904, 3591, 162, 2068, 3588, 1503, 1430, 2784, 4912, 3361, 1438, 3421, 3099, 1397, 2251, 3575, 4840, 3485, 4081, 3455, 4401, 3490, 3891, 304, 4444, 1785, 507, 3737, 277, 3791, 3153, 4702, 1128, 2064, 2264, 3233, 638, 4606, 2245, 206, 4728, 4424, 1578, 1958, 1006, 1162, 2183, 2811, 4088, 4804, 493, 2257, 78, 3434, 938, 3006, 4516, 631, 3106, 732, 3449, 4633, 3572, 3347, 3912, 1408, 2066, 2265, 3412, 1473, 3803, 4151, 3401, 3202, 2561, 1023, 3131, 946, 2678, 1361, 3433, 2979, 3178, 608, 4707, 3562, 3659, 4483, 3512, 1071, 171, 529, 2302, 2905, 3961, 1159, 1467, 3623, 241, 549, 3949, 4458, 1676, 703, 4830, 4902, 4495, 4773, 3923, 4573, 614, 2729, 3408, 388, 4325, 2986, 1811, 453, 3168, 2919, 1740, 3363, 3548, 293, 3190, 602, 3732, 1279, 1742, 2637, 2446, 722, 3046, 1711, 3333, 944, 540, 2609, 4906, 599, 2506, 4620, 2229, 3594, 908, 260, 1470, 1048, 1314, 3468, 1691, 3651, 189, 4086, 500, 3724, 3376, 3796, 1915, 2429, 4069, 2033, 4632, 615, 2269, 843, 4227, 747, 3289, 3676, 765, 1905, 3155, 476, 521, 2359, 1921, 3199, 719, 1082, 4753, 636, 3751, 3478, 230, 67, 4700, 4466, 1007, 29, 1577, 4182, 2337, 2067, 1106, 2417, 197, 2304, 1663, 4344, 2655, 1631, 4631, 4806, 3894, 1104, 2675, 3908, 3056, 3245, 4891, 2300, 4222, 3610, 1824, 2661, 3389, 1928, 1042, 4910, 2728, 4256, 597, 2529, 2267, 179, 1212, 2861, 3007, 1352, 2860, 4985, 475, 1963, 2876, 713, 3477, 218, 2260, 1834, 766, 1813, 2454, 4968, 668, 4626, 482, 4056, 2360, 1377, 1303, 425, 1114, 240, 383, 3511, 1490, 174, 1151, 2937, 4995, 1534, 1810, 894, 4858, 4223, 109, 675, 306, 2288, 1094, 4080, 4159, 3938, 1138, 4116, 2475, 2664, 973, 4399, 333, 94, 980, 2997, 2717, 948, 4686, 4451, 4990, 4373, 4670, 4450, 2301, 34, 4814, 107, 3666, 2495, 1144, 4252, 2167, 370, 3136, 4309, 2713, 1084, 4842, 3286, 4989, 2987, 672, 1280, 2689, 1709, 3700, 2541, 4724, 4719, 2036, 1536, 3038, 2983, 1728, 4797, 1772, 1255, 4975, 1700, 4756, 2782, 3381, 4392, 1069, 3718, 859, 1587, 2539, 246, 3580, 2920, 1396, 3715, 2334, 1275, 1855, 429, 786, 594, 133, 4032, 366, 1251, 624, 1694, 1302, 3023, 2570, 4178, 2291, 878, 1754, 69, 3646, 858, 3185, 4385, 3973, 410, 1887, 1229, 4501, 4812, 1248, 2227, 1496, 395, 1558, 4745, 2055, 3645, 1140, 2719, 262, 3164, 1878, 4676, 389, 288, 1286, 3747, 3619, 898, 2562, 3196, 2477, 1966, 198, 3849, 4511, 643, 379, 1077, 1686, 4688, 4388, 4177, 3931, 3188, 4194, 2449, 4045, 508, 801, 2517, 2059, 2902, 2490, 3831, 4164, 2964, 3018, 4135, 4033, 187, 4969, 1289, 1008, 3773, 3756, 4131, 3073, 1393, 4519, 4905, 167, 4097, 776, 2630, 1245, 4264, 3997, 3349, 2839, 4836, 3984, 1763, 2234, 749, 3284, 1868, 4802, 2696, 4735, 3072, 4680, 4729, 1682, 2261, 3066, 2779, 2751, 1727, 1353, 4460, 3451, 2366, 245, 1704, 1526, 3872, 2530, 4627, 200, 772, 4244, 129, 1420, 3223, 427, 2560, 2065, 1602, 3946, 4342, 4485, 840, 4714, 1451, 407, 4319, 4655, 4475, 3868, 1717, 3744, 2677, 2968, 3425, 4982, 2262, 3765, 3738, 3527, 287, 4957, 1889, 762, 4241, 2147, 2043, 285, 3822, 1978, 2950, 3826, 3922, 1967, 4198, 1695, 955, 3764, 2824, 85, 4395, 2944, 1633, 3993, 3860, 3085, 4404, 2379, 1680, 4037, 2256, 2510, 3679, 741, 852, 3967, 3079, 4622, 4012, 3687, 3874, 1299, 2930, 456, 3513, 4174, 365, 1256, 2900, 4217, 46, 2205, 1198, 2030, 657, 2899, 2357, 4945, 4800, 4132, 1780, 264, 755, 2653, 4739, 3326, 2062, 4561, 2410, 1988, 2781, 4629, 2104, 5000, 1158, 1735, 546, 2425, 4251, 1227, 3439, 3855, 780, 13, 452, 60, 1474, 2108, 3805, 1725, 1910, 3043, 1118, 1667, 2913, 4943, 1426, 2084, 4087, 2416, 4338, 3005, 4434, 1292, 4393, 671, 1969, 4207, 199, 4524, 2746, 3104, 2691, 103, 2057, 281, 4790, 1550, 4167, 2808, 3415, 4053, 1389, 1940, 1133, 3883, 3605, 3607, 2924, 4608, 2034, 596, 1528, 738, 35, 4955, 2440, 477, 3470, 4548, 2872, 4927, 1163, 4549, 1363, 899, 4768, 5009, 1944, 2951, 2702, 4066, 1827, 2112, 2421, 1820, 2732, 958, 2312, 3543, 1600, 3606, 3709, 720, 1999, 796, 4949, 4282, 2807, 2354, 1519, 1203, 2890, 2864, 4498, 3050, 4935, 1527, 991, 3311, 2367, 2107, 2077, 116, 921, 239, 2392, 2940, 1943, 443, 3399, 3665, 331, 3157, 3901, 2007, 2098, 1137, 4206, 701, 1783, 4335, 4654, 4247, 685, 2714, 160, 2796, 949, 4505, 4512, 17, 1664, 1054, 2181, 1894, 4367, 795, 503, 3636, 1665, 1551, 1795, 1380, 3205, 2049, 3547, 4897, 705, 3841, 1791, 65, 2010, 2191, 4555, 2770, 2701, 2053, 3343, 2772, 2598, 3152, 2006, 433, 881, 2852, 2317, 2738, 1760, 2749, 1808, 1954, 2284, 856, 829, 4094, 571, 3861, 3488, 355, 1186, 3920, 2547, 375, 1745, 345, 864, 3802, 428, 4801, 750, 2046, 4571, 229, 48, 2189, 4403, 4610, 1454, 1623, 4459, 4710, 4701, 2296, 2693, 2502, 1115, 216, 2356, 1986, 4929, 2339, 2243, 2571, 3748, 4878, 678, 2585, 4673, 4717, 2438, 263, 2635, 3690, 317, 2766, 1698, 1892, 2318, 1784, 1692, 2237, 4928, 1603, 2144, 238, 2123, 36, 307, 689, 2047, 4284, 704, 1316, 2927, 4660, 4924, 1427, 415, 1924, 1849, 286, 2121, 2790, 4576, 437, 3720, 4723, 4515, 2969, 12, 972, 2087, 2051, 3990, 4693, 1041, 1231, 1143, 242, 2232, 3297, 714, 2633, 652, 3613, 3293, 4956, 662, 75, 2154, 494, 413, 4354, 1072, 1020, 4236, 2212, 2346, 119, 1200, 3194, 4235, 2939, 2384, 1058, 3886, 1995, 1701, 2150, 1936, 1081, 3373, 4867, 530, 4331, 387, 2097, 2765, 4936, 2814, 4172, 4313, 1897, 1384, 3051, 1246, 3126, 411, 727, 1167, 2754, 2399, 1736, 830, 4257, 3839, 1657, 790, 3186, 4154, 4270, 3329, 2168, 3450, 2921, 1254, 2216, 1358, 2078, 1510, 1033, 1239, 1521, 1804, 3753, 4916, 3150, 1413, 236, 3561, 4271, 2372, 4059, 4129, 2750, 4951, 4064, 1089, 3609, 2081, 2235, 1261, 3429, 538, 1912, 3994, 4623, 514, 3642, 3357, 1131, 234, 2608, 2863, 885, 3008, 1425, 4803, 478, 1014, 4597, 2452, 4068, 3443, 1169, 49, 2626, 4853, 2777, 2179, 2253, 3294, 4755, 1666, 2001, 4202, 774, 1984, 1429, 1568, 434, 2115, 79, 2869, 2328, 4819, 3898, 2155, 1722, 2378, 674, 3834, 1216, 4113, 3825, 232, 697, 1803, 4034, 4650, 723, 3754, 3113, 2471, 2545, 2610, 3301, 276, 2837, 1421, 3022, 2599, 4075, 3427, 1462, 1269, 4041, 3942, 1013, 2996, 2854, 952, 3663, 2679, 3299, 1548, 585, 884, 1907, 3989, 3400, 186, 1579, 965, 4417, 2105, 3391, 525, 4128, 1236, 805, 2263, 3510, 149, 3275, 3084, 1121, 1848, 515, 2266, 1525, 2473, 740, 4725, 3976, 435, 3124, 3141, 2231, 1564, 3452, 4248, 777, 4040, 250, 2035, 131, 1199, 3761, 2966, 249, 2836, 2748, 4369, 4119, 927, 2767, 4918, 7, 1731, 492, 1543, 3525, 4496, 487, 3612, 10, 1837, 2895, 1757, 3453, 4845, 2436, 632, 438, 3162, 3558, 2031, 314, 1604, 1609, 4816, 3516, 4835, 4260, 2499, 3570, 676, 1765, 2535, 2419, 504, 3375, 256, 2045, 3031, 4468, 402, 2518, 2336, 172, 1357, 769, 647, 2165, 3929, 3184, 1964, 2109, 868, 1499, 2481, 3995, 1679, 3644, 598, 3025, 2226, 2795, 182, 954, 3215, 2690, 2249, 1374, 2222, 1775, 474, 4531, 3092, 2080, 2385, 4295, 904, 1982, 1294, 4609, 1950, 2914, 424, 1586, 4747, 5008, 359, 4074, 3549, 2845, 2250, 4809, 2753, 3499, 3067, 1402, 4328, 2945, 1002, 1204, 922, 3727, 4221, 2739, 1155, 861, 3337, 2665, 3492, 999, 4763, 4005, 2667, 807, 2509, 31, 4575, 3040, 1703, 2273, 4991, 3158, 2755, 4634, 4504, 2225, 4738, 915, 349, 3647, 882, 3654, 3521, 4216, 2355, 2020, 3660, 381, 3204, 1240, 1733, 3239, 2386, 637, 5, 1293, 2022, 1953, 788, 4556, 865, 3628, 2497, 733, 3001, 2052, 2185, 4507, 3851, 3219, 1329, 2682, 4305, 2565, 3166, 1308, 2050, 1491, 3269, 4983, 16, 967, 4474, 422, 988, 1833, 799, 4564, 4442, 4220, 3972, 3402, 1956, 3420, 3114, 1871, 1957, 2853, 1044, 488, 4506, 4224, 3671, 4886, 3197, 3564, 89, 385, 1394, 4383, 135, 4667, 1132, 4938, 1557, 4908, 257, 1272, 431, 4398, 2061, 3559, 2828, 875, 855, 259, 4646, 1130, 1653, 2816, 4898, 1715, 2602, 1301, 155, 2673, 1437, 3887, 4971, 2290, 4144, 4970, 2543, 2888, 535, 1444, 2575, 4530, 1977, 2011, 1060, 2403, 2911, 3464, 3586, 4586, 997, 3505, 2567, 2544, 3683, 2316, 1378, 577, 354, 4100, 2422, 3273, 3241, 56, 3458, 344, 4934, 2650, 312, 3251, 267, 3243, 583, 3482, 2685, 1263, 2037, 2431, 3264, 692, 2785, 3900, 886, 3368, 4193, 2932, 1365, 3214, 3741, 2142, 3335, 4600, 2580, 3087, 1997, 3313, 4965, 1142, 1708, 4402, 2170, 235, 729, 3672, 448, 3358, 2324, 4598, 1923, 1538, 1414, 1107, 4922, 3344, 2280, 1971, 1228, 1843, 1346, 584, 4001, 4930, 1342, 2197, 1522, 3078, 3621, 3256, 392, 4148, 4292, 3595, 3618, 3853, 2928, 3267, 376, 166, 3403, 3069, 3364, 658, 3341, 2118, 897, 2871, 4017, 3423, 4426, 2040, 2589, 2230, 106, 587, 2444, 2019, 2623, 1792, 706, 1037, 555, 3405, 1424, 2974, 2075, 1618, 2586, 1266, 3019, 4791, 3816, 2348, 450, 1674, 2306, 4852, 4022, 1860, 1349, 3120, 969, 2338, 3495, 1030, 665, 1274, 2193, 4265, 4569, 157, 913, 4699, 329, 4742, 1183, 2094, 3532, 4351, 1766, 2564, 3372, 3742, 327, 123, 537, 4761, 1052, 3775, 298, 1581, 1939, 1582, 2439, 771, 1445, 3987, 4941, 1500, 4545, 2556, 2406, 3809, 951, 2858, 4452, 252, 1489, 1206, 1532, 753, 1687, 1629, 191, 4536, 4926, 73, 3115, 4849, 2145, 3952, 2978, 2965, 441, 271, 1168, 595, 4240, 4940, 4760, 1326, 2478, 960, 2611, 2332, 1514, 2935, 3554, 76, 412, 4288, 3250, 778, 1262, 4291, 3829, 1832, 3230, 728, 2455, 2333, 3602, 358, 3247, 4090, 3531, 2415, 3502, 4004, 3926, 330, 499, 1468, 1908, 1561, 2555, 3302, 4192, 2756, 1399, 1821, 96, 4176, 4469, 4980, 4484, 609, 1649, 4143, 1863, 3960, 4964, 3342, 998, 1273, 4279, 536, 3731, 1829, 4330, 4299, 2069, 3870, 4281, 3481, 746, 3811, 2716, 736, 419, 4472, 4020, 165, 2054, 527, 876, 4453, 3259, 2581, 2734, 3713, 2956, 1095, 4947, 3568, 3652, 4911, 1460, 3788, 1379, 984, 5004, 2778, 679, 603, 4210, 1149, 297, 556, 3016, 971, 2325, 47, 4026, 3300, 644, 1652, 2504, 4657, 1091, 4976, 72, 2128, 3142, 467, 501, 883, 757, 1268, 3539, 4525, 2962, 1179, 2221, 1523, 2703, 730, 4229, 1861, 1678, 3585, 95, 2345, 3049, 2903, 1109, 2493, 4438, 2479, 4139, 4343, 1993, 651, 3237, 115, 3033, 3757, 4770, 3986, 1306, 1933, 612, 1893, 2663, 1407, 1188, 3392, 2857, 3388, 3947, 1594, 3167, 461, 2601, 4334, 2519, 1165, 2024, 2487, 320, 3430, 4366, 4873, 2277, 2214, 1646, 2255, 4952, 1626, 1472, 1867, 2849, 754, 1596, 1160, 5003, 4909, 2992, 192, 768, 3170, 4107, 449, 4071, 4642, 101, 659, 3127, 3413, 1979, 4596, 1124, 1669, 3782, 2192, 3677, 5006, 4126, 4690, 3328, 4716, 8, 4427, 3032, 205, 4333, 1919, 3707, 2922, 3117, 3632, 495, 2171, 1935, 4732, 2015, 3921, 625, 1768, 405, 589, 351, 3089, 794, 1234, 2405, 2597, 4009, 3059, 2093, 2240, 4028, 4253, 83, 1987, 2724, 2802, 2088, 3515, 4903, 3234, 121, 605, 4152, 2133, 4751, 2259, 2578, 3074, 4160, 2218, 3688, 4520, 2489, 459, 844, 517, 348, 1327, 707, 3551, 3218, 1025, 2281, 3209, 814, 994, 3463, 4406, 1207, 1309, 4793, 219, 1300, 1446, 2844, 2278, 1375, 3352, 1493, 1000, 4979, 2215, 580, 3461, 4653, 3701, 3655, 1574, 4727, 20, 1376, 4720, 4518, 1815, 1202, 93, 4289, 903, 4219, 1635, 1946, 3835, 1364, 4901, 2424, 889, 1900, 1547, 708, 4684, 4648, 4363, 2353, 3086, 4191, 3411, 3736, 1911, 1192, 2188, 681, 2016, 4821, 2335, 2769, 203, 933, 209, 800, 1865, 3061, 961, 2722, 2652, 1518, 1105, 3260, 1405, 4243, 1108, 1318, 839, 3246, 3771, 2299, 3371, 1502, 2491, 4007, 3845, 2859, 1777, 3444, 677, 3589, 635, 3965, 4542, 2461, 2970, 1972, 2398, 3231, 560, 1051, 1630, 1354, 4035, 14, 937, 148, 2576, 3998, 3966, 2762, 2557, 2298, 1483, 2620, 4612, 81, 1640, 451, 4851, 3195, 4972, 2815, 4868, 1087, 731, 1298, 153, 4389, 3296, 1148, 3424, 2975, 2712, 2437, 3047, 871, 2486, 2636, 3933, 4364, 2375, 611, 4209, 4900, 3905, 2073, 53, 3039, 1161, 2553, 1330, 1853, 3885, 4656, 956, 3955, 1350, 2507, 963, 4019, 2172, 2537, 2319, 1146, 1962, 102, 2657, 2136, 1774, 3576, 2622, 854, 627, 4568, 2605, 4994, 4409, 4711, 3719, 1458, 2791, 2850, 3675, 2508, 640, 4771, 3469, 1090, 3878, 709, 1927, 2912, 3631, 1152, 4913, 2615, 2151, 1706, 823, 1869, 3438, 4489, 2305, 2178, 1636, 4348, 3781, 4443, 4121, 3951, 3319, 1615, 3014, 4528, 308, 1356, 2875, 4058, 1417, 895, 3846, 4607, 255, 1877, 233, 1513, 4394, 4266, 928, 2146, 3145, 3200, 4788, 3598, 4996, 2482, 2658, 2308, 2584, 2528, 4146, 3125, 4030, 2, 2314, 4106, 4213, 4315, 1799, 779, 1884, 3010, 4682, 3058, 2113, 254, 4565, 2798, 1026, 1181, 80, 3519, 3410, 3583, 1992, 2466, 1891, 1479, 3735, 4893, 1881, 3587, 4822, 1241, 2148, 4419, 978, 4362, 1539, 2820, 2331, 4669, 1453, 396, 920, 3897, 4140, 1643, 970, 168, 1050, 783, 4618, 3983, 188, 1400, 2042, 1027, 3726, 3556, 4883, 2434, 687, 2026, 2878, 3779, 237, 3865, 1628, 3807, 4591, 1864, 3895, 90, 1847, 3198, 1608, 715, 1177, 4277, 4696, 4184, 2430, 2783, 352, 2946, 251, 1113, 4397, 2752, 3648, 4371, 294, 1362, 1607, 4370, 3309, 378, 1707, 1761, 673, 4125, 190, 4605, 683, 1888, 3437, 2099, 4482, 1816, 3616, 2244, 3471, 2175, 4461, 4408, 2577, 273, 462, 3954, 716, 626, 1320, 3656, 2376, 2877, 4685, 1080, 667, 2474, 1219, 4999, 773, 1153, 1325, 3356, 1599, 745, 1654, 3212, 1565, 3867, 3919, 824, 1083, 506, 303, 4109, 4405, 4421, 2952, 169, 4563, 4580, 1001, 4625, 2590, 4439, 1840, 4572, 2846, 1428, 4214, 1836, 1668, 4582, 1851, 1875, 877, 319, 918, 756, 4301, 2341, 1981, 2959, 4153, 2401, 2412, 1337, 1625, 3346, 4541, 44, 880, 4096, 3340, 3501, 3593, 3472, 403, 2174, 4368, 1812, 3207, 3474, 2617, 3890, 4741, 4587, 1423, 3377, 4349, 2390, 3680, 907, 1043, 3579, 4948, 2540, 1769, 4200, 1902, 2032, 1492, 4180, 3151, 2404, 3365, 4376, 781, 893, 2573, 3504, 1494, 2176, 2180, 1562, 4960, 1076, 1465, 4039, 125, 4712, 1079, 1, 2009, 1721, 2060, 3686, 827, 4317, 1537, 4824, 2025, 1127, 1724, 3943, 1906, 3163, 1433, 2660, 3180, 4920, 2934, 1509, 4105, 3318, 870, 2462, 3446, 1546, 3306, 3640, 4297, 3283, 1989, 3882, 2723, 4666, 4374, 1501, 3083, 2775, 1288, 3624, 4643, 531, 1662, 1431, 568, 3848, 3316, 906, 552, 1650, 4077, 1621, 4211, 104, 3823, 3928, 3633, 1497, 4407, 2127, 3493, 3339, 2394, 4872, 220, 3520, 2698, 3503, 2208, 2448, 1282, 2898, 2091, 2358, 23, 491, 2309, 4967, 282, 4534, 1097, 473, 1620, 321, 292, 4695, 3238, 4652, 194, 4581, 3172, 4838, 3804, 1487, 4539, 4358, 408, 1533, 2559, 4715, 867, 3508, 4815, 2163, 2705, 1841, 2206, 1243, 2349, 516, 2326, 3370, 3836, 445, 3597, 1334, 4003, 3465, 1260, 4663, 890, 3524, 3266, 518, 3353, 278, 4659, 3892, 1723, 2558, 1584, 1683, 2184, 3394, 3324, 1996, 3780, 3596, 4526, 1391, 1469, 4794, 3936, 497, 1324, 3048, 2886, 4197, 4992, 834, 4101, 464, 512, 3034, 108, 1955, 3065, 1370, 1925, 4018, 439, 3881, 1624, 4161, 3776, 3625, 558, 1895, 3812, 645, 1392, 1435, 693, 328, 382, 1345, 2764, 2773, 2909, 4360, 1595, 2220, 150, 1710, 502, 2531, 183, 2217, 143, 1171, 1321, 1752, 3036, 279, 305, 3390, 4445, 1341, 3213, 2223, 2727, 261, 340, 3657, 3854, 1032, 2119, 3497, 2362, 1938, 1702, 887, 3298, 1009, 4306, 4324, 3819, 4513, 4554, 1770, 3030, 4355, 791, 3906, 2285, 943, 3189, 2158, 1034, 4314, 3899, 4786, 217, 3888, 2720, 1388, 3507, 1086, 509, 3035, 2848, 3971, 3691, 3924, 380, 1839, 4492, 3528, 1291, 4635, 1485, 1193, 629, 2186, 4807, 4098, 3673, 4462, 3378, 760, 1233, 3290, 3517, 432, 3476, 896, 3858, 4274, 4546, 3122, 97, 6, 4562, 27, 1879, 711, 3064, 962, 4651, 959, 670, 2536, 1965, 2973, 2166, 3307, 2910, 468, 221, 4187, 4078, 3308, 442, 4175, 341, 1003, 1764, 156, 3808, 3484, 4933, 2157, 2383, 1117, 770, 2342, 3634, 2960, 4630, 1036, 4942, 4185, 2524, 648, 180, 211, 3169, 2027, 3934, 4455, 2809, 1605, 3447, 2323, 3123, 2943, 3, 3739, 4689, 1554, 1901, 2427, 337, 2831, 4163, 100, 1569, 910, 912, 1471, 3187, 173, 4879, 618, 4237, 394, 122, 3229, 1312, 1852, 208, 4145, 2538, 4345, 3692, 3116, 3514, 846, 3165, 2114, 2246, 1484, 3338, 3529, 2736, 11, 2407, 4378, 4396, 3985, 1059, 4875, 3814, 2941, 3174, 2409, 62, 2213, 3730, 4166, 4440, 4961, 3533, 1619, 3480, 919, 4692, 3053, 2219, 3777, 3028, 767, 1583, 924, 1092, 4759, 1762, 4006, 3475, 3958, 1257, 2563, 2687, 4418, 3685, 356, 498, 2388, 812, 52, 4208, 4249, 1934, 682, 3944, 1947, 2369, 2592, 1040, 270, 4860, 3017, 3785, 4490, 702, 1759, 4658, 4974, 3101, 4487, 3291, 3366, 2704, 3875, 465, 2793, 4021, 4329, 4111, 1788, 61, 2382, 1917, 334, 1508, 2468, 4234, 463, 4122, 2522, 3211, 4036, 215, 3077, 641, 1295, 4412, 2799, 3978, 3280, 1874, 1922, 1866, 374, 4186, 4592, 3697, 3699, 225, 3801, 3615, 2028, 3684, 2329, 4887, 4776, 1559, 2202, 2916, 4263, 524, 3838, 4588, 3842, 4311, 2456, 4843, 2472, 4736, 3999, 4060, 128, 2600, 2195, 1398, 4043, 4500, 4749, 110, 4339, 1809, 3041, 4706, 490, 4616, 2554, 4757, 2780, 2865, 2656, 364, 649, 3262, 3760, 3417, 570, 4679, 2887, 2856, 1835, 3374, 1336, 2595, 1031, 1359, 2116, 4687, 4537, 3080, 1447, 2825, 1994, 748, 2272, 4137, 4602, 2757, 3745, 4276, 2879, 592, 1047, 202, 485, 3254, 4869, 964, 3940, 4584, 2908, 41, 541, 4846, 3160, 2389, 1571, 1022, 968, 4709, 1259, 2923, 1134, 2140, 3889, 2423, 436, 3156, 112, 3317, 941, 4294, 1614, 4789, 4884, 3111, 2072, 815, 4881, 2985, 2742, 2149, 1606, 4639, 2228, 686, 3557, 161, 4538, 170, 4435, 1661, 2433, 1540, 2512, 2671, 3956, 3258, 4050, 3409, 2761, 4015, 4150, 3494, 1916, 3103, 4664, 3496, 2443, 4540, 953, 2917, 1705, 2085, 4697, 1903, 1542, 2374, 2058, 828, 1985, 3639, 4049, 3681, 4203, 588, 4067, 4617, 4356, 1570, 2070, 2236, 4818, 1639, 4726, 3387, 604, 3884, 4114, 2594, 204, 4766, 817, 551, 4436, 1632, 2788, 782, 373, 420, 735, 1297, 105, 4831, 1862, 4718, 4783, 3380, 2805, 4420, 4411, 2873, 3582, 4350, 4173, 2274, 2373, 505, 4110, 4681, 836, 460, 1882, 2999, 3810, 2588, 2124, 1617, 724, 4817, 2733, 2483, 4583, 2287, 857, 1926, 1343, 210, 4112, 3222, 2803, 559, 742, 4796, 3935, 4382, 2786, 1572, 3728, 2953, 539, 1482, 2207, 1100, 553, 4103, 4865, 3762, 2103, 70, 3793, 543, 3334, 656, 4195, 1017, 4997, 2521, 810, 2279, 4566, 2095, 3055, 3336, 1858, 513, 2102, 447, 1800, 4327, 3682, 2936, 3950, 4683, 353, 523, 1166, 4029, 2523, 775, 3070, 1129, 3128, 2041, 4937, 1677, 2822, 2120, 3974, 360, 1553, 1347, 860, 1983, 3755, 1767, 3147, 283, 1170, 1475, 2247, 1535, 4820, 2457, 1798, 4065, 2071, 2327, 4750, 4380, 1156, 2520, 3107, 37, 2662, 621, 1019, 1506, 751, 3265, 2684, 2957, 1601, 759, 2089, 996, 3694, 3969, 1385, 151, 66, 3907, 2295, 1720, 547, 87, 3042, 323, 2851, 3220, 2718, 2368, 2715, 1242, 1732, 77, 86, 3795, 3536, 1448, 50, 4441, 2686, 1778, 2330, 2659, 421, 1382, 4517, 1681, 4823, 990, 3279, 4914, 1235, 4862, 1276, 1078, 3857, 784, 4010, 3248, 158, 1098, 4346, 2201, 2484, 2976, 3137, 1355, 4782, 979, 2501, 4431, 2708, 1730, 2470, 1057, 974, 2003, 1305, 2039, 3722, 3664, 3537, 3436, 372, 4962, 1714, 4303, 3348, 2445, 526, 3824, 2745, 377, 1790, 3909, 2892, 4254, 2079, 1252, 316, 2819, 2426, 630, 3354, 146, 1039, 1322, 3541, 2135, 4136, 1647, 3130, 1173, 2002, 1287, 628, 274, 4578, 284, 1419, 533, 3827, 19, 4839, 1517, 227, 1201, 700, 2834, 1931, 3569, 406, 1217, 4464, 1313, 4523, 3992, 4778, 3487, 4777, 4168, 3553, 2813, 3235, 721, 2806, 30, 4977, 21, 2642, 3330, 3611, 4644, 40, 2794, 3500, 3386, 3573, 654, 939, 362, 1747, 1872, 2676, 4758, 3216, 4744, 1854, 1278, 3930, 819, 2303, 3082, 3305, 4425, 4661, 367, 1976, 2463, 1395, 3702, 848, 872, 393, 1339, 646, 936, 3456, 4201, 4340, 2552, 4574, 1613, 1876, 2591, 2315, 3419, 1232, 2258, 4931, 1061, 1684, 3242, 837, 2621, 1826, 2458, 2420, 2634, 145, 2774, 3282, 1771, 2414, 1271, 4287, 4231, 1176, 4917, 798, 1304, 4352, 1477, 484, 3261, 1942, 4104, 1749, 739, 2480, 1125, 33, 1222, 3418, 3303, 3830, 2012, 2654, 3689, 4147, 1466, 2847, 2759, 4672, 336, 1053, 983, 2018, 3416, 2614, 582, 1693, 2160, 4599, 57, 2340, 4239, 1418, 2810, 1139, 3132, 2550, 1642, 2741, 139, 310, 1123, 3649, 1011, 4326, 3274, 3506, 82, 2835, 2021, 4280, 386, 3404, 1205, 4880, 1056, 1507, 25, 3489, 3097, 58, 916, 4532, 726, 1990, 4611, 3740, 4754, 2310, 2627, 1111, 4024, 3975, 752, 1959, 4601, 1520, 54, 3154, 2768, 1941, 3181, 1498, 1748, 178, 3678, 3818, 1555, 3385, 1005, 2017, 32, 992, 144, 2100, 4463, 4156, 1870, 1070, 4359, 3716, 2152, 2514, 1511, 228, 3626, 2789, 140, 4577, 4871, 1794, 2735, 2933, 3004, 175, 3062, 1064, 4827, 2549, 3797, 655, 4558, 409, 91, 1998, 2485, 3638, 835, 3325, 4671, 1449, 3108, 2680, 2918, 4181, 2122, 2593, 3913, 2833, 803, 350, 1817, 177, 1968, 3135, 4499, 3538, 1696, 3121, 1476, 1221, 4641, 2628, 3442, 2224, 4286, 63, 4095, 808, 634, 2763, 1612, 322, 2897, 718, 2993, 3276, 3800, 2533, 544, 1802, 2534, 2321, 3281, 4298, 975, 2241, 2447, 1331, 2526, 390, 4300, 4218, 4870, 164, 3962, 141, 1145, 4337, 2286, 3614, 4987, 4885, 3105, 1697, 3322, 2132, 1627, 1247, 4047, 343, 2982, 4944, 520, 2862, 371, 1713, 892, 1776, 347, 1101, 0, 1738, 3902, 1067, 1951, 3996, 3102, 3770, 2967, 3758, 797, 1046, 118, 444, 3658, 695, 299, 243, 4183, 3096, 3749, 1655, 3201, 1281, 3109, 1210, 4008, 2971, 743, 3941, 4585, 3750, 2706, 1333, 1589, 1360, 1016, 2488, 1637, 3257, 2651, 3192, 947, 2838, 4205, 4559, 2023, 1024, 3866, 3479, 369, 3263, 2855, 2998, 3332, 3139, 2631, 3026, 159, 4361, 2747, 607, 793, 4674, 4447, 483, 4785, 2958, 4011, 3710, 3708, 3118, 3578, 1441, 4847, 2961, 4091, 2829, 548, 4746, 869, 4589, 561, 591, 1859, 3203, 2868, 2644, 3110, 2038, 335, 1154, 1456, 4473, 1012, 4179, 2320, 1045, 3630, 2361, 1141, 1319, 4946, 3980, 1096, 2991, 4828, 2925, 845, 2313, 3208, 809, 3768, 3991, 4954, 4027, 2182, 4668, 1208, 804, 1793, 4978, 4063, 1495, 4134, 564, 4375, 2649, 2402, 3988, 3228, 3398, 2582, 3440, 4792, 3161, 2548, 4267, 3176, 458, 1949, 4593, 3179, 3604, 888, 3601, 1544, 1644, 114, 4117, 3012, 3314, 207, 3044, 1850, 2527, 642, 4829, 4415, 1831, 361, 2632, 586, 1801, 4925, 574, 1348, 4677, 3367, 2694, 226, 2270, 1250, 4904, 2841, 3914, 15, 2139, 4488, 1593, 1065, 4638, 486, 1372, 3714, 4014, 1344, 2525, 4888, 84, 989, 3873, 1187, 430, 511, 4866, 43, 126, 522, 1323, 5005, 4046, 3397, 2948, 3173, 3384, 4529, 802, 2391, 2494, 1440, 2596, 1404, 1952, 4042, 4708, 1116, 2907, 2239, 147, 1190, 1750, 1830, 4861, 4743, 3431, 2381, 2885, 2827, 4479, 5007, 3918, 3191, 3013, 3627], "validation": [5083, 5139, 5099, 5311, 5391, 5106, 5344, 5341, 5108, 5434, 5318, 5104, 5198, 5263, 5142, 5442, 5348, 5051, 5407, 5061, 5247, 5416, 5156, 5346, 5468, 5429, 5328, 5110, 5107, 5177, 5120, 5085, 5316, 5301, 5068, 5102, 5044, 5373, 5276, 5125, 5123, 5079, 5423, 5259, 5444, 5228, 5024, 5041, 5394, 5067, 5305, 5419, 5466, 5153, 5242, 5476, 5383, 5435, 5385, 5506, 5417, 5217, 5163, 5200, 5309, 5092, 5405, 5252, 5073, 5266, 5112, 5055, 5128, 5082, 5345, 5213, 5130, 5012, 5090, 5460, 5406, 5443, 5312, 5230, 5014, 5136, 5308, 5264, 5148, 5020, 5098, 5441, 5410, 5143, 5260, 5033, 5498, 5469, 5314, 5165, 5269, 5428, 5065, 5330, 5179, 5286, 5352, 5032, 5254, 5257, 5224, 5222, 5226, 5172, 5117, 5207, 5183, 5399, 5063, 5411, 5489, 5251, 5160, 5299, 5184, 5389, 5448, 5388, 5335, 5295, 5488, 5386, 5155, 5089, 5310, 5503, 5115, 5425, 5043, 5060, 5487, 5218, 5071, 5454, 5424, 5035, 5371, 5433, 5306, 5084, 5440, 5212, 5297, 5453, 5367, 5240, 5418, 5220, 5233, 5291, 5422, 5166, 5245, 5402, 5467, 5445, 5349, 5146, 5478, 5162, 5332, 5376, 5353, 5039, 5398, 5025, 5203, 5274, 5234, 5509, 5015, 5151, 5248, 5323, 5280, 5465, 5101, 5204, 5206, 5088, 5038, 5054, 5303, 5451, 5253, 5109, 5292, 5481, 5211, 5181, 5430, 5307, 5180, 5244, 5302, 5258, 5384, 5135, 5455, 5507, 5250, 5235, 5064, 5463, 5149, 5116, 5427, 5199, 5216, 5505, 5461, 5059, 5267, 5325, 5141, 5360, 5236, 5368, 5091, 5114, 5241, 5329, 5365, 5080, 5321, 5284, 5227, 5168, 5030, 5118, 5232, 5193, 5047, 5298, 5040, 5196, 5462, 5016, 5010, 5018, 5126, 5358, 5185, 5473, 5122, 5239, 5062, 5372, 5400, 5296, 5313, 5456, 5281, 5036, 5272, 5273, 5255, 5175, 5134, 5492, 5377, 5058, 5458, 5268, 5439, 5202, 5415, 5294, 5057, 5357, 5337, 5479, 5480, 5119, 5152, 5261, 5359, 5355, 5449, 5191, 5013, 5022, 5472, 5452, 5395, 5331, 5095, 5197, 5320, 5249, 5374, 5493, 5096, 5028, 5237, 5137, 5017, 5042, 5278, 5438, 5271, 5049, 5097, 5366, 5347, 5491, 5275, 5363, 5437, 5457, 5031, 5171, 5300, 5446, 5170, 5050, 5164, 5223, 5127, 5205, 5093, 5420, 5077, 5243, 5221, 5046, 5219, 5354, 5401, 5094, 5324, 5140, 5370, 5187, 5277, 5403, 5500, 5231, 5111, 5066, 5011, 5189, 5215, 5288, 5351, 5045, 5504, 5392, 5342, 5356, 5178, 5100, 5496, 5113, 5201, 5486, 5413, 5499, 5161, 5124, 5075, 5056, 5287, 5103, 5475, 5138, 5053, 5283, 5474, 5121, 5338, 5336, 5290, 5387, 5173, 5396, 5037, 5210, 5379, 5190, 5326, 5432, 5208, 5078, 5450, 5285, 5436, 5322, 5133, 5378, 5317, 5414, 5262, 5339, 5380, 5023, 5087, 5145, 5265, 5167, 5334, 5074, 5214, 5159, 5246, 5282, 5270, 5464, 5158, 5421, 5256, 5471, 5350, 5408, 5340, 5470, 5362, 5229, 5176, 5150, 5397, 5364, 5412, 5304, 5021, 5157, 5459, 5390, 5495, 5431, 5182, 5327, 5484, 5069, 5501, 5144, 5447, 5393, 5076, 5188, 5194, 5225, 5293, 5019, 5132, 5477, 5494, 5483, 5034, 5195, 5490, 5105, 5426, 5192, 5072, 5508, 5369, 5027, 5497, 5502, 5052, 5279, 5174, 5186, 5026, 5129, 5154, 5482, 5070, 5048, 5319, 5333, 5238, 5382, 5131, 5343, 5169, 5375, 5086, 5029, 5361, 5289, 5209, 5404, 5409, 5485, 5315, 5081, 5147, 5381], "test": [5805, 5786, 5534, 5742, 5740, 5671, 5922, 5826, 5603, 5785, 5548, 5943, 5652, 5641, 6004, 5672, 5793, 5921, 5780, 5867, 5528, 5658, 5758, 5600, 5757, 5639, 5836, 5684, 5888, 5913, 6006, 5728, 5730, 5794, 5715, 5842, 5737, 5800, 5670, 5819, 5609, 5936, 5859, 5792, 5935, 5547, 5955, 5552, 5683, 5905, 5979, 5980, 5797, 5931, 5772, 5945, 5966, 5630, 5554, 6002, 5844, 5869, 5575, 5988, 5717, 5588, 5563, 5923, 6007, 5868, 5767, 5735, 5829, 5787, 5788, 5673, 5647, 5625, 5820, 5763, 5598, 5583, 5880, 5932, 5879, 5941, 5877, 5812, 5950, 5573, 5782, 5893, 5997, 5985, 5653, 5900, 5994, 5546, 5521, 5856, 5884, 5518, 5898, 5783, 5914, 5958, 5636, 5770, 5604, 5553, 5982, 5688, 5918, 5564, 5960, 5882, 5662, 5677, 5903, 5582, 5637, 5597, 5699, 5706, 5537, 5605, 5532, 5939, 5714, 5686, 5996, 5839, 5713, 5957, 5771, 5704, 5947, 5520, 5690, 5902, 6009, 5881, 5703, 5986, 5616, 5990, 5946, 5542, 5870, 5959, 5579, 5540, 5775, 5613, 5890, 5851, 5811, 5618, 5987, 5578, 5843, 5892, 5560, 5975, 5760, 5710, 5634, 5602, 5934, 5850, 5645, 5611, 5590, 5558, 5712, 5845, 5841, 5852, 5748, 5910, 5648, 5993, 5535, 5925, 5887, 5917, 5897, 5821, 5862, 6001, 5894, 5872, 5620, 5920, 5814, 5661, 5619, 5891, 5768, 5627, 5968, 5751, 5810, 5835, 5813, 5650, 5574, 5858, 5628, 5567, 5572, 5833, 5896, 5919, 5617, 5530, 5817, 5610, 5761, 5865, 5827, 5984, 5555, 5864, 5622, 5739, 5687, 5718, 5665, 5733, 5964, 5629, 5801, 5640, 5823, 5983, 5536, 5977, 5933, 5949, 5749, 5834, 5571, 5912, 5832, 5724, 5904, 5721, 5697, 5660, 5956, 5963, 5791, 5831, 5529, 5759, 5876, 5523, 5510, 5806, 5907, 5974, 5527, 5784, 5790, 5764, 5696, 5753, 5873, 5581, 5595, 5747, 5766, 5944, 5961, 5594, 5734, 6000, 5599, 5586, 5970, 5580, 5591, 5657, 5678, 5663, 5769, 5854, 5938, 5828, 5519, 5863, 5995, 6003, 5928, 5725, 5777, 5612, 5906, 5883, 5942, 5659, 5962, 5654, 5802, 5926, 5889, 5726, 5846, 5526, 5720, 5576, 5746, 5776, 5855, 5649, 5682, 5651, 5668, 5992, 5694, 5515, 5973, 5524, 5989, 5691, 5693, 5729, 5857, 5561, 5675, 5818, 5708, 5981, 5809, 5908, 5698, 5773, 5874, 5700, 5969, 5633, 5948, 5565, 5838, 5709, 5899, 5848, 5722, 5875, 5971, 5539, 5562, 5674, 5680, 5837, 5549, 5723, 5861, 5803, 5895, 6005, 5623, 5952, 5965, 5924, 5589, 5807, 5666, 5676, 5911, 5860, 5816, 5744, 5830, 5642, 5667, 5531, 5731, 5789, 5840, 5614, 5655, 5754, 5752, 5795, 5566, 5592, 5743, 5695, 5585, 5716, 5953, 5601, 5885, 5522, 5951, 5927, 5705, 5991, 5762, 5878, 5954, 5596, 6008, 5689, 5551, 5606, 5765, 5525, 5685, 5853, 5584, 5707, 5607, 5656, 5745, 5550, 5568, 5909, 5779, 5778, 5781, 5646, 5799, 5545, 5621, 5577, 5516, 5871, 5998, 5587, 5533, 5679, 5692, 5798, 5638, 5738, 5644, 5937, 5824, 5972, 5999, 5635, 5556, 5719, 5643, 5915, 5511, 5569, 5774, 5755, 5557, 5681, 5741, 5626, 5815, 5711, 5632, 5513, 5608, 5727, 5615, 5976, 5732, 5822, 5849, 5701, 5593, 5940, 5702, 5808, 5624, 5559, 5631, 5901, 5756, 5886, 5736, 5543, 5538, 5825, 5929, 5541, 5978, 5967, 5514, 5544, 5669, 5796, 5664, 5847, 5750, 5930, 5916, 5866, 5570, 5804, 5512, 5517]}, {"train": [3968, 1767, 3736, 113, 4076, 2562, 941, 4458, 3553, 2543, 2280, 3453, 2953, 3376, 2365, 91, 3199, 3729, 4216, 3947, 849, 1401, 1467, 2311, 3079, 3830, 4373, 4949, 1444, 4341, 4703, 735, 1713, 2270, 3305, 306, 2703, 896, 1983, 2834, 1239, 741, 800, 3291, 3649, 859, 1332, 1856, 4060, 787, 3829, 2998, 1971, 2888, 2087, 1896, 3544, 462, 4103, 768, 2192, 1816, 2077, 2429, 3676, 698, 424, 812, 25, 1905, 4339, 1993, 3679, 4143, 2943, 3787, 4214, 4324, 1745, 4700, 757, 2056, 3238, 4525, 2450, 758, 481, 4247, 1918, 3938, 2854, 1486, 1729, 2741, 3121, 2481, 3710, 651, 2051, 4335, 1733, 954, 3841, 482, 1451, 3498, 3846, 1455, 2596, 1062, 2878, 4233, 1132, 234, 3096, 830, 1171, 4526, 518, 3919, 2969, 3995, 1852, 2374, 969, 1912, 2964, 106, 3297, 3114, 856, 3709, 1018, 2723, 4555, 4785, 2922, 4343, 4728, 2273, 2355, 1958, 4685, 2792, 4174, 4556, 1814, 174, 780, 418, 1838, 206, 1824, 2018, 3200, 2856, 115, 2786, 883, 220, 3765, 601, 4169, 3550, 2721, 4584, 857, 343, 1285, 1069, 388, 3848, 2391, 2896, 632, 3207, 2839, 321, 2889, 64, 2887, 2848, 419, 4441, 1663, 4580, 1715, 2743, 3175, 607, 347, 1457, 2455, 823, 1026, 3586, 1532, 2974, 4570, 2497, 623, 4470, 1996, 2609, 2111, 3936, 4746, 2832, 1251, 456, 70, 2322, 47, 4378, 130, 137, 1441, 2622, 2272, 2230, 4896, 2036, 3514, 3523, 2034, 506, 2868, 1650, 159, 2333, 4384, 1942, 4347, 4386, 2342, 988, 2523, 450, 1664, 1159, 211, 4003, 3348, 2983, 407, 2089, 4006, 2561, 2396, 1554, 4136, 3358, 4208, 3233, 691, 3670, 3088, 4649, 3820, 3390, 1350, 3144, 3638, 1136, 2091, 3317, 1953, 1012, 1413, 218, 4417, 4469, 405, 4985, 2850, 2695, 4787, 4892, 4621, 3959, 549, 4977, 3166, 60, 587, 1635, 2104, 2435, 4695, 749, 713, 162, 4858, 2206, 1181, 1601, 2737, 18, 2155, 635, 1985, 734, 1022, 1501, 972, 1949, 71, 620, 4915, 1567, 330, 3543, 3476, 2897, 3587, 1165, 4663, 262, 5006, 1321, 3281, 737, 3491, 4564, 2545, 1626, 805, 4968, 2197, 4599, 1683, 3382, 4694, 464, 2347, 2057, 4478, 4847, 2630, 1168, 2067, 4007, 1390, 2222, 4568, 4186, 1393, 3269, 4890, 4162, 4987, 2788, 190, 4973, 1201, 3562, 4705, 1232, 4761, 1726, 327, 682, 1704, 3026, 917, 1347, 861, 1939, 3661, 4551, 3569, 4391, 4823, 4405, 460, 2870, 1212, 363, 739, 1638, 3788, 1032, 2637, 3378, 743, 2635, 3665, 353, 1778, 4911, 1108, 385, 3127, 2755, 2734, 4683, 3249, 4460, 4488, 1483, 4765, 4722, 1440, 4037, 4020, 2092, 3493, 317, 3125, 1060, 634, 1819, 1183, 1552, 3023, 4034, 2281, 4399, 616, 238, 4485, 359, 4588, 3822, 1972, 2515, 3870, 681, 3872, 3559, 1917, 3844, 4014, 3773, 1982, 1348, 3732, 4154, 2440, 3738, 4707, 1463, 4092, 4783, 1474, 3312, 403, 356, 4877, 222, 1326, 4796, 3240, 745, 4110, 4431, 3370, 4971, 1103, 2237, 2975, 1495, 3474, 124, 3, 3886, 39, 4329, 1512, 3894, 329, 1562, 4062, 114, 2917, 2914, 3651, 1310, 4146, 4981, 3064, 1013, 683, 3950, 2200, 1133, 3416, 2031, 2254, 3883, 1322, 2500, 2029, 3163, 3499, 2408, 318, 2768, 4050, 2716, 1139, 0, 5, 2475, 4312, 3656, 2127, 80, 4193, 4640, 4704, 3274, 738, 282, 1471, 2563, 4710, 1785, 3216, 3362, 785, 4967, 3463, 442, 2173, 811, 3433, 1613, 263, 769, 4651, 2772, 4625, 146, 3696, 1684, 1346, 3524, 3916, 285, 4770, 959, 3346, 4611, 453, 3843, 2844, 4101, 750, 82, 4846, 2163, 3974, 1624, 3805, 4199, 2940, 1817, 1779, 4609, 3296, 242, 4596, 1301, 877, 3567, 1105, 1085, 3241, 3798, 4659, 4997, 1267, 4411, 62, 4889, 858, 775, 3724, 2298, 1725, 4295, 1055, 3006, 3785, 3525, 1708, 659, 3643, 2689, 2592, 4465, 142, 4350, 1872, 2312, 1740, 1462, 2292, 221, 3235, 4542, 3467, 636, 690, 2449, 1096, 3148, 2044, 1706, 3674, 1782, 4248, 2137, 2118, 3734, 2634, 4606, 4750, 649, 4259, 3141, 4268, 973, 259, 439, 522, 2362, 3633, 4463, 1991, 3059, 979, 1534, 1575, 1138, 1330, 1460, 1357, 332, 1114, 4344, 4241, 4185, 2973, 3607, 463, 1154, 118, 1857, 3430, 2244, 3616, 3594, 584, 3890, 1647, 673, 1488, 3537, 2612, 3812, 1637, 2164, 4094, 1071, 3617, 2306, 28, 1137, 3048, 4048, 2808, 4046, 1790, 4133, 4719, 949, 2190, 732, 1397, 1438, 3242, 4244, 2720, 3231, 1689, 457, 5002, 2951, 110, 3534, 3108, 1645, 4131, 3909, 1480, 194, 3721, 4404, 2462, 1812, 695, 2899, 2764, 4674, 4717, 4492, 2485, 1907, 58, 1906, 3512, 26, 3712, 755, 2234, 3806, 86, 394, 4922, 1941, 744, 2795, 2661, 2646, 3266, 2468, 4548, 3299, 216, 4956, 1002, 2165, 1546, 2577, 1098, 461, 4684, 4763, 4409, 3113, 514, 1314, 3056, 1345, 1630, 2776, 1752, 1051, 2296, 3725, 271, 3735, 4280, 4576, 3455, 2815, 1272, 2143, 2106, 2659, 4313, 2551, 4281, 1066, 1842, 760, 483, 378, 1428, 1732, 1217, 1175, 1439, 4991, 2101, 4632, 4083, 1177, 1489, 138, 4336, 4989, 2346, 1493, 4105, 4380, 1621, 1083, 4797, 4298, 3404, 2183, 3602, 569, 3605, 3695, 1585, 186, 1037, 1569, 2431, 1549, 3948, 3716, 3462, 3047, 1776, 3475, 3420, 4950, 1730, 3365, 3989, 4346, 542, 3272, 4497, 3827, 1540, 2891, 4944, 235, 4926, 1516, 552, 3809, 1775, 24, 4505, 2103, 4294, 1268, 3979, 2620, 1193, 3389, 2828, 1023, 4916, 3436, 2842, 1091, 3162, 4932, 2706, 2068, 4898, 2217, 2806, 3093, 3766, 1519, 1255, 527, 4289, 4245, 4934, 4927, 3852, 3020, 1926, 3813, 3914, 3688, 2220, 841, 3364, 860, 1152, 1791, 1244, 1116, 331, 1219, 3327, 2152, 2919, 3134, 2041, 4835, 2658, 3746, 3654, 1378, 937, 2587, 4318, 2208, 2862, 4489, 1142, 3786, 4490, 3386, 939, 1987, 1384, 4714, 4980, 93, 4905, 1016, 2062, 3833, 3191, 3284, 4163, 1225, 2999, 2132, 2993, 287, 3173, 1029, 4677, 554, 1470, 2439, 2297, 1561, 2128, 4107, 2553, 4691, 2718, 2241, 4434, 4493, 3030, 2861, 4852, 3740, 4544, 2176, 487, 4598, 4904, 3790, 415, 3061, 333, 4590, 48, 4724, 2195, 4617, 2330, 2628, 2765, 3359, 3211, 4666, 3040, 2205, 4743, 1109, 23, 406, 3035, 4090, 477, 4718, 3338, 4025, 1705, 605, 4520, 44, 4863, 2890, 376, 3140, 4913, 4487, 1368, 4955, 1063, 4713, 3383, 2799, 4065, 3956, 4308, 1333, 4941, 1864, 655, 1048, 4974, 474, 1588, 1072, 2606, 326, 803, 2033, 4435, 551, 4540, 1477, 3571, 3631, 2629, 2569, 1570, 2136, 3445, 4293, 4082, 4348, 3097, 747, 4128, 305, 2339, 2519, 2602, 1359, 210, 1833, 3722, 3406, 3900, 783, 197, 2669, 4315, 2187, 268, 1017, 3644, 3169, 854, 4086, 4402, 2203, 1753, 2881, 4330, 3884, 4917, 1003, 2934, 2294, 1850, 1296, 2153, 2284, 59, 498, 4636, 1144, 232, 3915, 1582, 4317, 4023, 1808, 2997, 827, 627, 1572, 3842, 4480, 2704, 2472, 3328, 2525, 1853, 4494, 4660, 2521, 4591, 2061, 2638, 1088, 3221, 2930, 366, 1205, 68, 656, 3578, 2097, 4699, 1510, 1035, 2100, 2466, 290, 3262, 338, 2875, 2925, 4400, 4030, 1632, 3560, 3940, 2476, 428, 67, 1748, 507, 374, 4963, 286, 1952, 209, 2209, 4709, 1844, 2301, 2248, 3412, 541, 990, 199, 2193, 3244, 3254, 868, 449, 1042, 721, 310, 1900, 4844, 1247, 3574, 2447, 4159, 5001, 3176, 2464, 1676, 1671, 3417, 1710, 3877, 1281, 4529, 4377, 3557, 1680, 1878, 874, 4711, 4972, 2423, 4029, 4237, 4935, 4988, 3957, 3339, 3110, 617, 3288, 4583, 1594, 2229, 3345, 4302, 3693, 2663, 3245, 714, 1543, 4372, 660, 1121, 2926, 2738, 893, 4732, 1526, 2269, 3333, 1641, 4675, 1214, 2112, 4802, 370, 1674, 2351, 3742, 4870, 255, 2202, 131, 470, 4795, 4817, 2761, 75, 1410, 396, 1679, 955, 736, 4323, 4459, 2304, 4447, 4275, 3558, 3743, 742, 123, 806, 4543, 4946, 2715, 2191, 3332, 626, 275, 846, 1666, 1382, 3818, 1150, 2291, 571, 1709, 4078, 3349, 17, 1014, 3684, 4662, 3156, 2614, 1823, 1153, 2093, 435, 3871, 2988, 272, 2139, 4173, 2811, 1788, 2550, 3322, 4790, 1573, 4622, 1300, 3095, 4198, 43, 4473, 2655, 227, 909, 2928, 1191, 1207, 4390, 2707, 1727, 3913, 3618, 693, 4767, 4522, 4263, 4778, 4553, 3619, 2593, 1934, 2287, 3518, 431, 692, 2821, 4144, 2460, 1667, 2787, 5000, 4418, 3092, 144, 771, 3276, 705, 2948, 2336, 716, 335, 1837, 1086, 3016, 203, 2585, 599, 1595, 1598, 3932, 2144, 2399, 386, 1182, 3626, 1388, 4612, 4843, 3415, 1340, 3840, 1276, 4696, 3992, 2547, 852, 1954, 1913, 781, 864, 2418, 986, 3589, 4532, 2198, 3637, 4097, 4655, 3503, 4398, 3628, 3599, 2913, 950, 29, 1981, 935, 1914, 3797, 3513, 1968, 1161, 1741, 295, 1672, 4450, 1974, 2782, 2912, 3428, 1047, 4309, 2662, 3085, 1506, 2904, 2708, 4731, 4919, 2656, 472, 603, 1742, 3565, 2412, 520, 4784, 176, 2074, 1008, 3975, 1075, 4224, 4148, 195, 2810, 3931, 1261, 1910, 4472, 1185, 2935, 1986, 1174, 249, 2156, 2700, 4678, 3535, 4203, 4260, 4642, 754, 383, 1282, 3985, 3889, 730, 1894, 1652, 2544, 1735, 1059, 1067, 832, 4521, 160, 3764, 4457, 3910, 4491, 2474, 921, 1929, 2714, 3120, 4670, 4504, 57, 479, 1442, 3903, 325, 974, 2504, 4087, 3011, 1077, 1160, 1885, 2377, 4618, 946, 951, 3160, 2884, 2729, 3925, 4834, 3062, 1027, 3887, 3954, 2990, 1766, 4690, 1963, 1172, 3515, 3135, 1292, 2463, 4031, 2918, 4305, 729, 654, 1771, 4271, 1871, 1596, 4093, 3730, 1723, 2539, 1548, 4866, 2677, 3683, 1612, 4194, 1131, 4002, 3478, 4213, 4211, 1327, 4862, 3965, 2290, 4809, 3668, 4112, 4933, 1367, 426, 360, 3157, 867, 797, 2207, 4772, 581, 2134, 4650, 2778, 4853, 1720, 3895, 2454, 2927, 157, 3393, 4782, 563, 2324, 1386, 2598, 1416, 2607, 3972, 2434, 1855, 644, 4885, 577, 2212, 1522, 3675, 1454, 215, 4697, 2168, 3375, 1427, 1843, 3468, 4210, 4021, 1932, 3597, 3811, 590, 485, 429, 3795, 3217, 1233, 2172, 4184, 2946, 3043, 2223, 4342, 2599, 2632, 1719, 4498, 4656, 3532, 2038, 3071, 193, 1143, 14, 1245, 2432, 4741, 2952, 3377, 4874, 4109, 2219, 1082, 1135, 2843, 128, 1385, 942, 1690, 1101, 3334, 2507, 2469, 2327, 3751, 2849, 3392, 3528, 2189, 796, 1104, 1215, 2305, 2909, 3556, 1497, 2712, 1579, 3826, 4142, 1449, 4111, 4413, 3539, 4924, 4550, 2747, 3832, 2005, 2275, 4607, 88, 2503, 4072, 2299, 701, 3536, 3294, 4938, 4894, 2276, 3366, 4152, 3247, 27, 1198, 1717, 3858, 2080, 4004, 4395, 3640, 1795, 2063, 2211, 4331, 3899, 2820, 2916, 872, 1827, 3964, 2508, 3115, 2490, 4070, 2340, 1286, 4533, 2473, 4220, 4168, 3789, 3441, 4044, 1128, 15, 2070, 2052, 4091, 121, 4661, 4754, 2728, 1358, 4385, 2566, 4069, 1383, 4918, 4236, 4921, 1976, 2595, 3962, 1653, 3983, 1039, 50, 2529, 4001, 2672, 2379, 2256, 766, 1714, 1555, 1892, 2371, 230, 1421, 4975, 4840, 3351, 639, 3545, 3480, 1453, 1539, 4993, 719, 2393, 1465, 967, 3017, 4948, 3219, 1542, 2370, 981, 3584, 291, 4773, 3836, 2586, 1000, 2185, 2784, 4912, 2873, 3104, 3726, 1514, 4879, 2086, 609, 665, 247, 427, 253, 594, 535, 1826, 2274, 2491, 4243, 4813, 4748, 2668, 2407, 2679, 4049, 4401, 2835, 2000, 624, 2356, 2249, 1170, 1609, 3072, 1737, 2421, 1919, 2950, 2446, 1757, 996, 2060, 134, 3448, 684, 1312, 2392, 4939, 105, 4122, 445, 1772, 4755, 3220, 2611, 543, 2945, 3967, 4601, 1503, 4645, 2262, 906, 3492, 1627, 3865, 3330, 4789, 3703, 1828, 2748, 1033, 2415, 3456, 3432, 345, 879, 3511, 2030, 3179, 2725, 3615, 4760, 2065, 1636, 2816, 3396, 3625, 2349, 30, 1887, 4085, 3856, 4197, 1734, 4535, 299, 94, 1998, 207, 185, 3776, 1523, 4448, 922, 1070, 3664, 3933, 3256, 2841, 566, 432, 2300, 1762, 3815, 2404, 3573, 1020, 2942, 3714, 1789, 2372, 420, 919, 1903, 2756, 1323, 2167, 3044, 2639, 3718, 4451, 790, 793, 3443, 358, 1602, 4906, 916, 3984, 4419, 3236, 4942, 2384, 4742, 4039, 814, 274, 786, 1200, 4042, 455, 4836, 2885, 1479, 924, 983, 2117, 1633, 3782, 1820, 213, 4806, 3489, 1258, 4387, 3111, 1129, 2426, 2255, 710, 4586, 2829, 1863, 2171, 2053, 1890, 4880, 3756, 1521, 3669, 1587, 1100, 977, 175, 519, 500, 3831, 3259, 328, 4560, 1498, 882, 4856, 3124, 3708, 1360, 2902, 889, 4774, 1975, 643, 3081, 892, 1603, 2147, 926, 4196, 3958, 4379, 1464, 4643, 3183, 3620, 4382, 4648, 3853, 4814, 1898, 1500, 3804, 2874, 2572, 2109, 3951, 4735, 1210, 4242, 3379, 2282, 2064, 416, 1697, 3601, 2600, 1288, 4369, 76, 2882, 706, 772, 2008, 340, 3180, 1811, 1271, 3647, 4635, 116, 3036, 1711, 3275, 1891, 1093, 2160, 4630, 4359, 1847, 2932, 182, 2138, 77, 1227, 4000, 1893, 1550, 2838, 985, 1094, 3538, 1050, 369, 3624, 1372, 1487, 3138, 1331, 2338, 49, 4500, 653, 886, 4832, 3807, 228, 4771, 717, 3592, 4270, 4788, 384, 689, 495, 1197, 1228, 2487, 1763, 782, 179, 4333, 614, 4833, 2732, 513, 545, 4081, 169, 2798, 3052, 314, 4688, 516, 1685, 1718, 2228, 2075, 3000, 4187, 1576, 2452, 2516, 4054, 166, 731, 4443, 2605, 1834, 3510, 3694, 4365, 3411, 1803, 3015, 3880, 402, 2631, 658, 2733, 4080, 3132, 4466, 2436, 1344, 2574, 4682, 3158, 842, 3147, 1371, 4096, 4624, 1466, 2992, 3891, 4433, 1329, 881, 4901, 4982, 1908, 3898, 4965, 2770, 3875, 1886, 4509, 2288, 313, 4132, 3707, 3203, 1269, 2813, 2050, 1299, 2570, 1678, 243, 476, 2901, 2652, 3315, 4077, 36, 4744, 4672, 997, 2511, 1391, 2766, 918, 1110, 3107, 1617, 1155, 4764, 1513, 1646, 3777, 2066, 4503, 4952, 440, 4306, 3034, 943, 2382, 1524, 3250, 4251, 537, 2957, 1102, 3851, 2387, 2767, 663, 3087, 3226, 1634, 4951, 1870, 2352, 2422, 2116, 1533, 4453, 938, 4620, 1675, 3739, 1459, 357, 4075, 2531, 3189, 2675, 4456, 3727, 1849, 1468, 4615, 4602, 3194, 2578, 4462, 2548, 398, 1754, 3402, 3547, 2115, 3882, 2131, 3623, 1930, 3164, 3750, 910, 2433, 1180, 993, 2401, 1097, 3170, 2321, 3907, 372, 1591, 298, 4442, 3237, 1298, 3779, 3917, 1156, 2378, 982, 2624, 367, 4791, 2789, 3819, 829, 671, 2549, 3025, 373, 4440, 145, 1901, 3759, 1625, 4800, 1259, 4334, 4816, 2157, 3223, 1107, 818, 4356, 4886, 4392, 1615, 219, 4374, 4423, 4153, 621, 2800, 2697, 2032, 2225, 2148, 2224, 3555, 1404, 1169, 3343, 1940, 4140, 3099, 1692, 4371, 4264, 4994, 3257, 109, 99, 4117, 368, 2014, 2073, 4872, 3042, 4524, 934, 3609, 4777, 2386, 1302, 409, 724, 1234, 547, 3731, 2218, 3295, 1656, 4561, 1284, 929, 3982, 4272, 4437, 2277, 652, 1800, 3680, 847, 862, 2343, 2285, 948, 2702, 1380, 89, 79, 4171, 7, 3902, 1206, 1574, 1134, 2903, 3413, 2451, 4358, 149, 1787, 2944, 4326, 337, 2341, 3324, 2444, 3049, 2831, 2467, 1373, 2019, 3993, 3116, 4693, 4121, 4883, 4739, 4825, 3737, 1935, 826, 2962, 37, 4477, 4738, 4056, 4118, 1434, 2453, 3178, 2991, 2325, 1769, 3767, 3461, 3666, 4740, 1883, 4826, 2785, 1764, 5005, 4928, 1364, 4067, 3697, 4282, 1507, 312, 1001, 81, 4036, 1406, 511, 2636, 319, 1275, 4781, 1290, 2783, 3501, 2135, 3372, 4, 4947, 151, 38, 1149, 4729, 2107, 2837, 820, 3313, 3128, 1250, 1604, 4995, 3854, 139, 4068, 2559, 3387, 3258, 4410, 3969, 3077, 1496, 4730, 1425, 975, 1399, 4319, 585, 764, 97, 1056, 3548, 2911, 699, 66, 390, 2012, 4983, 2400, 2337, 1162, 2760, 4723, 2568, 2627, 512, 4310, 2096, 3611, 3862, 4135, 2201, 895, 3715, 1581, 899, 3103, 3193, 2985, 3298, 723, 4041, 3973, 2027, 1648, 1374, 1909, 2580, 4238, 2809, 589, 1415, 1236, 1030, 3892, 1957, 1858, 1253, 2791, 3912, 3070, 323, 3391, 2717, 4249, 728, 1770, 1794, 844, 2657, 1873, 546, 1263, 2130, 1303, 740, 167, 2908, 1294, 69, 202, 84, 1061, 348, 2915, 2380, 2231, 2740, 3970, 2709, 10, 2996, 4484, 1115, 2866, 3520, 4978, 3007, 3847, 850, 642, 668, 3440, 3459, 700, 2722, 1610, 4536, 4513, 709, 4038, 2169, 4325, 4545, 3874, 2430, 3437, 1429, 1178, 544, 3374, 1273, 3159, 1080, 2685, 1592, 2320, 2933, 1868, 2007, 2098, 2673, 1525, 3425, 2496, 756, 2571, 4925, 303, 3289, 4902, 1065, 1611, 4273, 3479, 1145, 304, 1405, 1011, 4028, 34, 72, 995, 561, 1396, 3878, 3356, 2546, 3032, 4156, 1541, 912, 4759, 3066, 853, 1759, 148, 437, 863, 2388, 245, 3381, 226, 4579, 3192, 3209, 1031, 1422, 984, 1736, 1057, 2604, 4332, 3781, 3091, 4698, 679, 4969, 501, 608, 4793, 1584, 837, 3893, 2054, 4195, 1402, 4893, 3582, 2015, 4204, 1164, 6, 532, 2520, 2329, 4455, 720, 4900, 3941, 3371, 1862, 558, 2124, 1213, 923, 777, 2456, 3058, 3143, 4221, 4664, 3845, 4486, 2526, 1279, 915, 1928, 4362, 4155, 2698, 4394, 801, 4962, 4531, 397, 289, 161, 1977, 1163, 4960, 4976, 3939, 2699, 2633, 904, 2126, 478, 4467, 4045, 3187, 773, 676, 3945, 4861, 1758, 4228, 2994, 4051, 3293, 526, 2055, 3488, 1747, 3112, 4644, 78, 261, 3484, 3660, 2735, 3435, 362, 3137, 1937, 1435, 2264, 3243, 1476, 4831, 1120, 1362, 4882, 855, 4016, 4954, 3835, 4285, 3699, 2501, 3409, 880, 4167, 502, 2781, 4074, 3397, 2643, 4157, 1052, 4786, 4720, 890, 4297, 504, 1249, 4354, 3487, 361, 2931, 2621, 3278, 3308, 4287, 3911, 4992, 2894, 4810, 3689, 2647, 349, 281, 4420, 1473, 1927, 1774, 2121, 4010, 53, 4546, 4569, 2591, 2583, 2252, 4610, 1936, 2035, 1784, 3039, 3466, 1966, 2442, 3325, 2437, 1798, 2004, 646, 1825, 1786, 1407, 3151, 597, 3888, 3646, 2409, 1829, 4296, 2295, 615, 3053, 3286, 1130, 3054, 2987, 2527, 4338, 1876, 3704, 364, 4959, 4232, 4370, 3987, 4910, 4120, 1111, 1875, 382, 1877, 1840, 1140, 3460, 251, 4022, 3719, 1848, 3990, 4775, 4160, 1308, 4032, 3320, 3784, 913, 4594, 3622, 533, 133, 154, 3214, 4766, 1607, 4669, 4108, 2047, 3857, 2749, 1484, 294, 3723, 267, 3706, 3671, 3310, 2350, 1058, 2869, 4149, 341, 1099, 2541, 696, 3980, 3576, 2283, 534, 802, 1423, 494, 95, 2853, 4276, 1447, 2682, 1118, 4113, 1293, 4328, 1517, 1270, 3473, 1558, 258, 1528, 4506, 2166, 2253, 568, 2360, 141, 3763, 4019, 2654, 2345, 2642, 2267, 277, 2972, 3118, 90, 270, 1686, 4255, 976, 2239, 4176, 4501, 3181, 2150, 283, 4284, 3065, 4653, 3519, 4530, 602, 1176, 3419, 3060, 2871, 1881, 1553, 4747, 2479, 2390, 1389, 2923, 2502, 3760, 2924, 3380, 1529, 399, 799, 3920, 3130, 3422, 1755, 2397, 1231, 3636, 4043, 1600, 2078, 2383, 3336, 2644, 52, 3429, 3783, 2818, 4865, 4827, 4079, 4205, 4571, 1805, 964, 894, 4752, 1915, 4449, 1973, 2665, 273, 680, 1045, 1365, 4565, 2043, 4353, 1527, 4471, 1173, 9, 3635, 3570, 2011, 3355, 4702, 1306, 2674, 604, 4291, 2470, 45, 4821, 469, 1620, 3961, 2125, 4452, 2879, 3074, 3791, 55, 4623, 1437, 1005, 4170, 85, 1537, 3634, 2601, 2204, 2575, 1649, 2742, 3711, 324, 1916, 1353, 1946, 562, 4277, 4582, 4151, 4158, 1328, 3755, 3186, 2617, 3949, 3304, 1559, 3167, 1818, 578, 4129, 31, 4929, 3859, 465, 2513, 3442, 2921, 346, 1924, 4628, 4084, 4026, 444, 1997, 2099, 1203, 3754, 1452, 2414, 998, 2289, 1122, 866, 3926, 302, 316, 4127, 2691, 2754, 1377, 496, 1325, 3353, 1961, 4283, 807, 606, 3588, 2238, 2510, 3521, 1307, 4851, 1256, 4027, 956, 3069, 2814, 3215, 371, 1662, 3572, 2499, 1141, 1902, 3775, 434, 205, 4515, 4657, 1211, 300, 3960, 2215, 4475, 625, 508, 2967, 2977, 761, 3486, 4986, 1087, 1628, 4849, 901, 4172, 120, 4626, 1366, 1845, 2937, 1944, 4141, 408, 960, 1411, 2560, 4240, 4502, 2058, 4222, 1184, 1933, 143, 4327, 4961, 1744, 2565, 2159, 11, 897, 2140, 4943, 3057, 1614, 1226, 1938, 2958, 3608, 4868, 231, 3677, 4779, 2013, 4527, 352, 3367, 1266, 573, 515, 4383, 1955, 8, 3227, 4095, 4518, 4819, 1780, 3551, 2398, 2506, 1801, 4671, 3041, 1673, 250, 3691, 4757, 2046, 3717, 2865, 1408, 3314, 1699, 1054, 798, 3471, 1640, 1375, 1895, 2505, 1804, 2162, 4592, 958, 763, 2461, 1376, 3682, 4715, 4712, 4639, 2705, 309, 4603, 2020, 4188, 2025, 4278, 3149, 46, 2518, 2650, 4547, 1021, 389, 256, 1209, 3863, 1431, 751, 4286, 2774, 930, 4482, 3896, 1343, 4229, 1695, 2863, 2858, 748, 1355, 3579, 727, 379, 4053, 2893, 2745, 3196, 4552, 1880, 952, 4848, 2886, 1962, 3076, 3022, 129, 3002, 4897, 4180, 4936, 822, 2079, 968, 2576, 4483, 4604, 4884, 1010, 1038, 1044, 2872, 1693, 870, 3155, 3942, 3663, 3067, 3161, 4178, 4908, 1822, 2394, 3780, 2573, 4855, 2696, 2424, 4689, 2119, 5003, 1948, 3698, 831, 991, 2026, 3923, 965, 3073, 4202, 468, 4581, 2016, 2257, 3702, 1283, 4794, 4235, 433, 135, 3323, 1701, 1475, 637, 2328, 3685, 3352, 214, 1295, 3287, 2110, 3410, 3400, 4274, 181, 2590, 1187, 1369, 1316, 2907, 1535, 3662, 4288, 936, 4838, 1352, 4139, 4225, 1920, 3369, 56, 1888, 3943, 1970, 4686, 4206, 2037, 2759, 3966, 2775, 4537, 828, 4481, 3050, 2683, 3963, 672, 3212, 1311, 2332, 4523, 4314, 311, 1419, 945, 4183, 1988, 2535, 2961, 1995, 903, 4389, 767, 1992, 2769, 1682, 183, 2258, 4914, 1716, 1807, 4262, 947, 51, 712, 3613, 2108, 3234, 5007, 4499, 2363, 784, 3306, 2651, 3552, 2213, 3102, 1882, 1746, 2402, 2181, 339, 4017, 3405, 531, 2326, 4534, 4953, 4539, 4970, 1074, 2373, 3860, 1802, 770, 2790, 3267, 1810, 4337, 1578, 3316, 1670, 1334, 3741, 612, 212, 3768, 170, 4246, 4573, 2542, 4388, 1566, 3302, 1338, 1179, 3331, 3672, 180, 3452, 3581, 2959, 4320, 2736, 1700, 73, 902, 2910, 2771, 2597, 633, 3673, 567, 297, 588, 484, 3495, 177, 2727, 354, 4554, 2686, 1448, 1015, 1832, 3566, 1830, 4340, 715, 1248, 132, 196, 2941, 999, 1123, 2459, 4563, 3713, 1265, 726, 3500, 3546, 1846, 927, 3384, 3184, 410, 3004, 3363, 1749, 4239, 970, 4123, 3136, 1618, 809, 2960, 4627, 1651, 1456, 3357, 2049, 2797, 4008, 2214, 1702, 2966, 565, 2232, 252, 3177, 2719, 4226, 4231, 753, 125, 87, 391, 550, 3652, 320, 4567, 41, 1068, 2313, 875, 2364, 4279, 525, 3335, 2984, 579, 3265, 2210, 3667, 2537, 2610, 629, 4808, 3593, 1565, 3150, 2227, 1577, 2671, 4608, 315, 3606, 1028, 1220, 4468, 446, 4559, 4015, 4430, 3632, 2242, 421, 1839, 1739, 4519, 1019, 217, 517, 3185, 3650, 4984, 392, 4445, 576, 538, 1262, 2836, 3008, 3300, 3598, 4360, 33, 4820, 4597, 20, 4408, 804, 2441, 1073, 2492, 2817, 1090, 4845, 2448, 1199, 1224, 4013, 2251, 2822, 2188, 575, 4381, 2184, 4585, 3248, 4439, 5008, 1204, 1511, 661, 618, 971, 4137, 430, 4514, 940, 3063, 4745, 1196, 4716, 3563, 1951, 813, 1631, 156, 4479, 4511, 4421, 703, 3542, 3927, 4223, 3529, 645, 2555, 1688, 4839, 2381, 4432, 4130, 3340, 2827, 2840, 4680, 978, 1657, 4762, 3206, 3361, 3464, 3648, 1304, 957, 3145, 488, 2588, 4495, 1815, 1831, 817, 2001, 104, 5009, 280, 598, 2981, 2353, 3301, 12, 2690, 3801, 1320, 2457, 2692, 3290, 4587, 4807, 208, 1761, 4751, 4161, 1874, 4600, 3477, 2623, 2129, 4227, 3084, 2494, 3263, 2653, 4422, 1481, 1556, 4256, 344, 3153, 3604, 13, 3123, 674, 2010, 3998, 3078, 3154, 4138, 2405, 164, 96, 2088, 1235, 3246, 1943, 2403, 422, 873, 3045, 3350, 878, 112, 869, 1698, 1356, 1751, 3946, 1446, 1677, 1084, 1046, 4830, 2199, 1691, 835, 2278, 2807, 2812, 411, 412, 2120, 1041, 2603, 3408, 610, 2072, 2618, 3009, 2676, 963, 2178, 16, 3224, 1188, 471, 2235, 1189, 3014, 4252, 4637, 1738, 3653, 2375, 765, 3772, 2649, 4115, 4706, 3749, 989, 4429, 2438, 103, 4708, 1354, 2236, 4363, 3861, 3881, 2905, 2357, 108, 3426, 2395, 2307, 4979, 4792, 1947, 560, 1619, 1505, 4089, 4166, 2731, 1127, 127, 1563, 1237, 2680, 885, 3230, 236, 2955, 3991, 1547, 3109, 4182, 1313, 2976, 2141, 2261, 497, 3778, 2517, 1004, 1222, 3255, 3621, 4476, 789, 1472, 2133, 404, 4557, 2488, 2864, 2989, 3368, 631, 3658, 2530, 2978, 414, 1796, 3028, 2081, 3225, 586, 2857, 2471, 4726, 3799, 2458, 2293, 4035, 725, 2179, 3385, 3051, 3509, 4496, 2006, 2532, 2589, 3549, 1931, 987, 2486, 2316, 4191, 1146, 4909, 2180, 375, 1387, 591, 3994, 992, 2314, 1167, 3808, 1869, 2615, 932, 2833, 200, 4876, 246, 4931, 1485, 1809, 4290, 1560, 1660, 1081, 1994, 1504, 4756, 1064, 492, 377, 2687, 2090, 2113, 4780, 1922, 448, 292, 3745, 4269, 32, 1395, 2536, 4397, 3591, 1009, 600, 570, 4634, 1287, 2773, 3814, 2310, 3018, 1989, 237, 4066, 792, 3885, 1394, 172, 3645, 2048, 3944, 3838, 4150, 2425, 1297, 3450, 4415, 3642, 539, 2445, 4629, 490, 3934, 1867, 933, 3868, 2411, 381, 3762, 3204, 3929, 2939, 2309, 3282, 3373, 42, 3261, 3089, 1124, 4541, 3595, 2406, 3401, 3540, 2711, 342, 1186, 3897, 1518, 3678, 184, 3271, 2302, 2260, 788, 3469, 2986, 556, 1007, 4646, 152, 961, 1531, 4033, 4428, 4438, 301, 1793, 1597, 2701, 2095, 2318, 4958, 3769, 3823, 4073, 74, 2979, 1509, 4654, 1238, 2059, 3454, 3222, 1544, 3418, 1076, 489, 4798, 2028, 1494, 884, 2489, 150, 3904, 4875, 3997, 1095, 4009, 493, 1884, 669, 4217, 638, 4349, 1092, 3195, 2920, 4292, 1969, 2317, 2308, 401, 4842, 3483, 486, 1836, 4461, 1622, 2823, 3090, 3190, 3277, 1398, 4734, 4631, 3774, 4930, 4768, 3508, 3360, 1379, 4562, 3577, 2805, 4873, 2777, 1274, 2825, 2348, 3482, 3971, 3977, 393, 3744, 2367, 1861, 1264, 1731, 3490, 2478, 3202, 1865, 3253, 454, 2315, 4230, 1190, 365, 265, 1147, 914, 4920, 2045, 2616, 4641, 1904, 821, 746, 816, 2084, 2498, 1230, 4088, 4679, 510, 1654, 2319, 1655, 380, 308, 187, 1694, 4106, 876, 1414, 1797, 2483, 2243, 2427, 4753, 3494, 630, 3930, 1593, 2640, 1792, 241, 4572, 1728, 447, 1157, 3554, 1644, 4181, 2071, 2480, 2123, 4964, 3444, 3251, 1223, 4733, 2846, 2667, 3005, 4805, 3098, 810, 1520, 1608, 98, 808, 1639, 865, 4937, 2226, 2744, 4811, 4367, 920, 1242, 1243, 2938, 4209, 3828, 887, 1241, 3981, 2022, 3414, 524, 3285, 2847, 1899, 3705, 3347, 3758, 3172, 466, 4510, 4595, 2266, 1424, 521, 3342, 1400, 1125, 441, 1445, 3610, 2376, 1571, 4888, 3600, 2681, 843, 4175, 3292, 2512, 1756, 158, 225, 4589, 3232, 400, 480, 1743, 4055, 3522, 1025, 2971, 3575, 4265, 4633, 2534, 4725, 35, 4368, 2146, 3451, 4566, 1866, 1040, 3470, 733, 3210, 3117, 3019, 3821, 1036, 687, 92, 2540, 4996, 1721, 248, 4667, 54, 3174, 4424, 3901, 2161, 1950, 3423, 4605, 640, 168, 3388, 815, 4307, 3864, 3507, 4828, 4957, 3687, 4316, 3876, 307, 1696, 178, 233, 1835, 2753, 1342, 4064, 1583, 3504, 666, 264, 795, 1216, 3530, 1923, 3341, 704, 1879, 4638, 2965, 3213, 557, 3752, 4776, 21, 3122, 4375, 2758, 4692, 4867, 1436, 4804, 1195, 2247, 776, 1078, 1851, 1278, 2752, 2495, 3309, 2713, 224, 350, 845, 1750, 548, 3922, 4364, 2626, 1192, 1945, 1536, 3457, 619, 1246, 3083, 3403, 1309, 2268, 2368, 4881, 1616, 4436, 336, 4361, 3446, 1658, 2420, 1979, 2263, 1538, 3502, 3657, 3816, 4528, 1841, 2533, 3873, 3527, 3496, 1590, 848, 1254, 4322, 1482, 2688, 3803, 686, 4474, 107, 2040, 165, 4258, 2246, 2906, 173, 4818, 5004, 3031, 3129, 4376, 1318, 3793, 536, 4304, 4903, 1964, 4613, 1959, 2855, 4321, 3198, 3817, 1034, 288, 1768, 3924, 2186, 1458, 4860, 1623, 3283, 2410, 1530, 3407, 4512, 3012, 2739, 622, 1260, 4857, 2009, 3627, 201, 3580, 702, 257, 2670, 2724, 1586, 459, 708, 3431, 2929, 2385, 101, 1461, 4177, 1659, 1351, 4687, 2389, 2678, 1799, 3055, 3802, 928, 3568, 559, 574, 523, 4058, 3988, 4234, 3497, 1564, 4668, 4250, 3918, 4179, 2465, 3800, 3152, 2867, 260, 4018, 1551, 4355, 3825, 126, 3208, 3603, 4215, 711, 3037, 1361, 925, 871, 4998, 2182, 2757, 284, 4114, 1089, 4758, 2582, 4647, 3303, 718, 269, 1921, 4126, 791, 1545, 2359, 4011, 2982, 1412, 4416, 1280, 4508, 4549, 3659, 1339, 3307, 3761, 189, 3094, 1432, 4207, 63, 3728, 3131, 1956, 3583, 452, 2876, 1418, 3686, 2076, 1765, 664, 1324, 4673, 3614, 825, 355, 2963, 4057, 3421, 2804, 1305, 3629, 4366, 4219, 2710, 3700, 759, 836, 762, 529, 1430, 4345, 3720, 3105, 3329, 3439, 4192, 678, 2660, 2995, 824, 473, 4218, 641, 1773, 1202, 3585, 778, 2970, 1335, 3596, 3733, 2796, 4899, 670, 580, 2416, 2895, 2852, 2608, 4266, 2581, 1557, 2968, 3630, 1417, 171, 4406, 3182, 1580, 2240, 2477, 3928, 4558, 1661, 1336, 838, 583, 2145, 2102, 1724, 3280, 898, 4059, 2892, 1854, 4727, 2335, 4052, 4407, 2898, 254, 1126, 4737, 4593, 2779, 564, 839, 1079, 4507, 1999, 136, 4614, 697, 4352, 3046, 2265, 1341, 3399, 2824, 2514, 3205, 4619, 4859, 3996, 2216, 2666, 4444, 244, 1252, 395, 3086, 3168, 1978, 155, 4446, 3590, 1363, 2956, 1897, 1967, 1426, 3311, 3472, 2105, 1984, 2564, 3792, 2413, 2177, 4871, 3747, 1599, 4665, 2017, 3839, 4517, 2794, 117, 4024, 4907, 4261, 22, 3033, 2726, 1106, 911, 2826, 3564, 3771, 1433, 100, 3879, 334, 4301, 2954, 530, 3770, 2175, 3639, 774, 2303, 1925, 4736, 4878, 1151, 1049, 4966, 1148, 3082, 707, 694, 4403, 966, 458, 1381, 1687, 2484, 279, 3753, 1257, 451, 1665, 1859, 4616, 4891, 3866, 198, 140, 438, 4850, 722, 3935, 3906, 4676, 3458, 2552, 3541, 3757, 675, 647, 4047, 1208, 3867, 119, 1722, 2331, 3810, 1349, 980, 1240, 1821, 2664, 2354, 2142, 3197, 1158, 204, 4119, 3561, 4257, 4801, 2883, 4681, 2851, 3952, 1508, 1980, 2094, 2023, 2344, 582, 3260, 3101, 3681, 1965, 3001, 3344, 1681, 4575, 1629, 509, 147, 3424, 1568, 4427, 3869, 3003, 851, 1642, 2196, 1113, 3641, 1492, 891, 2554, 4040, 553, 1024, 2801, 2528, 2443, 2114, 3252, 2900, 3465, 2557, 2802, 4990, 2083, 611, 4100, 3837, 1043, 3270, 2584, 662, 657, 3986, 2042, 65, 4267, 752, 2271, 2594, 122, 4414, 3321, 953, 593, 4357, 2082, 4254, 4574, 1777, 293, 905, 2366, 3692, 4145, 3447, 1490, 2751, 4396, 685, 3505, 2641, 2509, 4190, 2684, 387, 2936, 2730, 3024, 2085, 3268, 351, 4864, 2762, 1478, 2250, 3100, 1229, 2746, 3146, 1053, 111, 188, 2860, 650, 3229, 413, 4300, 3398, 4464, 4652, 3038, 1813, 4577, 3029, 2245, 2648, 2361, 2579, 1783, 4538, 2151, 1605, 467, 2154, 1806, 3013, 1712, 667, 4104, 4824, 3834, 4212, 648, 1707, 4063, 2002, 1669, 1166, 4099, 2174, 192, 4749, 3318, 3526, 3506, 2619, 528, 3106, 2830, 688, 491, 3855, 4887, 2556, 4012, 3908, 3337, 1443, 3218, 3796, 1502, 417, 1760, 2369, 3533, 2334, 1960, 3133, 833, 1668, 4426, 3612, 1781, 1006, 2419, 3394, 944, 4412, 505, 4854, 3201, 2694, 163, 3279, 677, 1990, 4803, 2158, 2417, 4869, 2194, 2358, 2149, 2625, 4124, 1515, 1194, 4822, 840, 2558, 4201, 1491, 3264, 3119, 962, 239, 4351, 3701, 1370, 2428, 503, 931, 4454, 628, 229, 2323, 423, 4658, 3794, 3655, 2949, 2522, 4299, 1589, 4999, 3850, 2003, 1911, 2524, 3690, 1319, 3481, 2567, 3080, 4799, 3978, 888, 2538, 3139, 1337, 1703, 4164, 2259, 2859, 595, 2845, 1112, 3319, 3142, 2877, 2947, 1469, 2693, 4165, 266, 475, 3188, 1315, 1499, 3068, 1317, 900, 4116, 1450, 2819, 1289, 4005, 2482, 443, 3239, 613, 2880, 3999, 3395, 436, 296, 240, 3171, 834, 4945, 4769, 3010, 2069, 322, 1889, 153, 2279, 3905, 40, 83, 2170, 3937, 61, 1860, 3326, 3449, 819, 3748, 4923, 1221, 2613, 2803, 1218, 4578, 592, 3027, 4125, 4721, 3955, 3849, 2645, 2122, 3485, 4303, 191, 3228, 3021, 1606, 4940, 4841, 4516, 2039, 278, 4098, 2221, 555, 4061, 4701, 4829, 4895, 907, 2493, 779, 3354, 4311, 4837, 3516, 499, 3517, 794, 3438, 2286, 540, 2, 3531, 3273, 572, 3824, 4425, 3921, 4253, 3434, 19, 1117, 1392, 4071, 4134, 3075, 2024, 3976, 2980, 4393, 2793, 4147, 1, 2021, 1403, 1277, 4815, 596, 425, 2750, 1119, 3126, 4812, 3165, 3427, 276, 1409, 4102, 2763, 2233, 908, 1643, 1291, 4200, 2780, 4189, 223, 994, 102, 3953, 1420], "validation": [5313, 5411, 5092, 5160, 5018, 5145, 5046, 5059, 5275, 5191, 5042, 5051, 5142, 5253, 5064, 5472, 5084, 5349, 5467, 5224, 5179, 5089, 5347, 5399, 5455, 5186, 5385, 5087, 5321, 5130, 5031, 5196, 5249, 5283, 5410, 5417, 5147, 5428, 5048, 5306, 5220, 5159, 5333, 5371, 5419, 5345, 5032, 5481, 5207, 5451, 5137, 5458, 5200, 5463, 5206, 5030, 5435, 5298, 5367, 5262, 5350, 5070, 5418, 5086, 5492, 5342, 5286, 5125, 5095, 5449, 5040, 5035, 5126, 5105, 5057, 5131, 5077, 5474, 5217, 5383, 5314, 5222, 5495, 5121, 5192, 5488, 5151, 5305, 5340, 5404, 5508, 5464, 5430, 5163, 5425, 5433, 5427, 5389, 5067, 5080, 5106, 5363, 5489, 5388, 5431, 5041, 5085, 5498, 5469, 5045, 5379, 5184, 5161, 5117, 5251, 5209, 5239, 5021, 5039, 5248, 5171, 5366, 5403, 5315, 5075, 5029, 5493, 5300, 5052, 5504, 5296, 5413, 5188, 5020, 5055, 5088, 5190, 5473, 5414, 5197, 5387, 5082, 5269, 5258, 5478, 5214, 5420, 5365, 5278, 5479, 5290, 5484, 5325, 5497, 5394, 5250, 5096, 5173, 5104, 5050, 5445, 5317, 5320, 5211, 5180, 5456, 5405, 5066, 5011, 5148, 5289, 5242, 5308, 5107, 5443, 5247, 5198, 5352, 5483, 5256, 5442, 5078, 5185, 5439, 5324, 5358, 5068, 5074, 5303, 5210, 5393, 5069, 5103, 5234, 5073, 5406, 5421, 5260, 5233, 5323, 5076, 5344, 5415, 5259, 5494, 5182, 5199, 5263, 5201, 5398, 5438, 5351, 5213, 5271, 5014, 5437, 5391, 5501, 5170, 5377, 5056, 5295, 5023, 5063, 5236, 5416, 5164, 5231, 5273, 5328, 5480, 5491, 5195, 5034, 5460, 5322, 5338, 5448, 5318, 5277, 5397, 5292, 5485, 5136, 5017, 5043, 5108, 5268, 5356, 5205, 5062, 5432, 5424, 5466, 5261, 5244, 5101, 5123, 5037, 5081, 5049, 5140, 5307, 5194, 5218, 5368, 5091, 5287, 5381, 5162, 5468, 5112, 5094, 5297, 5109, 5100, 5245, 5490, 5240, 5150, 5364, 5380, 5453, 5335, 5447, 5337, 5165, 5114, 5505, 5061, 5382, 5357, 5060, 5135, 5208, 5058, 5116, 5012, 5353, 5334, 5134, 5158, 5141, 5280, 5223, 5212, 5187, 5102, 5270, 5172, 5384, 5189, 5015, 5090, 5279, 5332, 5155, 5407, 5486, 5401, 5423, 5202, 5246, 5330, 5047, 5500, 5036, 5457, 5319, 5354, 5326, 5193, 5157, 5450, 5237, 5238, 5203, 5370, 5257, 5288, 5152, 5132, 5235, 5118, 5175, 5013, 5426, 5093, 5169, 5154, 5174, 5436, 5291, 5452, 5361, 5336, 5221, 5386, 5016, 5390, 5341, 5482, 5422, 5028, 5461, 5044, 5227, 5348, 5053, 5065, 5265, 5022, 5327, 5360, 5183, 5465, 5156, 5079, 5216, 5033, 5166, 5444, 5219, 5476, 5168, 5441, 5176, 5477, 5129, 5019, 5506, 5167, 5133, 5396, 5392, 5127, 5355, 5509, 5120, 5301, 5462, 5311, 5054, 5285, 5294, 5181, 5373, 5149, 5459, 5024, 5299, 5329, 5499, 5310, 5215, 5312, 5143, 5272, 5243, 5255, 5026, 5228, 5177, 5099, 5454, 5339, 5375, 5146, 5097, 5359, 5230, 5372, 5098, 5232, 5304, 5071, 5266, 5408, 5254, 5264, 5284, 5409, 5487, 5507, 5025, 5178, 5226, 5302, 5446, 5316, 5429, 5138, 5434, 5395, 5113, 5083, 5470, 5122, 5343, 5144, 5072, 5369, 5128, 5331, 5412, 5139, 5309, 5400, 5378, 5267, 5402, 5346, 5229, 5362, 5010, 5038, 5440, 5111, 5274, 5119, 5471, 5153, 5252, 5502, 5115, 5204, 5475, 5124, 5503, 5293, 5027, 5225, 5496, 5282, 5110, 5241, 5276, 5374, 5281, 5376], "test": [5641, 5952, 5666, 5949, 5729, 5963, 5917, 5769, 5571, 5892, 5582, 5732, 5897, 5748, 5773, 5851, 5889, 5995, 5700, 5575, 5527, 5609, 5738, 5573, 5605, 5544, 5802, 5854, 5643, 5796, 5809, 5811, 5933, 5734, 5862, 5627, 5904, 5712, 5542, 5735, 5656, 5908, 5595, 5513, 5629, 5621, 5613, 5879, 5512, 5878, 5697, 5720, 5730, 5867, 5574, 5931, 5655, 5788, 5681, 5564, 5940, 5852, 6000, 5598, 5764, 5722, 5760, 5886, 5793, 5789, 5836, 5761, 5603, 5657, 5726, 6009, 5705, 5618, 5939, 5999, 5518, 5614, 5590, 5634, 5699, 5557, 5759, 5934, 5975, 5519, 5711, 5589, 5687, 5898, 5891, 5957, 5713, 5682, 5716, 5937, 5567, 5637, 5771, 5968, 5708, 5777, 5520, 5791, 5902, 5667, 5668, 5998, 5774, 5709, 5549, 5831, 5594, 5795, 5909, 5569, 5556, 5815, 5813, 5545, 5592, 5728, 5888, 5844, 5808, 5733, 5984, 5663, 5584, 5915, 5974, 5986, 5633, 5665, 5750, 5562, 5546, 5901, 5693, 5869, 5560, 5914, 5510, 5532, 5947, 5833, 5540, 5581, 5962, 5766, 6007, 5741, 5865, 5871, 5842, 5958, 5916, 5814, 5971, 5673, 5676, 5533, 5688, 5803, 5806, 5792, 5981, 5928, 5662, 5859, 5779, 5746, 5847, 5768, 5762, 5991, 5976, 5801, 5601, 5754, 5890, 5820, 5845, 5745, 5586, 5642, 5883, 5785, 5602, 5942, 5596, 5816, 5787, 5870, 5625, 5672, 5721, 5778, 5858, 5839, 5946, 5543, 5972, 5626, 5910, 6002, 5521, 5548, 5723, 5857, 5747, 5610, 5822, 5912, 5653, 5992, 5783, 5985, 6004, 5714, 5781, 5840, 5770, 5561, 5980, 5522, 5650, 5876, 5990, 5670, 5799, 5756, 5927, 5987, 5921, 5948, 5572, 5690, 5719, 5776, 5689, 5635, 5826, 5804, 5834, 5691, 5945, 5704, 5969, 5872, 5959, 5583, 5528, 5920, 5775, 5978, 5800, 5896, 5524, 5577, 5943, 5737, 5906, 5960, 5838, 5835, 5535, 5983, 5541, 5739, 5669, 5516, 5913, 5786, 6005, 5608, 5989, 5628, 5757, 5929, 5919, 5552, 5953, 5830, 5658, 5856, 5743, 5537, 5797, 5798, 5925, 5529, 5622, 5563, 5651, 5866, 5647, 5875, 5807, 5636, 5780, 5675, 5922, 5692, 5671, 5884, 5846, 5725, 5887, 5530, 5794, 5966, 5607, 5923, 5585, 5515, 5660, 5973, 5703, 5517, 5686, 6006, 5638, 5930, 5677, 5694, 5588, 5695, 5579, 5855, 5553, 5591, 5782, 5899, 5679, 5932, 5832, 5550, 5767, 5784, 5905, 5698, 5684, 5615, 5850, 5881, 5680, 5950, 5965, 5612, 5907, 5763, 5707, 5620, 5970, 5525, 5982, 5526, 5568, 5944, 5724, 5597, 5558, 5639, 5623, 5758, 5790, 5736, 5661, 5951, 5731, 5895, 5727, 5749, 5674, 5559, 5863, 5843, 5659, 5744, 5936, 5825, 5646, 5645, 5894, 5961, 5604, 5864, 5523, 5630, 5772, 5555, 5683, 5967, 5821, 5539, 5534, 5911, 5956, 6003, 5753, 5988, 5954, 5873, 5648, 5715, 5593, 5874, 5860, 5576, 5606, 5818, 5979, 5877, 5861, 5827, 5994, 5997, 5664, 5580, 5752, 5924, 5882, 5880, 5841, 5685, 5955, 5554, 5570, 5805, 5717, 5926, 5538, 5652, 5977, 5536, 5996, 5819, 5702, 5696, 5993, 6001, 5823, 5829, 5837, 5740, 5765, 5893, 5755, 5853, 5565, 5617, 5706, 5587, 5903, 5600, 5810, 5941, 5578, 5817, 5885, 5701, 5531, 5654, 5812, 5935, 5900, 5824, 5828, 5644, 5710, 5964, 5599, 6008, 5678, 5938, 5511, 5566, 5514, 5640, 5849, 5624, 5611, 5848, 5632, 5616, 5742, 5631, 5649, 5868, 5551, 5918, 5718, 5751, 5619, 5547]}, {"train": [331, 4013, 3262, 2497, 146, 421, 3360, 998, 2596, 3180, 3492, 3189, 2215, 4705, 3463, 2648, 2269, 3525, 2793, 3340, 4799, 3693, 354, 4584, 3440, 4959, 3706, 3101, 2902, 1388, 3683, 2864, 3563, 2765, 330, 4515, 3629, 4676, 2775, 4719, 4355, 691, 4448, 417, 522, 3789, 4594, 1850, 917, 2503, 796, 4050, 1109, 988, 948, 3812, 4768, 4870, 4104, 2823, 4879, 2958, 3721, 2265, 4544, 1733, 3389, 3467, 2144, 98, 4596, 105, 2836, 943, 4101, 4917, 3828, 4704, 1518, 1682, 2738, 2029, 3951, 1340, 1936, 86, 306, 3254, 131, 795, 1422, 1402, 108, 2483, 4427, 1926, 1330, 2292, 1970, 4492, 539, 2001, 3777, 1871, 4576, 1979, 1288, 499, 2931, 4301, 2699, 4073, 312, 1868, 4113, 4956, 453, 3330, 2942, 828, 1393, 317, 3141, 4391, 4824, 2289, 2408, 2386, 1765, 1468, 3413, 875, 1753, 4285, 4479, 4412, 4194, 1144, 4672, 3813, 2428, 4002, 1001, 3477, 4150, 1758, 2332, 4481, 2519, 265, 1027, 2139, 2085, 3208, 4972, 4273, 677, 4837, 2634, 664, 188, 4170, 3314, 879, 4916, 2990, 2401, 4122, 2264, 4658, 4434, 845, 867, 3073, 4307, 3650, 604, 3328, 4543, 2910, 1062, 3043, 3839, 1897, 2947, 481, 3329, 695, 54, 3618, 228, 723, 927, 452, 4569, 4410, 2252, 3881, 3837, 4192, 801, 1741, 1347, 2546, 3157, 1888, 2076, 4707, 960, 4291, 2498, 3369, 4387, 1957, 2217, 2133, 937, 678, 2397, 3943, 1750, 3859, 2584, 1483, 2150, 1956, 533, 4342, 4174, 1886, 3679, 322, 3446, 360, 4680, 1112, 1491, 3298, 1745, 1414, 3197, 2973, 1282, 2472, 4839, 4974, 1511, 570, 4041, 4375, 4615, 2698, 3283, 3957, 3861, 616, 726, 755, 1380, 2683, 1966, 36, 4875, 4376, 3623, 4734, 2307, 2098, 3702, 4452, 1646, 2113, 280, 2208, 4692, 2176, 1219, 3654, 2093, 3619, 822, 1902, 262, 1125, 1289, 2218, 1136, 1415, 1121, 4054, 1560, 1976, 2660, 2964, 4049, 1130, 3593, 4449, 1489, 2854, 3883, 992, 4812, 316, 1730, 2609, 2682, 4118, 552, 1597, 4856, 3891, 2963, 559, 4570, 967, 912, 638, 3709, 4900, 4332, 397, 1119, 2826, 4690, 926, 3394, 2547, 3626, 1297, 1313, 1887, 1619, 2776, 2855, 4277, 3615, 4828, 1959, 1254, 4236, 1381, 4042, 2184, 4628, 4165, 1689, 607, 2954, 1108, 3898, 2801, 564, 1276, 3082, 4129, 3050, 4055, 2273, 4991, 2249, 2670, 3690, 31, 975, 2748, 804, 2309, 2363, 2925, 3243, 4033, 4598, 3035, 4831, 2887, 1836, 2542, 2571, 128, 4169, 2912, 34, 4825, 461, 2284, 4000, 2972, 1543, 1318, 1534, 3085, 2589, 1503, 3282, 162, 1760, 3422, 3462, 4121, 4354, 1513, 844, 4286, 332, 4044, 232, 250, 3663, 3751, 2878, 3059, 904, 4288, 1031, 4437, 4630, 702, 4878, 3705, 4018, 4361, 3974, 3004, 4153, 4497, 3411, 457, 1137, 965, 3532, 5009, 1068, 3845, 3774, 2466, 1581, 3381, 4936, 2114, 3796, 3052, 1476, 2749, 2638, 4335, 4302, 2017, 4976, 936, 3666, 508, 3799, 308, 1321, 2415, 589, 61, 1411, 2341, 4256, 910, 4181, 949, 4498, 2559, 1909, 3072, 3426, 3609, 3263, 870, 495, 1780, 2687, 2201, 4961, 3423, 4030, 4747, 3380, 1346, 67, 130, 2060, 859, 961, 3135, 138, 3794, 46, 1257, 1631, 1640, 846, 2033, 3409, 1146, 21, 3756, 4975, 2342, 3117, 3295, 2630, 4140, 4215, 3234, 343, 218, 4222, 310, 2162, 3969, 2493, 3204, 1221, 121, 3734, 3029, 1077, 3373, 3625, 2044, 4227, 3370, 3291, 395, 2399, 3105, 3171, 722, 2147, 1398, 2606, 3031, 1161, 1629, 4435, 1231, 526, 525, 1778, 3498, 2888, 4770, 1271, 1171, 2623, 1389, 4726, 510, 797, 422, 3617, 2320, 2812, 1736, 1310, 1587, 3338, 1555, 1087, 1016, 1044, 126, 2276, 2362, 4611, 2091, 1859, 3648, 2817, 1667, 1571, 4832, 4056, 538, 4897, 191, 2156, 892, 2458, 2510, 639, 2811, 4451, 1857, 4263, 3466, 3998, 4043, 4081, 3769, 1978, 3987, 2155, 3427, 3636, 3089, 1174, 2674, 4706, 2572, 2372, 3224, 3316, 1664, 2283, 1391, 1224, 3184, 2535, 776, 4644, 3550, 4089, 4468, 2690, 504, 2848, 1054, 1872, 1532, 3304, 3750, 1327, 3387, 1460, 4714, 2077, 2845, 893, 3510, 1382, 204, 1485, 3086, 1779, 2479, 2202, 62, 2445, 4271, 3401, 411, 2527, 4563, 1514, 955, 3244, 1683, 736, 2747, 3079, 2579, 1502, 2908, 2444, 903, 1999, 3543, 456, 1786, 1694, 4067, 1200, 3518, 2419, 3783, 3713, 1735, 1038, 1367, 4720, 57, 4461, 3557, 3627, 2173, 3695, 4463, 4367, 341, 132, 205, 1098, 2904, 4748, 2356, 668, 3970, 1196, 2374, 683, 2025, 1160, 1088, 3070, 1326, 3407, 1700, 378, 2327, 1526, 3358, 1427, 3034, 1129, 3364, 2058, 3233, 3511, 896, 3365, 2443, 2314, 4902, 4814, 2039, 1420, 3863, 2006, 4889, 821, 3584, 459, 741, 1390, 4845, 2597, 3418, 4742, 3621, 4364, 3183, 154, 863, 848, 3545, 1603, 1792, 2978, 791, 2442, 577, 3938, 349, 2681, 3657, 1212, 3092, 1157, 2071, 3212, 3982, 1362, 3897, 1614, 3730, 2745, 2515, 2677, 4167, 4418, 629, 619, 2804, 3044, 435, 425, 4201, 3168, 487, 2421, 4782, 817, 39, 665, 1801, 2692, 2261, 1688, 4372, 3248, 1352, 3476, 3470, 3408, 2381, 2423, 3994, 4854, 1654, 506, 2022, 3760, 4701, 2865, 690, 1585, 3084, 1242, 2663, 3444, 3293, 4166, 3541, 2680, 3469, 4652, 4218, 925, 2819, 799, 4945, 2704, 4986, 2711, 3704, 1329, 3081, 1566, 1800, 657, 1474, 4471, 630, 15, 1690, 772, 3428, 3502, 3781, 3992, 326, 333, 4818, 4266, 984, 1419, 2180, 3356, 808, 1985, 2014, 1069, 443, 1177, 923, 3155, 3098, 633, 4904, 1091, 3366, 1494, 2255, 1598, 4475, 2467, 1564, 874, 4767, 1403, 4193, 1521, 661, 1548, 1848, 4985, 3016, 1937, 2578, 1453, 2206, 3169, 3255, 2743, 3712, 1409, 2839, 2719, 891, 1991, 1208, 219, 3038, 3748, 3722, 2927, 207, 1974, 2453, 74, 3915, 340, 2013, 1204, 1206, 212, 3454, 93, 4, 3689, 3864, 1357, 1663, 635, 763, 1237, 1643, 4445, 141, 176, 3013, 2446, 3519, 4741, 4811, 2254, 2191, 3847, 1003, 3941, 1372, 3063, 19, 419, 2349, 2086, 4754, 3919, 759, 3860, 2980, 3166, 4171, 4393, 1594, 897, 2779, 1433, 71, 3718, 3589, 1387, 1873, 116, 3535, 4950, 3429, 1951, 2248, 3150, 237, 740, 2929, 3442, 3495, 2694, 2951, 159, 4152, 712, 2569, 3479, 3527, 4148, 4579, 2588, 4232, 3923, 2045, 4733, 4673, 464, 2459, 1710, 168, 1610, 4090, 1343, 3192, 1960, 3523, 4695, 4765, 2684, 1465, 2712, 4506, 3138, 3852, 3481, 3872, 3528, 4716, 4773, 234, 3571, 2735, 4406, 197, 245, 4243, 2266, 815, 174, 959, 4390, 4894, 655, 4686, 1809, 3763, 4184, 2207, 3333, 4836, 3165, 376, 4176, 1712, 1757, 916, 765, 4689, 3026, 2962, 2895, 4807, 222, 2391, 3447, 497, 2917, 1542, 372, 4454, 4955, 2922, 1334, 951, 5004, 4763, 1455, 1353, 3524, 3968, 2174, 4595, 2695, 1211, 4457, 1900, 313, 1931, 3517, 2311, 4235, 3643, 1910, 446, 1479, 2782, 3692, 2395, 4373, 348, 3927, 437, 4130, 84, 2989, 2088, 3033, 4505, 744, 1721, 981, 4509, 3963, 479, 1827, 120, 4186, 2431, 2794, 4281, 10, 1358, 3988, 514, 1338, 520, 3895, 3885, 611, 1218, 2604, 101, 1775, 1498, 1063, 3386, 4137, 4151, 225, 2807, 2212, 1379, 1017, 4910, 2482, 4550, 3139, 1967, 4937, 1671, 1006, 1480, 2756, 701, 3438, 1870, 680, 2041, 38, 3741, 1828, 1256, 4336, 2829, 3483, 414, 2125, 2259, 292, 649, 1424, 1185, 4388, 1134, 1233, 99, 3950, 3131, 2669, 3461, 3219, 1413, 708, 1086, 3450, 3040, 1759, 3049, 3986, 2759, 4906, 4546, 954, 1841, 2137, 2394, 4679, 3754, 1429, 1529, 2560, 2763, 4655, 1530, 732, 1562, 2661, 873, 3490, 4597, 1005, 2586, 4500, 3624, 3607, 1172, 4780, 2545, 3425, 793, 56, 1636, 831, 2872, 1316, 427, 720, 4188, 156, 4895, 4123, 3884, 4022, 3616, 509, 2797, 4687, 3638, 1540, 4554, 3025, 4941, 3410, 2556, 2856, 3173, 3420, 1094, 1829, 4450, 2438, 297, 1263, 267, 3851, 2805, 4211, 2410, 2652, 379, 2701, 2465, 3268, 3798, 3965, 1047, 979, 2532, 1167, 3200, 1697, 4944, 2339, 4657, 309, 1783, 3598, 3814, 5003, 2003, 2037, 486, 1046, 1225, 4933, 3395, 2676, 493, 1624, 1707, 3820, 829, 4062, 2837, 4988, 2781, 4290, 3940, 4407, 323, 182, 3921, 2347, 4331, 1447, 3496, 500, 1622, 5, 3313, 2461, 4786, 3888, 1878, 727, 97, 952, 2360, 2822, 4909, 2808, 2645, 689, 124, 1625, 4347, 229, 3087, 1830, 1842, 1165, 244, 1799, 2907, 1250, 4321, 885, 4872, 1386, 2957, 2235, 4537, 737, 4269, 4775, 1787, 663, 3931, 209, 4651, 1651, 1448, 432, 3611, 4248, 4486, 220, 3831, 3201, 1356, 4970, 51, 3339, 1720, 583, 2488, 1486, 4326, 3649, 4932, 3287, 1963, 4245, 4265, 377, 2862, 2293, 996, 1767, 4647, 685, 490, 3582, 4578, 2451, 3346, 3610, 4559, 4485, 4548, 3901, 438, 2650, 3807, 4493, 3614, 2190, 239, 534, 2718, 4764, 3857, 1093, 147, 4969, 2277, 4693, 651, 724, 3512, 2242, 2183, 606, 660, 1095, 2866, 1294, 4958, 4082, 4759, 2195, 1350, 1158, 3653, 2267, 2335, 3514, 4565, 2945, 2329, 2081, 2234, 3714, 1766, 3170, 4581, 4210, 2696, 4887, 1058, 840, 4306, 3715, 2506, 2587, 2338, 4738, 473, 4968, 3935, 3646, 1335, 3816, 1732, 4005, 3195, 37, 505, 2625, 4947, 1544, 2863, 597, 1849, 812, 3202, 2403, 1812, 2409, 1644, 3124, 3827, 2721, 4092, 4059, 4650, 529, 1814, 1884, 2679, 3914, 4914, 3185, 4128, 1553, 3989, 1986, 363, 2182, 3586, 4816, 119, 103, 2414, 1734, 2192, 289, 2350, 2761, 1972, 3199, 2112, 3767, 3494, 2424, 3652, 424, 4601, 4159, 1194, 3635, 1847, 82, 4423, 4641, 2965, 1057, 591, 3491, 636, 1840, 1299, 2220, 2272, 2136, 2600, 1430, 1244, 2046, 4774, 3405, 3359, 1755, 1784, 448, 434, 4885, 1764, 1726, 1, 4607, 1028, 4551, 2275, 426, 4852, 3671, 4217, 2992, 1034, 4070, 2302, 2219, 1551, 3499, 2932, 2884, 2416, 782, 4627, 1626, 2626, 4674, 4131, 3090, 3406, 2330, 430, 2657, 704, 3331, 3067, 2232, 4026, 4088, 719, 1305, 557, 4861, 196, 2898, 3720, 1586, 1724, 637, 2420, 3848, 1798, 4229, 2976, 1964, 4381, 2737, 3639, 4591, 2544, 957, 58, 336, 2916, 1013, 1272, 733, 3392, 4526, 682, 2436, 249, 339, 261, 4494, 4416, 4283, 2263, 284, 3707, 1076, 2328, 2364, 3140, 3862, 444, 3094, 2575, 403, 1920, 4064, 3596, 60, 4566, 527, 4913, 4117, 1794, 1078, 4234, 3677, 4588, 314, 1605, 807, 1181, 3822, 4527, 127, 582, 22, 1401, 2970, 282, 2152, 964, 1470, 496, 546, 2177, 2464, 4725, 899, 1022, 4926, 3645, 3564, 3047, 3209, 4654, 1110, 809, 4973, 725, 687, 4316, 2370, 2533, 610, 4614, 4834, 4094, 3776, 3600, 398, 1769, 2146, 3893, 3015, 2227, 384, 231, 4145, 2244, 676, 623, 4783, 2256, 3595, 3198, 76, 329, 429, 4612, 1035, 4308, 3955, 2172, 4119, 2344, 706, 4608, 85, 3177, 3900, 1912, 2960, 4138, 1896, 4132, 2062, 842, 2020, 3787, 749, 406, 3024, 2849, 2094, 73, 2601, 1445, 563, 4736, 4383, 1523, 2247, 408, 3587, 1983, 1493, 3182, 3886, 4930, 709, 3691, 876, 2074, 14, 2741, 401, 1475, 3500, 684, 4182, 3879, 2852, 2203, 1359, 2480, 1496, 4313, 123, 3232, 2968, 2228, 1696, 1790, 1015, 2010, 115, 1908, 990, 1568, 2160, 3377, 1522, 673, 110, 173, 3656, 4262, 3890, 3154, 4126, 2299, 999, 2070, 189, 2352, 4369, 1259, 4723, 710, 3020, 256, 850, 3071, 1845, 2672, 3902, 2306, 2145, 1457, 4580, 4015, 667, 3651, 4984, 4396, 1459, 4158, 3791, 2413, 2080, 1074, 3809, 2002, 1867, 3602, 4943, 4125, 4484, 4562, 455, 4605, 4487, 2876, 2643, 2375, 2268, 734, 3700, 3318, 2175, 3810, 735, 2251, 1824, 1658, 2496, 3349, 1152, 3372, 400, 3310, 849, 2967, 4796, 3782, 268, 221, 1984, 2541, 4793, 143, 1577, 3419, 3976, 1822, 2117, 3604, 104, 1940, 946, 3817, 2407, 2780, 3911, 1260, 5005, 263, 1082, 1306, 986, 4069, 918, 1345, 1084, 1659, 614, 718, 2641, 413, 248, 2509, 976, 3674, 1020, 3613, 3739, 2523, 2454, 1132, 2901, 1563, 713, 4549, 3468, 2986, 2732, 802, 4883, 2754, 3348, 4359, 693, 4490, 4785, 3606, 3780, 3603, 3191, 1452, 3605, 3488, 1565, 3350, 187, 2772, 4589, 4813, 1941, 3681, 4989, 2697, 2204, 3684, 2746, 300, 4324, 4907, 1037, 2396, 2149, 4694, 1032, 3475, 1481, 3732, 1508, 2843, 3196, 3542, 2762, 3106, 3565, 3210, 4567, 3269, 3620, 3676, 4826, 3825, 4666, 4922, 2005, 2319, 449, 1596, 3478, 2009, 4703, 2193, 4524, 3899, 2023, 1860, 1163, 2167, 2622, 1770, 1680, 2531, 2065, 933, 3238, 1550, 2784, 838, 3014, 2733, 264, 4634, 1591, 1818, 4466, 502, 4105, 1703, 4323, 4084, 3667, 1222, 4472, 894, 2824, 1446, 2538, 291, 4440, 909, 1579, 666, 4940, 160, 3574, 387, 4352, 4982, 3547, 2874, 1120, 1050, 4114, 3821, 1536, 2984, 866, 4667, 555, 3257, 4025, 3277, 2594, 1853, 4163, 447, 513, 296, 68, 2402, 2549, 2126, 605, 11, 4349, 4160, 2773, 2524, 3534, 1243, 4557, 585, 488, 359, 2786, 3065, 5000, 3193, 1660, 8, 2262, 4046, 4574, 766, 2205, 2585, 3672, 2274, 2526, 2418, 255, 4829, 1241, 1209, 1131, 1451, 90, 3634, 1252, 3388, 3529, 383, 153, 1484, 2798, 2880, 5006, 175, 1373, 2024, 4079, 2040, 2393, 1360, 3041, 2398, 2494, 2233, 1839, 1927, 626, 4164, 2087, 1018, 3775, 4264, 1524, 1996, 2722, 1917, 3497, 1092, 2389, 24, 2760, 3849, 1101, 4433, 4345, 1053, 2452, 1988, 3729, 2668, 243, 1251, 2340, 1133, 1692, 3097, 1140, 1425, 1281, 1549, 1899, 751, 3341, 3319, 4702, 3659, 1183, 1440, 3120, 3588, 1572, 1592, 4661, 2481, 3906, 200, 3317, 1002, 352, 402, 1880, 195, 833, 43, 4743, 134, 679, 4284, 1273, 4542, 201, 1569, 4066, 3074, 2127, 515, 617, 2447, 1337, 645, 1768, 4873, 4305, 1186, 3352, 4086, 3967, 301, 2877, 2486, 2886, 254, 1527, 4297, 4337, 895, 521, 4934, 1229, 2994, 4718, 365, 1740, 4662, 1893, 2631, 350, 2570, 342, 3866, 418, 1816, 4213, 1366, 4006, 4536, 4599, 1064, 2109, 1679, 258, 1807, 4519, 3655, 4535, 4571, 1247, 4625, 1623, 1309, 4618, 4223, 270, 3832, 4155, 3122, 4419, 3536, 4810, 1258, 1739, 2110, 203, 2667, 3924, 1156, 4353, 1763, 4389, 3046, 4087, 2337, 627, 1706, 4333, 1727, 1023, 4365, 1351, 1444, 44, 2708, 327, 2440, 482, 4109, 1709, 165, 1126, 346, 4619, 2468, 480, 1246, 2432, 2287, 579, 2026, 2198, 152, 3448, 731, 1856, 392, 1693, 547, 3265, 738, 3939, 686, 3838, 1395, 785, 2891, 394, 1939, 4547, 1201, 2752, 4603, 2116, 1588, 1541, 4162, 6, 798, 3962, 834, 4120, 2868, 3778, 4212, 3149, 246, 3114, 2194, 4202, 2926, 4143, 953, 1400, 2795, 266, 2185, 454, 2073, 1124, 2662, 3770, 1004, 353, 646, 1036, 2304, 3320, 1993, 814, 1067, 3306, 3551, 2593, 1892, 1083, 344, 2861, 3583, 1118, 588, 4370, 2516, 2104, 658, 2686, 3642, 4343, 858, 3137, 4061, 3398, 1324, 3815, 2511, 3206, 877, 1162, 4965, 2870, 2367, 4659, 3747, 1418, 3567, 4777, 1159, 1106, 3905, 4820, 4197, 4745, 2052, 199, 2334, 783, 1932, 3928, 2427, 4228, 4637, 3351, 380, 4713, 4057, 1268, 2290, 3371, 3835, 2072, 1439, 2580, 2118, 2707, 1406, 2298, 4080, 295, 1176, 2324, 3680, 2568, 599, 279, 4518, 2477, 3309, 1661, 3508, 3249, 524, 4779, 3363, 1987, 3176, 1155, 4908, 278, 857, 2995, 2909, 1582, 3275, 4587, 109, 2665, 1923, 4156, 4224, 1192, 1492, 2305, 1686, 2426, 4729, 1223, 1009, 4366, 2108, 4688, 3241, 4631, 973, 3178, 3051, 739, 4881, 3460, 2280, 2959, 1882, 1545, 746, 3930, 578, 962, 273, 2066, 2353, 2043, 4278, 1537, 4276, 966, 642, 2565, 4357, 1188, 294, 3078, 2257, 4476, 4299, 1142, 4827, 941, 1190, 2047, 1815, 2751, 2858, 3055, 1438, 4681, 2315, 1178, 1096, 760, 3220, 3300, 1749, 2379, 100, 1127, 554, 3064, 4052, 4624, 1235, 911, 748, 2971, 2050, 3321, 756, 3877, 2790, 3368, 888, 1877, 1813, 1516, 4757, 3660, 4864, 4746, 2889, 17, 87, 4444, 1684, 1434, 470, 1228, 3728, 4282, 4877, 3424, 1704, 79, 4534, 2154, 2688, 1945, 2997, 2885, 2567, 1810, 4456, 3880, 4206, 211, 1719, 977, 1249, 3362, 1874, 1230, 2950, 2757, 3647, 2294, 3480, 367, 3459, 714, 1620, 1375, 1312, 2774, 4823, 3009, 671, 386, 3240, 4255, 1348, 3773, 20, 3942, 2952, 777, 3473, 2157, 3673, 1723, 4575, 4963, 3053, 2051, 3516, 2358, 1426, 4737, 2921, 1039, 4643, 3696, 2297, 913, 705, 1559, 609, 4538, 3766, 2159, 304, 590, 2938, 1311, 778, 4149, 4709, 3112, 4432, 4295, 4847, 507, 1948, 2373, 4226, 3985, 2054, 2429, 4004, 3788, 3278, 3484, 271, 285, 396, 436, 4107, 142, 4896, 4918, 3631, 3095, 4261, 2385, 2966, 1210, 549, 4994, 3591, 2724, 816, 4287, 1270, 3802, 4296, 2336, 2881, 3841, 2500, 2106, 3006, 2075, 45, 1567, 4426, 622, 523, 4477, 2603, 3762, 3225, 3399, 3378, 1647, 2899, 3443, 1958, 2101, 4172, 3037, 3289, 1378, 1746, 3846, 3829, 2019, 2656, 3698, 4857, 1685, 3127, 994, 111, 1628, 4830, 3601, 328, 2935, 1698, 2188, 764, 2032, 2240, 95, 1612, 784, 4923, 528, 1539, 4436, 1831, 4784, 9, 856, 2079, 4259, 1202, 3909, 3302, 4496, 3099, 2348, 787, 593, 3936, 1825, 135, 3929, 4755, 3727, 4874, 2613, 1621, 4556, 2322, 148, 4442, 1463, 4507, 2873, 4717, 2924, 1747, 2097, 3622, 370, 2143, 2591, 1666, 4403, 3281, 4697, 423, 1469, 52, 1383, 1275, 3010, 2893, 4750, 2282, 703, 2627, 839, 882, 3977, 4053, 2592, 4465, 2456, 4020, 3913, 1611, 1227, 853, 2983, 42, 3568, 1302, 3956, 644, 1456, 3594, 4835, 3737, 3125, 2301, 3973, 3136, 351, 757, 4469, 4377, 3560, 3755, 3354, 3561, 324, 2463, 880, 4685, 381, 129, 3784, 4552, 890, 4892, 2573, 601, 1184, 656, 465, 3843, 3247, 410, 1404, 2551, 3121, 4303, 170, 4744, 4797, 3908, 2123, 2388, 1668, 3007, 3305, 3211, 288, 4315, 462, 1323, 2064, 2936, 1616, 3823, 968, 2132, 4077, 2539, 2501, 185, 4379, 368, 3436, 169, 1796, 1953, 4453, 1371, 2985, 1519, 208, 2103, 4205, 1952, 1135, 4014, 1322, 4358, 628, 3685, 478, 1921, 1731, 4671, 4721, 4312, 2487, 2940, 3999, 373, 1673, 2890, 1593, 1885, 3213, 1748, 1578, 3530, 1525, 3417, 2618, 4541, 1907, 2803, 1883, 2365, 2345, 1328, 980, 2647, 4480, 2473, 1718, 1195, 4441, 4792, 647, 1655, 1505, 3688, 2900, 4850, 3061, 1604, 1895, 2914, 1317, 458, 1308, 827, 1213, 3221, 238, 3749, 3069, 3439, 3953, 3726, 49, 1547, 3824, 2270, 2540, 1431, 4011, 2562, 4371, 4141, 3286, 1866, 4523, 4640, 2168, 3873, 1510, 2170, 571, 2536, 2624, 2412, 2828, 3058, 1913, 1890, 4653, 226, 4095, 575, 2897, 1421, 1117, 2271, 1995, 59, 2905, 517, 4478, 4115, 4776, 4275, 1669, 1187, 3947, 1994, 898, 4882, 253, 3892, 2664, 118, 3978, 983, 4179, 1331, 3267, 2122, 2411, 750, 2317, 531, 2673, 2522, 88, 2742, 2351, 4656, 3023, 3581, 1929, 729, 2659, 2991, 886, 66, 3383, 3811, 2469, 1742, 3018, 190, 3445, 4600, 2639, 4795, 1862, 2778, 3569, 4939, 2777, 4040, 472, 4698, 4341, 1238, 2561, 1320, 3252, 3062, 3612, 2629, 1728, 2490, 4616, 1916, 2813, 3903, 2253, 3251, 4564, 3396, 1267, 3686, 4100, 2842, 4187, 5008, 4530, 1894, 1635, 2323, 2706, 4629, 4821, 3264, 2250, 4199, 1554, 180, 4351, 632, 1090, 3546, 178, 1552, 3531, 935, 1180, 810, 4003, 2636, 2896, 1609, 3949, 819, 318, 2504, 1417, 78, 4846, 227, 4242, 3855, 1471, 3907, 2602, 800, 3717, 699, 1864, 3585, 2716, 2768, 117, 4508, 4464, 3506, 183, 1595, 615, 4060, 3259, 4582, 548, 1901, 1466, 2821, 2621, 451, 861, 3375, 3833, 2806, 390, 3230, 2835, 149, 4903, 3276, 3222, 2528, 1149, 643, 4838, 3844, 4244, 4204, 81, 3758, 361, 2955, 2499, 2141, 1332, 151, 3744, 721, 805, 3385, 179, 3533, 2359, 820, 1797, 4428, 2789, 1627, 4868, 754, 1166, 1754, 41, 3995, 4888, 1992, 2513, 311, 1168, 3271, 4620, 4924, 938, 905, 433, 1220, 2178, 2100, 536, 3513, 2061, 2505, 4246, 2213, 3057, 1914, 4103, 4252, 4008, 358, 2169, 161, 483, 3342, 2326, 4528, 4727, 1501, 4414, 2583, 4207, 3874, 3759, 3868, 2366, 1056, 4230, 4085, 4858, 3218, 674, 70, 1079, 4431, 4822, 3174, 1785, 2387, 573, 4063, 4708, 1365, 4808, 3797, 1197, 1855, 824, 753, 3179, 293, 4319, 1820, 572, 4753, 2987, 1226, 3878, 1922, 711, 3107, 2048, 5001, 2581, 355, 855, 2771, 2049, 503, 1854, 2035, 4996, 1030, 3223, 3292, 2078, 3190, 3048, 4678, 1973, 1099, 2753, 993, 3522, 2107, 3740, 3036, 2700, 484, 3001, 2321, 2974, 4646, 3403, 1528, 518, 2400, 2783, 512, 2059, 137, 4642, 931, 2392, 1287, 485, 4636, 2135, 3520, 4310, 2517, 3971, 4927, 2439, 4219, 641, 2368, 2260, 1374, 3658, 2599, 3486, 4966, 4635, 1307, 4761, 1576, 1245, 4529, 75, 1428, 1811, 530, 4401, 4842, 3738, 4292, 4558, 4320, 2111, 3315, 2119, 1761, 2241, 1203, 29, 3464, 2121, 869, 1653, 4622, 4019, 3434, 1139, 1314, 4144, 3088, 696, 835, 3735, 3865, 652, 3245, 2834, 3948, 1590, 467, 241, 1315, 2693, 3925, 3959, 2702, 3806, 2378, 4901, 752, 997, 2796, 1737, 1630, 202, 1634, 4051, 2799, 3261, 1517, 4977, 1354, 2067, 4200, 1681, 1930, 3068, 186, 2999, 969, 252, 1678, 4771, 4617, 4919, 1838, 2555, 2709, 2557, 613, 1458, 1339, 1364, 2258, 3130, 4078, 3258, 2246, 2031, 2171, 3323, 871, 742, 1377, 1990, 2316, 2115, 813, 3332, 3724, 4867, 2918, 3134, 4531, 2417, 3918, 2564, 3853, 428, 3335, 4039, 586, 900, 1638, 2130, 1515, 4809, 3507, 4099, 3882, 4378, 1008, 1341, 405, 420, 307, 1049, 4124, 1702, 1981, 1344, 114, 1394, 4034, 565, 3355, 4411, 2474, 3723, 194, 4790, 2295, 412, 3113, 112, 4532, 4272, 3415, 2158, 3570, 4843, 489, 3792, 2151, 4293, 4127, 3343, 3187, 2758, 3123, 2354, 624, 4185, 4512, 1100, 3580, 1708, 1677, 4254, 2457, 1500, 3279, 4623, 4621, 3465, 2815, 4992, 442, 1751, 210, 4446, 3376, 4384, 1278, 2941, 2021, 1397, 2281, 3786, 2478, 2099, 4731, 4732, 242, 2057, 1265, 774, 4397, 431, 35, 3779, 2628, 3768, 567, 2825, 2179, 4398, 4024, 69, 2993, 4430, 2448, 4097, 991, 3576, 1641, 3808, 1040, 3556, 3575, 2534, 3694, 1808, 2969, 2102, 1443, 3505, 1806, 3435, 4221, 4899, 1705, 4309, 1713, 4545, 3274, 4409, 4280, 216, 389, 2000, 3457, 3296, 2008, 3904, 728, 158, 1059, 3152, 1081, 4710, 595, 2016, 1725, 2946, 883, 1014, 4133, 1843, 4573, 3115, 596, 594, 730, 2437, 4300, 985, 4072, 2310, 4979, 1061, 1606, 3644, 3871, 3743, 3630, 4724, 1026, 2475, 2069, 3675, 1919, 107, 1007, 532, 3474, 106, 1879, 2961, 773, 4993, 2164, 1898, 2433, 878, 3311, 4700, 2892, 1711, 3471, 2574, 47, 4740, 3019, 4920, 4459, 2610, 3325, 4886, 2725, 3472, 2346, 1935, 2920, 2525, 7, 4460, 276, 2189, 1771, 1608, 928, 233, 1292, 3391, 1969, 4368, 4399, 3441, 3214, 1408, 1449, 3711, 4521, 4669, 2883, 1791, 2653, 823, 1971, 3972, 1283, 3573, 1782, 768, 4520, 3996, 670, 4670, 1947, 1982, 901, 33, 4239, 4660, 4735, 3215, 1858, 4425, 4863, 409, 818, 171, 3151, 779, 77, 1490, 1285, 2739, 2004, 598, 4083, 847, 4036, 4334, 3290, 2216, 3736, 3284, 930, 681, 2767, 3458, 854, 970, 3917, 286, 865, 2148, 391, 2943, 4136, 2915, 1656, 2278, 3964, 1717, 3765, 3489, 4997, 963, 4009, 3297, 3045, 3357, 2200, 2308, 4668, 1691, 12, 3294, 1716, 1851, 4960, 4610, 3242, 4978, 3400, 1239, 2948, 1674, 4699, 3096, 3509, 1148, 4175, 1410, 4593, 4154, 140, 5007, 3819, 3077, 3353, 2211, 3162, 631, 2015, 537, 688, 4231, 1214, 1925, 4880, 1714, 2975, 1153, 315, 4096, 3344, 4038, 1432, 393, 4322, 2142, 4844, 2728, 3005, 4928, 1019, 4028, 1423, 40, 2903, 2934, 4443, 4108, 494, 4473, 3997, 1253, 4178, 1284, 1274, 2831, 1918, 163, 1198, 561, 1363, 2030, 662, 4649, 1788, 3103, 2361, 230, 1405, 4639, 1835, 416, 501, 1632, 3933, 1729, 4513, 618, 4995, 4609, 1943, 275, 1141, 1911, 3682, 2703, 1279, 837, 592, 364, 697, 3733, 1029, 4250, 1652, 1071, 4871, 2554, 1580, 1280, 1145, 2558, 1193, 1789, 781, 1105, 2635, 1123, 1290, 3954, 978, 2846, 4404, 1376, 1385, 3237, 3347, 4191, 283, 3393, 1043, 4413, 399, 4606, 2691, 1392, 1722, 25, 3990, 2238, 3260, 1557, 707, 3011, 3818, 277, 4356, 3027, 3945, 298, 4891, 3854, 4912, 640, 1538, 745, 2582, 3158, 3142, 4586, 2710, 1147, 3664, 700, 889, 4967, 3066, 1639, 4482, 747, 290, 580, 4157, 3834, 1819, 1533, 145, 3772, 4438, 3830, 2720, 4730, 3590, 2607, 2617, 4195, 3227, 1826, 321, 4016, 4911, 2507, 545, 2138, 1965, 2042, 1904, 2502, 956, 4715, 1450, 3637, 2882, 3402, 3390, 2441, 1997, 167, 371, 1863, 1066, 1738, 974, 4677, 302, 2288, 3632, 1107, 1111, 3493, 3451, 4859, 55, 2553, 4749, 235, 4241, 2303, 1793, 3039, 1138, 4632, 972, 113, 3312, 4180, 2550, 3731, 906, 305, 3559, 786, 4395, 2646, 4220, 1021, 574, 2140, 2181, 404, 4374, 2520, 13, 4329, 3143, 3975, 881, 4007, 4769, 989, 4921, 2838, 1802, 2919, 3979, 1349, 1701, 1875, 2956, 2120, 4106, 3132, 3145, 2982, 2068, 4253, 862, 2210, 3266, 3030, 1199, 2369, 600, 2744, 553, 1633, 902, 4048, 543, 1687, 4147, 1795, 3129, 407, 1865, 1080, 2518, 4739, 4483, 4420, 2462, 1065, 3102, 4279, 3836, 1869, 1637, 1558, 4561, 843, 1989, 2512, 2906, 934, 477, 2649, 2131, 3246, 4540, 374, 3100, 2651, 3628, 347, 3188, 3840, 91, 2734, 4495, 1217, 3449, 2225, 3404, 1075, 4517, 2612, 1933, 3716, 794, 4504, 1467, 3670, 3553, 4865, 5002, 3285, 4328, 1662, 758, 4001, 4803, 2611, 269, 1743, 4386, 3307, 3042, 4851, 944, 4346, 2871, 3952, 1488, 3699, 4683, 4602, 568, 3501, 1676, 2007, 2084, 1236, 3083, 2096, 2566, 3867, 177, 3633, 1762, 303, 4514, 4382, 192, 1103, 1277, 475, 971, 1461, 3641, 2129, 2471, 476, 3944, 1487, 1325, 4029, 1495, 1024, 4139, 4318, 1097, 4402, 2685, 28, 3803, 1715, 3144, 2166, 3701, 4339, 1051, 4298, 762, 2165, 1336, 4499, 1033, 2090, 469, 775, 544, 4971, 4684, 2894, 806, 4489, 769, 3167, 3703, 1025, 3416, 1944, 4848, 4778, 1744, 2820, 1657, 3540, 4935, 1286, 3752, 1104, 1113, 648, 3022, 3075, 2678, 2460, 4613, 1477, 1891, 4075, 4962, 299, 2859, 3515, 3002, 4890, 4458, 2875, 3159, 995, 4884, 4058, 2949, 4074, 1384, 3093, 1905, 3912, 4183, 2376, 1618, 3361, 4638, 2937, 2128, 1648, 4626, 259, 3910, 1175, 272, 3887, 1216, 3433, 2787, 919, 1169, 1232, 3382, 2300, 3327, 2529, 1497, 569, 1607, 2036, 3250, 4338, 4268, 4869, 940, 4987, 3753, 2809, 832, 612, 3431, 4931, 4257, 1304, 4560, 767, 4455, 4664, 2988, 4568, 3456, 3452, 2055, 2713, 23, 3697, 2614, 4665, 3572, 1650, 4400, 4583, 3231, 224, 491, 3537, 2214, 2792, 2655, 1437, 4804, 4203, 4394, 788, 466, 560, 1416, 4791, 4990, 1248, 558, 1182, 2750, 4360, 3662, 2083, 1151, 3414, 471, 2632, 3091, 3785, 1881, 3876, 1955, 4327, 2450, 1506, 388, 492, 1615, 3710, 2851, 1583, 441, 4633, 2430, 206, 96, 836, 4135, 139, 4091, 3539, 2953, 4111, 942, 2705, 4951, 2633, 3367, 4815, 1938, 4691, 4675, 1150, 3128, 562, 3207, 2913, 4112, 4304, 30, 1803, 3203, 1962, 3554, 3345, 2377, 3764, 2620, 4805, 939, 3126, 4841, 1045, 3147, 1599, 2644, 1823, 4116, 1601, 1399, 1889, 675, 3109, 2449, 3771, 3384, 3118, 4648, 2222, 3922, 3028, 1368, 3665, 2764, 3108, 3842, 542, 3228, 439, 2357, 3597, 1089, 1861, 516, 4447, 4957, 1269, 3116, 4794, 3337, 2853, 2197, 2124, 4553, 4866, 3205, 1950, 4766, 2514, 2425, 3937, 4422, 274, 1649, 3579, 1642, 4462, 4999, 841, 4340, 3226, 4047, 2226, 2318, 4294, 2615, 2382, 3566, 4161, 4981, 4240, 3870, 2012, 3578, 907, 3503, 3991, 3056, 4555, 1817, 2857, 2027, 4330, 4311, 2930, 4392, 1370, 450, 1464, 4260, 2616, 4952, 2053, 4491, 1114, 2237, 247, 3960, 2343, 2714, 3, 2605, 3133, 2239, 4798, 908, 1499, 2981, 4247, 1122, 1472, 4408, 1570, 3805, 1556, 172, 1670, 3229, 4938, 4760, 64, 3111, 2056, 3983, 445, 3746, 4012, 1473, 2476, 2723, 3412, 4214, 1975, 4198, 3793, 338, 4533, 3640, 3932, 155, 1189, 4862, 4032, 83, 2537, 2944, 864, 3599, 743, 1961, 825, 924, 4522, 4146, 4362, 3017, 1115, 620, 2286, 3194, 3745, 1613, 1507, 3801, 914, 1589, 1300, 634, 4592, 337, 4237, 3526, 1012, 2333, 1773, 4274, 92, 1846, 2236, 519, 2231, 2018, 2658, 4267, 4953, 3164, 2134, 4983, 2530, 3032, 3485, 166, 2850, 2209, 144, 770, 1143, 4853, 4238, 50, 2755, 761, 2608, 566, 4350, 1645, 94, 3076, 2590, 1052, 4251, 2223, 2717, 1946, 2422, 1301, 3432, 463, 4439, 4216, 2939, 251, 53, 193, 4855, 3800, 1291, 2484, 1407, 375, 2595, 415, 541, 1116, 1173, 3110, 2034, 356, 2404, 240, 811, 958, 4325, 4511, 2285, 2996, 2788, 4525, 3592, 1949, 2187, 1777, 2406, 382, 625, 2727, 319, 3334, 3181, 4474, 2740, 3397, 150, 3562, 884, 4590, 921, 0, 4317, 2199, 2552, 4898, 771, 1834, 1319, 334, 4860, 1412, 345, 2979, 498, 550, 1756, 3487, 385, 1903, 4817, 3889, 4177, 2355, 2548, 1179, 4800, 3984, 2785, 2245, 1781, 440, 369, 4270, 214, 1072, 3374, 1462, 1617, 1478, 1998, 2911, 1675, 1774, 4249, 3000, 2860, 715, 3993, 602, 468, 1876, 257, 4068, 1584, 4344, 3958, 1303, 1821, 4840, 1832, 2730, 4696, 4071, 1954, 4905, 2390, 4027, 4093, 698, 32, 2977, 2092, 2371, 4045, 2841, 1699, 1042, 2879, 4980, 1154, 3153, 4756, 2279, 1980, 3555, 2619, 4915, 2063, 3934, 3856, 4893, 2818, 717, 540, 3175, 4417, 4010, 2766, 4787, 1262, 4501, 1695, 4168, 125, 3299, 1396, 4429, 16, 2791, 4314, 887, 4762, 2830, 3021, 2933, 1295, 1504, 2331, 2313, 4711, 2543, 1060, 3544, 669, 2325, 789, 4801, 826, 2224, 3326, 3253, 3060, 2455, 1600, 1085, 474, 1837, 460, 4802, 3080, 3708, 3668, 3054, 2495, 4208, 3217, 65, 3308, 4712, 3725, 48, 4385, 1011, 4948, 3379, 3336, 320, 3280, 1531, 63, 1361, 133, 3549, 1942, 4604, 3301, 366, 2434, 584, 3669, 4516, 2243, 576, 1672, 3148, 1772, 556, 3012, 851, 4098, 4017, 236, 2196, 3303, 932, 947, 3273, 653, 2082, 1128, 3235, 4925, 2291, 2577, 1844, 4380, 4209, 1055, 1852, 716, 2827, 2810, 2770, 102, 868, 4076, 1509, 3163, 2598, 3961, 3981, 136, 4363, 4467, 2491, 3608, 650, 1752, 3270, 3119, 2800, 2492, 281, 3980, 3003, 1575, 2689, 982, 1073, 2485, 4102, 2163, 3239, 2312, 2802, 4682, 1928, 3186, 3661, 2095, 929, 608, 3216, 1355, 3926, 3678, 122, 4806, 3966, 511, 3504, 3894, 945, 325, 3896, 2816, 659, 4233, 2715, 2, 1240, 4189, 3437, 198, 213, 1164, 1264, 2998, 3804, 4289, 1234, 4572, 1369, 621, 4110, 4758, 3790, 2844, 4788, 2489, 3538, 1010, 4037, 4849, 535, 3719, 581, 1266, 1512, 3552, 1805, 654, 157, 1207, 3869, 3850, 2642, 2928, 3875, 3548, 860, 4728, 1342, 852, 3288, 1573, 4645, 3104, 3742, 2521, 2675, 2405, 1435, 2508, 2028, 2769, 4503, 223, 357, 2840, 4964, 1546, 4772, 3757, 1665, 1070, 2726, 1915, 3272, 181, 1934, 1833, 2640, 830, 1776, 4663, 1804, 872, 335, 2161, 1048, 3172, 2576, 4021, 1436, 2221, 4035, 2105, 4722, 2296, 1535, 4415, 587, 2923, 3687, 26, 3008, 2671, 3236, 4023, 790, 4752, 922, 184, 1924, 4781, 3453, 1442, 2186, 2038, 2384, 164, 3156, 1261, 4946, 4196, 1170, 2383, 4502, 1296, 1293, 4833, 3482, 3146, 215, 2230, 3160, 3161, 1482, 1191, 3920, 2654, 4819, 27, 803, 3858, 2435, 1968, 1520, 1906, 260, 4225, 4789, 4348, 4190, 4942, 2380, 1441, 217, 4065, 3826, 4876, 1977, 1454, 72, 3916, 2153, 4031, 3577, 4173, 4134, 18, 4405, 4258, 4954, 3521, 4470, 2563, 3421, 4751, 3322, 2470, 2637, 80, 89, 287, 4949, 792, 4142, 3795, 4539, 362, 672, 4585, 4577, 1205, 2731, 2833, 3946, 3256, 3558, 2869, 2736, 2867, 2832, 950, 1215, 2847, 1561, 4488, 4510, 1298, 1102, 987, 4424, 4421, 2729, 1574, 4929, 1000, 692, 915, 3324, 694, 3455, 1041, 2666, 2814, 3430, 1333, 2229, 603, 2011, 4998, 920, 551, 3761, 2089, 780, 1255, 1602], "validation": [5171, 5078, 5235, 5200, 5012, 5239, 5444, 5371, 5044, 5267, 5475, 5255, 5198, 5061, 5115, 5424, 5381, 5492, 5319, 5033, 5132, 5287, 5463, 5165, 5217, 5448, 5167, 5477, 5420, 5296, 5204, 5456, 5332, 5290, 5186, 5270, 5373, 5360, 5289, 5372, 5221, 5292, 5473, 5230, 5099, 5488, 5380, 5085, 5432, 5427, 5264, 5147, 5391, 5052, 5418, 5181, 5168, 5376, 5250, 5101, 5109, 5260, 5045, 5244, 5281, 5029, 5124, 5491, 5266, 5141, 5313, 5036, 5021, 5459, 5035, 5392, 5086, 5022, 5259, 5193, 5384, 5357, 5219, 5139, 5299, 5184, 5443, 5476, 5358, 5030, 5237, 5095, 5067, 5024, 5144, 5480, 5108, 5368, 5500, 5020, 5293, 5401, 5450, 5416, 5176, 5261, 5485, 5309, 5343, 5305, 5118, 5243, 5507, 5188, 5303, 5248, 5207, 5096, 5407, 5213, 5073, 5435, 5320, 5502, 5446, 5324, 5140, 5224, 5429, 5134, 5345, 5104, 5479, 5472, 5214, 5307, 5340, 5442, 5398, 5256, 5208, 5486, 5011, 5215, 5367, 5161, 5349, 5120, 5185, 5127, 5467, 5226, 5170, 5504, 5163, 5254, 5508, 5137, 5283, 5499, 5318, 5308, 5306, 5062, 5482, 5092, 5251, 5059, 5469, 5414, 5383, 5236, 5274, 5060, 5483, 5364, 5334, 5375, 5071, 5049, 5253, 5327, 5173, 5034, 5438, 5048, 5425, 5402, 5156, 5333, 5083, 5388, 5363, 5014, 5487, 5501, 5206, 5088, 5316, 5273, 5338, 5080, 5325, 5387, 5154, 5162, 5258, 5164, 5355, 5466, 5196, 5399, 5413, 5348, 5468, 5312, 5352, 5126, 5462, 5013, 5280, 5091, 5301, 5291, 5417, 5390, 5317, 5116, 5263, 5342, 5362, 5326, 5178, 5493, 5423, 5460, 5404, 5015, 5056, 5410, 5400, 5452, 5490, 5439, 5374, 5471, 5228, 5257, 5496, 5351, 5405, 5365, 5146, 5412, 5262, 5074, 5454, 5346, 5322, 5129, 5302, 5354, 5177, 5051, 5449, 5090, 5105, 5064, 5382, 5031, 5192, 5311, 5453, 5353, 5145, 5252, 5111, 5190, 5377, 5419, 5112, 5054, 5347, 5210, 5018, 5023, 5378, 5505, 5415, 5294, 5232, 5038, 5385, 5148, 5465, 5182, 5218, 5330, 5379, 5341, 5328, 5447, 5047, 5069, 5286, 5275, 5394, 5395, 5157, 5421, 5152, 5138, 5039, 5426, 5411, 5103, 5288, 5066, 5225, 5430, 5143, 5359, 5201, 5233, 5356, 5437, 5436, 5058, 5075, 5079, 5135, 5155, 5428, 5441, 5278, 5249, 5113, 5272, 5106, 5457, 5040, 5094, 5130, 5043, 5422, 5072, 5068, 5197, 5017, 5136, 5160, 5503, 5397, 5408, 5209, 5445, 5282, 5131, 5222, 5175, 5369, 5195, 5484, 5084, 5199, 5238, 5223, 5298, 5070, 5203, 5433, 5315, 5128, 5121, 5098, 5455, 5114, 5025, 5331, 5110, 5100, 5242, 5451, 5458, 5234, 5323, 5057, 5119, 5097, 5509, 5220, 5117, 5409, 5246, 5284, 5265, 5434, 5481, 5279, 5016, 5187, 5276, 5227, 5093, 5149, 5151, 5212, 5310, 5268, 5337, 5339, 5053, 5037, 5089, 5205, 5295, 5010, 5506, 5087, 5081, 5123, 5470, 5055, 5497, 5277, 5027, 5389, 5321, 5489, 5125, 5271, 5304, 5247, 5366, 5065, 5431, 5041, 5102, 5028, 5063, 5361, 5478, 5440, 5032, 5107, 5386, 5189, 5314, 5180, 5076, 5474, 5046, 5153, 5297, 5231, 5370, 5042, 5166, 5336, 5202, 5269, 5174, 5494, 5335, 5240, 5026, 5393, 5194, 5403, 5179, 5183, 5216, 5133, 5495, 5396, 5344, 5406, 5158, 5498, 5169, 5142, 5241, 5329, 5159, 5464, 5077, 5150, 5461, 5285, 5229, 5172, 5300, 5019, 5191, 5082, 5245, 5350, 5050, 5122, 5211], "test": [5768, 5971, 5652, 5881, 5908, 5929, 5677, 5912, 5946, 5977, 5687, 5573, 5747, 5670, 5749, 5516, 5772, 5584, 5636, 5742, 5887, 5547, 5604, 5671, 5630, 5621, 5890, 5649, 5706, 5574, 5655, 5582, 5879, 5684, 5622, 5953, 5583, 5870, 5702, 5889, 5546, 5957, 5850, 5656, 5697, 5606, 5631, 5548, 5737, 5721, 5909, 5869, 5640, 5856, 5710, 5751, 5592, 5673, 5632, 5987, 5681, 5907, 5688, 5801, 5833, 5559, 5613, 5752, 5799, 5624, 5984, 5936, 5934, 5858, 6002, 5599, 6000, 5644, 5740, 5674, 5611, 5976, 5931, 5880, 5865, 6007, 5597, 5803, 5566, 5967, 5823, 5901, 5949, 5643, 5620, 5572, 5921, 5876, 5832, 5813, 5595, 5541, 5845, 5738, 5530, 5795, 5847, 5615, 5724, 5843, 5862, 6001, 5550, 5810, 5774, 5545, 5605, 5730, 5811, 5919, 5788, 5534, 5993, 5963, 5591, 5940, 5777, 5586, 5564, 5900, 5781, 5754, 5570, 5918, 5828, 5690, 6009, 5577, 5840, 5669, 5651, 5580, 5756, 5991, 5910, 5727, 5988, 5531, 5662, 5950, 5871, 5700, 5535, 5753, 5726, 5596, 5986, 5829, 5635, 5864, 5549, 5576, 5787, 5748, 5755, 5685, 6005, 5837, 5556, 5776, 5629, 5980, 5711, 5888, 5642, 5538, 5982, 5694, 5997, 5981, 5844, 5699, 5761, 5680, 5733, 5529, 5831, 5790, 5707, 5589, 5913, 5791, 5544, 5874, 5868, 5861, 5945, 5609, 5846, 5784, 5510, 5817, 5562, 5906, 5809, 5848, 5637, 5826, 5646, 5944, 5842, 5551, 5852, 5954, 5561, 5878, 5731, 5835, 5625, 5926, 5793, 5958, 5964, 5587, 5805, 5607, 5608, 5658, 5602, 5917, 5567, 5645, 5758, 5600, 5638, 5623, 5785, 5569, 5939, 5771, 5955, 5885, 5743, 5896, 5539, 5691, 5563, 5639, 5659, 5762, 6006, 5897, 5915, 5937, 5923, 5947, 5525, 5565, 5654, 5628, 5979, 5678, 5603, 5528, 5524, 5933, 5704, 5775, 5927, 5935, 5779, 5590, 5975, 5766, 5959, 5841, 5983, 5716, 5526, 5634, 5692, 5994, 5898, 5712, 5666, 5571, 5718, 5616, 5859, 5830, 5825, 5941, 5579, 5818, 5739, 5989, 5518, 5866, 5760, 5720, 5816, 5973, 5696, 5895, 5664, 5668, 5581, 5893, 5857, 5657, 5855, 5675, 5767, 5763, 5523, 5679, 5875, 5555, 5653, 5536, 5517, 5663, 5735, 5922, 5849, 5786, 5619, 5782, 5797, 5553, 5877, 5960, 5867, 5911, 5647, 5588, 5834, 5780, 5814, 5633, 5522, 5798, 5560, 6008, 5521, 5974, 5972, 5686, 5601, 5932, 5800, 5682, 5626, 5717, 5527, 5769, 5614, 5783, 5956, 5765, 5883, 5514, 5610, 5990, 5992, 5557, 5860, 5745, 5863, 6004, 5695, 5698, 5892, 5750, 5938, 5886, 5854, 5744, 5873, 5568, 5719, 5930, 5575, 5661, 5578, 5532, 5703, 5789, 5741, 5920, 5515, 5759, 5916, 5794, 5928, 5593, 5942, 5815, 5802, 5820, 5552, 5520, 5996, 5853, 5773, 5689, 5951, 5511, 5543, 5839, 5995, 5970, 5978, 5924, 5723, 5542, 5708, 5998, 5701, 5948, 5650, 5969, 5722, 5618, 5821, 5736, 5660, 5985, 5894, 5683, 5891, 5999, 5812, 5667, 5728, 5914, 6003, 5778, 5882, 5558, 5952, 5851, 5598, 5709, 5757, 5968, 5627, 5585, 5824, 5903, 5537, 5714, 5533, 5770, 5796, 5672, 5764, 5746, 5612, 5665, 5554, 5806, 5513, 5819, 5676, 5827, 5792, 5804, 5905, 5715, 5729, 5725, 5617, 5705, 5838, 5807, 5884, 5904, 5899, 5872, 5962, 5902, 5836, 5540, 5693, 5943, 5519, 5648, 5961, 5594, 5713, 5966, 5822, 5734, 5641, 5965, 5808, 5512, 5925, 5732]}, {"train": [4949, 797, 1318, 4160, 417, 1046, 878, 1764, 2892, 3448, 1322, 353, 4524, 2152, 4802, 3493, 621, 782, 111, 1776, 790, 3688, 1770, 2087, 4085, 1496, 3053, 2229, 1098, 4166, 157, 4119, 4505, 2553, 1655, 336, 884, 3513, 3640, 4435, 2981, 4759, 3516, 4179, 2051, 1144, 70, 3412, 4375, 84, 3222, 4447, 2730, 1735, 2150, 377, 835, 1421, 3721, 1751, 4641, 2499, 3681, 972, 64, 3732, 2933, 1297, 1619, 4570, 2661, 215, 3814, 2633, 332, 1543, 473, 4788, 2207, 1199, 2812, 579, 1954, 2114, 3575, 4240, 4100, 2327, 4668, 534, 4579, 2983, 2808, 3775, 4933, 1862, 4319, 1231, 3322, 1510, 1110, 490, 1620, 2838, 3531, 902, 3874, 1174, 1248, 2239, 2427, 3484, 810, 3283, 4608, 2704, 3123, 3873, 267, 2911, 4331, 4167, 634, 606, 4443, 3976, 2918, 2084, 532, 3066, 969, 1700, 1039, 66, 2443, 4927, 1653, 4783, 3217, 1618, 1345, 4099, 4513, 4172, 4535, 1207, 173, 657, 2024, 4121, 2963, 3180, 1443, 3773, 641, 4073, 1189, 2621, 1574, 97, 2111, 2681, 3403, 2995, 3130, 3882, 483, 3871, 4303, 4083, 3141, 2668, 2228, 3269, 4094, 4454, 3179, 4503, 512, 286, 3344, 322, 4973, 1165, 136, 2107, 2391, 1882, 3294, 4091, 2338, 4609, 46, 768, 3754, 4276, 120, 3680, 1040, 4377, 4958, 3547, 3771, 2530, 851, 2971, 127, 92, 470, 3604, 3580, 4714, 89, 4066, 3477, 1592, 1474, 4142, 4966, 4740, 1834, 998, 3174, 787, 3893, 3068, 1866, 4578, 562, 1675, 1438, 1640, 1397, 2408, 4573, 1491, 1230, 3411, 3795, 1749, 3829, 1371, 689, 3800, 990, 3603, 1479, 1941, 3801, 1398, 3512, 4826, 2573, 3930, 4581, 654, 109, 4101, 258, 1314, 1418, 1656, 2109, 3494, 1427, 272, 199, 755, 2237, 3643, 2824, 583, 4262, 1765, 1614, 3467, 757, 1573, 692, 2796, 1724, 2131, 3284, 240, 3189, 3039, 5006, 3557, 4168, 4598, 2637, 278, 910, 3195, 2104, 796, 4500, 3069, 1319, 2752, 3572, 3538, 541, 4137, 2197, 2852, 879, 4148, 3695, 1123, 4663, 3799, 789, 2886, 3849, 2156, 683, 1494, 2514, 690, 1774, 694, 1556, 4939, 4661, 3371, 1535, 3101, 2597, 1748, 445, 1773, 4396, 4968, 3997, 4631, 3107, 1431, 968, 2555, 3746, 3478, 476, 4151, 738, 4755, 3365, 1303, 2053, 1918, 2767, 780, 687, 2147, 1875, 3785, 1604, 991, 941, 538, 1403, 953, 2794, 3030, 3385, 4590, 1253, 1616, 3993, 1038, 2897, 3889, 2460, 3556, 4509, 3270, 2600, 4657, 4222, 3027, 312, 4012, 2300, 673, 2849, 4252, 231, 4602, 3577, 1676, 2094, 791, 2271, 615, 4658, 2191, 1901, 712, 1545, 4817, 2909, 2923, 4136, 1130, 1857, 238, 3823, 3142, 2065, 3315, 1366, 2372, 2657, 2118, 2649, 3129, 3120, 4964, 1003, 1184, 3301, 4736, 4176, 4340, 702, 2305, 762, 162, 1096, 3383, 646, 1300, 4376, 1000, 4029, 1471, 2829, 4165, 2562, 4719, 7, 2285, 1800, 2045, 2795, 2119, 4058, 889, 859, 2013, 4778, 2424, 2117, 4241, 838, 2515, 581, 2129, 4347, 2906, 3726, 1146, 2069, 4915, 177, 1499, 4931, 3482, 3064, 4853, 2586, 2414, 4200, 1310, 4232, 741, 4007, 4479, 3530, 933, 2090, 2373, 732, 574, 1695, 224, 1867, 3313, 3987, 2004, 221, 3196, 2433, 2501, 3700, 3251, 2491, 100, 4812, 3589, 4499, 1992, 2254, 4246, 1714, 3638, 2377, 345, 3646, 584, 4750, 1650, 369, 2982, 3048, 4440, 2747, 3740, 3553, 4821, 2844, 302, 4688, 4848, 2826, 3434, 3992, 626, 4074, 1211, 2400, 3445, 3298, 1069, 4036, 262, 98, 1994, 3518, 2867, 4288, 3017, 4243, 776, 298, 2511, 4298, 2304, 3999, 1993, 1536, 4979, 1824, 3661, 4482, 3613, 4278, 813, 3955, 4536, 3268, 4900, 4193, 788, 1205, 1278, 3712, 1332, 704, 2038, 447, 1737, 451, 3292, 3914, 2822, 1268, 915, 10, 2885, 3333, 3015, 3710, 774, 2050, 2738, 3983, 3741, 2582, 5000, 3457, 4356, 3737, 2290, 4838, 4779, 4465, 3972, 877, 3061, 4831, 4553, 3092, 2975, 637, 160, 1497, 1229, 1530, 115, 3892, 2356, 4771, 1551, 15, 1727, 3738, 1260, 1232, 1923, 2823, 3272, 3019, 1842, 4304, 2830, 1821, 4704, 4139, 3641, 405, 2984, 2173, 346, 4521, 1649, 3145, 1356, 3133, 1204, 4582, 1795, 3783, 3862, 1420, 3855, 478, 965, 4564, 2306, 3062, 1623, 2733, 4889, 3628, 3444, 4096, 913, 1880, 4002, 4042, 2493, 3805, 3299, 1220, 2647, 1677, 4147, 41, 841, 3234, 2157, 3674, 3933, 1460, 3404, 4448, 2789, 597, 4522, 2382, 3233, 4453, 1236, 4135, 770, 3994, 4947, 3968, 4260, 3635, 3476, 543, 4634, 4568, 2341, 2659, 2576, 1422, 5009, 3250, 2927, 1114, 4342, 266, 684, 1280, 3248, 4093, 3758, 1456, 3326, 3083, 1822, 2635, 497, 2240, 4359, 4271, 439, 1121, 1160, 1731, 4650, 1208, 905, 3989, 3915, 2620, 214, 1720, 1760, 3859, 263, 3161, 2857, 254, 2273, 4458, 1057, 2286, 1027, 2346, 4019, 4133, 3318, 103, 2744, 1150, 448, 2843, 2585, 2820, 3847, 3178, 1387, 477, 744, 1434, 1874, 306, 1557, 2097, 1486, 3927, 4227, 1833, 4868, 3988, 3339, 4995, 81, 392, 2816, 3929, 4423, 4184, 456, 3662, 535, 4980, 665, 85, 3401, 4242, 733, 2563, 1429, 487, 60, 4004, 4936, 4776, 1070, 4269, 2760, 4988, 3070, 4738, 2270, 1940, 1501, 3731, 116, 2195, 3591, 2531, 347, 701, 520, 53, 547, 2915, 502, 799, 3839, 2602, 104, 4845, 872, 4594, 1642, 573, 3508, 3843, 836, 1115, 682, 553, 2815, 3620, 2512, 2052, 2545, 1423, 2002, 1289, 3216, 4277, 4965, 4040, 2210, 3124, 3013, 4810, 4925, 2311, 829, 739, 3605, 3200, 430, 2828, 901, 3595, 986, 414, 857, 3788, 2850, 420, 4962, 3717, 2739, 3416, 3374, 899, 4586, 331, 2776, 577, 4097, 1137, 4456, 4957, 4424, 1513, 4899, 1979, 1411, 4954, 4892, 1793, 807, 792, 571, 3755, 305, 4512, 1936, 2914, 3202, 2098, 387, 3245, 1950, 2010, 3816, 2042, 380, 1107, 916, 3481, 2908, 2890, 2502, 1538, 3535, 727, 970, 4878, 3612, 3194, 4950, 3769, 1651, 3673, 180, 3571, 300, 2944, 2840, 1405, 492, 4318, 1407, 366, 406, 1011, 1809, 1902, 868, 2393, 1980, 1016, 3813, 320, 1377, 1621, 4772, 3682, 3280, 2775, 951, 2076, 603, 1087, 4233, 2386, 1457, 3521, 2367, 3615, 1031, 3961, 3138, 1593, 4293, 3550, 129, 4827, 1084, 2626, 1572, 4442, 1030, 4296, 751, 3305, 1996, 2967, 1364, 1763, 181, 2781, 1981, 3616, 1852, 1570, 2891, 4758, 462, 2505, 3031, 3442, 2449, 1514, 3139, 2578, 39, 2140, 2358, 3949, 3652, 4881, 80, 290, 1348, 2766, 3405, 2619, 4182, 2977, 2179, 869, 2986, 2962, 1408, 3479, 4613, 1062, 1962, 1808, 950, 1976, 3684, 3858, 1082, 1255, 3687, 506, 3967, 161, 42, 826, 2453, 1013, 643, 248, 2720, 530, 4323, 327, 870, 1504, 904, 672, 984, 2066, 2158, 4577, 1848, 4353, 3698, 4706, 4654, 2132, 4487, 805, 3132, 43, 2589, 4876, 1139, 1270, 1101, 2965, 2163, 2590, 230, 3622, 259, 661, 432, 4841, 2614, 3214, 568, 4055, 2836, 3672, 2151, 1171, 4867, 3542, 827, 3867, 1361, 4150, 4273, 4667, 2751, 1895, 1946, 3916, 4067, 3594, 640, 501, 4362, 1682, 2643, 4486, 3588, 2718, 2039, 3568, 4633, 2686, 4473, 2269, 2315, 4134, 2009, 1723, 3067, 1609, 4404, 3329, 2860, 3356, 4523, 2543, 2565, 1106, 2342, 624, 3497, 2265, 3902, 2521, 119, 1825, 1389, 1978, 4952, 4464, 2482, 1424, 1388, 187, 4557, 3639, 4039, 169, 3295, 808, 4963, 3263, 358, 35, 3307, 4189, 4671, 4928, 3008, 3856, 354, 2674, 2435, 3347, 570, 2093, 324, 304, 1708, 2743, 2921, 4384, 3342, 4205, 572, 275, 659, 2650, 2061, 228, 242, 907, 3778, 3846, 3384, 2504, 1413, 142, 1832, 1085, 3429, 939, 625, 1327, 4416, 1495, 3970, 3906, 3704, 4152, 2937, 2037, 669, 1132, 3655, 858, 989, 4030, 3352, 3936, 2727, 2298, 4190, 2399, 3621, 2606, 1922, 3423, 4627, 2212, 3822, 3828, 4255, 622, 1249, 2508, 2309, 200, 1641, 2323, 4194, 4388, 364, 3290, 1001, 1896, 1192, 937, 2422, 4369, 4439, 967, 761, 4345, 1394, 559, 3116, 2756, 4123, 3148, 3380, 4648, 1326, 107, 1437, 1738, 4103, 2651, 3707, 3418, 1602, 2855, 947, 2322, 3766, 1234, 809, 2216, 4917, 4231, 4405, 135, 4203, 493, 159, 3496, 2124, 1511, 1644, 1858, 4455, 730, 2247, 4095, 3977, 3188, 618, 2281, 2869, 1712, 2036, 1899, 3614, 3787, 3583, 4589, 1554, 400, 3522, 793, 4259, 1060, 4621, 4530, 3043, 1758, 372, 4478, 234, 189, 1419, 3598, 3304, 1263, 779, 1262, 2366, 975, 4741, 1344, 4629, 114, 297, 4628, 2853, 2707, 706, 452, 3834, 671, 2509, 1131, 743, 2903, 1401, 219, 647, 960, 4696, 323, 713, 1113, 4432, 1390, 179, 4666, 3947, 4937, 390, 465, 3857, 245, 396, 1219, 1313, 1019, 453, 1711, 5, 4811, 1673, 3353, 2510, 3302, 729, 1008, 4265, 4328, 1142, 2221, 3519, 150, 2567, 995, 2488, 1755, 1316, 2103, 3790, 3581, 1336, 4289, 4529, 1910, 3861, 1343, 4164, 3310, 4595, 2469, 3057, 4561, 1017, 3077, 122, 1217, 4953, 40, 1067, 2404, 1417, 2979, 1349, 589, 3100, 882, 4198, 3441, 2684, 293, 209, 356, 2047, 3420, 4549, 3243, 203, 352, 6, 4075, 4774, 4798, 4017, 1680, 588, 1590, 1881, 225, 1473, 2405, 4031, 1780, 1100, 38, 4948, 2552, 522, 4221, 3397, 2856, 1654, 178, 2636, 2475, 62, 4844, 198, 3099, 4670, 1180, 1854, 1840, 475, 4896, 536, 2701, 3679, 4498, 3118, 2615, 3078, 811, 900, 2326, 767, 2295, 2554, 426, 3471, 3281, 2064, 4216, 533, 731, 1664, 3806, 3964, 1239, 1406, 321, 2249, 2079, 1585, 3218, 4491, 2105, 3094, 2190, 1198, 442, 4245, 2696, 4335, 163, 3458, 1926, 3819, 3235, 368, 4550, 269, 4287, 270, 2608, 3126, 1482, 4793, 4050, 4976, 4803, 140, 1182, 2226, 3981, 2397, 2754, 1384, 1342, 2167, 291, 328, 361, 1757, 1713, 660, 4294, 1815, 3534, 1639, 141, 602, 1103, 2764, 3165, 3169, 419, 4417, 4606, 1112, 2870, 325, 4327, 3469, 834, 1560, 1404, 1697, 2467, 2442, 1451, 4725, 1872, 663, 91, 335, 90, 4794, 918, 2225, 1291, 479, 4705, 837, 3841, 2925, 1957, 217, 4865, 1358, 299, 3506, 3573, 349, 3546, 4726, 846, 3748, 3890, 2980, 4234, 68, 1871, 134, 650, 3786, 1025, 3818, 2198, 3540, 1850, 1311, 3240, 2030, 4765, 3255, 2777, 3831, 2725, 350, 598, 2473, 3090, 50, 1894, 2948, 1801, 4732, 3117, 631, 2403, 2950, 604, 3653, 1050, 418, 3049, 2604, 3485, 2877, 3566, 1071, 2566, 1399, 1819, 1247, 4041, 627, 3443, 4693, 4072, 1869, 2955, 4158, 4102, 2537, 4617, 2325, 11, 2580, 130, 4731, 1689, 1006, 3147, 1816, 3899, 1741, 4649, 1730, 2640, 4274, 662, 4614, 4391, 4483, 2742, 1804, 4930, 4538, 1638, 2330, 2490, 1195, 3877, 3736, 4665, 438, 1293, 1652, 3056, 1402, 3881, 4981, 2807, 3238, 2145, 1257, 3289, 2861, 1986, 611, 1516, 2461, 4883, 3511, 1307, 897, 5008, 2646, 1561, 1903, 2412, 582, 1589, 206, 2876, 2116, 1706, 2127, 1920, 3201, 3239, 1330, 3866, 3252, 3063, 1143, 2895, 4563, 3103, 3121, 1868, 3046, 1628, 1943, 449, 4496, 3509, 3395, 2057, 655, 2355, 3701, 3692, 2146, 2196, 3168, 1670, 3140, 1679, 2092, 31, 1214, 2096, 4576, 1120, 3678, 32, 1428, 1439, 1292, 3433, 2220, 4700, 4001, 3671, 1829, 3364, 252, 3931, 1591, 3137, 645, 3697, 4565, 3080, 3913, 822, 3183, 3010, 2691, 3220, 1883, 2575, 2055, 71, 2724, 3937, 1341, 903, 2450, 1190, 2966, 4213, 133, 680, 4999, 1048, 4332, 3709, 1462, 4047, 3400, 2075, 1933, 4016, 4301, 2219, 4862, 1624, 1766, 4054, 2206, 3011, 1915, 4805, 1887, 798, 1668, 1372, 875, 384, 2025, 2165, 2865, 3724, 1024, 2929, 516, 2818, 4751, 1478, 1582, 326, 1539, 1502, 4068, 1287, 4082, 2472, 1577, 3912, 3601, 2153, 397, 2308, 9, 3170, 2772, 3782, 3073, 1725, 4256, 4351, 1707, 1446, 1490, 218, 2535, 4370, 4437, 1884, 1383, 3260, 4307, 3422, 3105, 4942, 1844, 2706, 2928, 3920, 3388, 3627, 700, 4642, 593, 2312, 2611, 1512, 237, 4782, 1762, 3975, 1272, 1603, 4394, 2113, 4175, 503, 3943, 961, 201, 314, 1806, 4336, 3770, 3382, 710, 3417, 1360, 1391, 4226, 4809, 1035, 966, 4727, 4449, 4300, 3327, 51, 4425, 4643, 481, 1159, 359, 3026, 4261, 1518, 4186, 292, 4745, 2431, 2456, 192, 2379, 3925, 1223, 3796, 3041, 4669, 628, 2507, 2866, 4697, 3275, 608, 747, 3827, 2889, 693, 726, 1912, 3006, 2660, 3386, 1540, 3844, 2534, 424, 57, 515, 3285, 2904, 4840, 233, 644, 2932, 2919, 2139, 1444, 3743, 202, 3685, 688, 2301, 3206, 691, 1240, 1241, 3587, 3669, 1351, 4429, 3745, 2700, 4028, 235, 2347, 1065, 2902, 2102, 2940, 1266, 72, 4254, 3560, 2063, 3033, 2538, 4787, 4104, 4908, 186, 4129, 4299, 257, 3537, 3016, 1309, 1381, 1315, 2080, 1890, 3398, 4402, 3296, 3574, 3003, 4733, 4383, 4141, 3261, 2282, 2956, 1415, 2406, 4410, 2381, 3971, 3864, 4215, 3751, 759, 2384, 1072, 170, 2365, 3675, 3104, 3887, 4127, 1702, 2697, 1752, 1544, 2452, 1791, 3297, 2234, 557, 2202, 4717, 1919, 4820, 777, 3264, 718, 334, 464, 188, 927, 165, 468, 4188, 2238, 4786, 2969, 842, 830, 3579, 985, 605, 891, 196, 721, 2134, 4144, 4105, 4459, 4877, 4638, 4934, 1294, 3362, 3088, 3282, 3324, 2722, 3763, 1156, 208, 1321, 4887, 1212, 1104, 1400, 1224, 3963, 1527, 1362, 914, 679, 3231, 311, 2230, 1925, 2833, 2949, 61, 3059, 1119, 2313, 2451, 3065, 1226, 1951, 1827, 194, 4421, 4734, 2378, 3021, 4987, 2474, 4645, 3918, 3907, 4210, 652, 3022, 2666, 4837, 4914, 1953, 4250, 962, 1044, 699, 2683, 4080, 1465, 3435, 3744, 1201, 548, 2936, 1243, 3723, 946, 4567, 2161, 3265, 3358, 2437, 2896, 2665, 4560, 2750, 4461, 1037, 2370, 1775, 1141, 3505, 4525, 4839, 2471, 1454, 1674, 295, 1468, 2183, 4664, 4086, 2465, 4807, 74, 2172, 1683, 3112, 56, 3729, 3909, 2288, 3424, 2430, 4813, 3151, 2845, 1584, 3888, 1273, 2410, 1078, 3791, 1991, 2274, 2632, 4310, 4599, 3153, 3166, 3173, 1990, 1863, 2241, 2819, 2788, 4051, 183, 2426, 2976, 3461, 2351, 2893, 2922, 340, 2884, 4882, 309, 4951, 554, 3076, 2658, 931, 1353, 2593, 482, 4592, 4913, 2003, 3483, 4495, 2785, 4326, 2108, 4400, 2525, 2478, 113, 4275, 3759, 3325, 1015, 2618, 3034, 3186, 1736, 2029, 3084, 3109, 3812, 845, 4748, 1719, 4544, 2068, 4767, 935, 2421, 919, 1786, 4992, 2492, 4207, 681, 4427, 3407, 2041, 4574, 4379, 226, 1053, 3833, 4149, 3431, 4835, 940, 4140, 943, 2429, 1813, 2144, 3555, 2790, 4324, 2863, 2786, 4477, 2486, 1744, 3018, 4130, 2557, 4403, 4543, 4159, 2639, 4757, 4728, 1295, 279, 609, 1811, 1149, 4618, 4419, 4169, 1812, 496, 4092, 4180, 1136, 4201, 3962, 540, 685, 2328, 2017, 318, 4792, 1636, 1033, 1393, 374, 783, 4625, 3402, 1004, 4011, 3436, 1285, 2303, 1798, 1209, 1448, 1459, 285, 1481, 909, 2392, 2945, 4248, 545, 2882, 3253, 920, 2712, 561, 3686, 1608, 1699, 1541, 2804, 1196, 4970, 2970, 3898, 2464, 2780, 3631, 3789, 1600, 3317, 29, 2170, 4580, 3377, 1627, 3244, 1526, 610, 1081, 1099, 546, 4991, 4385, 1734, 416, 4368, 4015, 3527, 3237, 528, 4619, 4519, 1781, 4926, 4009, 255, 1227, 2343, 2813, 1567, 4514, 1837, 4497, 2825, 2182, 4308, 1550, 2248, 3232, 4683, 2837, 310, 1036, 2972, 1949, 4864, 4264, 3869, 772, 1252, 1904, 330, 2735, 987, 394, 2495, 2519, 507, 992, 1797, 1732, 2648, 3606, 550, 1170, 4893, 1449, 3473, 47, 4886, 4032, 1555, 2717, 1701, 1722, 2001, 3733, 1088, 4527, 3958, 211, 3719, 1568, 4502, 294, 3009, 3489, 2810, 4438, 4707, 154, 1648, 2627, 3623, 3590, 2277, 1515, 1509, 2058, 1897, 3335, 4407, 1010, 3001, 4305, 268, 246, 1646, 2951, 3276, 4662, 3539, 12, 4010, 3446, 431, 4806, 1203, 4183, 2985, 4312, 1576, 4537, 3095, 4218, 360, 4711, 4316, 3609, 4346, 3868, 4904, 78, 3376, 4660, 190, 578, 564, 1921, 3023, 4088, 1783, 3772, 2546, 4117, 1693, 4399, 4833, 2428, 4170, 1630, 1135, 3291, 153, 59, 1153, 852, 4493, 4333, 754, 3921, 881, 1569, 1378, 2336, 1790, 4971, 4532, 2526, 2887, 3155, 2851, 587, 3192, 2757, 1076, 1889, 1092, 1851, 4526, 222, 2034, 2178, 1860, 3455, 3928, 4546, 3052, 629, 4272, 2748, 3193, 3853, 4653, 3164, 1988, 4065, 2142, 151, 4851, 1124, 3160, 167, 982, 3735, 958, 1968, 4713, 511, 3762, 2533, 510, 4285, 3127, 3390, 3629, 3242, 2122, 2485, 2974, 2910, 4382, 2539, 2293, 3465, 3426, 2770, 4187, 4764, 3072, 1888, 2480, 4858, 1277, 3878, 82, 3254, 4024, 948, 2159, 4773, 3517, 2166, 997, 3449, 463, 4890, 1164, 166, 551, 2663, 3956, 2420, 2411, 1086, 3716, 1715, 3359, 4823, 105, 1733, 4281, 873, 2333, 2694, 4337, 1305, 1810, 1704, 2401, 3410, 13, 3468, 3600, 3047, 1849, 2413, 3880, 2136, 3331, 3903, 307, 4918, 4156, 1155, 861, 3154, 3323, 461, 1717, 450, 4114, 2625, 3699, 1505, 2141, 3425, 4554, 4854, 4545, 2714, 191, 4106, 1056, 3089, 2834, 3792, 1913, 2345, 276, 146, 4996, 3596, 1524, 666, 3370, 1283, 1571, 2574, 2398, 2033, 4178, 4762, 4922, 674, 288, 341, 4698, 2446, 3348, 3399, 3664, 2439, 2558, 1324, 580, 2564, 2692, 3012, 544, 1613, 2459, 1581, 3757, 2652, 2992, 983, 3204, 2235, 2835, 980, 1034, 4480, 2218, 1742, 5004, 4334, 4923, 4724, 1029, 256, 3453, 769, 4742, 3850, 411, 3190, 4320, 402, 4253, 3706, 139, 531, 4911, 87, 4237, 2520, 2736, 1461, 2028, 4365, 3213, 4329, 1879, 3551, 1093, 4352, 3676, 3271, 892, 697, 1930, 407, 736, 1066, 2419, 277, 4476, 1286, 4060, 4686, 2316, 2272, 4894, 4177, 4819, 4644, 3810, 1942, 123, 1965, 2959, 4219, 2049, 994, 4816, 1455, 1645, 3406, 942, 781, 1691, 4956, 4775, 4434, 1147, 922, 73, 149, 4945, 803, 2470, 4855, 3345, 3586, 2268, 339, 1596, 4637, 4197, 854, 2317, 4270, 4284, 1908, 1175, 195, 2046, 3608, 34, 2577, 4682, 3821, 333, 213, 814, 4035, 1855, 4145, 399, 2021, 2746, 3510, 2085, 2688, 1905, 4481, 2989, 4547, 614, 513, 3666, 800, 1328, 2737, 244, 833, 1769, 373, 2503, 1647, 2938, 867, 1929, 1635, 24, 2605, 2957, 3780, 182, 1703, 241, 2188, 1002, 3060, 4238, 1891, 3802, 301, 3793, 3582, 1453, 2487, 4856, 698, 2340, 4392, 623, 4311, 36, 4506, 978, 2099, 4398, 4615, 3619, 575, 4763, 2702, 2332, 908, 1601, 3097, 938, 3038, 1507, 2440, 518, 2019, 1907, 315, 2506, 3014, 1441, 1794, 1051, 1971, 2749, 638, 2497, 4874, 3157, 4674, 2839, 4061, 2561, 2275, 1073, 3414, 1432, 435, 2559, 33, 3905, 3665, 1475, 4217, 658, 1826, 4157, 4258, 1074, 753, 4282, 4374, 1681, 4756, 3979, 4515, 4381, 443, 4230, 2801, 2809, 2798, 676, 3000, 44, 2803, 4463, 1692, 2483, 1964, 2568, 3028, 75, 1696, 4023, 2802, 1728, 480, 2917, 4888, 3734, 1218, 1018, 720, 4542, 2123, 3725, 2699, 2848, 2276, 2176, 4181, 632, 3203, 3656, 1337, 4832, 3578, 4257, 4107, 329, 3541, 282, 917, 612, 725, 2126, 22, 2991, 4977, 3087, 1063, 2168, 3749, 1595, 1886, 1718, 1533, 2935, 3896, 2423, 3872, 4111, 3437, 3703, 3910, 4199, 4825, 2623, 3091, 977, 1202, 1433, 3781, 4492, 1452, 4387, 2873, 3221, 2872, 617, 4961, 871, 3338, 2827, 3567, 1238, 2953, 3715, 1562, 4959, 3439, 4469, 1435, 1935, 466, 4989, 766, 156, 4292, 2572, 2677, 794, 773, 370, 455, 3798, 508, 2517, 112, 4699, 2362, 595, 4358, 4408, 2476, 832, 1658, 1843, 795, 3286, 1386, 1873, 55, 4364, 1354, 898, 3691, 2407, 1325, 4313, 2584, 3256, 1279, 936, 239, 4155, 2916, 4932, 2741, 4777, 4752, 1975, 3024, 4584, 4909, 2769, 4709, 4386, 4214, 3054, 1631, 4753, 1995, 2121, 1661, 1185, 4935, 1304, 4677, 746, 860, 3341, 1607, 4006, 2782, 4946, 2005, 3451, 4494, 952, 2841, 656, 771, 620, 4836, 1820, 2587, 1625, 3960, 3657, 2549, 2628, 1966, 4607, 93, 1392, 2672, 2071, 4895, 2792, 1245, 4636, 3207, 529, 2364, 4824, 2726, 1870, 844, 3, 2264, 3648, 4297, 1564, 3514, 2581, 371, 260, 749, 2598, 1519, 4471, 3520, 52, 4279, 3004, 28, 1412, 274, 1042, 4472, 3029, 1091, 3647, 3045, 3848, 1927, 4847, 3501, 1376, 2644, 1128, 446, 4451, 1177, 2854, 1308, 4640, 3528, 2283, 1317, 3815, 2337, 4428, 2334, 2266, 2847, 3549, 460, 2528, 2942, 4916, 357, 3246, 3303, 3728, 1532, 2763, 3075, 2184, 383, 4, 2601, 434, 1440, 784, 2394, 2089, 4743, 2106, 2898, 3336, 2518, 1485, 3241, 363, 1023, 2630, 1853, 1952, 925, 2417, 5002, 4646, 4415, 4818, 2299, 27, 4084, 271, 4283, 1970, 3372, 18, 4314, 4295, 1222, 527, 1909, 4744, 3226, 815, 526, 1487, 3944, 4173, 3569, 316, 2592, 125, 1169, 3838, 4466, 1787, 4348, 2015, 3082, 4202, 2579, 4501, 485, 4076, 2162, 4692, 404, 1599, 2018, 4703, 1469, 4411, 4306, 1606, 1158, 2070, 253, 2203, 999, 3779, 2641, 880, 3626, 457, 3953, 2548, 4112, 1242, 4059, 433, 3430, 2774, 2060, 4508, 2369, 1109, 2961, 4804, 3765, 4003, 2496, 4780, 1678, 1802, 4679, 2243, 1365, 3840, 494, 1508, 3825, 3985, 2348, 249, 745, 3495, 1097, 1928, 3722, 3396, 1058, 4322, 491, 2560, 4013, 2349, 639, 3696, 3176, 4069, 4330, 3492, 4290, 3922, 2679, 3836, 117, 2481, 1672, 1660, 1547, 3219, 3211, 886, 88, 2232, 2267, 4280, 3636, 4026, 250, 850, 519, 3894, 4587, 4110, 2489, 131, 2032, 1745, 2550, 3957, 3660, 607, 296, 4124, 2211, 3941, 4575, 1626, 1521, 4716, 4857, 1610, 338, 2466, 4192, 171, 890, 1534, 1464, 586, 504, 3897, 4880, 2350, 542, 2000, 3119, 2947, 4079, 2231, 973, 3718, 1368, 2719, 775, 2359, 4128, 3694, 1357, 1470, 3945, 2784, 2529, 4746, 2713, 2256, 3863, 2043, 4695, 4014, 3472, 2734, 102, 1197, 1823, 3670, 429, 3050, 2532, 1369, 1161, 4611, 2222, 2445, 3025, 4025, 232, 2527, 849, 1012, 1235, 3942, 1525, 1157, 3677, 3809, 4739, 2817, 2354, 3951, 1233, 54, 19, 284, 2994, 1859, 2246, 4087, 4929, 4651, 1959, 1237, 2978, 1972, 831, 887, 3908, 2993, 4622, 4344, 4339, 2705, 1663, 415, 4860, 168, 971, 1898, 1028, 1367, 3456, 1694, 2958, 3720, 2331, 4343, 1637, 1771, 4309, 3392, 3330, 1665, 3450, 3191, 3842, 1095, 2814, 1963, 1476, 748, 3645, 4390, 2081, 3346, 3668, 308, 3820, 1221, 4115, 1563, 1756, 1480, 1788, 4822, 3340, 1183, 4710, 4985, 896, 1947, 1789, 3375, 3205, 2939, 4797, 3040, 4571, 4676, 742, 1284, 2653, 848, 2880, 4849, 355, 3350, 1716, 2291, 4941, 1022, 728, 2294, 337, 2444, 714, 386, 3564, 4341, 894, 3760, 2669, 2616, 1340, 2881, 212, 4474, 592, 2441, 3110, 2280, 3851, 1125, 1083, 853, 2074, 4445, 2912, 1445, 2541, 3466, 4510, 3209, 3158, 4209, 1984, 4070, 1122, 4924, 1154, 4555, 1617, 1565, 4722, 4485, 3143, 3837, 2842, 2208, 2279, 4879, 1259, 1246, 4350, 4534, 2447, 3387, 2112, 2438, 1578, 801, 523, 3870, 3020, 1877, 401, 2941, 3262, 4846, 1782, 4418, 649, 1290, 152, 2859, 4708, 1213, 1225, 4057, 1188, 3499, 2135, 4046, 2, 1021, 4212, 4548, 4531, 885, 4768, 2324, 1055, 717, 1138, 565, 4090, 765, 143, 3811, 1916, 1426, 703, 412, 569, 3462, 3427, 351, 2934, 2048, 2731, 3320, 3475, 2011, 410, 2082, 1477, 2656, 3306, 3529, 2913, 1173, 763, 261, 3633, 176, 4206, 1075, 2551, 1140, 862, 2954, 4902, 265, 4162, 395, 1450, 1488, 1552, 4978, 2901, 3959, 3177, 4247, 2319, 3875, 2088, 4395, 3085, 2169, 1549, 4528, 2260, 121, 5007, 823, 3807, 3480, 912, 1803, 3852, 4647, 118, 2189, 3490, 2297, 3705, 379, 2292, 1489, 883, 4596, 2708, 3625, 828, 1960, 895, 1032, 1151, 1127, 4122, 2662, 4488, 3767, 3597, 25, 281, 1346, 3607, 4484, 4558, 1193, 3570, 1629, 96, 3093, 1685, 1079, 2455, 2607, 4796, 484, 4321, 2022, 2360, 2771, 596, 489, 1846, 3973, 1335, 2879, 495, 2536, 4630, 2783, 4960, 2416, 3860, 2687, 2787, 1684, 976, 3432, 4143, 2199, 2253, 664, 2542, 1967, 421, 3413, 1094, 2214, 3637, 1228, 289, 3408, 1740, 3086, 4267, 3649, 1977, 2259, 2667, 988, 954, 3156, 1772, 2314, 1531, 3316, 1998, 2811, 1594, 1667, 1999, 1275, 2729, 3711, 2409, 21, 806, 4600, 2494, 4781, 2654, 4612, 2023, 2864, 4056, 3924, 2320, 2594, 2878, 1914, 2067, 4910, 601, 2973, 4052, 4109, 2344, 2062, 2115, 2612, 3463, 3693, 3228, 4018, 2709, 2753, 945, 4702, 3824, 2599, 2990, 2395, 863, 3311, 1359, 1831, 3708, 4861, 4378, 2740, 1643, 567, 1633, 3438, 4413, 4556, 106, 509, 1254, 3650, 2138, 2988, 83, 3563, 670, 58, 4235, 1288, 1784, 3554, 1847, 2613, 4008, 4770, 4684, 4064, 2479, 2930, 3952, 2629, 2603, 4338, 1269, 1014, 2155, 4021, 2556, 1917, 4712, 1299, 1244, 633, 4834, 1105, 4769, 4422, 413, 3562, 2335, 4998, 440, 3278, 1537, 3774, 3599, 2289, 3876, 3689, 3300, 1932, 4433, 1759, 17, 864, 110, 3199, 1729, 4871, 365, 1768, 1828, 3459, 2624, 921, 4815, 3163, 2762, 1117, 4195, 2307, 675, 4795, 3503, 2759, 3491, 4994, 2091, 2596, 934, 4593, 1622, 2768, 3634, 3288, 1817, 4814, 3544, 132, 2242, 840, 3134, 1542, 785, 3198, 1961, 566, 0, 4053, 4161, 1611, 2318, 4138, 2498, 3558, 1974, 4969, 3452, 4828, 3923, 932, 3891, 3308, 409, 1841, 37, 2894, 1054, 1746, 376, 2374, 4655, 3309, 2059, 4975, 1586, 2678, 1191, 1312, 1047, 175, 4426, 3954, 3208, 3394, 4761, 2926, 1710, 2964, 4363, 715, 2352, 2164, 4098, 4229, 488, 1, 1425, 158, 1818, 1264, 1937, 1805, 4967, 711, 4118, 458, 2617, 866, 1835, 1587, 1830, 4470, 2385, 2946, 3488, 3832, 1298, 1778, 138, 1836, 1172, 2670, 4507, 2799, 2185, 4873, 3830, 4517, 1657, 3525, 928, 3181, 3337, 236, 2339, 3617, 1301, 2130, 389, 4912, 1546, 1796, 4224, 4354, 184, 137, 422, 4863, 3502, 3236, 4829, 4401, 1276, 865, 4371, 1061, 4044, 1579, 1333, 220, 2547, 3197, 2544, 2432, 1043, 2180, 3663, 642, 2710, 1997, 4397, 3277, 1503, 4552, 1162, 3667, 2900, 2698, 1522, 2609, 4585, 4872, 1598, 4974, 2371, 2200, 348, 2194, 1167, 1767, 3227, 1466, 3162, 2233, 313, 3081, 824, 1705, 2120, 3756, 4249, 427, 959, 3314, 3368, 4737, 3215, 3593, 1989, 2755, 4412, 4626, 193, 2227, 3135, 3854, 3167, 594, 3886, 164, 1500, 2516, 1064, 4089, 2634, 1911, 4154, 3543, 3990, 4038, 1747, 500, 2008, 4569, 3777, 2987, 4236, 1666, 2434, 2353, 1187, 148, 3249, 3986, 1743, 86, 4901, 923, 3938, 1839, 3224, 552, 3651, 2685, 3391, 2458, 343, 3584, 1807, 3035, 3642, 1566, 686, 23, 4990, 3768, 4921, 1845, 2287, 1216, 4907, 1020, 403, 1200, 3654, 2026, 2205, 1634, 2086, 247, 49, 1350, 1409, 3036, 4475, 4452, 635, 3349, 2591, 4562, 5005, 2012, 4291, 4108, 1049, 1955, 3826, 4729, 4431, 3343, 2027, 1206, 3079, 4869, 521, 3548, 3071, 108, 3321, 2125, 4191, 1934, 957, 3900, 2171, 1885, 2257, 4678, 996, 1005, 2073, 95, 4885, 3464, 391, 30, 4675, 696, 2223, 197, 1721, 2569, 408, 1588, 16, 3950, 4760, 1558, 216, 3504, 4938, 2765, 3901, 1520, 67, 2642, 437, 786, 1983, 1944, 3618, 2524, 1414, 821, 707, 1181, 3658, 3804, 4367, 3515, 4691, 517, 3470, 2457, 893, 2622, 974, 4357, 2797, 2368, 1690, 4843, 2217, 2262, 1007, 3002, 2143, 4034, 3835, 2160, 2468, 4436, 1306, 1605, 1265, 1580, 1878, 3713, 2181, 4315, 2014, 2858, 3965, 3803, 2078, 4196, 1987, 4062, 172, 1323, 1334, 428, 576, 2388, 3106, 2500, 1179, 3486, 1709, 3917, 1102, 1250, 1089, 1458, 1484, 2375, 1864, 4632, 4223, 874, 63, 210, 4446, 3409, 1416, 3487, 2716, 2645, 2321, 3042, 1258, 4380, 1302, 3259, 1355, 1785, 1553, 3561, 4983, 2258, 3978, 2149, 4749, 3185, 4800, 3114, 555, 906, 2361, 1982, 48, 1331, 4302, 2390, 1739, 2263, 1215, 4993, 2083, 342, 3935, 4355, 3630, 2278, 3055, 1178, 2693, 2875, 2302, 4604, 4984, 3447, 3125, 4659, 4790, 3784, 243, 2209, 229, 4723, 740, 2100, 251, 591, 1267, 505, 2732, 362, 630, 3817, 876, 367, 2056, 1523, 1575, 2690, 128, 4372, 1956, 99, 1168, 3230, 4000, 2721, 1726, 4373, 4467, 3378, 1380, 2800, 4027, 4516, 1379, 4457, 1777, 1045, 4132, 1373, 3904, 2675, 2261, 677, 4906, 2899, 3659, 4468, 436, 1985, 3742, 144, 3507, 1838, 1483, 2676, 524, 1352, 1688, 599, 651, 1256, 3727, 4972, 4685, 381, 4920, 3361, 4852, 2638, 4566, 4551, 1068, 280, 613, 539, 4588, 2761, 1893, 2806, 3998, 4349, 375, 3644, 1529, 2793, 3332, 3845, 3552, 839, 1900, 888, 716, 2101, 4866, 2031, 4033, 4701, 2703, 204, 4504, 317, 3098, 2998, 3415, 2723, 3102, 4460, 4785, 2040, 722, 3212, 856, 668, 2020, 590, 378, 4163, 1865, 2996, 2711, 3794, 3974, 2376, 4204, 3991, 4153, 444, 1615, 817, 3152, 4045, 469, 930, 2402, 4228, 4830, 4113, 3363, 3389, 3885, 319, 287, 816, 65, 3128, 4462, 2245, 1186, 4171, 2745, 2522, 4450, 964, 1009, 843, 4244, 205, 4620, 4120, 472, 2389, 525, 3565, 4982, 4715, 756, 3360, 4490, 3932, 3229, 1194, 708, 2462, 467, 3592, 3319, 4518, 1126, 3273, 1077, 4721, 3354, 2192, 4539, 1612, 1116, 667, 2133, 2363, 760, 3730, 4718, 3533, 818, 1467, 1430, 124, 2664, 2463, 4414, 474, 2888, 4125, 2213, 2174, 2007, 8, 2387, 2187, 3454, 3532, 1754, 3131, 1375, 1969, 2832, 4842, 2154, 1145, 3969, 2540, 585, 2968, 2862, 4789, 3460, 1939, 3523, 4919, 2868, 2997, 4211, 1463, 4043, 1274, 4220, 3044, 1753, 4656, 944, 3808, 1152, 385, 4409, 4239, 709, 4081, 4690, 94, 4389, 2251, 1338, 3524, 101, 4875, 4005, 454, 4940, 4635, 1948, 1134, 2215, 1779, 2883, 2758, 3279, 2513, 174, 1385, 2186, 3995, 4623, 719, 2671, 3150, 4131, 3797, 1282, 4850, 2224, 4225, 4680, 3776, 2874, 2477, 3258, 155, 4689, 2695, 3369, 1176, 393, 4572, 76, 3934, 1281, 4747, 5001, 4048, 3939, 4208, 1906, 4652, 2072, 4263, 4317, 1396, 1339, 425, 388, 2077, 3610, 3611, 499, 3247, 3440, 2148, 3373, 1059, 2779, 2110, 758, 4720, 3747, 4251, 3267, 4541, 4406, 1814, 1973, 1133, 3632, 4808, 1492, 4624, 563, 4730, 2252, 993, 2920, 2418, 924, 1251, 4441, 2296, 126, 4903, 820, 1792, 911, 812, 3739, 2523, 3287, 1945, 227, 4071, 4020, 459, 2791, 1671, 1597, 3182, 4905, 3980, 3926, 3366, 3210, 77, 4687, 737, 1363, 1090, 3752, 1669, 2380, 1493, 3753, 648, 4605, 1687, 3225, 1498, 3602, 2610, 1924, 3328, 4681, 2310, 3274, 2396, 1347, 3051, 653, 3312, 3113, 3966, 1686, 471, 3115, 750, 3122, 26, 4694, 2137, 3381, 4146, 1148, 3005, 4735, 2682, 3184, 3111, 1374, 4784, 3919, 3357, 4533, 752, 2177, 4286, 1698, 2454, 5003, 4266, 4361, 4540, 20, 2044, 1559, 3750, 2255, 14, 825, 4583, 819, 705, 804, 2631, 344, 1395, 1938, 3419, 3266, 1261, 619, 4955, 2673, 3702, 847, 45, 1108, 185, 423, 3146, 1329, 4943, 2329, 3428, 1163, 2943, 1662, 2931, 3096, 2905, 3007, 4591, 1659, 3764, 3187, 4791, 3761, 2924, 303, 1296, 3690, 1210, 3257, 2571, 4639, 79, 4520, 558, 4603, 3911, 2204, 498, 2035, 4597, 3984, 4022, 4420, 734, 981, 4037, 1632, 2284, 2588, 3883, 3223, 1517, 4870, 2871, 678, 3895, 560, 3293, 855, 1861, 3526, 4801, 3940, 4986, 2484, 2595, 2006, 3136, 4174, 4393, 4601, 207, 3948, 264, 979, 955, 3865, 600, 4672, 1447, 695, 2425, 2960, 3355, 4997, 4360, 3334, 2570, 4268, 1442, 382, 4078, 147, 1528, 2655, 3585, 4444, 929, 1876, 2201, 3144, 3037, 3171, 2715, 3058, 3498, 1583, 4891, 3879, 2831, 926, 3421, 1052, 1382, 4766, 3172, 2244, 2383, 2054, 4489, 3576, 4610, 4897, 3074, 764, 2095, 1410, 145, 1118, 4898, 2415, 537, 3032, 1856, 549, 1026, 4049, 949, 556, 4859, 802, 3714, 4944, 1750, 2689, 2128, 2728, 4673, 3559, 735, 3624, 283, 514, 2193, 486, 2448, 723, 1436, 4511, 4185, 778, 3175, 4077, 1320, 2583, 3884, 273, 2805, 3159, 2236, 956, 3545, 1370, 3982, 2821, 1129, 4126, 223, 636, 2999, 4063, 4799, 2436, 963, 2680, 3500, 3367, 4366, 1080, 69, 2846, 4559, 1892, 1166, 4325, 2778, 616, 3474, 724, 2175, 2773, 2907, 4430, 3379, 1548, 3149, 441, 1761, 4116, 1958, 3946, 1931, 3108, 4884, 3683, 398, 1799, 2357, 3536, 2016, 1506, 1271, 2952, 4754, 2250, 1041, 4616, 3393, 3996, 1472, 3351, 1111], "validation": [5307, 5264, 5452, 5123, 5402, 5467, 5330, 5106, 5161, 5449, 5320, 5182, 5381, 5436, 5021, 5083, 5163, 5115, 5095, 5051, 5219, 5371, 5413, 5377, 5165, 5224, 5181, 5438, 5385, 5200, 5175, 5354, 5428, 5326, 5145, 5183, 5277, 5065, 5315, 5169, 5194, 5461, 5265, 5079, 5162, 5432, 5453, 5321, 5353, 5039, 5185, 5474, 5372, 5463, 5296, 5047, 5302, 5444, 5068, 5284, 5248, 5100, 5111, 5489, 5399, 5433, 5063, 5085, 5316, 5309, 5170, 5124, 5096, 5069, 5130, 5019, 5351, 5038, 5343, 5214, 5434, 5379, 5134, 5244, 5352, 5460, 5064, 5108, 5191, 5318, 5440, 5138, 5101, 5298, 5342, 5152, 5114, 5011, 5478, 5246, 5401, 5310, 5477, 5204, 5275, 5040, 5338, 5020, 5424, 5421, 5308, 5119, 5227, 5304, 5102, 5057, 5247, 5239, 5192, 5391, 5091, 5272, 5028, 5128, 5380, 5490, 5485, 5280, 5404, 5415, 5135, 5362, 5174, 5014, 5360, 5386, 5289, 5370, 5032, 5254, 5480, 5495, 5419, 5396, 5201, 5180, 5090, 5365, 5286, 5497, 5312, 5029, 5285, 5323, 5429, 5253, 5505, 5237, 5403, 5186, 5335, 5164, 5046, 5319, 5156, 5361, 5017, 5459, 5250, 5207, 5468, 5016, 5410, 5110, 5339, 5155, 5198, 5015, 5363, 5212, 5213, 5300, 5226, 5301, 5378, 5117, 5420, 5086, 5076, 5043, 5416, 5446, 5061, 5506, 5266, 5317, 5171, 5297, 5209, 5240, 5367, 5074, 5056, 5167, 5140, 5498, 5073, 5041, 5314, 5206, 5151, 5112, 5066, 5422, 5327, 5347, 5122, 5491, 5451, 5048, 5022, 5139, 5067, 5369, 5042, 5267, 5159, 5037, 5400, 5368, 5030, 5018, 5188, 5397, 5233, 5010, 5349, 5414, 5443, 5447, 5273, 5479, 5045, 5331, 5218, 5487, 5070, 5121, 5398, 5283, 5269, 5393, 5305, 5223, 5274, 5143, 5060, 5299, 5137, 5427, 5216, 5026, 5157, 5270, 5282, 5136, 5257, 5493, 5290, 5268, 5147, 5238, 5364, 5228, 5255, 5263, 5078, 5430, 5120, 5389, 5177, 5077, 5387, 5333, 5328, 5094, 5234, 5024, 5166, 5344, 5488, 5036, 5054, 5062, 5483, 5178, 5262, 5475, 5113, 5023, 5243, 5482, 5366, 5187, 5476, 5287, 5425, 5454, 5055, 5153, 5466, 5071, 5081, 5278, 5087, 5418, 5494, 5148, 5131, 5107, 5053, 5408, 5098, 5141, 5412, 5313, 5199, 5470, 5340, 5072, 5295, 5411, 5132, 5080, 5259, 5341, 5324, 5329, 5044, 5348, 5256, 5472, 5481, 5375, 5052, 5496, 5197, 5154, 5336, 5388, 5502, 5417, 5507, 5383, 5271, 5235, 5455, 5504, 5293, 5176, 5205, 5179, 5492, 5129, 5356, 5473, 5089, 5445, 5249, 5050, 5345, 5105, 5406, 5232, 5217, 5210, 5196, 5501, 5471, 5184, 5158, 5457, 5508, 5118, 5242, 5190, 5220, 5382, 5281, 5035, 5215, 5358, 5193, 5058, 5189, 5245, 5350, 5059, 5337, 5097, 5230, 5033, 5500, 5276, 5325, 5168, 5456, 5144, 5322, 5025, 5049, 5261, 5294, 5448, 5012, 5203, 5435, 5462, 5394, 5311, 5390, 5258, 5431, 5376, 5499, 5291, 5075, 5279, 5034, 5409, 5373, 5306, 5150, 5172, 5355, 5426, 5442, 5303, 5082, 5104, 5126, 5146, 5288, 5103, 5149, 5109, 5441, 5458, 5125, 5260, 5509, 5464, 5116, 5013, 5384, 5231, 5241, 5088, 5031, 5127, 5229, 5195, 5332, 5503, 5099, 5486, 5423, 5484, 5252, 5208, 5405, 5225, 5222, 5346, 5133, 5357, 5359, 5395, 5469, 5084, 5093, 5160, 5334, 5202, 5092, 5374, 5142, 5292, 5450, 5437, 5173, 5407, 5221, 5392, 5236, 5439, 5211, 5027, 5251, 5465], "test": [5585, 5901, 5602, 5612, 5511, 5609, 5711, 5773, 5873, 5679, 5920, 5613, 5818, 5636, 5607, 5864, 5854, 5654, 5973, 5755, 5935, 5820, 5924, 5681, 5581, 5614, 5718, 5897, 5536, 5631, 5695, 5814, 5665, 5569, 5519, 5641, 5752, 5804, 5516, 5615, 5917, 5981, 5700, 5546, 5894, 5989, 5611, 5870, 5543, 5715, 5900, 5637, 5689, 5819, 5676, 5822, 5576, 5693, 5896, 5821, 5828, 5779, 5761, 5619, 5589, 5555, 5542, 5557, 5805, 5785, 5851, 5694, 5936, 5842, 5788, 5550, 5978, 5730, 5848, 5980, 5868, 5903, 5772, 5780, 5886, 5712, 5912, 5673, 5807, 5719, 5944, 5908, 5985, 5647, 5537, 5688, 5687, 5832, 5997, 5586, 5561, 5743, 5835, 5940, 5952, 5556, 6005, 6001, 5951, 5756, 5823, 6002, 5703, 5659, 5548, 5578, 5692, 5963, 5884, 5570, 5624, 5777, 5742, 5968, 5904, 5738, 5765, 5969, 5644, 5887, 5863, 5922, 5933, 5566, 5571, 5891, 5664, 5770, 5674, 5993, 5531, 5880, 5907, 5774, 5839, 5530, 5666, 5955, 5893, 5965, 5564, 5623, 5625, 5709, 5909, 5925, 5782, 5568, 5790, 5754, 5529, 5643, 5667, 5781, 5651, 5677, 5945, 5724, 5706, 5793, 5638, 5866, 5579, 5601, 5803, 5953, 5994, 5672, 5588, 5528, 5877, 5801, 5741, 5857, 5697, 5984, 5518, 5652, 5961, 5552, 5662, 5583, 5745, 5767, 5590, 5527, 5783, 5849, 5572, 5707, 5584, 5580, 5582, 5670, 5974, 5565, 5535, 5844, 5633, 5776, 5646, 5867, 5905, 5539, 5632, 5962, 5855, 5675, 6003, 5699, 5733, 5705, 5946, 5727, 5809, 5930, 5815, 5510, 5836, 5830, 5642, 5596, 5987, 5544, 5932, 5698, 5680, 6008, 5736, 5762, 5927, 5938, 5871, 5649, 5937, 5919, 5686, 5829, 5764, 5845, 5906, 5653, 5837, 6000, 5731, 5846, 5931, 5554, 5604, 5810, 5816, 5525, 5640, 5753, 5621, 5739, 5524, 5910, 5547, 5744, 5947, 5737, 5668, 5954, 5684, 5812, 5859, 5660, 5971, 5514, 5800, 5860, 5939, 5723, 5526, 5661, 5964, 5941, 5878, 5575, 5671, 5639, 5838, 5950, 5562, 5563, 5979, 5768, 5928, 5558, 5996, 5847, 5716, 5597, 5533, 5620, 5824, 5717, 5655, 6009, 5789, 5902, 5541, 5591, 5853, 5683, 5916, 5749, 5729, 5808, 5972, 5918, 5957, 5923, 5746, 5959, 5911, 5726, 5650, 5520, 5658, 5862, 5895, 5869, 5983, 5553, 5960, 5898, 5795, 5732, 5858, 5888, 5977, 5678, 5921, 5567, 5861, 5990, 5713, 5701, 5617, 5975, 5626, 5721, 5627, 5592, 5889, 5988, 5991, 5787, 5682, 5791, 5998, 5594, 5956, 5913, 5540, 5899, 5728, 5735, 5759, 5645, 5748, 5817, 5986, 5875, 5629, 5599, 5796, 5634, 5797, 6006, 5720, 5751, 5850, 5874, 5976, 5685, 5598, 5834, 5517, 5648, 5872, 6007, 5757, 5512, 5513, 5833, 5606, 5856, 5760, 5926, 5618, 5775, 5515, 5628, 5934, 5622, 5966, 5610, 5690, 5696, 5885, 5750, 5534, 5852, 5560, 5949, 5538, 5967, 5603, 5747, 5722, 5784, 5740, 5708, 5616, 5999, 5587, 5663, 5942, 5831, 5970, 5827, 6004, 5806, 5600, 5890, 5522, 5929, 5798, 5778, 5714, 5883, 5595, 5573, 5704, 5577, 5769, 5879, 5943, 5794, 5608, 5551, 5825, 5892, 5593, 5992, 5691, 5549, 5545, 5521, 5914, 5826, 5958, 5605, 5523, 5725, 5786, 5915, 5865, 5635, 5882, 5792, 5771, 5657, 5630, 5876, 5669, 5702, 5881, 5763, 5813, 5532, 5710, 5948, 5982, 5995, 5811, 5734, 5656, 5802, 5840, 5843, 5841, 5766, 5799, 5559, 5758, 5574]}, {"train": [1529, 509, 2210, 4777, 3896, 1108, 1723, 8, 4958, 4307, 492, 1639, 1211, 273, 4677, 3323, 3636, 2530, 29, 358, 4957, 1759, 4874, 1193, 2051, 3779, 2887, 728, 4792, 3767, 4908, 2554, 822, 620, 4207, 2877, 1274, 3949, 3927, 1911, 86, 1072, 3317, 1756, 4955, 4318, 2398, 420, 1569, 1170, 159, 4847, 2412, 198, 3437, 4691, 4228, 1024, 2895, 1327, 1894, 4625, 4038, 1295, 1785, 2799, 4494, 939, 3033, 2546, 102, 3445, 1520, 2984, 697, 1429, 1138, 4636, 1810, 1028, 1258, 1452, 2695, 1436, 3858, 36, 425, 3101, 1883, 654, 1846, 3580, 1220, 1895, 4294, 2514, 4561, 225, 4505, 683, 2372, 1803, 516, 2317, 4416, 1032, 553, 1754, 1147, 1944, 2610, 1958, 3080, 3125, 4744, 2697, 1550, 3605, 3848, 1331, 2666, 536, 4976, 2016, 4559, 427, 3439, 1101, 4116, 4366, 4982, 1672, 514, 4579, 906, 485, 1943, 707, 3943, 4411, 2855, 3776, 928, 1434, 1073, 1478, 4875, 2495, 4772, 4995, 1770, 3613, 3075, 4152, 1673, 3087, 598, 4659, 3480, 925, 2063, 2042, 2013, 3683, 393, 1932, 2214, 4180, 3016, 2188, 4858, 2014, 4861, 4191, 676, 4433, 59, 3072, 772, 177, 4132, 1423, 2377, 2173, 4612, 791, 4924, 27, 2655, 3107, 1471, 549, 2190, 2743, 956, 4564, 171, 3959, 1457, 2641, 920, 4010, 316, 3027, 3200, 1748, 3222, 1680, 792, 3460, 2845, 1576, 2772, 2934, 4953, 3348, 2106, 526, 3198, 1947, 3171, 967, 4002, 1095, 243, 2486, 2544, 2461, 1872, 3434, 669, 1730, 3102, 4729, 1008, 3473, 1130, 329, 931, 3050, 3745, 389, 56, 4882, 4594, 4635, 3791, 2801, 3368, 1657, 644, 1394, 2582, 3335, 3237, 3525, 578, 2816, 1705, 4956, 286, 1373, 4657, 2928, 2625, 3880, 475, 1850, 1624, 622, 3423, 1289, 3581, 3993, 3783, 3402, 1527, 3005, 3963, 3232, 3768, 1268, 3265, 2466, 2682, 2821, 1343, 2300, 1620, 4707, 657, 4409, 4215, 489, 130, 2782, 3639, 3821, 308, 4568, 2971, 1708, 1155, 3188, 165, 801, 3202, 2916, 1821, 241, 943, 415, 3328, 3446, 1057, 4949, 888, 3577, 1386, 3071, 5006, 2230, 2746, 4049, 4262, 158, 869, 832, 1980, 4394, 1069, 3930, 3259, 2734, 3856, 139, 1865, 914, 99, 92, 3216, 1907, 628, 1917, 2778, 1235, 2161, 2601, 4136, 4768, 1325, 4821, 3230, 1308, 4263, 3100, 4282, 4072, 1798, 58, 357, 2298, 2763, 2917, 1124, 1441, 3780, 4671, 2253, 1747, 1326, 3092, 901, 845, 3214, 4629, 1963, 970, 1719, 4731, 135, 3367, 176, 3643, 1297, 2380, 3703, 2863, 3792, 2039, 2138, 2692, 3431, 4784, 4115, 2266, 2048, 3576, 4434, 2848, 4914, 4061, 3919, 3400, 2648, 4696, 974, 1494, 3738, 3283, 2068, 4840, 1849, 295, 94, 4445, 1568, 1179, 2085, 466, 4932, 937, 3622, 3793, 4830, 41, 1300, 3536, 639, 3264, 3038, 849, 2999, 910, 981, 3526, 4195, 3365, 3546, 4171, 1704, 2264, 2261, 3926, 3161, 3093, 4137, 4447, 3479, 1896, 68, 2940, 2336, 2788, 436, 4076, 3231, 3441, 1025, 3516, 4827, 4670, 2528, 1926, 1863, 1772, 1174, 2316, 1162, 4832, 524, 4235, 14, 474, 2365, 4003, 1533, 2833, 4422, 1814, 290, 2080, 561, 1278, 4611, 2494, 629, 3455, 591, 999, 3874, 2326, 3814, 2574, 266, 352, 3022, 501, 2878, 3047, 3778, 991, 1053, 3753, 2881, 1371, 1960, 3873, 2899, 1153, 4880, 4468, 1514, 1292, 2531, 2405, 3073, 3018, 4478, 3999, 1201, 895, 1854, 174, 3211, 4720, 3853, 3174, 777, 4548, 2200, 1618, 1125, 559, 2728, 4672, 3744, 1253, 4841, 1496, 2501, 851, 2694, 4991, 3077, 4746, 221, 1768, 3169, 4059, 3015, 4507, 2721, 3290, 4685, 818, 471, 1037, 1788, 2981, 2547, 1173, 3937, 275, 1941, 3127, 3004, 4149, 973, 2419, 1691, 1141, 441, 4460, 2727, 1640, 3697, 2066, 798, 563, 2345, 900, 2879, 3014, 4785, 2524, 4190, 4160, 3131, 3987, 167, 412, 1925, 3915, 4410, 3130, 2257, 2818, 3284, 4042, 1497, 4492, 4165, 4743, 2117, 507, 4267, 4965, 4607, 2910, 3111, 2432, 2078, 3687, 3808, 2021, 3181, 205, 4123, 1753, 4188, 2656, 2744, 461, 3235, 4164, 2473, 1975, 3986, 652, 1118, 1238, 4303, 370, 1450, 4892, 4000, 3747, 3660, 569, 4419, 1881, 2760, 1005, 4586, 2465, 1847, 3063, 3984, 3596, 4577, 797, 4842, 1998, 4243, 2657, 1055, 2660, 404, 3249, 4554, 3673, 1263, 3818, 2593, 4940, 7, 4292, 596, 3165, 2397, 1195, 62, 3641, 505, 3961, 271, 2646, 3286, 4395, 3878, 788, 1855, 4020, 2722, 1285, 2111, 1054, 476, 4067, 4145, 1903, 887, 2992, 1349, 4977, 1778, 1498, 3034, 609, 4945, 3621, 3275, 1595, 2752, 48, 4372, 1808, 2404, 4896, 4993, 3089, 4536, 3132, 2199, 3765, 4873, 1123, 3137, 1350, 3889, 3064, 2049, 3496, 1246, 4011, 4446, 4079, 2114, 450, 4295, 2094, 1418, 3857, 4647, 4148, 4021, 4780, 1244, 2011, 3150, 2351, 3589, 4379, 1816, 1868, 3657, 1480, 4036, 2973, 3640, 933, 2731, 2185, 825, 5002, 3029, 919, 3694, 2089, 211, 2375, 85, 2415, 1071, 1546, 2354, 3924, 356, 1137, 3258, 678, 3247, 4721, 2416, 4106, 2858, 4246, 2969, 3129, 375, 1646, 574, 1877, 141, 4583, 2925, 4966, 1379, 127, 2174, 3341, 30, 1140, 4336, 2780, 1757, 1731, 748, 1767, 288, 2458, 2277, 4682, 1715, 4937, 455, 4764, 1843, 4759, 4452, 477, 381, 4486, 4948, 321, 2155, 4645, 2463, 1296, 1584, 210, 2158, 2135, 992, 2038, 4466, 2152, 2140, 3995, 3655, 3412, 1593, 3046, 3500, 3701, 2168, 2823, 3992, 586, 4797, 4376, 483, 3013, 686, 93, 980, 335, 1271, 3868, 4331, 3635, 554, 602, 4095, 4296, 4162, 1936, 4512, 3386, 13, 2335, 4541, 3797, 4733, 178, 3854, 2236, 368, 1329, 3876, 306, 3147, 2054, 4565, 1764, 486, 4438, 840, 4337, 1983, 618, 3076, 3081, 1119, 3083, 4769, 2523, 2753, 3982, 270, 4490, 575, 3049, 3572, 4898, 5000, 4384, 1901, 903, 2170, 3342, 1656, 3030, 438, 4506, 1779, 303, 2700, 3989, 3816, 235, 1321, 4387, 4998, 617, 3757, 337, 2815, 4713, 922, 2292, 708, 1942, 1562, 1182, 4111, 117, 4545, 1956, 2966, 2422, 2127, 1151, 4288, 2002, 2131, 1479, 6, 1972, 948, 3194, 3088, 786, 2556, 154, 3139, 1641, 768, 4382, 824, 4278, 2553, 1038, 4587, 3193, 4897, 4026, 4600, 3741, 3907, 2639, 2276, 4808, 1103, 1040, 2209, 3699, 2476, 2153, 1991, 5005, 1241, 4109, 2736, 4501, 2960, 2263, 4575, 3750, 2600, 3207, 3451, 776, 1159, 1414, 3909, 2680, 1148, 106, 204, 4242, 4619, 3524, 63, 4001, 3590, 232, 2246, 1145, 1765, 3471, 1389, 4553, 4687, 3140, 262, 2652, 635, 3057, 2654, 4269, 1718, 3311, 3718, 4037, 1880, 1540, 1570, 454, 2888, 4668, 17, 4700, 3852, 330, 4238, 1818, 443, 2275, 2724, 4144, 2390, 4787, 2081, 3908, 934, 2262, 1165, 468, 2588, 3486, 4656, 2628, 2477, 3638, 3491, 3839, 4015, 4408, 4271, 4867, 1824, 1729, 735, 630, 209, 3588, 4835, 2272, 4973, 1228, 2705, 3823, 256, 1913, 898, 3487, 689, 361, 4537, 2344, 4265, 1487, 1617, 4281, 2674, 457, 1287, 724, 2929, 1312, 2584, 4383, 4714, 3003, 2596, 3870, 4809, 3623, 3715, 1398, 1218, 3817, 680, 793, 4798, 2805, 2951, 720, 4302, 1428, 872, 589, 2814, 3344, 3597, 4200, 3185, 3547, 1120, 2505, 2035, 2120, 3582, 3899, 2311, 2825, 3938, 3318, 380, 4688, 3124, 1523, 3677, 2933, 834, 4362, 2433, 2175, 3932, 216, 3241, 4365, 4185, 3890, 1212, 379, 4899, 2373, 2889, 2426, 1043, 4719, 866, 297, 3094, 3035, 4913, 2172, 2548, 2226, 1832, 4223, 4227, 1791, 40, 3316, 2487, 2297, 2765, 2015, 1509, 1194, 3550, 3728, 4457, 1893, 187, 138, 2564, 719, 1804, 2162, 2503, 897, 2703, 3681, 3393, 2028, 1800, 3556, 3809, 2291, 4754, 2249, 1802, 4900, 3812, 1976, 2937, 3702, 1052, 930, 1826, 1488, 2883, 4865, 4325, 854, 4960, 90, 1864, 2254, 1465, 632, 1701, 821, 360, 1464, 1363, 3587, 23, 1092, 2997, 3459, 2719, 142, 3355, 1997, 1107, 478, 2044, 2800, 2759, 3794, 3567, 4703, 3947, 3599, 1964, 667, 392, 641, 166, 3799, 4748, 263, 1796, 2865, 2787, 3059, 267, 4701, 3495, 714, 1333, 952, 803, 2268, 835, 2952, 4910, 2309, 2387, 3670, 3262, 3511, 3371, 4736, 467, 1675, 688, 2856, 1721, 3706, 462, 672, 4196, 2525, 960, 4836, 556, 2119, 2087, 3879, 2358, 2640, 4403, 2427, 3734, 1597, 3836, 4675, 317, 1215, 2839, 2062, 206, 4197, 4673, 913, 615, 2795, 4628, 1378, 815, 1582, 3619, 1034, 1834, 207, 3469, 2217, 4939, 2288, 2653, 650, 456, 3906, 2479, 2203, 133, 4558, 421, 3278, 4333, 1644, 284, 2661, 3163, 2830, 247, 769, 3294, 4884, 2819, 4024, 2720, 995, 955, 4046, 3182, 2837, 3695, 4813, 3411, 4726, 2192, 2725, 1062, 4806, 1755, 190, 3902, 4752, 55, 4760, 4054, 2526, 1594, 4709, 2998, 1345, 3305, 4216, 2522, 522, 1775, 1127, 191, 2408, 4154, 105, 1242, 3179, 4161, 865, 1083, 4692, 4751, 1695, 4943, 2796, 3277, 1161, 4975, 1067, 10, 4141, 2623, 3935, 3334, 3410, 3846, 2891, 237, 2809, 1342, 3144, 3489, 1388, 322, 4739, 3112, 2842, 299, 1144, 1131, 2290, 2549, 4904, 1982, 3190, 1875, 2241, 1484, 3796, 4755, 1882, 4488, 3251, 601, 2346, 231, 3123, 15, 4034, 137, 4786, 1293, 4534, 785, 1988, 3382, 60, 3394, 1996, 2211, 1914, 1954, 4467, 634, 3978, 1018, 3869, 4214, 3835, 1833, 4283, 1365, 2143, 1301, 1226, 2194, 3680, 3425, 593, 668, 2079, 3109, 2613, 1401, 2181, 4529, 1011, 1338, 3195, 2585, 4437, 1966, 2551, 880, 4822, 1280, 1961, 2906, 2126, 3564, 1539, 433, 1856, 734, 2798, 3535, 4083, 1952, 3246, 324, 1688, 3036, 3845, 4887, 1267, 802, 3299, 28, 4155, 4540, 1874, 4251, 1234, 1114, 659, 4253, 950, 1097, 962, 3632, 4444, 3898, 3648, 4824, 1807, 2446, 2541, 3563, 3600, 1752, 2733, 1634, 951, 3630, 4585, 2986, 3509, 1105, 1871, 4872, 4610, 264, 1115, 873, 2176, 2032, 83, 1769, 32, 448, 1749, 2361, 4626, 4404, 2207, 1987, 4504, 3820, 4436, 3614, 2318, 331, 2570, 2337, 2677, 2950, 2706, 3158, 4805, 4314, 1665, 2616, 546, 3829, 38, 1706, 4487, 625, 3363, 1146, 3607, 4491, 1959, 1045, 1007, 2022, 2493, 4329, 808, 4870, 3408, 3267, 2913, 3732, 195, 2739, 1224, 3624, 4694, 2258, 1009, 4582, 98, 4885, 1417, 3990, 4717, 2868, 4890, 3688, 402, 4869, 1050, 3615, 2718, 344, 4448, 893, 1106, 3333, 4968, 4859, 1678, 4380, 4590, 2064, 673, 2949, 3482, 4041, 161, 807, 1440, 1559, 2294, 855, 4679, 2420, 3464, 2451, 3418, 3662, 371, 1247, 4735, 1049, 3802, 3905, 2903, 4845, 4825, 3566, 4741, 1283, 3352, 1886, 4249, 464, 1299, 4189, 867, 1571, 89, 1937, 4355, 3044, 4562, 4983, 4324, 4929, 944, 3391, 1542, 3099, 1869, 4762, 3148, 5001, 2740, 3998, 4718, 521, 1603, 1503, 3742, 2371, 765, 675, 4693, 4014, 4724, 1042, 70, 180, 3735, 1851, 2029, 2482, 18, 3593, 2470, 2907, 2870, 4639, 2958, 3499, 4101, 4443, 3505, 44, 1707, 3118, 1720, 4308, 2341, 2557, 3579, 302, 4608, 4297, 4495, 1664, 2407, 287, 2130, 2289, 1427, 1929, 2191, 3785, 577, 4169, 2595, 3031, 3178, 4632, 2366, 1116, 4170, 2861, 4057, 189, 1210, 298, 4138, 1243, 4931, 4525, 1889, 170, 2052, 4368, 419, 700, 2010, 1536, 4181, 4915, 2435, 2083, 39, 3851, 4524, 1216, 1836, 4795, 2941, 605, 1815, 853, 812, 4624, 1030, 4285, 2968, 1801, 1861, 1840, 590, 227, 1266, 1955, 1328, 2232, 1697, 3172, 1890, 1841, 3586, 736, 4370, 1393, 571, 3306, 3865, 1535, 3555, 110, 3472, 1933, 2905, 2754, 3515, 3134, 3759, 4862, 4963, 410, 2124, 1516, 4151, 1515, 1214, 87, 2410, 1126, 823, 2983, 2156, 3887, 3903, 3917, 3104, 1682, 2773, 3149, 152, 643, 1999, 2125, 1600, 4280, 3466, 4339, 3964, 2212, 4738, 1630, 4652, 2355, 202, 3575, 954, 2395, 4980, 540, 3679, 496, 182, 4092, 4073, 2436, 4634, 3752, 2831, 4615, 1466, 2575, 4349, 3419, 3762, 126, 1984, 3303, 1734, 767, 4147, 4532, 4761, 4258, 1376, 3969, 3052, 2003, 4654, 1065, 2869, 1456, 3826, 2472, 614, 1648, 4218, 2828, 520, 4199, 4346, 4649, 2006, 3067, 3405, 343, 3307, 2884, 1156, 213, 164, 4710, 3864, 2711, 883, 3691, 3583, 949, 1592, 3736, 4274, 532, 1048, 4252, 2183, 1160, 2165, 1372, 3813, 3892, 3128, 1773, 4584, 2571, 2073, 4473, 3217, 1408, 921, 1451, 1406, 4666, 607, 2308, 4902, 1403, 3424, 3644, 3271, 2452, 2142, 2040, 3700, 1586, 4789, 4510, 2944, 3824, 1916, 2237, 2267, 3897, 4574, 2348, 965, 2219, 1610, 4638, 4526, 4231, 3994, 3204, 4811, 4818, 695, 756, 2019, 2084, 1844, 610, 3196, 1015, 1064, 890, 3633, 2202, 3508, 4538, 3957, 4699, 395, 220, 3782, 4186, 3800, 2545, 497, 2392, 3443, 4569, 2148, 2491, 3380, 4758, 4053, 4775, 3983, 539, 3678, 842, 2096, 2149, 926, 1091, 3176, 411, 4750, 2280, 2964, 3048, 162, 1970, 2658, 2287, 150, 3233, 1809, 3454, 517, 2343, 4599, 3398, 3674, 1473, 740, 51, 2642, 2741, 753, 4837, 4236, 2579, 2017, 692, 2559, 1845, 481, 1230, 2892, 704, 2936, 4099, 3261, 3811, 4364, 3690, 3601, 4313, 43, 1460, 4472, 2970, 2481, 1572, 4309, 2076, 679, 2860, 3173, 2313, 4084, 1681, 658, 613, 2664, 3026, 495, 1513, 31, 1744, 3225, 763, 2976, 4747, 3457, 4017, 4523, 1934, 2469, 3324, 2843, 1099, 932, 656, 2817, 488, 4458, 4028, 327, 3296, 566, 3417, 3684, 1077, 3822, 3760, 3831, 4033, 1409, 354, 2729, 1736, 642, 3221, 915, 2164, 2406, 2157, 2154, 746, 3861, 1066, 1993, 3427, 4031, 4521, 3916, 511, 160, 1204, 2995, 4604, 3888, 502, 1088, 3477, 3788, 881, 1500, 3201, 2166, 4476, 789, 3886, 4878, 113, 699, 4248, 4077, 2453, 567, 4417, 2067, 2897, 4479, 1545, 889, 408, 4893, 2060, 885, 4961, 2590, 1150, 414, 384, 3686, 4328, 3948, 2132, 4078, 1176, 4489, 1167, 3250, 2912, 4790, 4518, 899, 4477, 3040, 3895, 3497, 2177, 2008, 4210, 3474, 5004, 1001, 942, 269, 2338, 3116, 1453, 4071, 285, 2935, 2853, 1776, 4794, 4678, 1256, 3422, 5003, 252, 3428, 3280, 2401, 4441, 3988, 281, 4204, 3203, 4428, 2409, 795, 2475, 3421, 2650, 3438, 3478, 2093, 1438, 4056, 3191, 3786, 1923, 1100, 4135, 4353, 2770, 4782, 671, 2220, 2775, 1200, 292, 2315, 3669, 67, 1444, 684, 1786, 4343, 2443, 439, 1564, 841, 1351, 1254, 2954, 4401, 4023, 1743, 2184, 4650, 2704, 3293, 2542, 4268, 470, 3146, 3037, 3314, 4172, 3270, 3467, 1313, 4413, 3962, 3239, 2606, 3272, 1227, 217, 666, 325, 645, 2840, 3096, 242, 3795, 907, 341, 2353, 982, 4060, 3537, 4279, 1422, 4912, 3301, 2669, 1093, 4567, 1694, 4959, 3470, 3543, 4686, 3631, 600, 2151, 2767, 1303, 2633, 391, 260, 4431, 4327, 1981, 764, 4298, 1733, 1822, 1714, 1505, 3151, 1525, 4356, 3432, 291, 529, 1237, 4705, 4930, 3985, 3843, 1279, 3358, 4907, 4451, 350, 1910, 1771, 4833, 2665, 2852, 927, 3731, 1662, 2043, 388, 2792, 2004, 4330, 1291, 4906, 2742, 1897, 4801, 367, 4552, 3729, 1741, 1649, 2216, 3533, 4704, 3175, 208, 4035, 2428, 2147, 1766, 1474, 737, 148, 186, 1262, 4698, 624, 4990, 1939, 2791, 4708, 2478, 850, 830, 233, 2444, 4996, 4543, 4175, 3756, 3229, 3098, 1208, 203, 4876, 2213, 1502, 3594, 1359, 4681, 655, 2459, 1735, 1307, 1555, 3329, 4193, 653, 75, 4219, 3025, 4496, 230, 858, 4142, 1188, 1469, 2388, 770, 4661, 902, 4839, 1306, 2675, 4776, 1135, 1783, 638, 2790, 3440, 4856, 2108, 2421, 4820, 4846, 1209, 3384, 1758, 1613, 201, 4133, 3008, 3584, 4669, 4596, 4250, 4779, 4440, 2389, 4578, 2228, 2146, 3758, 1281, 2369, 4938, 490, 2413, 564, 1442, 3726, 4831, 479, 2946, 3855, 3069, 2810, 4233, 181, 1598, 4728, 3645, 3904, 2224, 2517, 3234, 3602, 3119, 966, 4927, 758, 4997, 986, 4032, 739, 4712, 3611, 3248, 2122, 2684, 1614, 3375, 3152, 1507, 1443, 4300, 1304, 4391, 1862, 122, 2467, 2771, 363, 4581, 1551, 843, 2820, 215, 2712, 4390, 3710, 918, 2671, 4614, 1666, 2620, 2496, 2359, 4535, 1277, 251, 2573, 3369, 4418, 280, 582, 1702, 1690, 3521, 2592, 1712, 1259, 3084, 3045, 3273, 2961, 3226, 1341, 983, 2776, 1337, 2434, 1225, 3379, 4684, 4198, 323, 3534, 1059, 2589, 1945, 2560, 4816, 783, 2092, 4205, 592, 3255, 3532, 2260, 2967, 935, 4771, 3476, 405, 3549, 4981, 3724, 4167, 1390, 1670, 3721, 4350, 637, 4828, 3095, 4916, 4944, 1166, 2285, 3197, 1563, 3485, 2919, 1825, 623, 442, 172, 1552, 2196, 4988, 2532, 530, 1761, 1499, 2116, 3871, 2020, 362, 2668, 2898, 1222, 2376, 3522, 3806, 80, 1722, 3337, 3722, 4209, 3991, 3936, 1839, 3675, 3574, 25, 2962, 2826, 1823, 2320, 4985, 4225, 3504, 3133, 4174, 1888, 4742, 3349, 2362, 1566, 1971, 333, 705, 814, 4791, 2566, 3712, 132, 3708, 4868, 4911, 2103, 444, 804, 2920, 3291, 3009, 1732, 3739, 3929, 751, 377, 2921, 2223, 1104, 4358, 665, 1530, 4814, 3156, 2959, 2098, 3483, 4516, 718, 2364, 997, 4125, 234, 1612, 1581, 1948, 3090, 693, 726, 2909, 856, 1068, 3717, 2417, 4891, 4917, 19, 4978, 4695, 3560, 4426, 796, 820, 2057, 2699, 2321, 4950, 2506, 2611, 4259, 1951, 2112, 1876, 1781, 794, 4224, 998, 3372, 3154, 265, 2324, 2991, 2676, 1900, 2437, 2835, 2735, 293, 3967, 259, 2918, 1169, 1625, 2082, 2295, 2086, 364, 2708, 3980, 228, 3973, 3626, 4601, 636, 1687, 1346, 1797, 3877, 3256, 2569, 3559, 3295, 2091, 993, 957, 4620, 3507, 1463, 4194, 1742, 1860, 550, 2128, 2061, 4593, 2134, 1364, 2139, 4143, 2033, 336, 1526, 831, 4881, 2537, 1740, 2182, 4551, 4069, 4140, 3383, 4597, 3435, 3693, 4531, 77, 909, 146, 3351, 1631, 4465, 4311, 1669, 2622, 3751, 4807, 2894, 3308, 4377, 627, 1711, 4866, 4462, 874, 3553, 12, 1036, 2, 4153, 4571, 4519, 3637, 3530, 1000, 3020, 21, 891, 4888, 1902, 484, 3719, 4560, 1949, 2314, 1111, 149, 3298, 2645, 4016, 4425, 2342, 4415, 3723, 4119, 3801, 663, 20, 1519, 1957, 836, 1317, 2357, 33, 1831, 3996, 1654, 3807, 97, 608, 416, 253, 2137, 4385, 4130, 1318, 4732, 3720, 460, 1658, 179, 826, 2607, 3336, 1920, 2534, 4542, 2953, 1117, 1651, 4737, 4305, 4895, 81, 1905, 2562, 1739, 3114, 1667, 196, 2457, 3834, 3170, 1986, 1608, 4498, 390, 409, 1027, 4312, 1482, 1728, 1392, 2099, 5, 328, 1380, 3192, 214, 1994, 924, 1023, 4047, 2041, 732, 3743, 246, 1098, 2378, 1094, 2302, 2518, 579, 2781, 473, 4070, 309, 4557, 1490, 1431, 1607, 3882, 1638, 3042, 3357, 847, 3407, 124, 2942, 4019, 1887, 2957, 3159, 527, 4432, 4475, 499, 2931, 1604, 4689, 4609, 3126, 4183, 2915, 3608, 882, 2635, 1402, 4105, 809, 3774, 4573, 3356, 4844, 2626, 3707, 4918, 2768, 3664, 1221, 4004, 2786, 3199, 1113, 1183, 4618, 3551, 2511, 1677, 315, 4182, 3884, 3628, 1112, 3519, 4463, 2647, 729, 4499, 2270, 1857, 491, 4756, 3798, 151, 4883, 1158, 1354, 332, 779, 1578, 1585, 3313, 2271, 1249, 2400, 4520, 1102, 1385, 2659, 3713, 3252, 1716, 2050, 4369, 2189, 2234, 1676, 3850, 3205, 4690, 810, 1149, 2221, 2037, 2256, 2429, 2605, 2687, 1643, 4697, 715, 3951, 2900, 1668, 3803, 1622, 709, 3561, 3492, 3965, 1560, 1848, 3875, 4315, 2385, 3513, 3766, 1157, 2169, 155, 1004, 2030, 2278, 3010, 3373, 345, 2502, 378, 4843, 449, 4347, 3921, 606, 143, 2885, 255, 3011, 4853, 4514, 4277, 1142, 1035, 1335, 1978, 4854, 1483, 3946, 2259, 4470, 3569, 3979, 353, 4341, 4051, 4923, 1080, 1189, 742, 1698, 4074, 4979, 3975, 4179, 3168, 2850, 4393, 4815, 698, 2283, 4093, 2806, 3024, 626, 1696, 805, 2618, 301, 4103, 510, 959, 548, 1039, 445, 4287, 2100, 1199, 2568, 2310, 2716, 4800, 3078, 760, 766, 2638, 2160, 3266, 583, 2609, 4102, 3716, 857, 4213, 2749, 4272, 557, 3815, 4107, 1352, 223, 3086, 339, 984, 2932, 844, 1205, 3828, 4159, 3585, 3388, 958, 2109, 1835, 4266, 3206, 1322, 3847, 4110, 4090, 3833, 3711, 2615, 3220, 4471, 1288, 1577, 861, 3450, 3668, 3620, 1273, 1683, 4257, 244, 2301, 722, 1492, 2927, 4528, 3006, 1044, 2339, 3436, 1717, 904, 212, 2382, 4864, 1485, 4674, 2474, 4386, 1700, 506, 1006, 163, 4454, 4361, 1751, 3976, 430, 4040, 2847, 2693, 4030, 2328, 4050, 4096, 4340, 762, 4332, 3041, 4480, 4829, 426, 1229, 3913, 2360, 2802, 3245, 4375, 664, 2604, 305, 1908, 257, 3494, 2204, 1152, 800, 437, 1187, 1472, 552, 877, 1524, 2804, 3545, 1780, 4414, 2363, 1252, 4967, 3218, 3772, 1501, 3527, 710, 2423, 3625, 1532, 515, 3297, 1660, 2627, 3390, 1047, 588, 4879, 4187, 4920, 2229, 2402, 4500, 2323, 1977, 713, 535, 3972, 1477, 2808, 4450, 1133, 2580, 1659, 1760, 1330, 2231, 4260, 424, 3529, 604, 2643, 1928, 3725, 64, 3019, 1891, 1493, 310, 533, 4342, 3570, 1076, 1109, 2922, 401, 1909, 1236, 1777, 4722, 2714, 4984, 562, 1601, 1122, 1467, 4270, 4662, 1178, 1475, 3304, 1336, 2053, 4129, 3224, 4941, 2707, 4555, 2245, 3000, 3219, 1510, 1270, 3183, 4091, 2890, 3610, 3838, 1858, 2197, 741, 2069, 4734, 1430, 4992, 1435, 1685, 2334, 1190, 1575, 3392, 1082, 1412, 35, 4664, 373, 2756, 2930, 1139, 3647, 3804, 3650, 744, 4630, 1486, 2789, 452, 1425, 3254, 852, 2284, 848, 4745, 3442, 2462, 3573, 2240, 2441, 711, 3894, 2330, 886, 3331, 1061, 4972, 2070, 168, 4075, 4566, 4237, 1016, 595, 528, 4852, 1573, 3378, 2972, 4173, 1405, 4903, 2088, 2499, 119, 2206, 2279, 4563, 3523, 2667, 2757, 4773, 1419, 4423, 194, 3364, 1852, 2418, 875, 551, 2340, 4007, 413, 4616, 1143, 4184, 1356, 2911, 3769, 725, 4064, 3359, 4114, 258, 2198, 964, 3860, 3429, 4849, 3338, 2632, 1320, 3960, 2425, 2876, 648, 451, 813, 2144, 2893, 4921, 976, 3740, 1332, 754, 3557, 3618, 3781, 778, 4550, 2755, 1041, 2179, 4360, 1912, 2046, 2238, 1650, 183, 2634, 4894, 4118, 806, 3617, 387, 1302, 3591, 4848, 4104, 3881, 1481, 2529, 2483, 1703, 2688, 1177, 3604, 2631, 4009, 1629, 4725, 4819, 2180, 340, 4936, 4043, 3453, 3292, 4080, 2026, 239, 4855, 46, 417, 1087, 538, 447, 4631, 946, 463, 4244, 4006, 1924, 3649, 4306, 2965, 545, 487, 3115, 2001, 4863, 1628, 4796, 1591, 245, 775, 3784, 3177, 1885, 383, 1965, 525, 1357, 3840, 4348, 2597, 123, 342, 2255, 4120, 631, 3326, 2539, 1231, 3353, 1517, 3361, 4344, 1784, 2747, 1206, 3819, 3366, 1449, 4946, 1689, 1251, 4767, 3502, 238, 428, 518, 4572, 3017, 4834, 101, 1763, 917, 1191, 603, 2874, 790, 3227, 4245, 1019, 945, 1020, 407, 2673, 750, 972, 4942, 2985, 1579, 2785, 268, 2784, 3595, 4483, 4648, 1154, 1361, 2587, 2764, 3339, 3730, 374, 294, 4146, 2242, 2988, 458, 2979, 752, 4781, 3257, 282, 224, 2982, 4971, 1223, 114, 2222, 912, 261, 4389, 1056, 2303, 4063, 4803, 4085, 4999, 1653, 4068, 3653, 587, 4402, 147, 584, 4112, 573, 4683, 2521, 1671, 1774, 118, 4055, 1558, 3705, 1828, 313, 50, 1172, 2520, 4299, 4113, 4139, 3465, 677, 1348, 4352, 4623, 2603, 91, 2274, 896, 3974, 1197, 2374, 4716, 860, 4508, 116, 1842, 3910, 1075, 3136, 1827, 2201, 3055, 3097, 2055, 2644, 1548, 2859, 3934, 197, 1806, 145, 4442, 4226, 696, 3399, 2538, 4770, 936, 2129, 612, 4765, 1063, 1979, 3397, 1652, 3449, 3540, 2273, 76, 3733, 3825, 938, 1968, 2281, 3381, 2581, 3462, 2307, 2766, 1198, 129, 871, 3416, 4128, 4291, 3510, 4646, 1309, 4058, 4580, 1377, 3805, 1404, 3243, 435, 3552, 3940, 1196, 277, 1437, 1537, 892, 4177, 3138, 2367, 1375, 4928, 3105, 3671, 1010, 2880, 4289, 731, 296, 2215, 2233, 4715, 4633, 312, 2576, 4319, 3160, 3263, 4121, 4066, 2978, 1829, 3520, 576, 1830, 3413, 397, 541, 2901, 1686, 4176, 1990, 4256, 3180, 4556, 4357, 863, 701, 1203, 2247, 4081, 1250, 1315, 4334, 727, 4485, 3528, 3309, 4351, 2104, 1245, 621, 3285, 745, 1360, 1489, 4740, 95, 670, 2489, 2807, 2689, 3012, 96, 193, 3844, 4412, 4461, 870, 3918, 2543, 2058, 1454, 2947, 4886, 3954, 307, 4430, 3841, 2945, 351, 1633, 3376, 2866, 16, 74, 3642, 4082, 3923, 4826, 2327, 4511, 2948, 4005, 4617, 1407, 743, 597, 4602, 2018, 611, 4255, 3319, 3370, 1257, 4399, 4592, 3274, 4396, 2024, 1931, 24, 941, 2578, 11, 1544, 4221, 2391, 1636, 4922, 4217, 755, 4293, 1180, 1512, 1745, 52, 2691, 4947, 3571, 4570, 3240, 1, 782, 2637, 4345, 4406, 4823, 2000, 1557, 3385, 3970, 2758, 2005, 1819, 320, 2803, 2829, 4131, 1962, 2329, 3488, 3493, 2629, 996, 3228, 1799, 1709, 1164, 2533, 1792, 640, 4642, 1611, 1411, 3612, 3484, 3568, 2873, 3770, 787, 3953, 2269, 4660, 691, 3676, 537, 4449, 3210, 2854, 2926, 2698, 1817, 2565, 2403, 988, 3501, 811, 519, 2550, 827, 3727, 53, 534, 240, 3654, 4310, 4643, 3761, 4591, 1879, 2508, 3315, 3714, 2500, 2536, 250, 4276, 2774, 4952, 1661, 4926, 1046, 4749, 4335, 1362, 1615, 2312, 2036, 730, 3448, 2077, 2670, 565, 3051, 4232, 1186, 49, 1795, 2386, 1074, 1794, 780, 2709, 3420, 3544, 4435, 1458, 1370, 3468, 3253, 4474, 2710, 79, 838, 702, 4680, 3039, 4799, 547, 4012, 3347, 694, 1171, 3481, 2107, 1314, 3704, 3374, 2299, 3406, 1811, 1459, 1561, 1553, 2872, 651, 5007, 2738, 78, 34, 531, 1746, 837, 3288, 2939, 3043, 112, 369, 4851, 4974, 4367, 2171, 504, 4964, 4539, 3646, 1995, 2993, 3377, 1400, 717, 2296, 1298, 125, 4008, 314, 4667, 1110, 3360, 134, 3053, 1580, 4901, 781, 1255, 1927, 153, 100, 1606, 558, 2115, 4812, 4397, 4502, 2333, 349, 399, 3506, 2924, 2440, 3066, 1353, 4163, 3332, 1021, 971, 2251, 3538, 2624, 406, 3141, 4317, 819, 3849, 2379, 2250, 2456, 649, 2619, 1416, 979, 459, 469, 1367, 226, 585, 2761, 2356, 2690, 908, 432, 1276, 3396, 3444, 3737, 4641, 3145, 3663, 107, 304, 1445, 3110, 3279, 4124, 26, 1233, 219, 2871, 418, 2027, 2822, 465, 2555, 3415, 1070, 4322, 839, 1033, 47, 4783, 4653, 2136, 759, 1424, 434, 1922, 1812, 3166, 4127, 2844, 1904, 1969, 254, 355, 2586, 319, 2492, 3028, 1026, 2095, 2490, 3866, 3189, 1017, 84, 289, 862, 2101, 3143, 2394, 4522, 1086, 1022, 2750, 1605, 2987, 3542, 2527, 512, 1873, 1699, 2471, 660, 1461, 1556, 2105, 1387, 1003, 4969, 422, 4676, 894, 1762, 57, 3054, 2980, 4606, 690, 2813, 3920, 661, 879, 3763, 3698, 2567, 1655, 581, 131, 1029, 1684, 3977, 1596, 1132, 400, 1870, 104, 22, 423, 594, 2322, 2468, 1012, 681, 3956, 4025, 1583, 4097, 318, 1522, 1323, 1272, 3164, 4621, 3517, 3260, 3606, 1623, 3603, 2383, 1867, 326, 4877, 1541, 3056, 1248, 2450, 3058, 3070, 2726, 3242, 1609, 376, 4429, 4723, 2679, 2227, 3911, 1319, 3548, 109, 4407, 1647, 1310, 3754, 276, 4637, 394, 1820, 169, 2414, 1078, 3389, 829, 2838, 4919, 4420, 3789, 4909, 1793, 4651, 2509, 3755, 2352, 3215, 3514, 799, 662, 2963, 1415, 1264, 3452, 3689, 1381, 71, 1930, 2074, 4398, 3157, 2009, 272, 66, 2399, 1679, 3320, 4817, 2205, 3859, 4934, 1790, 3121, 1547, 156, 3108, 2460, 3837, 385, 2851, 3350, 3968, 682, 2007, 2513, 4027, 4935, 1079, 1898, 2368, 963, 1213, 4598, 3153, 4363, 1853, 4086, 3062, 2990, 429, 5008, 1627, 4763, 1953, 4925, 4603, 3085, 2349, 3142, 1725, 1491, 1383, 3002, 3952, 1207, 2464, 1395, 4157, 1439, 2023, 2777, 1511, 2662, 2519, 300, 4802, 4589, 3236, 1899, 2678, 3322, 4022, 65, 4655, 69, 706, 817, 994, 1128, 4168, 1432, 4509, 1476, 4230, 905, 4374, 4994, 3900, 2350, 2393, 1355, 2384, 784, 1621, 3944, 2882, 2886, 3103, 2195, 3651, 3658, 4517, 4388, 4301, 1366, 2563, 4860, 1081, 4605, 953, 1950, 3565, 2193, 2591, 120, 3340, 1462, 1358, 3928, 3764, 4018, 1090, 3842, 3541, 2685, 4048, 2411, 859, 493, 4264, 3746, 2159, 3061, 4212, 2864, 878, 1543, 128, 1305, 2347, 2438, 4469, 115, 2686, 3863, 2449, 1565, 4513, 334, 1470, 685, 3867, 2797, 3007, 2431, 2447, 2943, 3652, 3135, 3578, 2663, 4089, 3810, 916, 864, 3268, 4987, 2485, 1884, 2497, 4094, 4392, 2745, 103, 1260, 1265, 236, 278, 1588, 3827, 2834, 1374, 4359, 2121, 2769, 3787, 1420, 774, 2034, 1599, 513, 3609, 1602, 2651, 1813, 1096, 482, 2045, 3777, 1985, 1324, 2123, 2572, 2857, 4354, 1397, 733, 2218, 4986, 2723, 248, 4658, 1058, 4838, 1261, 1616, 2332, 1518, 2832, 4778, 1060, 2594, 3748, 2841, 2133, 4503, 4421, 0, 3958, 947, 1938, 2512, 2974, 1789, 4261, 3458, 2118, 3395, 816, 4481, 4201, 747, 4229, 2681, 188, 9, 347, 3950, 721, 2325, 1275, 1645, 1446, 440, 1181, 2636, 2702, 3981, 3931, 1447, 1136, 2621, 4013, 3925, 4850, 1399, 3627, 4326, 2827, 2319, 523, 1992, 1085, 348, 3696, 2110, 4459, 2701, 3244, 646, 1468, 4156, 544, 3282, 4371, 403, 2783, 989, 4117, 2480, 3661, 4192, 1967, 4088, 3287, 4533, 4663, 969, 1413, 3238, 1918, 4100, 580, 88, 1240, 3554, 1693, 396, 2208, 2488, 1632, 3403, 3362, 4381, 3404, 1973, 3966, 4087, 1663, 2715, 2381, 2793, 372, 1219, 4546, 4484, 940, 3447, 3213, 1915, 218, 45, 4951, 3409, 3518, 4933, 3531, 3346, 4320, 3667, 4766, 4239, 2293, 3430, 4793, 1713, 2612, 868, 2914, 2282, 1014, 200, 1286, 1642, 2090, 616, 543, 4029, 192, 2445, 3912, 1528, 4203, 2430, 773, 4323, 4039, 446, 3941, 3310, 2248, 4706, 1859, 37, 1391, 2617, 1692, 1637, 2867, 4788, 1567, 2065, 3512, 2649, 2331, 4871, 4527, 1013, 3276, 3773, 3659, 4544, 2113, 2849, 1554, 687, 4062, 723, 4065, 4857, 4284, 2908, 3558, 3091, 2071, 2510, 1989, 1334, 4378, 3155, 1635, 2955, 2178, 4234, 3539, 2846, 4774, 3208, 4702, 1290, 3685, 398, 3300, 1921, 3281, 3914, 2902, 3885, 1878, 4889, 978, 568, 4, 3598, 1369, 2252, 633, 2186, 4316, 1384, 1163, 1538, 2484, 4052, 1455, 1269, 619, 3302, 4970, 4453, 1549, 3433, 3209, 346, 42, 4178, 4753, 599, 929, 4810, 647, 2012, 2989, 1974, 4158, 3325, 157, 4321, 4208, 1192, 961, 72, 1504, 4515, 3592, 4455, 1282, 3616, 4222, 3672, 2975, 3414, 2163, 1866, 1129, 1587, 3167, 3387, 2072, 4424, 2794, 1935, 3401, 1421, 3463, 2602, 923, 2938, 1726, 4400, 1232, 3830, 498, 4134, 3327, 4405, 3490, 73, 1674, 1121, 2923, 386, 1892, 1838, 2243, 2075, 2540, 2102, 572, 3162, 3321, 1185, 4905, 2239, 3032, 2225, 2996, 2717, 833, 144, 4098, 4241, 494, 3074, 3120, 2047, 279, 3065, 4122, 911, 1626, 4549, 480, 1940, 173, 2141, 4338, 4150, 2994, 1521, 3186, 2515, 3955, 4464, 500, 876, 4757, 1508, 2304, 4427, 508, 431, 3872, 4044, 3933, 4166, 975, 4627, 3269, 761, 2306, 2577, 82, 184, 2875, 1239, 968, 3862, 2558, 1574, 884, 365, 2150, 3343, 4211, 3212, 140, 3117, 2904, 1619, 1340, 3682, 3023, 3187, 4202, 3122, 3634, 366, 1347, 2552, 1368, 1433, 2779, 3629, 4530, 311, 4304, 2145, 2448, 4126, 1344, 1506, 555, 1396, 2762, 2031, 4273, 3113, 3692, 54, 3922, 1134, 2235, 2097, 1724, 1710, 1727, 1294, 2977, 1217, 1750, 4727, 738, 2696, 3893, 4804, 4045, 828, 2056, 3354, 1284, 4220, 175, 990, 4644, 4989, 2751, 4493, 2713, 283, 985, 2439, 2730, 2732, 1410, 3426, 2561, 1202, 3790, 2683, 2812, 1089, 3184, 1031, 4456, 1919, 4547, 2059, 4373, 1837, 3345, 2862, 1168, 3021, 3461, 4640, 2824, 712, 1805, 3971, 1002, 4108, 2265, 2286, 222, 570, 674, 1787, 4588, 1426, 3666, 846, 2836, 3060, 249, 111, 503, 2370, 1339, 3, 3068, 4290, 136, 2598, 1448, 61, 359, 3223, 3709, 2504, 2305, 2956, 1311, 2507, 453, 4665, 4497, 4595, 716, 1534, 199, 2442, 1590, 3330, 1382, 2187, 2614, 4613, 3656, 4254, 703, 2424, 5009, 1737, 121, 2167, 3883, 2583, 2516, 2535, 2672, 1495, 338, 3939, 4730, 1589, 3997, 3562, 771, 3498, 3475, 2811, 4206, 4247, 757, 2498, 472, 3945, 1946, 3106, 2025, 4482, 542, 2608, 2244, 560, 749, 1782, 3775, 4622, 1051, 229, 274, 2896, 108, 4240, 2454, 3289, 3771, 4275, 1175, 3079, 3001, 4576, 3312, 2630, 1316, 1738, 1531, 1906, 3901, 4439, 2599, 2737, 3456, 3665, 987, 977, 2455, 2748, 3082, 3503, 2396, 3832, 1184, 4954, 4962, 1084, 3942, 3891, 4711, 4286, 3749, 185, 382], "validation": [5347, 5319, 5201, 5207, 5509, 5427, 5283, 5367, 5455, 5327, 5162, 5270, 5274, 5399, 5261, 5348, 5377, 5229, 5196, 5282, 5090, 5099, 5257, 5331, 5420, 5192, 5416, 5141, 5440, 5358, 5157, 5376, 5235, 5237, 5185, 5238, 5171, 5350, 5016, 5460, 5110, 5499, 5418, 5067, 5074, 5318, 5154, 5202, 5411, 5444, 5271, 5306, 5459, 5222, 5146, 5205, 5190, 5031, 5489, 5355, 5023, 5098, 5404, 5021, 5381, 5351, 5290, 5495, 5496, 5195, 5346, 5299, 5012, 5097, 5294, 5062, 5316, 5317, 5391, 5449, 5150, 5436, 5142, 5469, 5357, 5045, 5486, 5059, 5313, 5370, 5020, 5273, 5389, 5152, 5156, 5272, 5352, 5126, 5292, 5466, 5015, 5010, 5212, 5027, 5504, 5037, 5433, 5147, 5070, 5406, 5505, 5206, 5278, 5088, 5200, 5129, 5030, 5369, 5024, 5018, 5264, 5120, 5478, 5079, 5106, 5424, 5028, 5232, 5438, 5093, 5057, 5199, 5158, 5034, 5267, 5084, 5506, 5307, 5431, 5378, 5419, 5387, 5108, 5311, 5233, 5269, 5286, 5134, 5373, 5413, 5055, 5437, 5426, 5341, 5225, 5159, 5361, 5056, 5490, 5280, 5194, 5240, 5465, 5310, 5501, 5422, 5349, 5128, 5040, 5133, 5356, 5036, 5439, 5442, 5388, 5014, 5285, 5398, 5092, 5452, 5502, 5405, 5032, 5344, 5035, 5076, 5470, 5451, 5148, 5102, 5390, 5243, 5414, 5155, 5210, 5456, 5497, 5379, 5174, 5114, 5080, 5400, 5172, 5217, 5354, 5477, 5164, 5026, 5183, 5485, 5125, 5329, 5208, 5182, 5039, 5191, 5268, 5323, 5131, 5429, 5322, 5061, 5204, 5447, 5069, 5289, 5335, 5266, 5332, 5127, 5263, 5434, 5482, 5308, 5188, 5412, 5236, 5107, 5392, 5247, 5138, 5487, 5130, 5402, 5250, 5403, 5281, 5288, 5231, 5077, 5170, 5253, 5277, 5063, 5380, 5219, 5220, 5160, 5109, 5167, 5408, 5228, 5397, 5293, 5112, 5115, 5441, 5468, 5303, 5073, 5476, 5168, 5149, 5360, 5334, 5491, 5254, 5091, 5051, 5362, 5359, 5143, 5049, 5336, 5337, 5314, 5457, 5415, 5330, 5116, 5372, 5480, 5494, 5058, 5136, 5184, 5462, 5119, 5224, 5096, 5326, 5432, 5325, 5198, 5083, 5301, 5365, 5417, 5177, 5041, 5054, 5047, 5473, 5249, 5320, 5463, 5153, 5137, 5382, 5507, 5394, 5339, 5180, 5068, 5103, 5038, 5179, 5324, 5276, 5475, 5025, 5216, 5113, 5503, 5017, 5298, 5118, 5401, 5013, 5395, 5087, 5445, 5295, 5481, 5105, 5472, 5111, 5458, 5407, 5428, 5072, 5165, 5022, 5333, 5248, 5302, 5064, 5173, 5052, 5258, 5046, 5048, 5193, 5043, 5384, 5241, 5175, 5239, 5132, 5163, 5050, 5085, 5066, 5101, 5383, 5145, 5296, 5140, 5209, 5104, 5078, 5135, 5368, 5291, 5214, 5374, 5363, 5328, 5493, 5117, 5223, 5479, 5065, 5197, 5218, 5421, 5181, 5256, 5500, 5386, 5245, 5166, 5371, 5309, 5409, 5213, 5464, 5396, 5246, 5215, 5315, 5124, 5338, 5227, 5366, 5343, 5265, 5122, 5454, 5474, 5176, 5393, 5161, 5086, 5230, 5089, 5094, 5279, 5242, 5244, 5492, 5275, 5287, 5019, 5081, 5186, 5340, 5467, 5255, 5123, 5121, 5169, 5305, 5044, 5144, 5139, 5060, 5461, 5042, 5364, 5151, 5178, 5075, 5304, 5187, 5211, 5471, 5435, 5375, 5221, 5483, 5100, 5410, 5226, 5260, 5095, 5450, 5345, 5033, 5053, 5498, 5484, 5011, 5284, 5262, 5189, 5488, 5071, 5029, 5425, 5453, 5448, 5259, 5252, 5508, 5297, 5385, 5234, 5446, 5423, 5321, 5430, 5353, 5443, 5251, 5300, 5203, 5312, 5082, 5342], "test": [5528, 5997, 5804, 5513, 5811, 5854, 5632, 5721, 5866, 5531, 5965, 5816, 5592, 5776, 5906, 5677, 5520, 5857, 5595, 5622, 5783, 5958, 5676, 5764, 5822, 5855, 5643, 5758, 5745, 5608, 5976, 5639, 5927, 5814, 5894, 5763, 5770, 5516, 5808, 5837, 5962, 5945, 5538, 5747, 5856, 5671, 5874, 5895, 5795, 5947, 5780, 5757, 5910, 5954, 5748, 5755, 5567, 5778, 5987, 5568, 5775, 5657, 5933, 5876, 5864, 5749, 5839, 5926, 5966, 5908, 5948, 5598, 5550, 5593, 5552, 5740, 5634, 5537, 5624, 5991, 5546, 5887, 5842, 5714, 5626, 5868, 5888, 5602, 5539, 5896, 5696, 5847, 5968, 5710, 5913, 5967, 5548, 5613, 5964, 5730, 5788, 5534, 5742, 5827, 5853, 5601, 5953, 5665, 5943, 5699, 5988, 5551, 5883, 5720, 5585, 5799, 5524, 5588, 5636, 5517, 5845, 5920, 5562, 5794, 5635, 5560, 5664, 5587, 5631, 5940, 5786, 5979, 5865, 5824, 5569, 5667, 5875, 5781, 5978, 5941, 5576, 5807, 5977, 5914, 5925, 5817, 5986, 5625, 5672, 5684, 5580, 5890, 5638, 5530, 5647, 6006, 5838, 5765, 5815, 5899, 5884, 5726, 5738, 5989, 5633, 5650, 5961, 5561, 5840, 6007, 5819, 5718, 5900, 5575, 5982, 5619, 5992, 5825, 5949, 5675, 5628, 5627, 5584, 5760, 5797, 5971, 5869, 5596, 5912, 5983, 5862, 5994, 5820, 5937, 5916, 5573, 5558, 5687, 5693, 5663, 5915, 5830, 5732, 5511, 5801, 5597, 5777, 5605, 5803, 5934, 5570, 5881, 5583, 5586, 5541, 5695, 5713, 5921, 5615, 5614, 5719, 5621, 5974, 5604, 5800, 5559, 5563, 5769, 5911, 5774, 5834, 5902, 5851, 5787, 5844, 5739, 5673, 5970, 5557, 5932, 5724, 5589, 5637, 5611, 5581, 5678, 5944, 5993, 5999, 5510, 5873, 5841, 5829, 5706, 5835, 5878, 5973, 5918, 5733, 5806, 5549, 5609, 5743, 6004, 5929, 5702, 6003, 5805, 5882, 5566, 5798, 5599, 5905, 5554, 5951, 5521, 5658, 5812, 5985, 5779, 5980, 5860, 5957, 5694, 5725, 5898, 5885, 5711, 5565, 5606, 5646, 5849, 5861, 5659, 5679, 5996, 5771, 5892, 5682, 5704, 5698, 5790, 6008, 5600, 5616, 5722, 5975, 5512, 5923, 5728, 5796, 5893, 5610, 5809, 5935, 5843, 5891, 5691, 5655, 5813, 5669, 5712, 5772, 5828, 5872, 6001, 5735, 5928, 5955, 5653, 5708, 5753, 5750, 5727, 5544, 6002, 5863, 5789, 5846, 5848, 5618, 5950, 5532, 5574, 5879, 5897, 5535, 5640, 5649, 5645, 5810, 5547, 5737, 5648, 5577, 5867, 5939, 5852, 5660, 5518, 5731, 5942, 5717, 5522, 5564, 5578, 5909, 5746, 5556, 5995, 5734, 5870, 5607, 5762, 5683, 5705, 5729, 5886, 5642, 5768, 5880, 5904, 5751, 5692, 5984, 5701, 5690, 5715, 5919, 5591, 5533, 5641, 5756, 5952, 5674, 5754, 5680, 5523, 5877, 5782, 5527, 5620, 5716, 5936, 5741, 5666, 5907, 5981, 5572, 5773, 5761, 5959, 5901, 5703, 5792, 5515, 5922, 5700, 5956, 5623, 5686, 5654, 5553, 5960, 5543, 5826, 5582, 5785, 5594, 5850, 5833, 5662, 5689, 5832, 6009, 5697, 5526, 5707, 5540, 5823, 5603, 5831, 5617, 5752, 6005, 5529, 5630, 5972, 5536, 5542, 5858, 5759, 5917, 5836, 5931, 5784, 5889, 5791, 5651, 5709, 6000, 5514, 5859, 5555, 5668, 5924, 5998, 5767, 5590, 5946, 5871, 5670, 5930, 5571, 5612, 5903, 5938, 5525, 5802, 5766, 5629, 5656, 5963, 5652, 5579, 5644, 5969, 5681, 5793, 5818, 5723, 5688, 5685, 5661, 5990, 5744, 5736, 5545, 5519, 5821]}, {"train": [1182, 551, 3462, 2677, 3911, 3567, 1449, 1506, 1961, 178, 2169, 3822, 3702, 2223, 2533, 1135, 3249, 144, 4414, 4375, 3265, 3164, 982, 15, 2708, 2549, 3393, 3080, 3347, 3475, 2395, 4285, 3639, 3303, 324, 4234, 3297, 546, 4674, 4283, 1351, 3675, 2044, 2126, 569, 198, 2277, 2601, 1249, 2116, 4335, 2152, 3830, 3789, 646, 3100, 3340, 4972, 1340, 2508, 4215, 2907, 2722, 414, 190, 4417, 4987, 1746, 1855, 1364, 68, 510, 2494, 930, 4525, 845, 377, 4257, 4727, 2201, 375, 2062, 2074, 1273, 941, 4827, 3281, 3468, 3338, 2214, 4539, 2063, 2516, 2844, 3522, 1852, 2742, 520, 810, 3211, 1369, 2932, 1682, 4673, 1063, 3632, 682, 3442, 3141, 2706, 3999, 1913, 2280, 2435, 4993, 4874, 3600, 2695, 4736, 1201, 3794, 4735, 2716, 2139, 4978, 612, 3998, 1094, 3955, 1433, 2631, 4357, 1763, 713, 1809, 2194, 3557, 1489, 2860, 4267, 2477, 3732, 1429, 918, 759, 991, 4408, 4239, 4953, 2108, 2278, 1196, 2283, 2412, 3169, 4301, 1220, 599, 4905, 2207, 1520, 3902, 4931, 203, 592, 4012, 4197, 828, 381, 2052, 4935, 3401, 827, 2714, 2661, 2537, 2310, 3864, 4570, 173, 1844, 2339, 2195, 1251, 2247, 1482, 2323, 3349, 4305, 348, 1771, 3590, 602, 162, 2809, 716, 708, 4996, 2225, 3758, 1024, 1084, 1409, 698, 1237, 4509, 256, 4891, 2971, 1092, 1787, 4984, 2151, 2355, 34, 3695, 750, 3801, 1179, 4080, 3157, 2166, 1661, 4073, 3667, 1079, 2838, 3985, 3752, 3872, 4825, 3461, 4471, 3026, 1130, 1654, 4988, 3833, 3666, 1732, 4708, 2467, 3634, 3612, 854, 1633, 2702, 2438, 4973, 138, 4038, 1799, 1243, 2038, 248, 3978, 2978, 859, 37, 3167, 3750, 4853, 986, 3242, 755, 2012, 3788, 1410, 1347, 3378, 1597, 4894, 1728, 4434, 4908, 3690, 2061, 2637, 4395, 583, 3781, 255, 4581, 2852, 4230, 3605, 589, 4331, 596, 405, 391, 1155, 1905, 1963, 931, 49, 2418, 531, 1121, 4599, 639, 2293, 2193, 3455, 4873, 3799, 3749, 4743, 314, 530, 3267, 2564, 2569, 1583, 1274, 4154, 768, 2097, 2723, 1622, 4517, 4386, 2855, 4466, 2599, 4634, 3958, 4625, 3670, 1906, 4258, 1891, 1421, 4565, 4262, 45, 1112, 2396, 2148, 497, 1898, 1091, 629, 4204, 883, 3631, 2902, 4270, 3686, 3354, 1564, 1265, 3744, 4208, 3029, 4798, 1866, 4213, 1808, 2856, 4482, 3467, 4149, 4540, 3291, 4314, 1235, 4750, 3160, 338, 1953, 1759, 4782, 214, 135, 143, 303, 781, 3284, 3090, 1014, 2861, 572, 3641, 1836, 3248, 4760, 4329, 822, 157, 677, 443, 4681, 3221, 4607, 3146, 1240, 1936, 2231, 1015, 4826, 3986, 4817, 322, 102, 1594, 1283, 4964, 165, 2240, 4909, 797, 459, 2871, 297, 4906, 3925, 3007, 2581, 4420, 3296, 747, 2175, 2574, 3315, 914, 3650, 3045, 669, 1310, 1227, 1511, 2958, 3855, 3881, 4777, 2228, 2835, 607, 3759, 1730, 1312, 3358, 3920, 2882, 4914, 830, 25, 1872, 4128, 4143, 4091, 3768, 2176, 892, 2232, 3678, 733, 180, 2548, 2758, 3711, 1481, 4753, 4327, 2667, 2686, 3927, 3808, 1317, 1850, 2524, 1911, 4389, 1194, 2660, 3374, 502, 3539, 4590, 224, 1287, 1425, 3064, 2698, 3457, 295, 3278, 684, 4476, 191, 233, 2170, 4957, 1503, 2511, 4831, 2848, 1887, 1760, 3046, 425, 574, 2379, 559, 3994, 1562, 1629, 4651, 240, 1713, 988, 2026, 2007, 2893, 3453, 1535, 2335, 2842, 680, 4028, 2068, 997, 4011, 1368, 488, 2720, 3163, 1739, 4883, 3821, 2406, 1097, 1824, 4341, 2826, 3553, 3206, 4180, 2623, 3940, 1635, 3936, 4784, 3276, 2397, 1272, 928, 4974, 4775, 1942, 1062, 1354, 3643, 3510, 3741, 4081, 2506, 1983, 3460, 220, 3743, 183, 4663, 4797, 2407, 2091, 2204, 3534, 2559, 4934, 2381, 3173, 4004, 1011, 2618, 3123, 4429, 1926, 2402, 2515, 2868, 412, 3848, 2356, 761, 2572, 813, 1613, 2456, 3844, 4221, 2669, 1374, 1670, 672, 1693, 4336, 1834, 3810, 362, 3451, 2802, 2976, 2365, 1874, 979, 168, 4843, 3715, 547, 119, 4141, 1631, 501, 2316, 1464, 4537, 3549, 4248, 3809, 3523, 215, 645, 549, 1342, 3474, 3517, 2810, 2719, 1302, 1964, 125, 3324, 3931, 16, 1658, 3599, 3210, 2004, 1994, 2050, 2891, 3627, 1970, 4926, 1007, 42, 4016, 901, 1803, 277, 440, 2626, 2759, 1102, 2527, 2385, 4755, 3223, 2104, 2748, 1775, 1810, 4980, 2943, 3573, 1200, 2210, 660, 2274, 3950, 2703, 2376, 3306, 2671, 200, 2471, 3884, 1309, 449, 1716, 4758, 3783, 4564, 3341, 3897, 4467, 2084, 4652, 3335, 3108, 3932, 905, 4569, 2946, 816, 3721, 3595, 4749, 3202, 335, 3816, 3638, 2513, 446, 1605, 363, 1415, 998, 1832, 222, 1869, 4717, 933, 2077, 754, 4400, 4123, 3391, 2545, 1344, 3436, 1720, 616, 681, 1125, 4347, 1199, 2929, 4024, 4835, 22, 3425, 1649, 259, 3790, 3996, 3687, 4694, 4039, 2120, 1776, 2510, 4737, 4211, 4752, 924, 2819, 1727, 416, 2366, 1443, 4754, 1218, 2022, 4114, 2977, 4662, 2143, 4611, 2925, 79, 4901, 4647, 1304, 3622, 4148, 4166, 1698, 644, 3263, 4018, 615, 1755, 1706, 1737, 1904, 257, 1949, 1291, 760, 2179, 3811, 289, 2577, 2188, 4260, 1050, 2560, 1022, 4048, 2854, 1163, 4068, 618, 4377, 984, 2789, 319, 4962, 3270, 3494, 584, 495, 4616, 3984, 67, 2970, 3896, 2588, 4063, 4558, 1532, 2752, 2130, 2744, 4614, 207, 1915, 3738, 4299, 687, 663, 1563, 1411, 2679, 513, 3660, 2408, 1162, 4615, 2279, 1137, 1617, 2156, 3961, 2834, 2224, 852, 3571, 1664, 205, 4688, 1507, 3652, 4100, 4082, 1571, 3128, 3656, 4097, 4235, 2067, 12, 4989, 1943, 2751, 1107, 88, 70, 4029, 3857, 2208, 123, 4108, 3724, 1250, 4868, 598, 4188, 956, 1054, 2219, 3075, 4106, 3261, 4014, 2921, 2260, 2487, 2197, 597, 3803, 1636, 1396, 2110, 1175, 2878, 1126, 3751, 1028, 1788, 4543, 1825, 3629, 3203, 3107, 2895, 1859, 4833, 647, 4816, 2503, 4247, 2244, 2398, 802, 2590, 4050, 1707, 4579, 3041, 1261, 1114, 4120, 3777, 3118, 4025, 3326, 3533, 2298, 1892, 3664, 1279, 284, 2734, 3653, 4040, 3490, 3683, 2066, 3529, 4741, 428, 4842, 1522, 480, 2592, 825, 3433, 3847, 2635, 260, 0, 2005, 4771, 3244, 2308, 2251, 283, 4897, 2822, 4966, 3392, 2864, 769, 1027, 3317, 4098, 4168, 1843, 1181, 3503, 3972, 2002, 1747, 2997, 2168, 467, 4322, 2266, 4145, 2879, 3960, 4459, 1293, 2024, 2239, 3256, 61, 751, 2823, 2546, 261, 909, 792, 2020, 2205, 1533, 4298, 1969, 2483, 904, 2393, 3806, 1493, 146, 1531, 1222, 483, 2321, 4409, 2150, 2843, 4985, 1696, 4390, 1882, 4001, 734, 1408, 2763, 4090, 24, 82, 195, 2655, 935, 1762, 484, 3942, 2158, 3904, 3764, 2562, 4070, 4436, 2964, 92, 3207, 3791, 2615, 919, 234, 3448, 604, 3588, 3528, 4813, 496, 2300, 2497, 1390, 3773, 594, 3352, 4224, 3189, 4137, 1679, 2380, 2338, 3579, 722, 3561, 4751, 2824, 603, 506, 3410, 4712, 2230, 1998, 2651, 43, 321, 4142, 1089, 3447, 4382, 1096, 3323, 150, 1861, 3018, 1471, 442, 3659, 1599, 3888, 4854, 1316, 1835, 2264, 1979, 700, 3587, 538, 3435, 1454, 1895, 581, 3429, 2458, 4254, 2917, 176, 674, 3825, 4861, 2304, 2898, 564, 3438, 701, 3589, 4650, 4679, 202, 4266, 3172, 2464, 1176, 1586, 2183, 2301, 1434, 1704, 1224, 3113, 3104, 4653, 1326, 4860, 2552, 1064, 3699, 2746, 384, 2317, 4096, 4474, 2665, 41, 4010, 51, 3977, 1986, 2682, 3418, 4893, 365, 4157, 1373, 1615, 2118, 3886, 3103, 486, 1593, 4828, 3176, 3096, 1802, 4992, 2979, 2291, 2867, 3195, 1903, 1873, 2736, 3024, 867, 112, 794, 514, 4948, 963, 1144, 2796, 3725, 3049, 1389, 2851, 608, 4654, 2911, 3099, 765, 2604, 4282, 2426, 3959, 1469, 1106, 2955, 756, 1610, 4177, 1753, 3406, 1133, 2694, 3796, 3566, 1038, 3604, 1982, 3098, 4933, 970, 2968, 199, 1717, 5002, 4425, 374, 349, 4725, 2967, 587, 4101, 965, 3530, 1951, 3407, 2547, 134, 4209, 2299, 1588, 1518, 4158, 1652, 4716, 1154, 2361, 1127, 1288, 2454, 3805, 3603, 213, 2160, 4053, 4591, 796, 1386, 179, 697, 1539, 2668, 392, 561, 2774, 850, 3876, 212, 1495, 758, 2904, 1473, 1857, 899, 3917, 4182, 1977, 2314, 3748, 1812, 2542, 1501, 3887, 623, 4555, 723, 3222, 4181, 3865, 4316, 1297, 29, 665, 4419, 3458, 3602, 2614, 2384, 1205, 1260, 4958, 791, 4226, 3059, 3795, 1414, 3982, 4597, 2794, 1702, 3524, 1590, 2351, 4952, 1697, 2180, 2866, 720, 164, 4061, 75, 3009, 4541, 3774, 3619, 1975, 166, 4232, 2501, 2625, 91, 1628, 1826, 1902, 2948, 2715, 4723, 1685, 885, 577, 4089, 2417, 2862, 1514, 3696, 668, 1908, 1086, 819, 1352, 1376, 1111, 2534, 368, 20, 2846, 62, 1044, 4739, 4288, 540, 2541, 3283, 3424, 4576, 403, 2740, 4374, 3739, 4850, 2422, 4220, 4478, 2129, 1333, 2579, 1580, 3004, 2367, 2673, 3798, 2849, 4457, 4201, 1385, 3658, 2594, 2332, 4243, 1886, 2344, 3642, 4851, 4849, 4885, 4189, 1246, 4083, 3039, 1078, 3282, 1743, 1525, 4491, 131, 1888, 4864, 3028, 2765, 2359, 915, 661, 2218, 4176, 1247, 4823, 177, 4546, 2713, 4140, 3519, 422, 231, 3713, 4164, 880, 555, 2969, 3439, 3445, 2329, 4110, 170, 4702, 2830, 4250, 89, 4026, 192, 4435, 2580, 4701, 1271, 3492, 2678, 4947, 3404, 2896, 4659, 3076, 4997, 3945, 4443, 1941, 2609, 4017, 1314, 130, 2246, 4045, 3476, 4135, 2445, 1445, 3243, 3901, 4302, 3456, 2162, 1001, 251, 951, 1042, 1073, 2347, 1055, 2434, 1128, 575, 4960, 1343, 2189, 4945, 342, 3852, 4763, 2816, 2049, 2226, 3212, 4981, 1159, 3962, 4170, 1379, 4119, 1660, 1991, 85, 2463, 1786, 4161, 770, 1584, 1184, 4121, 332, 4892, 1875, 2135, 316, 2059, 1056, 14, 3051, 3124, 4296, 1703, 4252, 585, 1767, 1109, 3432, 4046, 4483, 387, 2544, 4515, 1505, 1816, 1478, 106, 4545, 4339, 1853, 1801, 3967, 2981, 868, 2429, 861, 60, 2440, 519, 2776, 4326, 1954, 4664, 3730, 121, 4601, 2954, 2710, 3097, 2055, 4325, 1800, 3479, 2273, 2996, 408, 4394, 815, 1726, 3230, 719, 2241, 336, 971, 4830, 4162, 104, 4369, 330, 3197, 4363, 882, 28, 4002, 1098, 4982, 1215, 333, 4959, 2557, 3471, 1795, 1938, 76, 3058, 17, 1267, 4451, 3363, 4423, 108, 1632, 3933, 2124, 709, 311, 4994, 4344, 4940, 2167, 1264, 3181, 3384, 2079, 3454, 1418, 4518, 4133, 1214, 201, 1833, 2137, 591, 2270, 249, 2374, 1922, 746, 631, 4463, 3413, 1967, 900, 4303, 1145, 1185, 4241, 611, 1363, 1150, 3054, 4092, 702, 3375, 2803, 3548, 3861, 1559, 3020, 1336, 1332, 568, 1458, 626, 2252, 3929, 654, 2888, 4946, 184, 3606, 3414, 2689, 417, 1388, 688, 3841, 2805, 4444, 19, 1315, 3228, 3654, 994, 2863, 884, 3671, 3655, 4939, 2990, 4995, 33, 1442, 4732, 4829, 1914, 3969, 2787, 4222, 4631, 3162, 2233, 737, 1573, 4020, 2198, 163, 2949, 2229, 470, 4412, 1318, 1298, 2780, 3688, 2909, 3608, 3235, 3680, 1637, 1814, 1687, 4643, 1864, 2832, 175, 942, 4912, 4709, 1515, 4778, 3272, 3219, 4691, 4307, 4312, 888, 4122, 2284, 353, 1731, 1868, 4439, 389, 1880, 402, 487, 2770, 3362, 864, 3299, 2000, 2847, 3615, 3990, 4722, 516, 4449, 2821, 4686, 1996, 4521, 3331, 4103, 1002, 833, 2936, 617, 1116, 4656, 3325, 4951, 48, 1805, 2755, 1701, 155, 1148, 1648, 4042, 1822, 3342, 1065, 3767, 4844, 1901, 3209, 1362, 3770, 1447, 2800, 406, 4315, 450, 3077, 1105, 3081, 243, 1889, 3526, 2128, 1691, 4277, 4772, 1645, 2476, 2250, 401, 1989, 2437, 1428, 2292, 3304, 4726, 1278, 453, 3757, 2390, 1750, 3915, 840, 498, 3322, 1479, 4879, 1811, 2621, 4759, 1958, 3493, 1921, 1663, 960, 1337, 4225, 4586, 1398, 4780, 2595, 2874, 571, 3672, 3563, 3491, 1561, 978, 1988, 2025, 712, 3497, 3489, 2099, 438, 1233, 975, 4713, 1608, 4183, 2659, 795, 4684, 2627, 2982, 407, 4190, 897, 590, 3293, 1455, 3373, 3849, 2790, 31, 3676, 3116, 823, 773, 4745, 3594, 210, 4655, 4488, 1370, 3863, 1945, 3002, 3831, 3290, 2121, 2001, 4030, 1049, 4006, 1258, 4291, 601, 4313, 3023, 3525, 4649, 367, 2934, 366, 2349, 1818, 3395, 3633, 4657, 4937, 3135, 2257, 2428, 3477, 288, 3396, 4147, 1303, 5000, 4218, 4300, 4560, 4965, 3313, 2111, 2656, 1457, 2512, 1611, 2993, 966, 2507, 887, 4078, 1131, 706, 3006, 3012, 4859, 2897, 3782, 1968, 886, 2654, 3854, 557, 989, 4867, 2700, 3895, 2112, 1403, 1709, 3512, 2109, 3369, 4845, 3133, 2051, 621, 2553, 707, 2957, 563, 3450, 2212, 1723, 2535, 4604, 2220, 1777, 872, 3201, 2433, 4401, 4044, 1508, 3078, 4321, 1468, 1894, 3867, 3966, 1321, 4783, 1930, 1556, 3137, 2690, 2484, 3330, 140, 2640, 1460, 893, 3826, 1259, 521, 276, 2371, 1673, 1946, 290, 622, 4636, 98, 4544, 1893, 1263, 3292, 1644, 478, 1463, 2253, 2808, 1189, 2570, 1574, 1472, 3502, 3368, 2337, 4638, 5006, 4862, 923, 114, 107, 2750, 1666, 710, 1308, 762, 2209, 2666, 3495, 1529, 3273, 1380, 3582, 560, 4167, 2674, 3437, 4563, 376, 3218, 3227, 3560, 3585, 637, 1695, 1956, 2346, 4904, 832, 2913, 4837, 2436, 1032, 1778, 666, 1405, 2442, 4967, 3559, 2920, 3042, 2500, 3114, 1712, 2031, 790, 346, 2098, 3348, 1174, 2003, 4495, 3851, 2566, 4094, 4587, 3485, 274, 1683, 3036, 3707, 3989, 1456, 1876, 1757, 3236, 4379, 2585, 1236, 3231, 1570, 3712, 2504, 705, 3232, 239, 2988, 1484, 808, 2258, 2275, 1140, 735, 871, 1399, 4470, 2072, 2598, 2926, 3346, 1569, 3016, 3874, 136, 869, 4802, 4627, 1616, 2539, 4695, 837, 1324, 1510, 2753, 2624, 2217, 3329, 3151, 1480, 4730, 539, 692, 189, 4139, 2721, 4977, 1966, 237, 1081, 778, 1820, 1669, 1253, 2360, 3609, 1018, 2799, 1053, 158, 3684, 219, 1948, 3165, 1367, 1790, 3648, 2699, 4448, 4605, 1932, 4245, 4385, 3185, 3735, 1160, 451, 1319, 890, 4548, 3786, 4095, 2054, 3673, 4561, 3056, 4410, 3339, 2131, 350, 1400, 3131, 2028, 4405, 4876, 4007, 4349, 4896, 3913, 1736, 2103, 4711, 715, 1248, 193, 4281, 4202, 1974, 2235, 3092, 4624, 2449, 4000, 2836, 1592, 1320, 793, 1862, 281, 2364, 3057, 1322, 369, 3387, 3440, 1496, 4358, 2474, 855, 1153, 992, 469, 3957, 1300, 1254, 3309, 3905, 4160, 2773, 2285, 4789, 1566, 798, 235, 964, 2857, 4757, 772, 3134, 477, 1517, 2262, 4426, 1117, 309, 683, 573, 1846, 1797, 2961, 390, 1470, 1397, 2693, 4009, 4944, 4799, 3827, 3635, 3377, 2186, 1156, 2405, 4520, 2182, 543, 818, 4356, 2409, 670, 1192, 3390, 3733, 4461, 783, 812, 4715, 3943, 3257, 4383, 4968, 380, 4927, 1060, 3649, 2642, 4099, 4818, 4913, 3150, 4547, 4841, 821, 3224, 3246, 775, 3136, 357, 2157, 1681, 2306, 3175, 2550, 1792, 4975, 2016, 3625, 2164, 3275, 3179, 686, 2761, 160, 3337, 990, 1668, 752, 785, 748, 1768, 3584, 301, 983, 1335, 838, 784, 2093, 3069, 372, 895, 3428, 2455, 1238, 606, 1582, 2983, 3598, 2568, 2785, 4362, 2717, 3019, 3834, 2345, 278, 2087, 3734, 3093, 4838, 1356, 1000, 3449, 1110, 1037, 2414, 4907, 4421, 962, 2034, 87, 3569, 1195, 4714, 462, 1821, 4317, 1355, 2430, 1761, 78, 3701, 2923, 1885, 4287, 2741, 1375, 38, 3125, 3310, 2610, 2105, 3581, 4447, 209, 444, 4306, 373, 4376, 799, 4127, 4921, 3127, 4683, 1016, 3305, 1550, 320, 2944, 4499, 3552, 370, 473, 3547, 4274, 4676, 1120, 262, 3084, 3102, 4354, 1085, 3356, 2073, 3878, 3828, 4031, 2048, 3320, 3829, 1068, 2584, 628, 2269, 968, 4553, 2685, 4112, 2995, 3367, 122, 4464, 849, 2327, 1245, 1607, 3928, 782, 996, 4911, 399, 4371, 361, 2333, 4175, 1104, 4381, 4473, 94, 4454, 2127, 1831, 4432, 1764, 3578, 3900, 749, 2987, 3079, 4680, 331, 3403, 545, 1036, 2571, 1331, 3646, 2254, 187, 3775, 2122, 3411, 1005, 4619, 2737, 4228, 1957, 3388, 1099, 3376, 3359, 524, 2617, 2267, 1381, 1269, 1817, 2994, 56, 613, 946, 4796, 3073, 3555, 3111, 3318, 4889, 3178, 4489, 3444, 411, 3689, 1030, 2113, 556, 4294, 3792, 504, 2636, 292, 3463, 1807, 1630, 2999, 1554, 1270, 3089, 4468, 2058, 4472, 4836, 2450, 3890, 3949, 441, 4346, 3480, 4116, 3540, 4685, 505, 4440, 2391, 562, 3220, 2040, 508, 4308, 4728, 2453, 4396, 3705, 2766, 3817, 4398, 4384, 409, 3394, 3941, 2403, 2486, 4513, 4719, 4705, 463, 3184, 5008, 4617, 3700, 1095, 1499, 4207, 3504, 298, 1699, 567, 3142, 2870, 1101, 2724, 2975, 2792, 2963, 3946, 4690, 3062, 268, 1962, 2771, 4866, 1043, 3845, 3481, 2727, 2922, 2960, 2350, 2443, 3860, 1766, 127, 1166, 558, 1168, 2362, 3048, 3382, 3546, 80, 1995, 393, 2358, 3117, 1023, 2696, 2927, 1158, 1444, 1863, 2172, 57, 1420, 3180, 3427, 3593, 2469, 1784, 2330, 4399, 4596, 4422, 4609, 834, 925, 922, 3508, 2729, 379, 4648, 404, 3405, 3255, 4998, 817, 2650, 1751, 4323, 2683, 961, 1534, 340, 921, 3380, 1213, 730, 286, 4610, 2938, 2202, 4646, 481, 4276, 2767, 97, 2633, 3186, 771, 4159, 4790, 1057, 2518, 1432, 1985, 2728, 1686, 2, 4822, 739, 2784, 3565, 958, 2597, 2743, 4746, 267, 2791, 4199, 865, 3898, 1067, 3907, 4920, 2155, 3506, 4865, 4969, 1170, 1624, 8, 2945, 632, 36, 4612, 1289, 1688, 3499, 3350, 3536, 1618, 2952, 4566, 3122, 3147, 3839, 4309, 3879, 1591, 3415, 2989, 3858, 4720, 4512, 4955, 3551, 3229, 223, 4035, 4223, 2928, 3746, 3452, 4602, 2499, 3679, 4588, 169, 3544, 4848, 39, 578, 419, 4265, 1545, 1035, 4924, 156, 4072, 3314, 225, 3908, 943, 4812, 2473, 4523, 3005, 777, 2377, 2411, 2312, 2032, 325, 2672, 3970, 1950, 3951, 1066, 2692, 2187, 3562, 1944, 1754, 1051, 2607, 4877, 4370, 3171, 1033, 3818, 1039, 2328, 1923, 4494, 1829, 3910, 3013, 351, 4910, 58, 1620, 4203, 4542, 1404, 2010, 4839, 4481, 2071, 452, 418, 2326, 2419, 2203, 400, 300, 4286, 2305, 3088, 4986, 1625, 413, 4284, 1705, 1999, 2243, 1992, 4460, 500, 1313, 4791, 4781, 4372, 1223, 2769, 2783, 4902, 932, 2343, 4057, 3015, 2730, 5005, 1177, 3014, 2644, 1241, 3120, 2234, 4815, 4559, 4943, 86, 1277, 1338, 4983, 1972, 1840, 2331, 523, 2908, 3190, 1721, 3956, 3319, 4032, 2394, 1494, 1896, 2613, 2090, 554, 4134, 2675, 4115, 1465, 1870, 3527, 1059, 4806, 4104, 4721, 4682, 1061, 3082, 2447, 3663, 4872, 1793, 3621, 4899, 433, 866, 3470, 1729, 1225, 334, 3564, 3289, 3182, 420, 217, 4109, 809, 2806, 4297, 3647, 3053, 4805, 3067, 4786, 4514, 464, 1543, 3226, 5007, 3295, 3316, 1741, 3355, 4567, 3017, 3779, 3714, 3443, 863, 2930, 3980, 3824, 430, 4034, 4729, 2142, 2427, 4668, 2288, 3837, 3199, 3930, 894, 2354, 1598, 4219, 1973, 1029, 1069, 4707, 4623, 3465, 2446, 1164, 4508, 2132, 9, 3139, 4480, 343, 1301, 4278, 2649, 2011, 1523, 2488, 4186, 1, 4077, 605, 1357, 2353, 1219, 788, 1722, 1711, 279, 1544, 1959, 3166, 1041, 1204, 1878, 1093, 3793, 743, 1981, 4530, 3694, 1431, 2653, 145, 4592, 4600, 2647, 2786, 4178, 2287, 355, 3645, 3868, 1990, 1899, 4428, 1070, 4304, 4511, 2249, 1058, 1393, 4036, 3241, 4351, 275, 1372, 2015, 2827, 4364, 4756, 945, 3637, 1217, 1138, 4568, 3287, 4875, 3498, 4898, 2680, 1090, 3010, 1424, 2883, 2094, 1021, 3372, 2107, 2242, 2916, 1553, 3769, 1827, 987, 1046, 2772, 1796, 2237, 2451, 4918, 1815, 649, 1171, 3761, 1100, 490, 1860, 550, 3708, 879, 4452, 1924, 2529, 4507, 2119, 536, 3000, 1437, 1466, 2042, 643, 4915, 976, 2424, 347, 4261, 1609, 3836, 3345, 386, 3063, 63, 3514, 2875, 4538, 161, 3132, 4484, 4144, 1978, 3965, 2117, 1292, 3755, 515, 151, 1082, 2914, 2973, 4900, 1040, 609, 3478, 944, 1012, 466, 4490, 3361, 206, 757, 1671, 1646, 3919, 2039, 4416, 3974, 3843, 2088, 3252, 2078, 3754, 4675, 1960, 1735, 4814, 3192, 2265, 4403, 1427, 2325, 537, 2725, 4465, 2884, 397, 1475, 1718, 4880, 312, 1071, 4584, 4492, 2704, 4498, 3155, 3037, 142, 2707, 4251, 518, 1678, 4328, 1500, 1284, 4350, 326, 3371, 1136, 3288, 820, 4033, 4916, 1576, 461, 4433, 4630, 767, 3981, 3727, 4618, 2593, 4442, 1193, 1118, 4333, 3538, 4067, 741, 4273, 2388, 2858, 902, 435, 3802, 4475, 185, 580, 881, 4769, 2778, 3238, 4792, 3550, 2940, 3812, 3934, 327, 4200, 3521, 638, 624, 3370, 2297, 846, 4932, 1965, 272, 4003, 1209, 1325, 2762, 4295, 4233, 714, 1080, 188, 4054, 1604, 4882, 2701, 2643, 907, 4, 1548, 4118, 727, 4696, 2589, 694, 3177, 2833, 780, 456, 648, 2378, 4407, 55, 1202, 3611, 4941, 889, 1640, 4922, 197, 4527, 4293, 2754, 4360, 21, 3572, 4583, 3047, 3344, 4066, 3921, 1884, 542, 238, 934, 3923, 115, 1075, 1476, 878, 4192, 96, 4249, 3846, 3674, 194, 109, 2567, 318, 3034, 509, 1276, 1557, 4768, 1361, 4173, 1971, 3823, 570, 221, 3935, 2145, 1589, 2619, 1280, 877, 147, 2125, 3971, 3409, 5004, 1206, 4734, 1467, 1178, 3785, 1680, 3213, 4502, 4556, 4156, 482, 1228, 3112, 2227, 3237, 1407, 3624, 398, 6, 4088, 3693, 635, 954, 3086, 3032, 2859, 3636, 2296, 186, 4700, 831, 167, 2709, 1565, 1412, 860, 344, 2053, 4238, 1221, 2290, 2556, 1371, 4942, 4886, 7, 936, 1509, 3294, 3944, 4855, 3924, 2739, 4554, 396, 1804, 111, 2974, 2056, 526, 3662, 116, 4710, 2813, 3247, 1186, 3225, 1578, 955, 1441, 1849, 3577, 132, 1275, 3610, 3262, 4678, 13, 4146, 3952, 2986, 3140, 3008, 2670, 1485, 1528, 4438, 101, 294, 3420, 3379, 2886, 3532, 3483, 3106, 1077, 2586, 1575, 1521, 1837, 1108, 4832, 2021, 4846, 3918, 2211, 2174, 4469, 898, 2811, 1003, 857, 588, 517, 2818, 2629, 2583, 1527, 1585, 1928, 2801, 981, 1139, 3623, 4388, 726, 230, 1524, 1151, 1567, 244, 2444, 2102, 939, 995, 913, 3685, 3948, 2441, 2045, 532, 1601, 3158, 4353, 675, 3520, 2101, 2368, 4764, 4271, 118, 4834, 4169, 1226, 4642, 3704, 5009, 4019, 479, 1146, 3258, 310, 3011, 4397, 2764, 2035, 3095, 1748, 1847, 3583, 3697, 2947, 4366, 685, 2965, 1345, 2457, 2115, 3446, 732, 2817, 3718, 634, 544, 2095, 718, 4290, 3484, 641, 4807, 1602, 3188, 3500, 633, 2711, 4049, 3731, 120, 2149, 2178, 2481, 4194, 1392, 2096, 2837, 1378, 2461, 4259, 3717, 3620, 1334, 1749, 3903, 2972, 1623, 3501, 4185, 69, 911, 1765, 4129, 620, 3277, 3586, 3426, 3736, 4917, 3149, 99, 4506, 3866, 3937, 1854, 35, 271, 4990, 154, 3052, 3778, 4580, 4013, 1976, 4774, 1281, 1740, 4919, 1542, 2465, 529, 2415, 1772, 3912, 566, 2491, 1165, 1653, 1290, 1252, 4365, 3719, 1614, 4740, 3198, 1595, 196, 1536, 254, 522, 383, 2526, 4549, 824, 1842, 395, 4450, 829, 742, 2069, 699, 4991, 2793, 1440, 4125, 421, 2013, 2523, 1714, 4621, 4431, 4361, 4687, 2352, 3722, 306, 4152, 2383, 3518, 3001, 3797, 3399, 1502, 1758, 1210, 940, 1124, 1555, 1549, 296, 595, 1045, 2910, 2881, 653, 4516, 4795, 3486, 625, 2302, 2582, 1296, 4198, 4503, 3784, 2495, 1187, 1925, 4214, 4446, 218, 844, 1916, 2413, 3025, 1879, 1417, 32, 4808, 2775, 3083, 3511, 4240, 2086, 1242, 4415, 4531, 2532, 4205, 2222, 4626, 2263, 1690, 4761, 3300, 1791, 126, 1993, 1487, 3531, 1848, 2017, 2528, 378, 4672, 3301, 266, 4023, 2565, 4660, 3204, 3121, 2289, 4210, 2037, 4852, 1806, 4809, 2638, 3644, 2083, 910, 2551, 679, 2641, 2173, 1365, 4665, 1341, 627, 1477, 4441, 2502, 1448, 1858, 949, 2033, 4074, 1474, 4819, 2030, 972, 776, 4633, 3043, 3992, 2163, 870, 875, 3692, 3109, 3613, 3385, 1285, 1143, 181, 3516, 1920, 2980, 2448, 2041, 3333, 4887, 4970, 3327, 804, 2140, 1984, 1203, 2697, 3472, 4533, 4355, 1642, 1725, 437, 3626, 4950, 853, 1161, 2043, 3038, 4840, 4770, 2200, 4810, 3488, 3771, 2519, 3217, 3891, 4462, 1212, 429, 3027, 424, 2076, 4572, 1845, 2144, 1451, 2165, 3040, 1360, 4359, 54, 3804, 3840, 3251, 4505, 285, 476, 77, 3742, 3170, 640, 471, 1715, 4456, 2807, 2841, 2452, 1169, 2075, 764, 1419, 4677, 2478, 4348, 2889, 2369, 3963, 1781, 721, 273, 4244, 4963, 2812, 3570, 4598, 4418, 1147, 2080, 4005, 1940, 2691, 1416, 2047, 410, 1910, 3022, 1927, 2399, 3168, 4342, 4870, 3877, 4742, 3591, 3174, 2027, 553, 4639, 2657, 725, 4593, 1692, 4595, 1734, 3657, 3205, 100, 4165, 1216, 133, 2611, 2538, 3669, 4510, 159, 4055, 4021, 3541, 689, 2459, 728, 1634, 4124, 2493, 917, 658, 4881, 431, 3271, 2733, 4863, 2282, 1939, 1083, 4393, 2432, 1650, 2181, 2485, 1819, 23, 1823, 2756, 1724, 4689, 3869, 1009, 59, 3947, 2256, 71, 2340, 729, 1413, 4961, 4744, 763, 856, 3681, 2318, 3542, 2236, 2146, 740, 1708, 3154, 2271, 3311, 2416, 242, 1229, 4062, 250, 2630, 1917, 4938, 3618, 3138, 4155, 3850, 4534, 3336, 1008, 2466, 2942, 1700, 427, 4575, 1662, 2804, 1667, 2153, 4179, 3601, 3703, 2620, 1401, 1770, 4504, 3575, 4535, 2616, 2498, 3161, 3422, 3607, 4404, 4804, 3353, 947, 1612, 3762, 4731, 4574, 2100, 4801, 2777, 3991, 3954, 2578, 1329, 4052, 337, 848, 4936, 836, 576, 1694, 1349, 2196, 896, 1638, 2931, 299, 2185, 4064, 4076, 1394, 152, 2814, 766, 3885, 1756, 3892, 64, 3431, 1497, 2057, 2959, 485, 3419, 4131, 4532, 3193, 3787, 305, 1603, 999, 304, 2901, 841, 1294, 4193, 2490, 2106, 50, 2133, 4666, 30, 2865, 3882, 1752, 1152, 2768, 842, 2795, 4424, 2558, 1006, 3412, 1929, 2815, 969, 2687, 950, 3737, 4380, 3728, 3129, 1779, 582, 3200, 3400, 4628, 1047, 906, 3417, 4172, 803, 3208, 247, 3880, 745, 2735, 1780, 494, 1268, 952, 3776, 3661, 4785, 2372, 3756, 2336, 3975, 1919, 3720, 2829, 1839, 4692, 1330, 4794, 3815, 44, 3003, 4130, 371, 128, 2505, 507, 1672, 2966, 1395, 1402, 4589, 458, 3130, 678, 4320, 1306, 3466, 2576, 3434, 4766, 1387, 4216, 4632, 1172, 2520, 139, 1606, 3906, 847, 3060, 1142, 4227, 252, 2311, 1871, 4803, 472, 3033, 1641, 3274, 246, 4528, 948, 1596, 1208, 356, 3183, 1190, 4253, 3909, 4163, 1244, 528, 2141, 3152, 81, 1558, 2472, 3408, 1689, 3214, 2912, 2781, 3216, 2114, 1353, 657, 1311, 415, 1459, 3813, 650, 1639, 4496, 876, 1918, 3558, 3537, 1516, 1512, 826, 4255, 690, 2255, 3716, 1626, 4437, 2939, 1377, 512, 2554, 4670, 2839, 4697, 2828, 3987, 3473, 3266, 4811, 1305, 2962, 4718, 4242, 891, 4206, 1286, 973, 534, 3307, 245, 4703, 149, 1526, 1900, 2489, 1934, 1052, 1087, 1423, 1519, 315, 2238, 2404, 1486, 4929, 4738, 3835, 2065, 2890, 4637, 3682, 4767, 2046, 1856, 4536, 2664, 4330, 1538, 929, 3074, 2892, 774, 2370, 2525, 4126, 1010, 1897, 3030, 1931, 4324, 3723, 2213, 307, 744, 270, 3094, 2387, 1883, 652, 1579, 1980, 1600, 4269, 2885, 4150, 153, 4847, 3871, 3780, 4890, 1323, 1547, 1890, 1188, 4111, 1877, 2587, 527, 1952, 3087, 3893, 3870, 3351, 2014, 1643, 3302, 3726, 1257, 1490, 1239, 4292, 468, 263, 2319, 3328, 4263, 26, 1450, 1541, 3259, 1232, 1211, 141, 1851, 4608, 630, 3842, 2138, 4641, 2540, 18, 4954, 4065, 2439, 3698, 873, 3, 2779, 269, 1123, 974, 1530, 814, 2831, 2348, 3995, 4102, 4184, 2357, 4644, 4047, 2745, 3286, 2676, 1744, 4445, 3677, 4923, 4645, 4430, 4747, 2307, 4971, 2509, 1350, 4748, 4856, 3753, 2161, 4529, 4622, 656, 4582, 1183, 1774, 72, 4337, 2425, 1483, 308, 3568, 2522, 2575, 2634, 1266, 2853, 511, 3250, 3366, 651, 3665, 724, 4059, 4888, 1935, 73, 2530, 664, 423, 11, 3883, 1435, 1830, 2324, 3312, 228, 4229, 3988, 937, 753, 1684, 1017, 2918, 4022, 1551, 465, 1912, 3968, 4724, 3853, 552, 4195, 439, 3914, 3068, 287, 691, 2869, 3264, 1710, 2933, 3110, 1789, 2475, 2825, 2123, 4455, 2373, 1234, 232, 174, 2023, 46, 4500, 2423, 671, 4107, 3580, 3668, 835, 1436, 291, 673, 3556, 172, 711, 1197, 1742, 2082, 2731, 1513, 3308, 2905, 4332, 789, 323, 3496, 1619, 1358, 3772, 1552, 3554, 4413, 2496, 4706, 2386, 2375, 1738, 2295, 938, 2392, 2206, 2521, 2492, 105, 4869, 4406, 113, 1446, 4497, 2092, 2248, 1997, 1733, 4367, 2421, 1828, 4237, 1769, 2389, 264, 2089, 1013, 2561, 1406, 1785, 3597, 2606, 3187, 2070, 2303, 3285, 3740, 4486, 2662, 541, 2216, 2517, 3061, 4368, 3268, 4117, 2608, 1207, 2215, 4231, 358, 3509, 4136, 4606, 3233, 4402, 1933, 1391, 4487, 3423, 3576, 4698, 4903, 4884, 1798, 137, 2482, 4979, 1119, 211, 4275, 2294, 1838, 4051, 3993, 3889, 920, 3357, 2726, 4603, 1621, 229, 3115, 1173, 2658, 1719, 4340, 535, 182, 1773, 74, 1504, 1034, 1581, 2536, 957, 4093, 4373, 565, 4577, 4562, 4779, 1568, 4043, 2468, 3862, 3640, 3710, 1295, 3628, 2261, 3389, 3617, 806, 1255, 4800, 4693, 227, 2747, 2652, 1987, 313, 3875, 4153, 1677, 1129, 858, 3983, 4571, 1782, 4552, 2342, 3441, 2648, 3143, 93, 3543, 4762, 117, 2259, 3729, 2531, 2420, 1676, 2245, 4667, 360, 3279, 2309, 4196, 4857, 2820, 339, 4858, 1453, 3105, 282, 4256, 3334, 4594, 1651, 4280, 3766, 3507, 1675, 1587, 226, 525, 3807, 3464, 2596, 1191, 4352, 4733, 2401, 52, 2688, 2919, 2480, 2147, 1157, 2134, 1262, 2899, 434, 4411, 4949, 53, 4008, 2313, 1307, 3459, 731, 385, 110, 4787, 1282, 208, 2850, 2798, 445, 3832, 2136, 4522, 1339, 985, 4391, 4620, 3055, 4928, 2268, 2951, 1647, 3873, 3747, 662, 394, 4069, 3979, 738, 3126, 4453, 2221, 4550, 3416, 1947, 4551, 610, 3091, 4661, 4773, 655, 3916, 3899, 3215, 1657, 265, 786, 388, 2612, 66, 4578, 4976, 2645, 359, 2363, 95, 1132, 4477, 280, 636, 1674, 4037, 3859, 3021, 3398, 1865, 3298, 4519, 3596, 2591, 1134, 447, 2873, 345, 2900, 3630, 2953, 3260, 455, 1113, 457, 341, 2603, 216, 4027, 3614, 2937, 3332, 317, 1813, 2992, 4272, 3386, 1462, 695, 364, 993, 2184, 977, 953, 2029, 2880, 2924, 129, 2400, 3153, 3515, 4319, 1359, 2797, 3234, 5, 1025, 3196, 619, 3997, 454, 4212, 843, 436, 2081, 1627, 426, 3071, 3035, 2663, 2760, 382, 4015, 4820, 4132, 2877, 293, 4788, 1546, 1328, 491, 1488, 2276, 2845, 2192, 2191, 4345, 3535, 3814, 1422, 4105, 2639, 696, 3709, 3065, 2887, 5003, 2915, 2334, 779, 704, 2998, 676, 2410, 2064, 2382, 3254, 3487, 241, 2341, 3469, 2605, 1540, 3926, 1231, 3706, 460, 1072, 4113, 959, 4217, 3402, 83, 1299, 171, 1841, 4174, 2602, 4378, 4087, 1907, 4056, 1167, 1438, 2322, 2991, 1439, 839, 2985, 2646, 1230, 1088, 4187, 4236, 3070, 4793, 4334, 1103, 1665, 1452, 474, 2460, 4585, 2738, 2272, 2177, 4776, 3939, 148, 1346, 4999, 2941, 3240, 3159, 4925, 2154, 2006, 3101, 3482, 2009, 2315, 2431, 3397, 2712, 4658, 2470, 493, 2543, 489, 84, 3856, 3191, 1020, 4635, 4501, 1560, 4075, 1031, 851, 4338, 432, 2514, 1867, 2622, 2984, 1537, 3763, 2876, 1461, 4084, 302, 586, 3119, 65, 3513, 642, 1745, 2555, 3820, 2628, 903, 3592, 2286, 4246, 1430, 3800, 659, 2479, 2600, 2681, 1498, 2171, 352, 3922, 4085, 4311, 4310, 4264, 3894, 329, 3953, 1115, 4956, 258, 4824, 4895, 27, 253, 4138, 2159, 2036, 4343, 805, 4289, 693, 1955, 4458, 4071, 2199, 1426, 3072, 4526, 4041, 1383, 5001, 811, 1384, 3253, 1180, 2732, 2008, 3085, 2684, 1659, 3194, 3973, 1656, 103, 3360, 4671, 1348, 908, 2906, 4191, 4079, 3365, 1141, 4704, 3321, 4171, 90, 874, 4387, 1256, 1074, 3616, 807, 3066, 3938, 2956, 703, 801, 503, 1076, 3421, 1019, 927, 787, 1909, 1783, 3148, 3505, 912, 3269, 4485, 4392, 1048, 4086, 1149, 3364, 40, 4629, 980, 3574, 3745, 4878, 593, 4151, 800, 492, 499, 3765, 4060, 2788, 3031, 2018, 3964, 1794, 3050, 600, 4821, 1366, 2573, 1492, 4613, 2757, 1572, 204, 448, 2950, 4427, 2462, 548, 236, 2903, 736, 916, 3691, 2749, 1327, 2563, 717, 475, 3430, 4930, 4557, 4765, 2019, 967, 3383, 4479, 614, 2320, 3838, 3651, 3239, 4871, 4318, 1382, 3144, 1026, 3381, 926, 1004, 2190, 354, 2705, 2782, 3156, 3760, 1198, 4279, 4699, 4573, 579, 1881, 2281, 3280, 3343, 2060, 3245, 3545, 4058, 1491, 2894, 2935, 533, 4640, 124, 3819, 47, 1122, 3145, 667, 3044, 3976, 10, 2085, 862, 4268, 2840, 2632, 2718, 4524, 4669, 4493, 2872, 1577, 1655, 328, 1937], "validation": [5316, 5397, 5176, 5166, 5063, 5488, 5363, 5212, 5452, 5444, 5287, 5033, 5307, 5248, 5329, 5428, 5350, 5119, 5023, 5227, 5366, 5331, 5303, 5130, 5477, 5299, 5146, 5191, 5319, 5297, 5301, 5249, 5507, 5129, 5354, 5179, 5259, 5065, 5330, 5238, 5254, 5413, 5310, 5134, 5339, 5500, 5351, 5295, 5342, 5011, 5288, 5426, 5163, 5453, 5155, 5221, 5302, 5044, 5234, 5218, 5421, 5490, 5062, 5277, 5085, 5379, 5152, 5325, 5256, 5125, 5207, 5478, 5060, 5473, 5445, 5344, 5386, 5022, 5036, 5064, 5443, 5420, 5466, 5015, 5077, 5442, 5122, 5273, 5202, 5506, 5293, 5408, 5088, 5083, 5177, 5283, 5269, 5291, 5389, 5156, 5240, 5438, 5192, 5017, 5333, 5027, 5289, 5391, 5304, 5381, 5471, 5054, 5087, 5164, 5503, 5410, 5508, 5069, 5460, 5131, 5392, 5014, 5467, 5226, 5245, 5328, 5489, 5024, 5371, 5035, 5037, 5106, 5183, 5098, 5113, 5451, 5345, 5493, 5081, 5127, 5335, 5100, 5271, 5509, 5154, 5165, 5505, 5071, 5372, 5317, 5446, 5487, 5167, 5294, 5423, 5448, 5174, 5355, 5112, 5414, 5274, 5292, 5107, 5393, 5305, 5361, 5078, 5097, 5162, 5337, 5205, 5040, 5322, 5225, 5441, 5082, 5476, 5042, 5144, 5456, 5457, 5411, 5230, 5470, 5232, 5341, 5267, 5061, 5494, 5504, 5242, 5086, 5404, 5010, 5275, 5215, 5020, 5139, 5468, 5019, 5427, 5149, 5432, 5175, 5116, 5237, 5458, 5375, 5464, 5412, 5298, 5475, 5045, 5021, 5498, 5352, 5474, 5159, 5096, 5050, 5454, 5229, 5184, 5384, 5398, 5491, 5190, 5263, 5140, 5264, 5436, 5102, 5233, 5104, 5171, 5315, 5235, 5099, 5118, 5395, 5051, 5300, 5382, 5495, 5052, 5348, 5251, 5278, 5327, 5072, 5320, 5158, 5356, 5284, 5472, 5484, 5483, 5141, 5353, 5380, 5217, 5147, 5142, 5025, 5429, 5419, 5370, 5403, 5241, 5252, 5349, 5253, 5418, 5321, 5415, 5013, 5481, 5143, 5362, 5135, 5016, 5058, 5449, 5502, 5093, 5029, 5279, 5188, 5213, 5076, 5182, 5332, 5282, 5133, 5482, 5383, 5424, 5387, 5150, 5439, 5079, 5243, 5026, 5338, 5018, 5092, 5103, 5169, 5055, 5224, 5285, 5340, 5180, 5270, 5073, 5222, 5255, 5101, 5075, 5346, 5385, 5124, 5290, 5373, 5220, 5318, 5358, 5074, 5197, 5312, 5128, 5231, 5145, 5400, 5247, 5056, 5407, 5070, 5039, 5043, 5492, 5323, 5497, 5111, 5178, 5485, 5068, 5049, 5032, 5168, 5262, 5399, 5296, 5430, 5120, 5440, 5480, 5181, 5311, 5201, 5465, 5459, 5334, 5105, 5396, 5151, 5206, 5326, 5364, 5090, 5306, 5394, 5132, 5280, 5214, 5455, 5204, 5266, 5114, 5433, 5161, 5435, 5246, 5425, 5309, 5136, 5250, 5219, 5194, 5388, 5261, 5369, 5198, 5059, 5160, 5047, 5479, 5260, 5094, 5461, 5406, 5447, 5173, 5138, 5401, 5422, 5308, 5378, 5417, 5193, 5365, 5209, 5501, 5046, 5115, 5089, 5148, 5462, 5405, 5239, 5123, 5268, 5360, 5048, 5028, 5486, 5170, 5200, 5402, 5276, 5367, 5053, 5199, 5195, 5431, 5244, 5108, 5067, 5286, 5185, 5216, 5376, 5377, 5469, 5034, 5109, 5434, 5374, 5038, 5265, 5041, 5211, 5210, 5223, 5172, 5153, 5236, 5186, 5196, 5390, 5012, 5437, 5359, 5463, 5258, 5368, 5084, 5110, 5187, 5080, 5336, 5157, 5313, 5031, 5496, 5324, 5030, 5228, 5121, 5409, 5203, 5091, 5137, 5343, 5189, 5314, 5357, 5281, 5450, 5272, 5416, 5117, 5057, 5095, 5347, 5257, 5208, 5499, 5126, 5066], "test": [5893, 5784, 5773, 5959, 5604, 5563, 5816, 5775, 5519, 5814, 5725, 5666, 5976, 5729, 5674, 5779, 5769, 5667, 5551, 5697, 5908, 5924, 5582, 5742, 5817, 5691, 5741, 5616, 6008, 5590, 5948, 5774, 5584, 5726, 5949, 5588, 5658, 5651, 5576, 5859, 5973, 5839, 5611, 5890, 5672, 5988, 5787, 5544, 5838, 5686, 5995, 5712, 5840, 5989, 5738, 5878, 5550, 5998, 5957, 5561, 5652, 5875, 5965, 5671, 5573, 5668, 5904, 5705, 5996, 5591, 5844, 5866, 5740, 5733, 5571, 5653, 5717, 5530, 5895, 5527, 5639, 5772, 5798, 5546, 5809, 5592, 5796, 5511, 5597, 5636, 5850, 5905, 5690, 5664, 5913, 5734, 5715, 5622, 5522, 5799, 5776, 5570, 5678, 5877, 5851, 5887, 5971, 5518, 5867, 5871, 5745, 5617, 5557, 5765, 5679, 5669, 5727, 5750, 5746, 5543, 5578, 5762, 5939, 5855, 5525, 5911, 5860, 5598, 5628, 5615, 5841, 5832, 5595, 5861, 5797, 5629, 5897, 5941, 5700, 5791, 5964, 5637, 5565, 5954, 5872, 5835, 5831, 5610, 5581, 5892, 5822, 5951, 5768, 5757, 5868, 5914, 5857, 5821, 5900, 5940, 5626, 5684, 5699, 5599, 5523, 5589, 5974, 5559, 5979, 5977, 5985, 5783, 5903, 5602, 5987, 5952, 5659, 5714, 5937, 5613, 5882, 5537, 5623, 5731, 5800, 5739, 5624, 5927, 5916, 5970, 5645, 5980, 5805, 5540, 5755, 5531, 5577, 5743, 5600, 5528, 5620, 5709, 5554, 5556, 5990, 5744, 5771, 5858, 5981, 5788, 5728, 5960, 5560, 5607, 5912, 5930, 5972, 6004, 5935, 5896, 5795, 5601, 5767, 5947, 5642, 5722, 5766, 5665, 5997, 5759, 5901, 5778, 5539, 5688, 6002, 5569, 5541, 6000, 5532, 5828, 5517, 5933, 5943, 5719, 5702, 5655, 5986, 6009, 5969, 5833, 5758, 5685, 5910, 5730, 5635, 5574, 5854, 5605, 5925, 5529, 5929, 5723, 5804, 5845, 5966, 5735, 5680, 5962, 5812, 5968, 5710, 5663, 5853, 5512, 5737, 5782, 5842, 5535, 5761, 5585, 5921, 5747, 5534, 5934, 5806, 5683, 5931, 5514, 5694, 5999, 5928, 5898, 5899, 5834, 5646, 5614, 5552, 5846, 5515, 5793, 5777, 5732, 5994, 5634, 5609, 5808, 5862, 5824, 5926, 5967, 5548, 5802, 5718, 5984, 5953, 5883, 5770, 5568, 5603, 5657, 5963, 5555, 5849, 5982, 5721, 5803, 5612, 5693, 5708, 5789, 5562, 5606, 5922, 5662, 5579, 5885, 5618, 5695, 5945, 5586, 5583, 5869, 5961, 5567, 5932, 5545, 5526, 6005, 5549, 6003, 5915, 5754, 5643, 5649, 5627, 5874, 5575, 5909, 5889, 5654, 5801, 5711, 5864, 5827, 5907, 5716, 5703, 5847, 5753, 5625, 5992, 5785, 5696, 5650, 5944, 5675, 5781, 5687, 5856, 5689, 5792, 5891, 5698, 5823, 5751, 5632, 5938, 5848, 5720, 5780, 5881, 5748, 5587, 5879, 5958, 5946, 5644, 5978, 5794, 5917, 6001, 5520, 5564, 5993, 5673, 5837, 5950, 5815, 5648, 5572, 5608, 5906, 5852, 5752, 5749, 5880, 5656, 5619, 5955, 5919, 5670, 5873, 5713, 5524, 5701, 5661, 5843, 5736, 5676, 5902, 5836, 5647, 5810, 5876, 5764, 5641, 5825, 5547, 5975, 5510, 5533, 5724, 5790, 5918, 5763, 5936, 5811, 5829, 5786, 6007, 5596, 5521, 5706, 5820, 5566, 5886, 5707, 5542, 5536, 5991, 5630, 6006, 5888, 5983, 5516, 5865, 5677, 5513, 5704, 5553, 5920, 5593, 5633, 5756, 5631, 5621, 5638, 5580, 5863, 5830, 5956, 5594, 5884, 5807, 5818, 5660, 5942, 5692, 5894, 5682, 5813, 5819, 5870, 5640, 5558, 5538, 5681, 5760, 5923, 5826]}, {"train": [4297, 4579, 2142, 994, 1166, 1654, 4369, 3656, 3898, 2653, 135, 4315, 1091, 1736, 4325, 1843, 1149, 656, 4757, 362, 2592, 729, 428, 1649, 4913, 1129, 1807, 2263, 869, 4966, 4112, 2154, 235, 2796, 857, 3698, 444, 1983, 203, 369, 3925, 1509, 2729, 1984, 3408, 4866, 3791, 809, 3830, 2461, 4548, 3039, 2293, 2455, 3952, 4359, 156, 652, 3644, 2980, 4772, 1269, 4003, 4741, 183, 43, 599, 2705, 2667, 1048, 4735, 991, 3886, 1690, 3175, 4340, 4425, 3404, 494, 4462, 1538, 3836, 2472, 4763, 4822, 4547, 1020, 1465, 4711, 2272, 1909, 1547, 2236, 1625, 3356, 4986, 868, 3519, 2393, 1489, 1370, 319, 1591, 3249, 874, 3942, 4159, 496, 1165, 1698, 3181, 4170, 1347, 2750, 1672, 2584, 624, 1518, 1133, 4203, 2825, 4568, 3664, 2581, 1848, 4931, 4629, 4302, 3619, 3555, 3151, 1942, 336, 1275, 597, 1533, 3349, 331, 3604, 2420, 4730, 1600, 2654, 3447, 3178, 3040, 4621, 940, 776, 2194, 4051, 4980, 1822, 154, 2493, 4155, 4939, 3754, 1840, 2615, 4222, 2203, 3432, 4308, 2435, 4334, 982, 196, 265, 3341, 222, 928, 2604, 2273, 2408, 411, 748, 4954, 335, 1431, 960, 83, 2725, 921, 535, 498, 3895, 995, 2829, 4110, 1085, 962, 2606, 664, 3527, 3675, 4584, 2007, 1595, 25, 1609, 1144, 1893, 214, 2650, 4544, 2230, 3276, 4066, 2375, 4136, 855, 4377, 4883, 4500, 4983, 2769, 3499, 2499, 4416, 4078, 4590, 4435, 3603, 1322, 2959, 3133, 2376, 4709, 1394, 3646, 2607, 3654, 4862, 2539, 3469, 4742, 2372, 4448, 3286, 2773, 128, 2801, 4274, 495, 2452, 2898, 3479, 1317, 3319, 4881, 2768, 1879, 3167, 4641, 2181, 1973, 1357, 112, 3517, 3939, 3453, 1352, 1257, 2771, 4656, 2692, 1554, 2713, 899, 1794, 174, 2368, 1853, 2540, 1053, 3727, 955, 1935, 3110, 978, 4760, 59, 834, 3896, 1382, 2174, 668, 4138, 2400, 4238, 894, 172, 3547, 1349, 4504, 4201, 1865, 4137, 3087, 1436, 878, 3584, 3704, 4684, 2816, 1277, 2930, 273, 4860, 4792, 2225, 356, 62, 560, 3176, 4633, 1216, 1626, 2593, 773, 2357, 3018, 4295, 3799, 752, 2437, 905, 1030, 3897, 2708, 4491, 596, 1900, 1447, 3339, 539, 2833, 2430, 659, 501, 87, 116, 2156, 3640, 2676, 2427, 4497, 4188, 1715, 226, 1667, 2082, 828, 1945, 2192, 2342, 137, 2091, 3376, 1200, 1155, 3252, 3366, 354, 1835, 2856, 3180, 4279, 1232, 4595, 3537, 1485, 1151, 1027, 88, 3774, 2951, 3805, 1185, 5009, 2696, 525, 2363, 4093, 3015, 1982, 30, 2811, 1235, 3623, 3170, 4463, 2148, 613, 1855, 4452, 3801, 1885, 3112, 881, 2531, 3009, 4978, 1157, 977, 74, 355, 3364, 242, 4878, 1260, 1775, 1270, 269, 3195, 4937, 1765, 2624, 3816, 640, 2266, 4470, 816, 542, 4522, 2733, 2262, 3529, 760, 4906, 4399, 3451, 2737, 4762, 1487, 1419, 1905, 4973, 4831, 1240, 1947, 4855, 4283, 2723, 2227, 2896, 2167, 526, 1309, 3333, 2577, 1227, 4744, 1880, 1638, 2297, 1346, 4813, 4293, 2280, 1392, 4945, 4620, 1770, 3309, 1380, 769, 2, 3132, 1334, 1934, 3084, 3941, 1214, 785, 2244, 2391, 2256, 2343, 3669, 945, 1377, 4061, 2557, 666, 2764, 2106, 2957, 4033, 1785, 4165, 486, 430, 4963, 902, 4849, 1218, 212, 1606, 4350, 3545, 564, 3931, 2776, 4607, 825, 3209, 1800, 2946, 2410, 514, 185, 1297, 2450, 3784, 3108, 28, 1306, 1171, 1918, 2634, 812, 1384, 1818, 2956, 1094, 2098, 4129, 1639, 2580, 4815, 4714, 979, 3291, 1417, 1040, 2931, 3478, 323, 3288, 4285, 2837, 1332, 2568, 1018, 3037, 1594, 1728, 4701, 4046, 3981, 886, 2238, 1531, 4644, 950, 3159, 1612, 4573, 4892, 473, 2677, 1637, 121, 4249, 2864, 314, 1311, 2548, 4872, 2689, 1341, 751, 2474, 3482, 4126, 50, 3353, 1422, 1647, 3198, 193, 1205, 2287, 2909, 4825, 898, 1877, 2362, 618, 4867, 4353, 3150, 4512, 3107, 419, 794, 598, 3218, 1834, 2766, 2073, 2399, 4917, 3792, 1906, 3553, 3978, 3730, 2381, 1744, 2999, 3026, 4712, 2843, 4856, 1748, 2027, 796, 49, 3081, 2794, 1110, 93, 4383, 4287, 2693, 1412, 4734, 4337, 4612, 836, 3148, 3884, 4566, 2079, 2099, 288, 1522, 101, 1620, 2165, 440, 737, 3182, 1917, 1940, 517, 378, 3372, 1108, 1173, 3086, 792, 4608, 4546, 5002, 1636, 3131, 1052, 1459, 4842, 4060, 3576, 2331, 4193, 1356, 882, 4502, 1373, 259, 935, 4144, 2471, 4267, 1037, 3558, 2152, 1757, 4408, 798, 3398, 2874, 3, 858, 1437, 2943, 3284, 2118, 4692, 2215, 3829, 1565, 4128, 403, 5005, 2792, 3871, 519, 3514, 591, 3932, 2880, 3954, 2426, 626, 1004, 1686, 3331, 4700, 2143, 4225, 1339, 2687, 4876, 2649, 1116, 4140, 1204, 2694, 2295, 1820, 754, 1012, 4465, 1432, 2037, 1999, 676, 3235, 1679, 1000, 681, 4930, 4371, 2984, 3544, 4631, 2848, 2088, 4821, 3863, 1403, 3671, 1941, 592, 3044, 2204, 3047, 4529, 2361, 3912, 2524, 4074, 3783, 4613, 1718, 1239, 3314, 4877, 322, 3100, 885, 1075, 4788, 4261, 4368, 3393, 3231, 4771, 3065, 3723, 450, 2339, 677, 3561, 4614, 3687, 3770, 4671, 513, 4557, 1948, 586, 520, 2144, 463, 4715, 2778, 334, 1494, 1163, 3414, 4662, 3745, 1283, 824, 1645, 4281, 2775, 4594, 1546, 2735, 2901, 4784, 3568, 4519, 4169, 2449, 981, 328, 1631, 2441, 1032, 2703, 2746, 3769, 1617, 1119, 470, 2714, 4750, 5001, 3057, 1454, 4362, 3504, 1623, 2781, 2572, 3101, 1927, 327, 2496, 2226, 4736, 1582, 3913, 993, 3921, 1769, 4429, 2579, 1473, 3495, 4397, 4625, 2497, 4865, 3391, 2129, 3736, 1059, 3308, 1759, 2786, 1584, 3160, 1022, 4719, 2296, 2337, 2276, 1281, 4764, 4965, 3831, 97, 248, 2145, 1460, 4477, 3128, 2274, 3024, 1140, 4789, 1208, 2001, 2702, 787, 4273, 1583, 4751, 391, 1632, 4230, 4518, 2249, 4630, 2133, 4045, 1979, 1528, 4680, 4142, 2623, 3795, 889, 4934, 2492, 1752, 1836, 3025, 38, 3670, 4884, 4390, 4989, 3899, 607, 1466, 1198, 3720, 800, 1892, 1278, 4141, 3487, 4343, 3154, 2899, 1706, 4456, 4697, 1338, 2969, 115, 655, 4374, 2527, 3129, 390, 538, 2662, 2180, 1186, 3278, 1179, 4000, 2858, 1450, 1399, 4309, 3031, 63, 4214, 3224, 1202, 3613, 719, 3099, 4063, 739, 2535, 2110, 2655, 133, 826, 1296, 3689, 2657, 78, 603, 2281, 3881, 2445, 1381, 3797, 1362, 702, 3357, 2489, 4826, 3397, 969, 3847, 4839, 3612, 1678, 3375, 2902, 4123, 3301, 1369, 3775, 2683, 250, 4065, 4145, 2371, 3821, 1413, 4587, 4457, 3287, 3362, 3214, 3144, 4756, 1767, 1253, 4611, 24, 1439, 451, 3300, 4777, 1193, 641, 1430, 4459, 4812, 4008, 1215, 4844, 3094, 1295, 4699, 1856, 830, 4748, 4747, 4928, 4946, 1688, 3019, 3659, 2916, 2353, 2292, 149, 747, 2036, 3840, 359, 2412, 5004, 4704, 2312, 3767, 3407, 4410, 1078, 2183, 3400, 1438, 4054, 1128, 3856, 246, 1773, 3385, 4460, 277, 2658, 4215, 337, 2066, 2243, 442, 3449, 649, 731, 4256, 2151, 3458, 4403, 4352, 2989, 2854, 1217, 3445, 3042, 3915, 29, 2160, 741, 1717, 1955, 4146, 3386, 3960, 4489, 461, 3852, 4354, 2385, 417, 4921, 2709, 3498, 2015, 2928, 2070, 2190, 3007, 2283, 2882, 4833, 3423, 1342, 4453, 2119, 332, 557, 3197, 3833, 4180, 875, 1739, 3609, 3877, 2918, 901, 750, 2024, 1409, 1713, 3092, 1467, 1321, 4382, 3890, 2736, 4907, 3772, 3329, 4426, 3343, 4351, 5007, 1766, 2087, 4681, 3441, 3387, 745, 4367, 1443, 439, 4685, 72, 4728, 1505, 3700, 3648, 3551, 1579, 2261, 2976, 1630, 2189, 4034, 4576, 4042, 4601, 1652, 1024, 61, 644, 4886, 2454, 238, 4393, 2330, 1303, 4402, 3143, 2718, 2910, 1396, 3812, 3095, 778, 3102, 1889, 3415, 2086, 3425, 912, 2659, 3652, 1087, 3822, 4296, 2845, 4563, 2508, 4956, 499, 3876, 4564, 1446, 2026, 2822, 1435, 772, 3406, 3171, 1624, 1371, 293, 2631, 2269, 736, 3540, 1622, 4488, 3442, 19, 4981, 3846, 1732, 20, 4182, 2744, 4666, 807, 1660, 4720, 3541, 3351, 1873, 724, 289, 3918, 3156, 3751, 2514, 1014, 660, 3045, 2558, 1482, 414, 1162, 1729, 4935, 3211, 2576, 1968, 1635, 1254, 1545, 4029, 4603, 5008, 2698, 2358, 3591, 2402, 2965, 2434, 2968, 3850, 275, 2207, 4398, 2163, 3217, 4271, 3010, 3594, 673, 1212, 3905, 4897, 3904, 1123, 2201, 109, 264, 701, 1407, 3232, 2065, 4998, 3693, 2147, 3212, 4400, 3256, 3138, 4636, 4242, 1664, 2431, 1551, 3615, 1354, 888, 4824, 3982, 4968, 3800, 2745, 3970, 2242, 3471, 1127, 1034, 1017, 4667, 3825, 4571, 2712, 224, 4336, 3990, 3988, 4976, 3699, 4057, 820, 4696, 1070, 3058, 4212, 3762, 512, 721, 3997, 2905, 3029, 4069, 1148, 2157, 1866, 1726, 4708, 1895, 1393, 2005, 883, 2920, 528, 1793, 1812, 1282, 2047, 4807, 5006, 201, 3311, 3192, 351, 2356, 4338, 4967, 3322, 4540, 4952, 2533, 2020, 3277, 2721, 4149, 3111, 4605, 2341, 3789, 3022, 3973, 3050, 723, 420, 2237, 2100, 774, 1011, 2052, 4040, 4152, 3999, 3986, 4549, 2985, 2021, 4827, 937, 4471, 45, 1343, 4893, 2200, 2648, 4096, 4458, 1289, 4829, 2932, 3719, 1943, 3866, 3302, 4451, 566, 1991, 1902, 3679, 4132, 2944, 4586, 4808, 1168, 2314, 600, 3093, 472, 4944, 3731, 1753, 3917, 4200, 1190, 4905, 948, 2975, 712, 2795, 713, 3063, 2517, 3338, 3335, 425, 843, 2828, 4677, 4880, 241, 706, 1512, 3577, 2212, 2947, 3378, 1386, 1517, 3585, 907, 1864, 1451, 3930, 2846, 3571, 3520, 4257, 4432, 1374, 1395, 163, 1557, 1894, 1684, 70, 1901, 2549, 2219, 615, 3586, 569, 827, 3496, 2739, 3864, 3327, 2636, 3234, 3790, 162, 1586, 1969, 3096, 2521, 2594, 2407, 616, 728, 1657, 333, 2819, 1768, 2054, 4026, 3562, 4793, 892, 4272, 3703, 1201, 3903, 3793, 2424, 1714, 2855, 3394, 4650, 3173, 1057, 1139, 2791, 197, 4791, 932, 469, 3889, 3501, 837, 2962, 1046, 364, 2176, 4687, 4450, 2761, 3315, 4904, 315, 1887, 1213, 2395, 1588, 3035, 4298, 4080, 4975, 3934, 575, 4592, 3014, 3169, 4412, 1614, 572, 669, 1641, 2017, 1779, 793, 1550, 775, 1049, 200, 4164, 3426, 4053, 917, 4589, 2620, 31, 1411, 3976, 2961, 1300, 670, 3279, 4392, 4424, 1691, 1360, 2587, 2663, 1495, 2617, 3574, 4275, 920, 396, 231, 239, 1132, 4536, 1764, 181, 871, 388, 4861, 1510, 1098, 129, 3694, 4153, 2578, 4769, 4721, 637, 1977, 4781, 1693, 1849, 693, 379, 1916, 3446, 2978, 3350, 2728, 1760, 2182, 4598, 2599, 398, 1841, 1662, 4651, 1922, 585, 3194, 4890, 2595, 3989, 4555, 4388, 2675, 4332, 4250, 4987, 2810, 3463, 1476, 4202, 1740, 2726, 1076, 4517, 3403, 3080, 4947, 3079, 1471, 372, 220, 152, 3998, 1975, 3894, 1589, 3168, 4754, 2223, 4569, 909, 405, 4044, 4317, 1414, 4894, 3782, 1074, 1160, 1226, 860, 1147, 2053, 3316, 555, 944, 4011, 4838, 562, 3647, 2569, 3355, 2488, 4104, 3964, 1453, 1783, 2560, 4749, 1720, 1749, 413, 633, 3875, 632, 232, 2038, 3183, 4746, 2870, 1513, 343, 4805, 4609, 2532, 2500, 2872, 4637, 1525, 2078, 3389, 2334, 814, 1405, 3013, 1627, 4009, 4160, 3556, 4798, 1912, 699, 387, 3484, 1325, 1951, 64, 4050, 1618, 1498, 3104, 455, 1677, 3636, 2044, 2185, 4014, 1523, 904, 340, 726, 3034, 2063, 985, 1366, 426, 14, 1854, 389, 703, 1480, 3538, 1294, 1246, 4850, 3348, 3056, 3420, 1884, 658, 320, 4064, 2315, 3818, 1154, 1963, 1697, 4339, 3764, 3431, 429, 1605, 221, 4514, 2487, 3780, 2131, 34, 1659, 3206, 401, 3325, 4726, 1716, 2993, 173, 1899, 2335, 1156, 1868, 3936, 381, 942, 2303, 4208, 124, 953, 4027, 602, 4632, 139, 4652, 104, 1177, 653, 368, 1319, 4811, 1083, 1279, 1448, 4058, 3448, 3559, 4992, 3524, 691, 2034, 4413, 4113, 4004, 1141, 125, 2987, 2912, 4233, 4818, 3269, 3817, 4593, 844, 1145, 1891, 188, 515, 3744, 2972, 318, 1008, 3530, 548, 2503, 4542, 2697, 4236, 1353, 1468, 2250, 2097, 3752, 2413, 1252, 3853, 1274, 1585, 4770, 1068, 140, 4355, 1291, 4891, 1117, 924, 452, 2552, 272, 2359, 1661, 1665, 2028, 2802, 3190, 1897, 2800, 4847, 2935, 740, 3678, 1787, 3103, 581, 3440, 2130, 4938, 866, 4286, 980, 3347, 1463, 1762, 2278, 1511, 151, 2060, 2285, 2979, 3318, 3203, 280, 3667, 2550, 2911, 4206, 4427, 4537, 2355, 42, 642, 2940, 3444, 82, 12, 4523, 184, 4936, 4363, 829, 1064, 2105, 2529, 840, 838, 3460, 2010, 1326, 3910, 4349, 1331, 3855, 4835, 3980, 510, 2875, 3292, 4794, 2495, 228, 4533, 3787, 2960, 3565, 1137, 1042, 3140, 4118, 1857, 746, 1497, 1925, 2447, 2506, 2530, 303, 877, 4087, 252, 168, 311, 1026, 2374, 625, 2077, 1425, 4253, 3127, 150, 4727, 270, 3758, 3814, 3135, 3773, 4803, 3573, 1924, 1828, 2396, 4809, 103, 3691, 783, 2679, 2994, 4828, 2711, 3984, 1330, 86, 3711, 1821, 1010, 2199, 1777, 4265, 4481, 819, 4244, 4407, 3492, 2841, 4717, 347, 4911, 1780, 1153, 3771, 427, 36, 2668, 2460, 158, 735, 1180, 377, 1247, 4419, 1971, 1860, 3697, 4615, 671, 2239, 4464, 3788, 4581, 3947, 2058, 91, 3746, 3240, 383, 2317, 1345, 3210, 4041, 2127, 941, 4437, 630, 2929, 4739, 835, 4816, 4797, 4135, 1758, 412, 2538, 4389, 2195, 258, 4875, 3811, 4597, 3839, 2481, 3638, 2310, 1178, 448, 1061, 1365, 3261, 1286, 3340, 3714, 2401, 3539, 4319, 686, 989, 3464, 4545, 1564, 3916, 1041, 432, 4125, 3474, 1143, 1771, 2813, 2220, 3481, 3796, 2642, 3480, 422, 2319, 409, 1327, 2354, 870, 2486, 3382, 1104, 3283, 4023, 4689, 4870, 1524, 3060, 4600, 1944, 2680, 2992, 263, 1280, 4480, 1044, 146, 2790, 1055, 3618, 1231, 622, 4304, 522, 4305, 4703, 3199, 155, 1150, 2752, 1810, 2025, 385, 2325, 2755, 1920, 4773, 1089, 4020, 4376, 3803, 3756, 142, 4260, 1648, 2169, 2647, 4718, 4387, 1560, 3705, 3631, 3865, 3595, 4282, 733, 4628, 3786, 1692, 2462, 1932, 2808, 416, 90, 823, 4787, 490, 164, 3657, 2234, 1426, 4737, 3373, 4307, 3459, 2323, 1904, 1112, 4659, 810, 1700, 588, 3832, 3117, 3735, 2405, 846, 1839, 1888, 631, 3270, 1699, 271, 2554, 2881, 2740, 1778, 1789, 561, 1683, 4447, 2477, 4218, 2684, 2309, 1167, 2132, 1561, 305, 780, 3491, 3477, 3511, 4342, 4094, 1701, 1314, 2573, 2774, 3798, 2556, 3979, 468, 3668, 4702, 4016, 3508, 3849, 4454, 662, 4800, 4005, 4211, 3244, 3672, 68, 2906, 4582, 3627, 3606, 2618, 3489, 348, 2937, 1601, 1420, 1681, 296, 2344, 4848, 2389, 2415, 3388, 1913, 3531, 4198, 4879, 3139, 1081, 1859, 304, 1708, 2809, 4090, 4524, 2691, 1949, 4783, 3810, 2504, 576, 1376, 1529, 3344, 2629, 1182, 485, 6, 2890, 1570, 1229, 2803, 725, 573, 1689, 4604, 4617, 984, 60, 3660, 467, 891, 2787, 4035, 1082, 3204, 3345, 1062, 3616, 4845, 2893, 4516, 1610, 2977, 179, 4259, 3068, 3690, 1596, 1763, 867, 316, 4927, 2006, 2404, 2635, 4173, 4327, 4889, 1389, 3621, 4117, 939, 854, 4043, 4804, 4148, 4724, 4247, 458, 2596, 3001, 1598, 3369, 1367, 606, 2313, 4565, 2622, 4428, 1478, 4404, 3023, 1203, 4653, 2072, 1707, 2515, 4228, 1526, 3493, 4322, 266, 2253, 1556, 1829, 2939, 3826, 1100, 4216, 1575, 4, 3806, 1957, 2446, 2137, 4854, 1861, 604, 3975, 4753, 167, 502, 3290, 2887, 89, 2258, 2216, 4745, 3575, 779, 1080, 1427, 2945, 852, 1929, 3516, 1985, 2251, 148, 4209, 2638, 2772, 3275, 2730, 53, 2998, 2923, 1928, 3230, 1318, 2997, 1305, 107, 2029, 454, 2564, 3950, 1065, 1361, 1863, 1115, 667, 4341, 3201, 234, 4487, 166, 4234, 1106, 1911, 3200, 4705, 4509, 4107, 765, 225, 1174, 3257, 4100, 756, 3368, 3629, 2840, 4418, 2329, 4873, 4086, 1876, 4036, 1105, 3983, 4384, 4683, 276, 1532, 480, 2198, 73, 4474, 1406, 1009, 4116, 3938, 2378, 2483, 4220, 3454, 2967, 4217, 4440, 4394, 2804, 3733, 1592, 1358, 3421, 3906, 466, 614, 4577, 3528, 2651, 3560, 2571, 3843, 1576, 4324, 2383, 4185, 4914, 1188, 3193, 2096, 1242, 3360, 1470, 131, 1063, 505, 4530, 2863, 2351, 1673, 477, 4294, 4484, 2469, 3416, 3901, 1060, 4574, 3842, 1650, 3611, 1090, 3549, 4313, 2860, 1980, 536, 41, 4643, 4690, 2003, 3222, 3062, 1169, 3253, 1472, 803, 1298, 341, 2609, 4068, 2710, 4373, 346, 4896, 1921, 1397, 3332, 2582, 3972, 3041, 4836, 474, 2409, 194, 2760, 15, 3048, 1479, 4661, 2674, 18, 2417, 4356, 2042, 85, 4469, 786, 16, 127, 1817, 1187, 4885, 2059, 479, 872, 3686, 4912, 954, 475, 37, 2954, 3677, 3435, 4178, 2061, 1103, 4572, 2563, 421, 2566, 2423, 3267, 608, 3957, 2186, 1328, 2045, 3242, 2908, 2348, 516, 3323, 563, 2611, 3701, 2973, 249, 1307, 3280, 1310, 397, 4738, 3310, 2068, 698, 1234, 434, 1727, 1919, 4820, 1301, 3221, 532, 3949, 1267, 865, 1372, 1039, 3620, 738, 1196, 4942, 1567, 3948, 1170, 1379, 3052, 4639, 3802, 2324, 595, 3413, 3572, 262, 4733, 187, 638, 1023, 478, 1102, 2707, 4312, 556, 254, 763, 4786, 3926, 4962, 1875, 2834, 1953, 2457, 1676, 1288, 2641, 2512, 853, 1815, 3461, 3410, 344, 768, 2049, 3157, 171, 489, 2482, 4774, 4241, 2490, 1746, 934, 2756, 2009, 753, 504, 1923, 3750, 1161, 2048, 651, 211, 1261, 4224, 2299, 4105, 3658, 571, 1344, 3054, 1960, 2817, 2168, 33, 2224, 476, 4085, 3266, 321, 3467, 3709, 1721, 1490, 4691, 176, 709, 3313, 839, 2254, 4372, 831, 1262, 2545, 4495, 2838, 1725, 4623, 2672, 2126, 3152, 2071, 3582, 330, 2340, 2419, 690, 770, 46, 861, 2741, 4344, 67, 2678, 3067, 4543, 256, 2150, 4223, 717, 69, 113, 2544, 23, 3909, 3721, 191, 3396, 338, 3809, 3078, 1907, 2031, 2113, 1687, 972, 1363, 1316, 2798, 3061, 3411, 47, 1013, 3354, 2832, 2716, 2643, 4406, 393, 1444, 3610, 2484, 4067, 3911, 2380, 781, 3763, 1811, 3532, 1067, 4716, 2075, 3605, 1477, 4082, 3761, 4863, 3919, 4059, 3233, 3384, 3880, 1350, 956, 438, 2076, 2570, 1146, 1867, 1954, 3968, 1553, 44, 4288, 1181, 2255, 2877, 1126, 3122, 3996, 4853, 1402, 4187, 3295, 122, 240, 2349, 3465, 534, 2448, 4558, 1669, 2547, 567, 2720, 578, 3076, 617, 1003, 742, 3098, 2633, 418, 1577, 933, 873, 2050, 437, 2366, 1189, 4977, 2282, 3593, 700, 609, 3639, 1324, 2411, 4972, 4910, 4710, 565, 974, 1845, 3834, 2537, 1440, 3588, 1312, 661, 3367, 2862, 345, 4723, 3226, 4232, 4752, 4092, 4988, 117, 2439, 4370, 856, 705, 4999, 2814, 1323, 1504, 1852, 3994, 4289, 4531, 1844, 1273, 4508, 2719, 1228, 2555, 1655, 893, 1265, 462, 3012, 1223, 832, 1142, 4264, 1006, 3243, 4162, 3320, 789, 4616, 300, 688, 4663, 4103, 1355, 3158, 1416, 4106, 4300, 1593, 298, 2271, 206, 2387, 2302, 801, 3433, 1415, 1492, 120, 4503, 244, 2030, 3837, 627, 2995, 2541, 1534, 147, 2191, 4903, 2757, 3533, 4386, 1515, 2197, 2865, 2360, 3742, 2699, 1871, 325, 350, 3002, 958, 182, 3680, 2955, 3578, 1508, 2743, 4698, 1950, 3665, 2847, 1421, 2233, 4316, 720, 2316, 4073, 2035, 2597, 2826, 2205, 99, 245, 4347, 4175, 3120, 2321, 3008, 847, 76, 52, 1976, 3430, 665, 545, 2585, 2377, 4131, 1071, 1335, 3321, 1552, 558, 457, 1333, 1457, 4948, 1823, 190, 1401, 134, 1484, 1797, 2136, 4358, 4538, 2125, 4380, 1475, 2836, 208, 3873, 1230, 3683, 4765, 2586, 4255, 4405, 4492, 165, 2861, 4647, 2388, 1400, 913, 3011, 2228, 2671, 4378, 2211, 3216, 4391, 4161, 919, 2934, 4195, 2510, 4655, 3137, 911, 3962, 1368, 3130, 4974, 2406, 4513, 284, 3483, 3285, 1086, 2397, 3848, 2583, 3263, 2590, 2784, 1290, 2963, 3966, 605, 2109, 926, 2069, 1113, 4270, 930, 611, 1832, 2591, 1709, 718, 1408, 4580, 1266, 260, 4099, 1449, 349, 3245, 943, 4634, 1015, 788, 3924, 2221, 3379, 267, 3260, 707, 2338, 431, 114, 4646, 1441, 1572, 1237, 961, 1375, 2369, 1987, 1772, 3005, 4551, 1248, 1562, 3953, 4660, 2777, 79, 2878, 4900, 1674, 4599, 3946, 734, 471, 1802, 2518, 299, 1993, 3075, 66, 1056, 1114, 601, 3944, 3725, 1761, 4924, 4994, 1735, 1619, 4618, 3820, 4101, 3614, 1097, 2259, 2706, 500, 4258, 2603, 4291, 4482, 2494, 189, 744, 3223, 3992, 4675, 784, 2553, 2565, 3028, 3439, 312, 4668, 3422, 161, 2453, 2081, 281, 574, 2300, 0, 2218, 3716, 1491, 497, 2322, 2857, 4778, 2546, 3534, 2210, 554, 84, 2257, 1031, 3272, 1099, 2032, 1801, 1199, 584, 4396, 4002, 4326, 1938, 4949, 1989, 2722, 2513, 2333, 988, 4802, 4520, 2717, 1445, 4624, 2102, 1244, 1558, 3409, 1348, 4157, 925, 3706, 1816, 2895, 1837, 848, 947, 4985, 4127, 1915, 2608, 4768, 2613, 3328, 1646, 3712, 3457, 410, 4799, 1628, 92, 4167, 4919, 4360, 1502, 1077, 4494, 1028, 102, 2953, 4901, 3987, 1390, 570, 3273, 4151, 4834, 4266, 1874, 1723, 3522, 2734, 4028, 1965, 1824, 527, 2732, 2639, 1069, 2352, 804, 3933, 715, 3902, 2246, 4679, 1458, 797, 8, 132, 3548, 4498, 3088, 1195, 2850, 3503, 697, 1130, 360, 2727, 2991, 963, 2516, 123, 876, 1784, 3507, 376, 1001, 4970, 2505, 4433, 233, 743, 2543, 3456, 4032, 4439, 1329, 2476, 3361, 1998, 3662, 4466, 3118, 2738, 551, 4843, 4499, 4414, 4658, 446, 3749, 524, 4532, 1138, 4806, 968, 1599, 4192, 301, 3874, 268, 3336, 3395, 4984, 2498, 4941, 3717, 3630, 3760, 811, 192, 2094, 3637, 523, 3153, 3991, 2416, 808, 3844, 582, 218, 4521, 1455, 2478, 4578, 4755, 4743, 1838, 3743, 4539, 402, 1539, 4421, 4779, 2085, 3681, 3066, 3781, 1747, 4922, 2958, 2022, 4648, 4301, 1634, 3883, 936, 657, 2083, 4602, 685, 3684, 771, 4823, 126, 3692, 3661, 4864, 4996, 3241, 2950, 2758, 2117, 821, 4501, 1782, 4673, 910, 2291, 1175, 1050, 4417, 2917, 1503, 818, 3401, 3835, 1881, 3739, 1249, 4472, 749, 1858, 1846, 1966, 1514, 1587, 4385, 1340, 2107, 3303, 4184, 3073, 2138, 710, 4143, 3085, 4444, 4081, 4678, 4476, 3074, 2818, 1021, 2155, 580, 2187, 3136, 4591, 2793, 236, 897, 1882, 3149, 169, 2873, 1792, 4801, 4570, 2301, 3030, 1668, 2403, 2418, 1988, 77, 4031, 1656, 3358, 4682, 2103, 2602, 3298, 2196, 822, 4627, 4284, 3755, 1520, 3250, 727, 1096, 1206, 2305, 997, 1926, 371, 4248, 195, 3642, 3776, 2128, 2919, 1741, 3715, 2501, 4154, 3819, 5003, 4010, 1559, 4314, 3466, 1731, 2304, 2885, 1733, 4606, 4505, 2936, 1695, 1933, 1597, 2161, 2062, 4015, 3412, 2332, 1578, 2964, 3509, 2458, 2588, 3294, 1258, 363, 3718, 2432, 1788, 4243, 782, 2562, 4245, 4321, 687, 4553, 530, 4121, 3564, 1745, 518, 1245, 4333, 1221, 2046, 4213, 1703, 859, 2830, 1774, 4961, 3622, 3506, 1795, 918, 1682, 243, 4226, 999, 2888, 1464, 4619, 1336, 3069, 75, 2171, 4210, 4150, 361, 543, 1543, 1542, 1602, 3828, 952, 2064, 880, 2179, 2701, 2842, 3090, 3959, 636, 186, 1956, 1320, 4486, 247, 4239, 4664, 1043, 1994, 2135, 324, 790, 1207, 4415, 2519, 1486, 198, 4780, 2139, 4196, 3710, 4091, 2949, 4318, 2440, 3583, 2286, 4960, 965, 3473, 1724, 2041, 1136, 3337, 4479, 3497, 4622, 648, 2241, 4120, 1194, 1304, 4025, 2121, 2849, 2307, 1790, 2823, 2637, 2779, 3124, 2116, 1066, 380, 3854, 639, 3304, 757, 274, 4758, 3443, 3777, 3882, 2123, 4455, 903, 1814, 306, 2664, 2686, 204, 80, 2891, 3977, 22, 404, 643, 4098, 1908, 144, 223, 3049, 4089, 1914, 3757, 4348, 2140, 3870, 1219, 3106, 2245, 3753, 1404, 1931, 4556, 3324, 3643, 1124, 1483, 3653, 1568, 4446, 3208, 863, 3794, 722, 2879, 755, 900, 4858, 2886, 4670, 95, 4017, 4729, 4088, 4076, 2520, 1730, 3470, 2270, 3468, 2724, 1750, 3961, 1910, 1251, 3879, 4926, 1530, 1285, 2700, 3225, 17, 2867, 675, 2166, 2747, 4851, 2628, 1644, 967, 1434, 4888, 623, 1616, 4541, 2336, 3581, 283, 851, 3861, 931, 4001, 3602, 842, 386, 2456, 3732, 1391, 679, 4990, 4467, 1210, 2392, 568, 2656, 3162, 1990, 1666, 459, 2927, 2248, 2753, 3857, 199, 1896, 2742, 4468, 1651, 2894, 1970, 2428, 4585, 4024, 2820, 1118, 3032, 3105, 3688, 546, 2534, 1292, 1054, 689, 1663, 153, 3512, 1025, 2632, 2479, 4306, 3293, 2289, 3599, 3003, 2621, 3462, 3115, 313, 2365, 1019, 4859, 2436, 4506, 3020, 2644, 4925, 4964, 3891, 663, 3145, 970, 3590, 2467, 4693, 815, 4311, 1007, 4038, 3579, 415, 3017, 436, 1469, 3867, 2835, 3405, 3370, 1961, 3841, 1878, 4971, 3696, 3126, 2924, 579, 2442, 3247, 4268, 3077, 635, 1079, 4381, 507, 4920, 251, 4056, 4109, 4490, 3617, 4166, 577, 3779, 2178, 1633, 711, 1072, 2889, 1574, 1754, 1493, 3535, 1051, 3824, 1152, 4290, 2111, 3091, 2018, 1569, 4252, 550, 4183, 384, 492, 1791, 229, 3365, 1573, 4535, 294, 2789, 1870, 1946, 1005, 1830, 3596, 2491, 2859, 1850, 3097, 3399, 1642, 119, 2267, 3965, 3307, 2089, 4638, 4077, 399, 506, 4254, 2914, 3196, 4124, 4030, 1302, 896, 2320, 4814, 2422, 4840, 3033, 1236, 895, 3708, 3436, 1742, 1978, 4561, 3608, 3259, 1803, 923, 3089, 4932, 4052, 4012, 456, 2384, 2509, 1424, 2971, 2379, 4361, 2589, 1293, 5, 2247, 4766, 106, 1233, 3674, 4953, 552, 4507, 2039, 2831, 4375, 178, 758, 2386, 202, 443, 3937, 159, 2011, 1192, 594, 503, 732, 1962, 2884, 453, 559, 2853, 2346, 2214, 3893, 1519, 278, 4515, 1826, 1974, 1629, 2279, 2284, 3488, 4649, 2141, 2682, 4229, 1029, 1243, 3490, 279, 2897, 4048, 4483, 959, 1886, 1537, 983, 2311, 1271, 217, 447, 2240, 3566, 2235, 3380, 1540, 2990, 4672, 3187, 986, 3878, 2326, 3419, 1016, 2824, 3306, 4969, 2812, 1109, 3016, 1496, 35, 3186, 2922, 3428, 4916, 3437, 4013, 449, 1287, 4194, 849, 2762, 3515, 1613, 4102, 3246, 4174, 1847, 110, 1452, 2480, 465, 3262, 2019, 108, 672, 3191, 3768, 2080, 2318, 2688, 2000, 3166, 3174, 2016, 3383, 3004, 4991, 1566, 3888, 4357, 1256, 4331, 435, 2542, 2763, 2780, 2308, 2120, 1611, 3274, 2625, 4940, 2871, 996, 799, 2459, 3885, 4951, 3645, 1992, 583, 7, 4006, 3908, 65, 2425, 4156, 1035, 2468, 1972, 2043, 3650, 2056, 1581, 3570, 1544, 1359, 4401, 1433, 4047, 3109, 1986, 4610, 3860, 3381, 3059, 2551, 3417, 307, 3859, 4219, 1872, 4084, 4933, 2074, 1671, 1555, 3165, 3554, 4021, 3146, 3682, 987, 3778, 54, 683, 2805, 4950, 764, 205, 621, 1738, 3281, 4436, 2394, 3766, 3334, 777, 3955, 3869, 4583, 3248, 4423, 4346, 3392, 210, 4731, 3312, 2575, 4189, 3264, 4320, 339, 4111, 2522, 1122, 1101, 3922, 553, 1710, 3958, 3649, 2193, 1939, 4810, 884, 1172, 3635, 4588, 1781, 441, 3254, 2598, 2084, 2925, 2438, 2470, 2433, 4062, 2463, 2996, 3082, 2681, 3237, 3172, 4857, 290, 1981, 4550, 3239, 1398, 5000, 791, 992, 714, 2966, 1255, 1705, 3114, 4775, 4095, 4158, 57, 2797, 2559, 1541, 862, 2600, 4740, 2970, 1428, 3296, 370, 2876, 2421, 2264, 3974, 1806, 906, 3452, 3785, 3113, 2108, 1804, 533, 1796, 4022, 4277, 508, 4674, 2785, 3587, 1, 3748, 1423, 4686, 32, 2124, 3702, 4108, 646, 587, 352, 4007, 3592, 3729, 2660, 3371, 3823, 1958, 3589, 4665, 1351, 3155, 696, 1798, 3625, 4246, 841, 4122, 4130, 39, 3213, 357, 2525, 2619, 2023, 2217, 1388, 3634, 375, 4379, 1268, 2158, 2616, 813, 2921, 966, 4037, 4139, 3305, 4310, 4172, 914, 295, 1107, 4176, 678, 3542, 1002, 4070, 2002, 2172, 3985, 2328, 3724, 4496, 3740, 3377, 3808, 3951, 4552, 730, 4445, 3641, 3189, 3227, 2444, 209, 3747, 1197, 230, 3695, 3116, 4959, 1220, 1488, 3887, 927, 4449, 692, 4278, 1276, 3737, 4526, 4329, 1831, 2983, 1883, 4560, 521, 392, 3624, 3036, 3815, 4083, 143, 4918, 3282, 4837, 3920, 2690, 94, 2507, 680, 3179, 1176, 367, 2057, 3927, 4955, 4782, 1033, 2645, 2013, 2115, 4871, 261, 2134, 2008, 3121, 2398, 51, 9, 2014, 157, 1786, 1711, 4221, 1890, 302, 2601, 4958, 2347, 2685, 1580, 177, 3940, 4695, 1959, 3021, 3963, 136, 2104, 957, 1222, 2252, 975, 2112, 2783, 2988, 4018, 3043, 4795, 2465, 138, 3663, 908, 1308, 1743, 708, 2265, 4430, 3713, 1819, 4640, 695, 549, 4510, 4993, 3346, 1862, 4899, 1833, 2475, 833, 2904, 3390, 491, 180, 464, 4431, 645, 374, 3563, 3601, 286, 373, 4528, 1640, 4366, 716, 1670, 2306, 2443, 674, 767, 1250, 1211, 3929, 58, 3673, 2173, 4097, 759, 3500, 3900, 4335, 2754, 3051, 2170, 1808, 4019, 3238, 1121, 2626, 3838, 291, 2350, 1851, 1135, 4235, 215, 2288, 1111, 3862, 326, 2807, 2473, 2561, 3164, 4676, 2669, 2040, 1751, 4898, 4461, 2869, 4654, 2268, 3518, 2749, 2051, 3685, 55, 3494, 3046, 257, 3633, 2852, 1608, 2926, 1259, 1045, 3055, 2748, 1238, 973, 2414, 10, 4817, 3845, 2986, 3219, 3450, 3600, 3995, 850, 3935, 1702, 4534, 1507, 4204, 3177, 2327, 4493, 2209, 2184, 482, 2004, 2770, 509, 3342, 4079, 3971, 3552, 11, 2162, 2528, 227, 2827, 915, 4197, 1996, 4997, 4979, 287, 297, 1548, 2782, 2122, 4722, 628, 1604, 1995, 358, 4832, 1621, 170, 544, 2933, 684, 879, 4725, 4133, 4114, 3418, 3083, 2892, 1653, 4299, 2502, 4511, 4442, 3543, 4887, 3184, 4525, 292, 3147, 4982, 1694, 3722, 2900, 2213, 2866, 3969, 3872, 3676, 111, 1225, 619, 4420, 3475, 890, 3202, 2640, 2382, 2294, 4323, 3424, 3993, 3163, 2164, 4915, 593, 3943, 1058, 4929, 4345, 3707, 98, 3053, 2981, 96, 2903, 4263, 382, 795, 130, 4694, 2093, 342, 4181, 1696, 971, 2188, 3580, 2101, 3967, 3402, 2942, 990, 4115, 394, 3072, 1084, 2466, 3526, 1159, 2095, 2815, 1456, 3271, 4328, 141, 2948, 21, 2373, 647, 4251, 1125, 2485, 4177, 964, 2673, 1442, 2907, 806, 3268, 3363, 2974, 3429, 938, 4179, 400, 922, 1964, 1313, 3759, 3804, 4790, 160, 2665, 3868, 4796, 2666, 213, 1719, 175, 40, 4732, 2704, 2090, 3359, 1712, 2175, 2526, 1643, 481, 4785, 951, 2511, 2915, 3486, 219, 2982, 253, 864, 2370, 4995, 2652, 845, 1997, 118, 4957, 976, 3455, 2114, 916, 493, 1536, 3205, 2202, 547, 4134, 3628, 3006, 3851, 766, 1549, 1158, 3326, 4688, 2067, 2451, 423, 2298, 1936, 4049, 3119, 4199, 1038, 2092, 308, 1036, 946, 949, 4186, 1073, 1722, 285, 4657, 4422, 4207, 1737, 2883, 2799, 1756, 4642, 3598, 13, 4830, 488, 71, 4559, 3907, 2159, 3945, 3607, 2364, 2765, 802, 3728, 4706, 3434, 4626, 1284, 4191, 634, 487, 3632, 1930, 2429, 929, 1093, 4908, 3134, 2222, 2938, 4819, 2206, 2614, 3064, 3525, 4434, 3438, 4411, 3215, 1704, 761, 81, 3251, 4438, 3038, 4205, 1241, 1481, 3258, 3726, 4575, 1658, 2208, 2627, 4713, 4168, 3536, 1191, 2759, 2612, 1095, 2806, 4409, 3161, 3297, 1385, 4292, 3502, 4475, 1209, 4119, 1364, 1461, 2229, 3071, 1685, 3330, 1521, 4303, 26, 207, 2731, 2661, 1418, 1120, 3125, 4055, 4485, 2390, 1675, 2153, 317, 1805, 3557, 3142, 3655, 2868, 4276, 2646, 27, 4478, 694, 3513, 2523, 4527, 2630, 3188, 3472, 1937, 610, 2913, 365, 366, 531, 3626, 1462, 3546, 1184, 1535, 2345, 1088, 2367, 329, 3651, 2695, 56, 4707, 1967, 629, 4075, 998, 1571, 3741, 145, 4443, 2275, 2536, 3207, 1607, 4567, 3141, 1315, 1825, 2851, 4759, 3914, 3510, 2839, 1500, 1299, 817, 4869, 460, 2260, 2821, 2751, 4841, 445, 2012, 2605, 1131, 704, 4554, 2788, 3734, 1516, 1499, 887, 4852, 3765, 237, 2767, 4269, 511, 3299, 424, 589, 2232, 2941, 3123, 1092, 4262, 682, 2146, 3427, 483, 4882, 3928, 1776, 4874, 3220, 2574, 4645, 3229, 216, 1410, 2231, 1952, 650, 3827, 4596, 3352, 2149, 620, 3476, 805, 1224, 1755, 48, 4072, 2715, 3374, 484, 3597, 395, 4767, 1183, 1164, 4909, 1263, 3813, 4237, 4280, 529, 309, 4943, 4071, 2277, 3505, 3236, 2567, 3523, 762, 3228, 3027, 3923, 3185, 3255, 4669, 1603, 1387, 4846, 4895, 1264, 2610, 4227, 2952, 2670, 3289, 3265, 4776, 1563, 100, 1506, 1047, 1378, 4364, 537, 590, 4330, 3485, 4473, 4190, 1474, 654, 1429, 540, 4441, 1337, 1842, 3892, 1680, 4365, 3070, 2844, 1590, 1272, 3738, 282, 3807, 406, 407, 3666, 1615, 3567, 2464, 4761, 1898, 541, 4163, 105, 255, 1527, 4147, 3858, 1134, 4868, 3550, 310, 1799, 1734, 1501, 1869, 3569, 2177, 3317, 612, 3521, 408, 1383, 1827, 1813, 353, 1903, 2055, 4923, 2033, 2290, 433, 4171, 3000, 4039, 4635, 4395, 1809, 4902, 3956, 4240, 4231, 4562], "validation": [5219, 5147, 5228, 5190, 5413, 5437, 5328, 5045, 5239, 5381, 5295, 5419, 5238, 5469, 5018, 5034, 5073, 5315, 5102, 5303, 5161, 5432, 5195, 5013, 5452, 5496, 5390, 5148, 5242, 5135, 5321, 5085, 5017, 5116, 5423, 5027, 5046, 5156, 5257, 5339, 5042, 5100, 5305, 5286, 5172, 5221, 5417, 5306, 5246, 5108, 5262, 5500, 5081, 5145, 5170, 5086, 5072, 5415, 5201, 5088, 5040, 5292, 5434, 5498, 5206, 5066, 5244, 5308, 5279, 5363, 5426, 5255, 5418, 5010, 5280, 5208, 5337, 5122, 5203, 5345, 5247, 5059, 5015, 5273, 5402, 5207, 5168, 5025, 5360, 5427, 5014, 5327, 5504, 5092, 5296, 5336, 5314, 5077, 5317, 5491, 5386, 5250, 5142, 5234, 5058, 5401, 5220, 5114, 5261, 5183, 5316, 5335, 5370, 5192, 5166, 5070, 5414, 5098, 5410, 5371, 5477, 5267, 5179, 5218, 5435, 5442, 5075, 5509, 5309, 5274, 5375, 5487, 5141, 5369, 5020, 5361, 5016, 5022, 5384, 5409, 5297, 5281, 5047, 5350, 5265, 5095, 5453, 5210, 5212, 5374, 5440, 5236, 5104, 5340, 5341, 5276, 5087, 5083, 5356, 5398, 5243, 5508, 5393, 5277, 5278, 5404, 5101, 5119, 5188, 5146, 5069, 5176, 5140, 5325, 5137, 5187, 5362, 5171, 5307, 5447, 5115, 5227, 5302, 5464, 5359, 5299, 5194, 5400, 5211, 5403, 5320, 5150, 5159, 5298, 5484, 5291, 5475, 5173, 5344, 5450, 5331, 5425, 5388, 5493, 5149, 5024, 5379, 5467, 5263, 5368, 5338, 5431, 5182, 5373, 5134, 5105, 5039, 5167, 5256, 5021, 5165, 5224, 5136, 5118, 5061, 5490, 5446, 5031, 5193, 5215, 5439, 5353, 5043, 5438, 5259, 5131, 5266, 5062, 5037, 5456, 5214, 5322, 5332, 5107, 5258, 5124, 5454, 5406, 5132, 5448, 5112, 5164, 5304, 5429, 5407, 5312, 5162, 5181, 5470, 5056, 5237, 5330, 5333, 5233, 5254, 5125, 5364, 5264, 5111, 5063, 5222, 5465, 5078, 5223, 5126, 5068, 5382, 5245, 5231, 5290, 5483, 5186, 5420, 5499, 5028, 5376, 5049, 5029, 5198, 5372, 5054, 5036, 5035, 5482, 5311, 5139, 5494, 5196, 5486, 5416, 5269, 5329, 5445, 5310, 5011, 5394, 5026, 5230, 5199, 5313, 5342, 5103, 5235, 5502, 5064, 5143, 5260, 5354, 5489, 5366, 5113, 5080, 5485, 5323, 5492, 5358, 5421, 5090, 5216, 5457, 5074, 5383, 5275, 5351, 5057, 5449, 5030, 5154, 5396, 5422, 5506, 5133, 5501, 5408, 5460, 5294, 5151, 5157, 5117, 5444, 5424, 5355, 5189, 5123, 5507, 5241, 5287, 5462, 5497, 5378, 5191, 5289, 5060, 5217, 5346, 5348, 5249, 5253, 5110, 5270, 5038, 5144, 5065, 5433, 5468, 5288, 5023, 5476, 5293, 5285, 5082, 5200, 5300, 5091, 5109, 5272, 5153, 5050, 5067, 5389, 5084, 5044, 5079, 5129, 5213, 5071, 5352, 5155, 5158, 5271, 5478, 5106, 5252, 5053, 5395, 5343, 5284, 5052, 5503, 5411, 5229, 5163, 5365, 5175, 5055, 5251, 5319, 5130, 5232, 5120, 5240, 5505, 5209, 5268, 5169, 5160, 5197, 5202, 5185, 5451, 5174, 5032, 5184, 5099, 5094, 5283, 5318, 5138, 5428, 5481, 5180, 5089, 5474, 5412, 5392, 5033, 5458, 5405, 5096, 5128, 5204, 5443, 5377, 5367, 5347, 5387, 5357, 5326, 5301, 5472, 5093, 5041, 5380, 5178, 5097, 5282, 5019, 5495, 5473, 5479, 5480, 5459, 5385, 5349, 5152, 5121, 5455, 5324, 5051, 5466, 5436, 5399, 5488, 5461, 5471, 5441, 5012, 5076, 5248, 5205, 5463, 5334, 5397, 5177, 5127, 5430, 5225, 5048, 5226, 5391], "test": [5558, 6007, 5554, 5664, 5992, 5653, 5549, 5788, 5654, 5790, 5995, 5957, 5655, 5761, 5691, 5540, 5545, 5520, 5755, 5985, 5513, 5890, 5894, 5613, 5854, 5862, 5701, 5515, 5586, 5752, 5548, 5851, 5892, 5871, 5866, 5708, 5592, 5559, 5917, 6004, 5994, 5793, 5541, 5990, 5818, 5636, 5625, 5707, 5769, 5635, 5927, 5930, 5768, 5519, 5791, 5551, 5997, 5518, 5912, 5720, 5846, 5984, 5981, 5949, 5536, 5929, 5783, 5680, 5932, 5523, 5603, 5678, 5940, 5622, 5960, 5621, 5834, 5811, 5689, 5845, 5946, 5719, 5754, 5638, 5706, 5684, 5983, 6006, 5667, 5650, 5612, 5922, 5864, 5863, 5623, 5672, 6005, 5718, 5802, 5939, 5573, 5516, 5609, 5908, 5568, 5743, 5632, 5951, 5646, 5722, 5860, 5843, 5891, 5668, 5767, 5741, 5649, 5702, 5827, 5610, 5750, 5575, 5762, 5999, 5591, 5527, 5679, 5991, 5815, 5661, 5976, 5868, 5562, 5590, 5913, 5873, 5867, 5884, 5553, 5534, 5751, 5911, 5692, 5915, 5966, 5693, 5970, 5923, 5637, 5704, 5869, 5555, 5885, 5740, 5987, 5734, 5578, 5766, 5902, 5602, 5539, 5764, 5577, 5847, 5914, 5897, 5893, 5971, 5600, 5639, 5998, 5936, 5535, 5607, 5824, 5816, 6001, 5872, 5875, 5670, 5907, 5533, 5756, 5798, 5803, 5944, 5784, 5599, 5710, 5931, 5948, 5648, 5633, 5712, 5899, 5640, 5598, 5709, 5886, 5552, 5746, 5918, 5547, 5857, 5604, 5996, 5821, 5799, 5883, 5964, 5517, 5666, 5644, 5763, 5546, 5988, 5651, 5900, 5779, 5747, 5658, 5838, 5525, 5832, 5865, 5732, 5716, 5690, 5528, 5736, 5729, 5789, 5522, 5879, 5980, 5945, 5570, 5795, 5521, 5597, 5962, 5748, 5877, 5898, 5631, 5852, 5605, 5617, 5968, 5805, 5715, 5901, 5510, 5953, 5749, 5699, 5538, 5882, 5826, 5645, 5781, 5585, 5627, 5855, 5556, 5753, 5796, 5771, 5588, 5594, 5630, 5731, 5794, 5711, 5909, 5956, 5955, 5745, 5853, 5647, 5904, 5975, 5952, 5809, 5926, 5786, 5801, 5842, 5681, 5696, 5532, 5831, 5941, 5659, 5606, 5888, 5619, 5792, 5812, 5616, 5954, 6000, 5972, 5776, 5765, 5938, 5969, 5624, 5982, 5778, 5721, 5920, 5874, 5530, 5634, 5780, 5887, 5760, 5652, 5777, 5829, 5543, 5526, 5656, 5870, 5967, 5544, 5705, 5833, 5881, 5836, 5820, 5986, 6009, 5848, 5963, 5876, 5840, 5947, 5688, 5739, 5580, 5835, 5529, 5905, 5703, 5934, 5943, 5759, 5813, 5959, 5737, 5550, 5937, 6002, 5643, 5839, 5775, 5601, 5730, 5674, 5697, 5589, 5611, 5569, 5965, 5628, 5626, 5817, 6003, 5989, 5837, 5620, 5828, 5662, 5808, 5669, 5593, 5895, 5933, 5614, 5800, 5935, 5557, 5629, 5859, 5512, 5896, 5608, 5563, 5579, 5713, 5973, 5810, 5695, 5542, 5906, 5773, 5717, 5572, 5942, 5925, 5583, 5694, 5785, 5880, 5584, 5841, 5733, 5822, 5676, 5665, 5770, 5615, 5723, 5728, 5850, 5958, 5560, 5618, 5804, 5961, 5642, 5727, 5671, 5919, 5979, 5726, 5582, 5714, 5889, 5774, 5782, 5663, 5814, 5849, 5673, 5910, 5858, 5823, 5571, 5587, 5861, 5657, 5844, 5698, 5565, 5660, 5787, 5581, 5675, 5700, 5797, 5742, 5566, 5974, 5641, 5830, 5921, 5687, 5685, 5744, 5511, 5807, 5924, 5758, 5677, 5537, 5977, 5724, 5564, 5567, 5682, 5928, 5576, 5725, 5806, 5916, 6008, 5531, 5686, 5993, 5524, 5683, 5514, 5757, 5561, 5950, 5878, 5819, 5978, 5574, 5772, 5735, 5856, 5738, 5596, 5595, 5825, 5903]}, {"train": [3171, 4385, 2498, 544, 1991, 1349, 3243, 432, 27, 1332, 812, 2875, 2318, 910, 1100, 87, 4932, 3791, 1774, 3524, 1042, 2262, 3477, 3515, 98, 2458, 1500, 4609, 1654, 2742, 3106, 2953, 3816, 2154, 2216, 4171, 530, 3175, 1419, 477, 4490, 1658, 4339, 2792, 1423, 2338, 4088, 4872, 4518, 2468, 4978, 4097, 1057, 1642, 261, 3031, 4328, 973, 3085, 2197, 4099, 2150, 443, 168, 1182, 3079, 3173, 1, 420, 390, 853, 898, 2992, 3827, 850, 2484, 3199, 4567, 1971, 1547, 4607, 4389, 2296, 2570, 2997, 1935, 4203, 1004, 4669, 4464, 913, 3890, 253, 3358, 762, 2175, 1844, 2496, 1281, 3911, 3431, 4935, 1916, 3325, 324, 3211, 1505, 3406, 4874, 3266, 198, 2307, 4661, 825, 339, 4134, 2369, 469, 1950, 1961, 2324, 3351, 4639, 3466, 3714, 4044, 4706, 1622, 4289, 1536, 1018, 2405, 4002, 3218, 2690, 4235, 55, 4449, 3842, 1710, 1425, 26, 863, 1175, 4, 2898, 1944, 2892, 1236, 2492, 3220, 4361, 3510, 686, 2380, 3384, 3585, 1946, 2951, 4151, 1963, 4331, 4394, 2738, 1609, 433, 3201, 2542, 1313, 3652, 2715, 653, 1902, 326, 2532, 2159, 2105, 1896, 4593, 2985, 3137, 1202, 4153, 3779, 3747, 4212, 1365, 4692, 389, 4453, 4161, 4847, 638, 2316, 4163, 4760, 1435, 3658, 2873, 715, 4396, 2188, 2098, 982, 4201, 2766, 623, 1641, 3313, 571, 4710, 4060, 288, 2987, 2734, 3161, 2960, 3366, 58, 915, 3638, 453, 4528, 1193, 2223, 624, 3777, 4448, 4327, 4222, 3835, 4691, 1434, 3144, 570, 3915, 1566, 4805, 2583, 1846, 3847, 4390, 2081, 1473, 3736, 179, 3650, 1171, 1456, 3273, 1858, 1471, 3557, 2347, 1502, 867, 4863, 406, 1738, 2174, 4269, 4623, 4631, 2499, 2895, 4964, 2728, 683, 4559, 3840, 4814, 4591, 4206, 4832, 14, 3646, 661, 1565, 4992, 243, 4200, 4371, 107, 3014, 952, 711, 3780, 3571, 3839, 2170, 932, 127, 1264, 4788, 673, 4415, 1495, 4403, 3900, 19, 2938, 2016, 577, 3475, 2, 3329, 4705, 1644, 2705, 1375, 851, 3883, 3005, 2623, 2348, 3804, 4648, 1638, 4295, 3784, 970, 4335, 1356, 1317, 2152, 535, 103, 2392, 46, 3789, 3846, 1721, 2857, 3763, 497, 4209, 986, 4778, 1709, 1911, 362, 3990, 3938, 119, 2555, 3184, 1092, 3536, 4386, 3563, 2131, 4283, 474, 2986, 2839, 1847, 2331, 4783, 1301, 4602, 1534, 4759, 1406, 1632, 3686, 3430, 2477, 3797, 852, 2343, 1546, 876, 2252, 2201, 873, 299, 3134, 4929, 784, 2231, 2678, 3878, 1639, 3187, 2569, 2541, 4382, 3855, 4875, 604, 4862, 4685, 4400, 4971, 2695, 241, 3008, 1327, 242, 3803, 1879, 4315, 2434, 4167, 4954, 4024, 3393, 1734, 3203, 3356, 3377, 4753, 2090, 3655, 1132, 2257, 144, 2798, 3904, 3483, 990, 3613, 155, 2688, 929, 2656, 2333, 4588, 1653, 2730, 2994, 3095, 2089, 2533, 557, 1561, 2095, 4305, 1231, 705, 4730, 2487, 1688, 123, 891, 687, 4664, 1476, 1164, 727, 1733, 3300, 78, 4057, 2846, 2357, 1176, 580, 3286, 3688, 1237, 3500, 356, 925, 4831, 2387, 1131, 152, 2031, 3280, 3247, 3589, 3188, 927, 3368, 4933, 761, 2758, 3491, 380, 4889, 2850, 3642, 995, 371, 1445, 3495, 3263, 3285, 1744, 3983, 565, 232, 728, 1880, 1873, 814, 2446, 2637, 2248, 3924, 2361, 4098, 1933, 3966, 2937, 337, 4351, 2838, 4208, 2586, 2219, 2837, 3330, 1860, 552, 315, 3346, 2886, 24, 1979, 1964, 1723, 3884, 2620, 3112, 1929, 62, 4991, 3047, 540, 3025, 4646, 3772, 3649, 2351, 4183, 4748, 2802, 2297, 3719, 1390, 741, 2513, 2973, 3781, 700, 2078, 2706, 30, 2552, 2014, 1179, 3759, 149, 968, 186, 2910, 4195, 2204, 1663, 4082, 1725, 4017, 1574, 2529, 1321, 4555, 2962, 3754, 4582, 1116, 1010, 35, 4092, 1664, 2383, 2113, 4696, 1330, 121, 2205, 4437, 2367, 843, 2476, 989, 740, 1044, 1542, 3221, 3253, 3484, 1778, 4767, 1637, 3695, 2240, 4612, 1965, 2013, 471, 270, 2990, 1679, 4485, 4942, 2373, 3715, 3864, 3916, 2114, 1937, 4267, 4834, 1480, 1223, 1885, 1197, 4149, 1278, 4438, 1422, 4349, 4311, 4021, 3975, 1284, 3497, 373, 489, 924, 1753, 2700, 4063, 955, 1513, 4887, 2149, 3050, 964, 4565, 764, 3611, 1297, 3525, 2627, 2545, 2761, 2821, 2352, 1952, 2177, 1912, 4632, 1699, 2019, 1139, 1325, 1106, 4384, 895, 4974, 2146, 2390, 3035, 0, 3836, 1395, 4751, 2321, 2173, 3087, 3269, 860, 801, 3928, 259, 164, 2780, 4743, 3361, 2332, 4708, 141, 3790, 2589, 2935, 2436, 3181, 4517, 2210, 1151, 2003, 2245, 2033, 861, 213, 1295, 1531, 4651, 1359, 716, 4715, 1806, 4570, 17, 3320, 3244, 2136, 578, 1958, 4492, 2117, 3665, 4655, 4244, 1031, 2344, 1482, 1099, 4366, 3141, 3621, 1388, 1127, 2580, 1334, 3328, 4826, 3468, 1799, 4040, 576, 323, 1519, 2670, 4141, 3054, 3397, 3533, 204, 1548, 1294, 77, 1253, 1999, 1262, 1669, 4928, 91, 184, 395, 3538, 408, 478, 2270, 3954, 2309, 2121, 292, 1782, 1801, 448, 4987, 4304, 2725, 1906, 3532, 1599, 659, 3498, 4601, 4383, 4878, 4499, 271, 3895, 1000, 2096, 3893, 2256, 550, 112, 3685, 1227, 2807, 2233, 531, 158, 2221, 118, 1702, 246, 630, 470, 2395, 2311, 1268, 4516, 4650, 4239, 4439, 874, 3534, 4509, 4960, 526, 3012, 1355, 1840, 2765, 2198, 3775, 1766, 1095, 1259, 3874, 2399, 4280, 751, 862, 4864, 2671, 240, 348, 2916, 748, 466, 2112, 951, 3169, 220, 1417, 4860, 4041, 4956, 774, 2042, 4574, 3060, 3378, 2861, 2425, 1511, 984, 172, 573, 667, 4514, 4242, 15, 1324, 564, 4701, 4703, 4064, 2065, 4373, 1156, 4575, 4321, 2668, 709, 61, 4515, 1128, 2903, 4957, 3721, 3586, 2040, 4337, 1968, 363, 4630, 1047, 4468, 3906, 4034, 4296, 258, 2784, 2375, 427, 4343, 4643, 1063, 360, 757, 4848, 2356, 4260, 1101, 3572, 4914, 2362, 4487, 807, 4486, 1540, 877, 4341, 2085, 2624, 1146, 4225, 2779, 974, 2272, 788, 1086, 3733, 4892, 3445, 608, 2388, 3017, 4393, 1054, 3219, 1739, 4250, 3619, 1293, 1399, 4137, 1497, 701, 3564, 4885, 4421, 2250, 1978, 3991, 1178, 2060, 2082, 1820, 799, 1742, 2719, 467, 721, 4140, 3918, 5006, 3792, 3246, 4355, 3118, 4008, 4622, 3755, 560, 4232, 3817, 4709, 256, 2376, 4108, 900, 4597, 585, 2605, 93, 3575, 53, 2144, 3131, 465, 375, 3862, 2482, 2703, 4160, 1300, 3239, 4131, 436, 2008, 2312, 1836, 1420, 2277, 2346, 1503, 1591, 2021, 1662, 3454, 4475, 3076, 2181, 4193, 866, 3518, 3863, 976, 2168, 1855, 2934, 4869, 3801, 1041, 823, 4825, 4365, 865, 3133, 1610, 1486, 4642, 841, 379, 3082, 1024, 4181, 726, 1924, 4345, 4380, 4925, 4989, 4148, 4688, 529, 2505, 357, 794, 4427, 1009, 2943, 4276, 1222, 4270, 1650, 943, 2024, 2230, 3282, 3274, 473, 109, 4344, 4854, 4578, 1114, 1129, 147, 917, 3738, 958, 2371, 2786, 2225, 4012, 2741, 3771, 461, 3673, 1208, 2129, 2126, 2573, 4579, 2769, 4215, 4213, 3556, 627, 71, 349, 317, 1440, 1648, 2632, 1568, 4666, 2581, 1613, 4058, 2749, 2389, 1233, 1488, 501, 2051, 1326, 304, 277, 3183, 4129, 3355, 730, 429, 2454, 3674, 3989, 3357, 750, 3339, 3205, 2211, 4802, 3947, 1713, 4138, 2659, 4883, 4083, 4667, 1382, 2860, 3156, 4922, 2488, 3202, 3126, 4156, 1071, 4494, 4596, 2215, 4876, 3164, 2901, 3176, 3390, 3832, 599, 3216, 4318, 2342, 2890, 821, 1629, 20, 4285, 1244, 1363, 3496, 4746, 2653, 1926, 2176, 3967, 596, 1920, 5001, 1883, 4207, 290, 2419, 4851, 3699, 2646, 3637, 1341, 1383, 209, 3192, 1366, 1392, 1984, 791, 3559, 2325, 3240, 628, 840, 1191, 3268, 3998, 3456, 2294, 980, 1147, 3527, 4500, 4461, 919, 148, 2206, 2292, 3467, 3002, 318, 4263, 1418, 128, 3609, 1134, 425, 41, 539, 2385, 1093, 2641, 2756, 666, 1898, 572, 4121, 4784, 2393, 3770, 1243, 4742, 39, 3873, 512, 3726, 1496, 421, 879, 637, 1966, 3212, 202, 2217, 568, 3704, 3553, 4154, 636, 2229, 4677, 4888, 4356, 450, 3492, 2654, 4917, 983, 1492, 4025, 4313, 2326, 1037, 3344, 1835, 4744, 632, 3696, 1494, 536, 4790, 1649, 4323, 4940, 3209, 3877, 4868, 3727, 4850, 1361, 2897, 4884, 908, 1795, 3333, 3812, 922, 2980, 1533, 1354, 2677, 3452, 170, 1504, 3341, 959, 1614, 3420, 510, 4319, 940, 40, 639, 3200, 1032, 2531, 1133, 4595, 3604, 3503, 2972, 2407, 3728, 2707, 4145, 4572, 3276, 2506, 1537, 1219, 2411, 621, 90, 2023, 4030, 4822, 2156, 1429, 1235, 1705, 2731, 1587, 32, 365, 2746, 781, 2729, 4476, 1640, 2467, 2760, 868, 903, 3257, 3108, 1625, 765, 3322, 4674, 2630, 3340, 3004, 3343, 735, 3261, 846, 609, 1084, 3235, 1751, 3013, 4659, 2481, 3993, 34, 505, 3011, 3072, 1484, 4020, 1026, 2995, 556, 1022, 306, 2721, 2107, 444, 2161, 3504, 598, 3507, 4022, 1070, 888, 3215, 2080, 2976, 566, 1673, 4120, 117, 1016, 4835, 3622, 414, 3854, 4410, 1350, 139, 5005, 1652, 2212, 4489, 4910, 3852, 162, 1894, 4292, 84, 1142, 747, 838, 4481, 3894, 3382, 1891, 66, 1659, 3794, 4563, 4219, 617, 2814, 3132, 4626, 1209, 3494, 626, 3868, 4114, 4699, 1635, 856, 2616, 3584, 595, 4562, 2524, 592, 4785, 1651, 1249, 1605, 1925, 92, 3807, 3419, 4495, 4968, 3970, 4196, 302, 4843, 4927, 2748, 1982, 3679, 3514, 2974, 2516, 2982, 1754, 2453, 3519, 3228, 4647, 4078, 2642, 281, 1064, 1286, 400, 1046, 4871, 1772, 1543, 1013, 3092, 3006, 710, 185, 2209, 3119, 3317, 2823, 2285, 548, 1917, 4274, 2789, 2142, 547, 2629, 1067, 942, 4729, 2840, 4374, 4981, 1957, 4999, 4584, 4840, 1656, 1323, 2788, 1385, 605, 4243, 4750, 593, 3489, 2182, 1204, 2549, 1759, 3981, 3182, 3999, 4176, 4392, 4105, 1316, 1082, 2859, 3653, 3304, 1983, 2596, 4791, 2353, 960, 376, 130, 4549, 3224, 4711, 1094, 364, 368, 2194, 3398, 2699, 70, 3198, 4493, 340, 2273, 1708, 712, 2923, 1166, 2852, 388, 2349, 896, 2709, 2759, 397, 4890, 4532, 3932, 2465, 1040, 1433, 1312, 574, 3457, 3001, 4829, 2979, 1824, 670, 4217, 1451, 1945, 3594, 3113, 775, 4870, 1265, 4159, 771, 796, 4537, 3861, 3359, 45, 3177, 4031, 837, 2950, 629, 3882, 4479, 2577, 4606, 233, 3453, 1315, 3757, 2001, 2590, 3068, 3577, 100, 2774, 2242, 3046, 4762, 3464, 518, 2226, 4996, 1232, 4303, 3191, 746, 1895, 1017, 3724, 3189, 2928, 2615, 2919, 313, 1275, 768, 2675, 2863, 3444, 1941, 1203, 827, 3415, 2091, 2100, 2636, 1923, 928, 2339, 3937, 524, 4546, 1007, 689, 2011, 1329, 3365, 4633, 4198, 2163, 3701, 4635, 1117, 308, 4676, 1157, 725, 4402, 3703, 4891, 1245, 3607, 643, 3765, 805, 2066, 2888, 4322, 1126, 4754, 3370, 3037, 3731, 1773, 3410, 947, 1307, 2565, 2768, 216, 3412, 1170, 161, 2368, 4125, 3502, 2942, 847, 187, 2755, 251, 1527, 4372, 4119, 4908, 4007, 1368, 1137, 1814, 4959, 3213, 3800, 4867, 2455, 1636, 3259, 2776, 1886, 1353, 343, 1900, 4265, 2030, 2585, 1828, 4757, 4921, 234, 4704, 2634, 4447, 3217, 3741, 3506, 1512, 4939, 1823, 1255, 806, 2613, 4188, 75, 3099, 2920, 3956, 4828, 855, 4780, 2118, 1726, 2189, 2956, 1720, 1800, 2202, 3717, 2010, 3957, 4752, 101, 4794, 2132, 1338, 2397, 644, 2864, 3546, 2323, 4529, 503, 136, 2716, 4218, 4359, 1697, 2337, 2658, 2313, 1786, 2645, 108, 3428, 2172, 1975, 1588, 4737, 1107, 729, 2220, 495, 106, 2810, 3248, 2035, 2261, 3936, 4620, 3080, 1576, 545, 818, 4429, 1955, 1884, 2193, 3583, 4934, 1109, 124, 1903, 3537, 4526, 3449, 1079, 2069, 2685, 4407, 1360, 2829, 3545, 4259, 4519, 2433, 4893, 714, 352, 1510, 1001, 3597, 4444, 2679, 4697, 1120, 480, 4695, 4915, 219, 76, 3399, 1947, 534, 3986, 4445, 2086, 3170, 3152, 4091, 681, 755, 1153, 2115, 4000, 2278, 4277, 2070, 434, 1273, 22, 4253, 203, 3260, 3469, 1051, 4603, 4673, 3605, 561, 1285, 769, 645, 125, 1549, 3640, 4816, 3487, 1802, 3708, 372, 2234, 4938, 3167, 2301, 1490, 3850, 104, 4896, 3372, 538, 1776, 2028, 314, 4698, 3539, 3573, 1866, 454, 2025, 28, 2414, 660, 393, 3151, 2579, 795, 963, 2291, 3186, 2880, 3599, 1410, 4859, 59, 2969, 1698, 3528, 3089, 303, 2535, 4985, 2588, 2907, 4126, 1168, 1165, 4103, 3155, 1304, 140, 1796, 4080, 646, 2822, 1421, 3354, 2282, 4168, 521, 2830, 2203, 1552, 2000, 785, 1299, 442, 1678, 4122, 2382, 1149, 4652, 3045, 992, 255, 1162, 359, 904, 3128, 2306, 1409, 920, 886, 1172, 1163, 3745, 1015, 3995, 2286, 111, 2718, 704, 3332, 2631, 3049, 3723, 1189, 3386, 3433, 1195, 3700, 511, 887, 452, 1311, 872, 4233, 998, 1864, 1342, 1447, 475, 3830, 2052, 601, 4312, 4287, 1729, 455, 3149, 3735, 3048, 2891, 223, 4456, 4782, 1288, 1002, 3254, 2628, 4071, 3432, 2926, 1553, 3375, 586, 2267, 651, 382, 2006, 4727, 4264, 3857, 4775, 1781, 916, 2975, 1150, 2845, 3549, 2268, 2576, 2251, 4614, 4583, 4065, 3129, 4587, 1826, 2279, 1893, 2844, 4853, 2595, 3643, 3603, 1239, 1943, 2320, 4051, 3418, 38, 977, 4226, 1717, 2253, 1993, 1339, 2463, 2109, 1458, 4227, 513, 3411, 2470, 3312, 611, 1413, 4722, 2518, 2398, 4534, 1760, 2812, 2591, 591, 3567, 4179, 537, 3667, 4931, 4988, 4952, 3295, 2736, 2140, 4497, 1791, 1138, 4273, 2269, 3876, 1909, 1210, 1940, 567, 3929, 3676, 3130, 4467, 1258, 4969, 4399, 3078, 4256, 1008, 1811, 94, 1677, 334, 1538, 1087, 4855, 912, 1003, 4290, 3910, 3958, 1569, 1815, 1838, 1469, 4513, 2241, 603, 1921, 1336, 1230, 4050, 2848, 2582, 310, 169, 2012, 3194, 3323, 1408, 2341, 476, 201, 1188, 3523, 3542, 4477, 4580, 1478, 263, 3548, 1136, 1761, 1308, 4702, 366, 4144, 2793, 190, 4589, 1939, 3799, 3414, 4586, 1853, 3124, 4765, 3976, 1039, 1518, 268, 4617, 206, 3236, 2597, 4719, 4809, 3157, 1890, 8, 4388, 3302, 966, 2298, 3408, 894, 1220, 1025, 4053, 3786, 2558, 831, 224, 2350, 3223, 3802, 2724, 2479, 2762, 2509, 3234, 1403, 3083, 2791, 1857, 798, 558, 3798, 1384, 2657, 905, 4397, 3859, 4951, 1376, 3748, 1593, 4937, 4774, 3657, 2302, 486, 4571, 3825, 2125, 884, 4157, 3753, 2165, 4279, 3400, 3028, 4173, 1545, 4629, 3283, 4348, 135, 2720, 4842, 488, 2870, 1448, 996, 3683, 2723, 4947, 3547, 1221, 1686, 1348, 3691, 430, 2079, 4641, 2195, 353, 1386, 4916, 386, 435, 2015, 4833, 3291, 4037, 4404, 200, 3193, 160, 3262, 479, 1446, 2034, 520, 1872, 1287, 878, 2423, 145, 3796, 4458, 4541, 4249, 2819, 3121, 2538, 4912, 3509, 1821, 2944, 276, 3520, 901, 341, 1829, 692, 1887, 3127, 4841, 3214, 4075, 4417, 3297, 1211, 1827, 4637, 1792, 2026, 2374, 1586, 1416, 1144, 237, 4123, 174, 177, 3242, 4798, 3435, 3964, 3318, 786, 4714, 3326, 3684, 668, 4320, 779, 4360, 1274, 1140, 322, 4894, 3237, 4418, 4398, 4236, 657, 777, 1818, 2185, 3872, 854, 1822, 1731, 4358, 1083, 1608, 4330, 994, 4117, 4856, 2666, 849, 2427, 154, 3746, 2328, 4728, 579, 4551, 589, 2310, 3596, 4377, 63, 4758, 2993, 3866, 2952, 74, 1616, 2790, 2514, 2459, 3448, 3926, 402, 4457, 3978, 1089, 985, 4611, 2745, 4747, 1344, 2137, 658, 4231, 294, 2551, 1252, 3912, 217, 2893, 4716, 3720, 2072, 351, 1461, 459, 3744, 3287, 833, 1035, 3395, 2534, 3979, 3984, 1951, 1200, 3166, 1848, 2200, 1843, 4238, 1524, 2740, 3702, 650, 792, 2447, 3644, 4897, 2127, 4102, 3238, 3952, 1226, 3103, 4628, 4823, 997, 3059, 3474, 587, 1674, 4136, 2560, 1501, 1333, 3403, 1119, 1155, 2649, 134, 1856, 4342, 1143, 2424, 4143, 2998, 3391, 3917, 4272, 2908, 1596, 4470, 808, 2797, 165, 2305, 2077, 3438, 327, 4463, 1477, 4104, 1389, 2601, 283, 1718, 1318, 1072, 3334, 307, 3225, 782, 4491, 4994, 766, 2218, 3038, 1426, 1736, 3278, 4381, 1936, 3289, 2988, 3865, 4530, 3405, 3081, 1594, 272, 3687, 1715, 3074, 2363, 4015, 1598, 2665, 773, 2924, 1161, 4430, 1762, 3531, 2420, 4164, 3512, 67, 3980, 4434, 758, 1972, 1700, 2494, 4613, 4924, 3760, 2831, 4815, 2493, 4023, 1747, 3626, 664, 3270, 4387, 143, 381, 4049, 3316, 2733, 2486, 311, 3394, 3592, 4720, 2260, 612, 3174, 2504, 800, 3734, 2469, 3732, 4879, 156, 1914, 3032, 4214, 4943, 3601, 1050, 839, 2232, 305, 1055, 1837, 2878, 4503, 13, 2806, 4732, 64, 2103, 3029, 52, 902, 2525, 600, 1186, 1328, 2141, 4561, 2183, 993, 2913, 1981, 2643, 3713, 2805, 778, 2227, 1554, 2946, 2478, 1704, 939, 4422, 2957, 3711, 3061, 3065, 2032, 3102, 2778, 29, 3324, 1320, 244, 1053, 498, 780, 830, 82, 2701, 3036, 2912, 3301, 4723, 2489, 3100, 911, 4608, 2732, 3750, 175, 2162, 3179, 3740, 4432, 3349, 1158, 4649, 4090, 4055, 1272, 2138, 4585, 1499, 635, 2664, 1076, 3853, 2422, 1948, 3965, 2110, 2652, 4354, 4199, 820, 81, 2512, 528, 4334, 4554, 3018, 54, 3052, 3943, 411, 4615, 797, 3960, 804, 1528, 2123, 935, 115, 3125, 3544, 3848, 3109, 1058, 2045, 642, 2002, 2431, 2164, 4056, 2166, 1525, 4298, 3632, 2456, 4224, 451, 4375, 2049, 1689, 2497, 4118, 4357, 392, 4443, 4459, 423, 189, 3488, 1177, 3293, 3441, 3951, 2968, 3163, 4911, 4288, 4520, 3447, 151, 2772, 3195, 2239, 345, 5, 991, 4679, 504, 2029, 2692, 3027, 73, 405, 829, 625, 99, 2475, 4995, 4624, 3822, 588, 4043, 655, 3499, 3154, 3606, 3718, 3795, 3697, 4654, 1269, 824, 2648, 1967, 3416, 3921, 2554, 191, 10, 3178, 3306, 4724, 4350, 2556, 1377, 3369, 937, 384, 2501, 4877, 4777, 3634, 3371, 3844, 4109, 971, 3692, 969, 3019, 4973, 1011, 4738, 472, 733, 618, 1997, 4707, 173, 936, 3871, 3275, 732, 4741, 2495, 1901, 4566, 2933, 1415, 3633, 1215, 464, 4846, 1626, 3923, 3930, 4768, 3233, 3675, 2020, 4128, 2617, 1684, 3204, 4936, 649, 4682, 3434, 790, 4411, 3902, 507, 3574, 3319, 4573, 57, 1755, 4257, 1266, 4085, 4527, 4234, 1373, 1681, 1192, 816, 4820, 1187, 3647, 2625, 4764, 4306, 4836, 3231, 2503, 3309, 3660, 2743, 4035, 2283, 1805, 3614, 2553, 2409, 4184, 4812, 4781, 613, 3555, 3625, 417, 1555, 383, 2754, 724, 2644, 4556, 2584, 1081, 1263, 3526, 3716, 1291, 697, 3407, 4771, 1833, 2340, 4483, 1623, 3997, 3887, 4326, 1019, 4700, 1788, 2526, 1194, 1088, 2317, 2598, 3837, 4189, 4913, 1498, 328, 1748, 2473, 4818, 864, 1907, 3513, 1990, 1581, 1066, 4241, 1573, 2905, 523, 3120, 1225, 2622, 319, 1298, 3462, 5009, 698, 4552, 1927, 3055, 4116, 4902, 1038, 3043, 2238, 3767, 3348, 3615, 4594, 4926, 2698, 706, 1065, 1850, 695, 4469, 1310, 4512, 214, 4066, 2064, 4003, 1557, 3889, 3230, 4793, 1732, 3776, 2485, 4827, 3442, 1279, 1276, 4028, 1934, 2682, 1851, 3307, 2036, 3252, 3580, 1881, 3207, 238, 3381, 1183, 245, 4501, 2527, 554, 3897, 546, 3305, 3385, 3066, 1571, 1825, 2471, 1061, 763, 3690, 262, 65, 3056, 1190, 4726, 4059, 4484, 2735, 4886, 3940, 385, 1938, 3919, 3107, 717, 2246, 1633, 2599, 3015, 4569, 2128, 4547, 1374, 4472, 4657, 42, 3446, 1459, 2638, 4165, 424, 4761, 2394, 634, 2345, 1462, 4158, 407, 1722, 869, 4294, 3147, 2808, 4185, 2904, 3338, 669, 2811, 492, 5003, 2274, 2686, 1465, 4577, 934, 3451, 4070, 881, 1804, 1052, 344, 1889, 1693, 1337, 4175, 4538, 438, 4073, 4480, 2594, 4450, 3064, 1438, 1985, 4675, 4194, 2970, 2054, 4598, 3808, 4849, 1261, 3820, 274, 3206, 4038, 3680, 3645, 1431, 2927, 743, 1049, 4558, 1647, 1841, 1765, 3541, 4604, 2750, 2889, 4663, 965, 1340, 3988, 1206, 4955, 3030, 3953, 2714, 745, 2711, 333, 3756, 4543, 1558, 239, 3899, 3845, 458, 2055, 4909, 543, 1627, 2044, 122, 1267, 4653, 79, 1135, 1102, 525, 3905, 4803, 2681, 1624, 3026, 4107, 2959, 3101, 3766, 3463, 297, 4521, 2450, 4640, 3670, 361, 906, 1343, 4405, 2853, 1687, 3562, 3040, 4756, 1780, 4033, 4254, 192, 1427, 3693, 1634, 3110, 1393, 3342, 3034, 3707, 1351, 4317, 4813, 4205, 2867, 4625, 1391, 2180, 2947, 933, 1845, 2921, 2406, 3436, 4966, 3486, 3806, 2308, 3994, 3485, 1201, 4779, 260, 3620, 776, 1620, 3870, 3600, 4079, 3858, 3955, 3782, 2445, 1986, 2833, 1604, 18, 1977, 1224, 4997, 1277, 2592, 3232, 3421, 1485, 3521, 3709, 1387, 3091, 685, 1942, 1585, 113, 4795, 3298, 1690, 2403, 1607, 1645, 1522, 403, 1023, 3666, 1706, 2827, 4069, 1045, 2088, 2753, 2336, 280, 3896, 4425, 2408, 4452, 2057, 4127, 2258, 369, 1005, 3023, 60, 2856, 4202, 858, 2800, 979, 3093, 4424, 2244, 2854, 3712, 4680, 1405, 1489, 2523, 4068, 3471, 1167, 752, 3774, 1229, 3388, 1904, 1551, 2929, 2996, 1703, 1803, 3227, 3016, 1048, 678, 3142, 1770, 3117, 671, 4504, 1790, 4560, 2327, 3579, 2931, 1125, 2249, 4112, 4721, 1212, 2116, 2862, 2059, 615, 129, 3086, 3931, 760, 2662, 3226, 3920, 2158, 2651, 4408, 1730, 72, 4961, 647, 4172, 899, 597, 3374, 4845, 1218, 3345, 3470, 4920, 2989, 291, 4316, 1483, 2017, 404, 2213, 1256, 3490, 4252, 4683, 4949, 2966, 1369, 1655, 4182, 2214, 739, 2626, 1118, 4638, 742, 4792, 1174, 2799, 4077, 1816, 1892, 1077, 2602, 2145, 387, 412, 4739, 2865, 1508, 4498, 3737, 2451, 1631, 4918, 4799, 4046, 1247, 301, 975, 2918, 4953, 102, 1798, 1396, 3480, 69, 2835, 723, 4510, 1104, 2825, 2074, 2334, 1628, 1767, 3942, 1589, 4300, 4736, 2687, 4634, 2804, 4042, 3722, 2911, 3450, 2948, 978, 3051, 1603, 3439, 231, 1152, 3422, 1695, 1696, 3258, 44, 4369, 4251, 1428, 3950, 194, 4766, 817, 2809, 1813, 682, 4789, 2474, 1436, 3751, 3041, 1559, 1305, 3908, 1600, 2611, 3982, 2295, 4923, 3944, 2817, 3624, 3973, 2039, 1550, 3096, 1506, 3948, 4671, 33, 1464, 1439, 2539, 1248, 2421, 105, 4433, 1758, 4162, 2135, 870, 347, 3314, 2439, 279, 4772, 3426, 1400, 4773, 2564, 419, 2264, 2400, 50, 2971, 2794, 3977, 2647, 3769, 631, 330, 3424, 4980, 4684, 2932, 2160, 2571, 3165, 2781, 2561, 845, 3185, 3826, 227, 4255, 2661, 1779, 1181, 316, 2930, 3752, 3654, 142, 3160, 1606, 2983, 178, 2222, 4428, 756, 2815, 446, 3423, 1954, 3856, 4801, 1783, 2722, 1579, 3522, 287, 3265, 2618, 2461, 1544, 3829, 3148, 396, 4807, 563, 378, 1380, 4605, 3425, 2416, 285, 693, 4291, 4899, 2460, 2767, 2603, 4281, 1535, 892, 1441, 3639, 3190, 3971, 4694, 4806, 2567, 346, 4186, 441, 848, 3987, 2254, 4712, 298, 2694, 4882, 4133, 3208, 6, 1115, 4211, 4074, 4506, 4844, 3888, 3020, 2537, 4755, 4302, 3373, 235, 2530, 468, 4100, 4036, 4550, 1381, 3180, 377, 1869, 4965, 2063, 2076, 4531, 4600, 1913, 1228, 1745, 309, 4618, 4471, 3758, 4395, 3143, 236, 3623, 2546, 2122, 4627, 2702, 3071, 1370, 114, 1580, 3413, 4644, 4431, 1694, 4687, 2676, 3939, 2472, 2562, 4975, 2522, 1749, 2621, 3809, 120, 4423, 2438, 1877, 1618, 950, 1685, 3255, 2568, 3922, 2101, 197, 3479, 1995, 4462, 3279, 1743, 2866, 4523, 2354, 4246, 616, 1364, 1727, 4019, 3946, 3764, 4278, 594, 2359, 1987, 4391, 2949, 1680, 2945, 2984, 961, 1141, 1692, 713, 3962, 1714, 4261, 31, 4645, 449, 4958, 3122, 4150, 150, 222, 772, 2093, 519, 2169, 1028, 5008, 83, 4787, 1976, 2235, 1989, 3251, 3588, 4619, 358, 4903, 491, 2871, 3404, 4592, 2379, 4268, 2058, 3476, 2803, 1757, 1319, 2005, 2335, 1080, 1130, 4204, 767, 3662, 4880, 3053, 1582, 1290, 1830, 4353, 893, 3659, 2783, 439, 2073, 1752, 2826, 2448, 3554, 2520, 126, 2874, 4336, 12, 842, 3299, 4797, 1472, 4147, 1097, 2813, 652, 938, 3140, 3913, 440, 3094, 3153, 3501, 1768, 3879, 312, 3139, 153, 4665, 640, 3337, 1666, 2075, 3364, 4039, 5007, 131, 4011, 3135, 4096, 4262, 211, 3256, 2067, 553, 656, 815, 399, 2683, 1027, 3671, 3933, 3508, 3570, 229, 4370, 4072, 3590, 2785, 1832, 2521, 1449, 690, 3838, 3569, 2426, 1241, 2914, 3558, 3996, 4553, 2965, 3516, 3969, 1683, 1322, 252, 4482, 1507, 36, 4900, 684, 3150, 1728, 2196, 2563, 2004, 248, 719, 1160, 4539, 3090, 2842, 1612, 1509, 2751, 696, 1861, 3843, 273, 4026, 4821, 3831, 2319, 1994, 948, 2885, 2502, 3197, 803, 1260, 2669, 4146, 3636, 3292, 1567, 2386, 493, 1036, 2366, 4535, 3892, 676, 602, 2507, 4511, 3617, 11, 2097, 3805, 3389, 1207, 3631, 527, 3823, 2771, 56, 116, 4169, 2773, 4309, 331, 3271, 2330, 754, 2841, 4616, 3762, 688, 4873, 3098, 1871, 4717, 2464, 2048, 2941, 1424, 4473, 2877, 3648, 4734, 138, 4944, 508, 4718, 2108, 4132, 3664, 2977, 1250, 228, 923, 4858, 633, 3587, 882, 4367, 1169, 1452, 1110, 1577, 1756, 1378, 3814, 2266, 2148, 2444, 3321, 4364, 3818, 1029, 967, 221, 2708, 880, 4271, 3455, 2824, 1910, 3608, 416, 2442, 4678, 1854, 1953, 2650, 2255, 320, 2574, 2401, 3267, 1020, 4435, 1358, 3437, 4314, 3552, 445, 1675, 2050, 1103, 481, 4240, 2084, 2519, 2696, 4446, 1367, 2404, 835, 207, 3105, 1078, 2417, 2284, 744, 3901, 3009, 3311, 2763, 1859, 2094, 2547, 2795, 584, 2452, 1217, 3560, 1630, 2358, 2510, 1899, 1541, 2007, 2483, 4413, 3162, 946, 496, 4986, 4984, 1928, 582, 1691, 4837, 2191, 2816, 2836, 622, 1487, 3903, 1973, 2710, 897, 176, 4993, 4061, 2925, 3635, 1667, 4454, 890, 2281, 1808, 4749, 2120, 4013, 575, 2737, 3067, 2056, 4901, 1539, 3396, 1437, 3264, 4067, 3974, 4228, 1394, 1819, 2727, 4014, 2437, 1068, 2902, 2457, 2106, 4301, 3681, 2543, 2418, 1184, 3461, 3062, 506, 3010, 4838, 1668, 4599, 293, 2247, 3044, 3511, 4401, 1173, 1314, 3886, 2087, 1794, 2922, 2462, 4378, 2300, 2697, 2663, 2047, 822, 2713, 3376, 3582, 1475, 49, 4210, 4610, 1764, 391, 3033, 1481, 3992, 4333, 1283, 2958, 3941, 43, 163, 1105, 3022, 3402, 4419, 2184, 2157, 1516, 4656, 4113, 182, 2939, 2104, 1621, 410, 2849, 4548, 826, 1590, 3196, 4368, 1564, 3841, 2868, 1468, 1012, 3392, 2640, 4016, 398, 930, 1789, 1074, 4178, 3335, 581, 4230, 708, 4054, 2693, 3481, 2428, 4689, 4800, 4247, 4245, 1665, 4686, 325, 264, 3914, 3627, 3788, 4660, 3610, 522, 4340, 4142, 146, 3281, 654, 1060, 2999, 813, 247, 834, 2818, 282, 2674, 875, 2355, 1529, 610, 3761, 4769, 1719, 2018, 2515, 1474, 1145, 2500, 3427, 1611, 4745, 4220, 4522, 2572, 3880, 401, 3222, 844, 3787, 3465, 374, 3505, 3290, 802, 2672, 2550, 3612, 1362, 3849, 3296, 3909, 3084, 3867, 3945, 483, 2782, 4970, 3362, 4032, 1043, 1270, 1198, 541, 2280, 3158, 1466, 3725, 514, 4310, 1660, 4284, 329, 583, 4297, 4967, 4474, 1453, 171, 3249, 691, 4166, 4004, 4478, 1562, 2775, 1520, 962, 4174, 3677, 1296, 1213, 2046, 2378, 2896, 1180, 1777, 1254, 2604, 4441, 2139, 4329, 956, 3669, 1643, 1470, 4048, 4895, 1443, 871, 4442, 1797, 132, 4248, 1113, 3077, 4690, 1619, 2820, 2633, 3245, 4266, 3315, 3898, 3576, 1091, 2415, 4533, 2906, 2429, 972, 3783, 494, 734, 415, 811, 3661, 2370, 1602, 2187, 707, 1062, 2691, 2276, 1515, 559, 4155, 3429, 2894, 3473, 4930, 4275, 4881, 702, 1432, 462, 1111, 889, 1676, 945, 95, 4866, 2544, 3352, 1238, 679, 2757, 3730, 1530, 4047, 4576, 3039, 3616, 555, 3459, 1517, 3272, 532, 68, 2777, 1450, 4094, 4545, 4324, 2954, 4662, 4861, 3961, 1960, 4670, 4983, 3007, 2043, 2872, 3595, 215, 1852, 1412, 2037, 1849, 1379, 159, 914, 3088, 2739, 4420, 1670, 502, 3869, 4152, 3097, 4451, 699, 2322, 987, 1716, 4223, 428, 4557, 3694, 4505, 1905, 4130, 1787, 2224, 4819, 250, 300, 3925, 1442, 2610, 3815, 2955, 1479, 2432, 2851, 4808, 2192, 367, 4525, 2899, 1303, 269, 2377, 500, 2111, 3561, 1578, 1918, 1491, 770, 3363, 614, 2315, 1205, 4830, 1750, 3353, 3875, 1098, 2413, 23, 1335, 2449, 3417, 1251, 1526, 4672, 3111, 249, 3949, 1584, 3000, 195, 3057, 4621, 394, 4414, 3277, 1959, 2207, 199, 1401, 370, 4770, 787, 1701, 2787, 4948, 1242, 3116, 3550, 2608, 3350, 1724, 2587, 883, 4898, 1159, 3472, 4496, 4089, 819, 2619, 3380, 4776, 4006, 286, 4906, 4725, 3778, 2263, 2402, 1970, 1444, 2614, 731, 2053, 1280, 3327, 3743, 722, 4904, 953, 1199, 2151, 3630, 4308, 718, 4544, 2879, 2607, 926, 2391, 3383, 3628, 2289, 3824, 4027, 3114, 3682, 4307, 3618, 254, 737, 3729, 677, 4029, 85, 1069, 3793, 2287, 1454, 1834, 1812, 680, 4998, 1430, 3104, 2384, 265, 321, 662, 3602, 1746, 3773, 166, 2071, 3070, 193, 3651, 3136, 2435, 4379, 4416, 1563, 1346, 485, 2881, 413, 167, 4542, 3540, 133, 1006, 4101, 789, 1888, 1463, 1932, 4018, 457, 4465, 4124, 208, 418, 4325, 2909, 295, 3493, 1969, 484, 1302, 4508, 2559, 2360, 3229, 2412, 4062, 1615, 3821, 3042, 1560, 2259, 3288, 4135, 4540, 3785, 3063, 2528, 1863, 1931, 1257, 335, 2381, 4990, 2917, 4763, 1769, 1532, 426, 2124, 1014, 447, 1949, 2639, 490, 2243, 487, 456, 1876, 1240, 606, 4731, 3069, 9, 2365, 3834, 1817, 499, 2130, 3678, 2153, 4086, 1763, 3145, 1671, 2190, 1345, 196, 2940, 4524, 551, 3907, 355, 3347, 4338, 3663, 3598, 753, 4972, 2689, 4282, 2764, 4412, 738, 3860, 4347, 4977, 4346, 1737, 4299, 4536, 2271, 1572, 533, 4636, 4568, 749, 4502, 1108, 810, 3885, 2673, 266, 2275, 4962, 212, 350, 1956, 3551, 3968, 1021, 3972, 954, 590, 2752, 3819, 4693, 4488, 907, 4005, 2883, 921, 2540, 2847, 1371, 3656, 2092, 1154, 885, 4084, 3881, 3478, 1455, 180, 2557, 675, 88, 97, 4658, 431, 1784, 1514, 4045, 4824, 1740, 2575, 4945, 4332, 3891, 4740, 3543, 4187, 4352, 1657, 1234, 857, 2147, 1592, 7, 3458, 3749, 1809, 230, 4507, 463, 2167, 1216, 3811, 2061, 1402, 2208, 2009, 3742, 1874, 3828, 1460, 3123, 2293, 3, 2915, 1556, 2027, 2440, 1865, 949, 4216, 3073, 999, 2744, 836, 3529, 1357, 1998, 1842, 2228, 4363, 3566, 4564, 3401, 3058, 2609, 517, 1992, 909, 2186, 4177, 542, 2981, 3668, 4436, 267, 3336, 665, 1897, 4963, 2143, 2290, 3303, 2704, 3934, 2304, 4810, 2964, 422, 720, 1123, 4426, 4919, 2134, 1121, 562, 332, 4087, 4811, 157, 2288, 2083, 4115, 1331, 3443, 1282, 3985, 3440, 1059, 3705, 3310, 981, 2606, 3710, 137, 1601, 4052, 1741, 3833, 1148, 2178, 1309, 2364, 569, 2828, 1292, 4941, 2936, 4796, 2832, 1112, 289, 3360, 3593, 1372, 4946, 437, 918, 1196, 2660, 2480, 2655, 3387, 988, 2022, 3629, 3581, 1404, 338, 2858, 4170, 409, 4192, 3810, 1831, 1908, 1672, 16, 957, 1352, 2119, 2041, 3241, 278, 4106, 2593, 1661, 931, 1034, 509, 205, 607, 1922, 3210, 1771, 1570, 1271, 3672, 4907, 3706, 1867, 1974, 4839, 2237, 1707, 2099, 3168, 80, 3565, 3568, 257, 3768, 1875, 47, 5002, 89, 4110, 4950, 3963, 4221, 3250, 5004, 3024, 4455, 2199, 4406, 4905, 1033, 4076, 703, 4735, 342, 210, 460, 25, 3530, 1980, 1962, 4713, 2834, 4293, 4440, 1882, 2062, 4852, 3813, 5000, 3482, 1868, 1617, 2855, 1785, 2102, 4180, 663, 3075, 2869, 4733, 1988, 2680, 181, 941, 2443, 296, 354, 4191, 2900, 3003, 549, 1810, 4190, 3367, 1996, 3698, 21, 1839, 1246, 1862, 2410, 2466, 2430, 641, 2303, 2667, 1124, 3115, 4362, 1575, 3138, 2882, 4001, 2843, 4197, 1597, 1085, 4857, 2490, 1056, 2684, 482, 2314, 2963, 1793, 2566, 4093, 2536, 672, 2712, 1306, 48, 1398, 4466, 1735, 1347, 2236, 3409, 1930, 2770, 3294, 3159, 4095, 2517, 3021, 4237, 218, 793, 2155, 2876, 1712, 1807, 3308, 4681, 2991, 3927, 2441, 1457, 1870, 1775, 1122, 3535, 2329, 1289, 4817, 1521, 1075, 2801, 1878, 2511, 828, 2961, 1414, 736, 1711, 96, 1214, 2265, 2578, 110, 2038, 3578, 2372, 3172, 4081, 944, 859, 1096, 2600, 2884, 2068, 1493, 4976, 226, 1915, 3591, 4111, 1030, 2491, 1090, 4139, 2171, 4258, 648, 4979, 4581, 37, 1595, 809, 4376, 336, 3146, 275, 4590, 2299, 2508, 2796, 2717, 1073, 4865, 2396, 51, 3641, 515, 86, 1523, 1407, 4804, 183, 2726, 4460, 1185, 783, 4286, 3739, 4229, 4409, 4982, 2747, 674, 1583, 3460, 4668, 225, 2133, 4786, 694, 3935, 3517, 284, 3689, 619, 4010, 3379, 2635, 620, 1919, 1397, 3284, 2978, 832, 1682, 4009, 2967, 3959, 1411, 2887, 2612, 1646, 2179, 188, 3851, 2548, 759, 516, 1467, 3331], "validation": [5454, 5336, 5498, 5119, 5170, 5299, 5030, 5488, 5361, 5358, 5451, 5060, 5333, 5425, 5222, 5416, 5463, 5395, 5123, 5423, 5427, 5076, 5474, 5296, 5509, 5216, 5387, 5191, 5304, 5282, 5253, 5142, 5180, 5227, 5356, 5325, 5248, 5352, 5293, 5205, 5312, 5329, 5360, 5424, 5046, 5288, 5321, 5272, 5335, 5127, 5460, 5213, 5174, 5071, 5266, 5215, 5187, 5264, 5491, 5224, 5023, 5066, 5396, 5130, 5241, 5351, 5324, 5059, 5067, 5192, 5128, 5033, 5245, 5054, 5231, 5118, 5446, 5035, 5143, 5413, 5172, 5373, 5294, 5242, 5232, 5340, 5101, 5481, 5374, 5097, 5072, 5448, 5322, 5080, 5036, 5290, 5053, 5153, 5162, 5185, 5375, 5196, 5230, 5027, 5238, 5320, 5243, 5383, 5103, 5388, 5074, 5014, 5177, 5039, 5317, 5087, 5470, 5499, 5285, 5471, 5406, 5265, 5037, 5148, 5161, 5458, 5461, 5394, 5480, 5411, 5269, 5181, 5415, 5052, 5083, 5376, 5420, 5305, 5078, 5202, 5073, 5478, 5457, 5108, 5337, 5186, 5489, 5197, 5256, 5468, 5115, 5453, 5062, 5105, 5330, 5479, 5263, 5368, 5183, 5262, 5490, 5050, 5497, 5167, 5477, 5261, 5165, 5355, 5136, 5041, 5482, 5173, 5144, 5353, 5434, 5120, 5019, 5298, 5295, 5371, 5193, 5151, 5258, 5281, 5057, 5382, 5315, 5189, 5068, 5436, 5199, 5098, 5392, 5233, 5455, 5091, 5475, 5226, 5342, 5314, 5113, 5495, 5058, 5426, 5141, 5307, 5211, 5255, 5031, 5306, 5401, 5435, 5393, 5135, 5077, 5015, 5456, 5259, 5048, 5201, 5439, 5093, 5302, 5124, 5095, 5319, 5476, 5440, 5278, 5034, 5207, 5040, 5357, 5364, 5100, 5140, 5260, 5409, 5286, 5380, 5372, 5032, 5166, 5082, 5089, 5452, 5328, 5190, 5018, 5412, 5079, 5116, 5405, 5430, 5257, 5419, 5311, 5125, 5326, 5345, 5378, 5280, 5389, 5507, 5188, 5428, 5308, 5133, 5384, 5292, 5338, 5247, 5316, 5010, 5026, 5086, 5084, 5289, 5496, 5208, 5341, 5417, 5042, 5137, 5218, 5283, 5075, 5500, 5178, 5287, 5021, 5064, 5145, 5473, 5486, 5149, 5085, 5381, 5110, 5274, 5444, 5092, 5212, 5229, 5408, 5273, 5279, 5407, 5348, 5235, 5223, 5354, 5459, 5472, 5462, 5011, 5254, 5404, 5171, 5331, 5431, 5301, 5045, 5508, 5043, 5081, 5469, 5249, 5438, 5309, 5276, 5437, 5055, 5139, 5219, 5209, 5485, 5106, 5206, 5366, 5131, 5399, 5111, 5234, 5121, 5112, 5343, 5362, 5184, 5432, 5049, 5494, 5450, 5252, 5104, 5061, 5198, 5155, 5114, 5369, 5346, 5163, 5063, 5017, 5028, 5445, 5403, 5237, 5179, 5024, 5464, 5129, 5176, 5370, 5158, 5506, 5029, 5134, 5150, 5503, 5164, 5094, 5505, 5204, 5386, 5402, 5240, 5169, 5332, 5363, 5313, 5465, 5175, 5099, 5016, 5228, 5239, 5502, 5070, 5088, 5339, 5467, 5044, 5013, 5267, 5051, 5349, 5277, 5102, 5487, 5268, 5203, 5214, 5449, 5210, 5069, 5152, 5443, 5418, 5109, 5492, 5390, 5334, 5168, 5194, 5126, 5398, 5154, 5284, 5200, 5433, 5020, 5391, 5385, 5056, 5291, 5501, 5250, 5466, 5244, 5107, 5347, 5138, 5447, 5414, 5065, 5090, 5157, 5022, 5221, 5096, 5377, 5270, 5359, 5246, 5122, 5146, 5025, 5251, 5303, 5429, 5422, 5483, 5365, 5310, 5327, 5421, 5379, 5217, 5156, 5047, 5493, 5400, 5038, 5442, 5441, 5159, 5300, 5367, 5275, 5504, 5225, 5182, 5117, 5350, 5012, 5220, 5297, 5236, 5160, 5410, 5344, 5132, 5318, 5323, 5271, 5147, 5484, 5397, 5195], "test": [6005, 5982, 5579, 5860, 5901, 5696, 5769, 5905, 5547, 5567, 5913, 5510, 5575, 5758, 5530, 5556, 5749, 5621, 5785, 5859, 6002, 6000, 5625, 5748, 5827, 5916, 5815, 5799, 5674, 5970, 5765, 5591, 5811, 5857, 5864, 5702, 5574, 5909, 5716, 5639, 5581, 5724, 5669, 5842, 5812, 5960, 5962, 5598, 5563, 5796, 5844, 5961, 5770, 5793, 5642, 5814, 5736, 5511, 5688, 5950, 5527, 5602, 5904, 5613, 5956, 5698, 5715, 5938, 5514, 5695, 5566, 5756, 5899, 5647, 5889, 5902, 5753, 5774, 5728, 5878, 5675, 5745, 5646, 5536, 5692, 5824, 5884, 5876, 5849, 5976, 5596, 5648, 5914, 5949, 5791, 5512, 5597, 5789, 5600, 5537, 5594, 5964, 5520, 5863, 5953, 5940, 5890, 5874, 5668, 5740, 5801, 5585, 5709, 5993, 5603, 5760, 5969, 5933, 5779, 5927, 5777, 5826, 5946, 5531, 5612, 5737, 5858, 5981, 5762, 5570, 5792, 5544, 5908, 5803, 5957, 5706, 5809, 5732, 5634, 5921, 5532, 5546, 5898, 5958, 5830, 5767, 5590, 5794, 5725, 5822, 5986, 5832, 5615, 5541, 5852, 5661, 5671, 5787, 5549, 5624, 5932, 5855, 5872, 5918, 5607, 5979, 5907, 5971, 5704, 5664, 5571, 5823, 5568, 5846, 5968, 5896, 5892, 5733, 5535, 5722, 5578, 5712, 5965, 5562, 5903, 5653, 5773, 5926, 5538, 5601, 5689, 5747, 5534, 5776, 5599, 5754, 5805, 5564, 6007, 5619, 5937, 5853, 5604, 5930, 5955, 5611, 5763, 5699, 5694, 5650, 5719, 5707, 5735, 5681, 5729, 5764, 5554, 5643, 5951, 5897, 5942, 5573, 5790, 5656, 5882, 5635, 5978, 5623, 5633, 5833, 5881, 5854, 5742, 5750, 5944, 5838, 5818, 5786, 5744, 5522, 5659, 5888, 5780, 5684, 5701, 5752, 5939, 5850, 5912, 5552, 5726, 5545, 5977, 5816, 5550, 5877, 5606, 5680, 5630, 5810, 5670, 5526, 5821, 5757, 5713, 5717, 5649, 5952, 5652, 5731, 5542, 5720, 5798, 6003, 5560, 5788, 5972, 5886, 5985, 5577, 5879, 5627, 5771, 5662, 5636, 5843, 5963, 5931, 5875, 5691, 5917, 5743, 5751, 5529, 5856, 5880, 5559, 5683, 5990, 5587, 5586, 5759, 5644, 5654, 5925, 5778, 5693, 5943, 5561, 5910, 5819, 5572, 5802, 5983, 5589, 5906, 5515, 5730, 5616, 5868, 5915, 5565, 5995, 5989, 5766, 5605, 5781, 5518, 5519, 5929, 5584, 5705, 5721, 5746, 5948, 5992, 5703, 5775, 5869, 5825, 5551, 5959, 5708, 5513, 5994, 5588, 5945, 5557, 5797, 5521, 5663, 5610, 5543, 5666, 5807, 5828, 5817, 5637, 5640, 5806, 5682, 5924, 5626, 5525, 5677, 5795, 5975, 5865, 5966, 5690, 5954, 5911, 5673, 5723, 5614, 5638, 5672, 6004, 5620, 5999, 5784, 5593, 5887, 5540, 6009, 5679, 5711, 5687, 5524, 5936, 5871, 5920, 5894, 5984, 5718, 5582, 5569, 5834, 6001, 5974, 5631, 5632, 5919, 5595, 5839, 5548, 5885, 5891, 5555, 5866, 5870, 5700, 5738, 5741, 5808, 5837, 5553, 5847, 5658, 5655, 5845, 5628, 5516, 5608, 5893, 5973, 5923, 6006, 5867, 5804, 5900, 5609, 5991, 5755, 5617, 5987, 5996, 5657, 5539, 5685, 5862, 5576, 5727, 5831, 5820, 5583, 5761, 5645, 5836, 5988, 5528, 5883, 5840, 5651, 5622, 5592, 5734, 5848, 5922, 5517, 5895, 5533, 5772, 5660, 5629, 5928, 5851, 5580, 5800, 5997, 5841, 5768, 5829, 5641, 5873, 5697, 5835, 5967, 5676, 5665, 5710, 5813, 5934, 5783, 5618, 5998, 5667, 5935, 5558, 5947, 6008, 5861, 5686, 5782, 5980, 5678, 5714, 5739, 5523, 5941]}]
\ No newline at end of file
+[{"train": [2240, 296, 3901, 2563, 4157, 1106, 4743, 376, 1730, 1873, 393, 922, 3920, 995, 3821, 3419, 3135, 1229, 1547, 3197, 180, 2482, 2105, 2948, 3184, 1563, 3154, 3731, 1329, 695, 2186, 3291, 1398, 3831, 1567, 3740, 1747, 1970, 4185, 1407, 595, 2115, 1460, 2799, 1149, 15, 4446, 2199, 2022, 4108, 4955, 4866, 824, 2062, 708, 741, 2347, 2854, 3257, 472, 593, 3018, 3351, 1654, 3546, 956, 2517, 3752, 238, 3279, 4435, 3639, 711, 18, 3041, 3649, 3938, 1617, 1121, 3604, 874, 3563, 473, 73, 3646, 251, 891, 1405, 4086, 41, 294, 88, 1661, 2117, 798, 1238, 2604, 4820, 2907, 4207, 1773, 2179, 2095, 3128, 3877, 1027, 4905, 3031, 1077, 3666, 1878, 1400, 4008, 4036, 3602, 4760, 3472, 2541, 3694, 1576, 4600, 4747, 435, 1102, 1453, 2371, 742, 1339, 4016, 1283, 431, 3804, 2863, 2980, 331, 2600, 1274, 2740, 2821, 4287, 4246, 1147, 1353, 2867, 2010, 2720, 2143, 348, 3506, 1131, 1371, 155, 2994, 4278, 1394, 775, 558, 4131, 3998, 3843, 165, 2565, 1683, 4821, 2926, 3836, 630, 4930, 589, 2805, 1603, 3898, 649, 42, 427, 2205, 2418, 1596, 4993, 4434, 1995, 3178, 1855, 1579, 2754, 2748, 1769, 917, 2858, 2675, 3318, 4726, 4005, 1375, 2715, 3783, 3496, 1122, 1984, 3835, 1239, 2512, 836, 4975, 3309, 4095, 2487, 4169, 1447, 1302, 163, 4413, 2856, 202, 2043, 2352, 0, 3071, 2934, 4534, 723, 788, 4705, 4615, 1259, 1419, 1499, 4141, 4501, 2937, 1012, 1636, 22, 4248, 4893, 3263, 3275, 117, 1869, 371, 1922, 4661, 3163, 1554, 4430, 715, 4638, 1446, 71, 2904, 256, 61, 3656, 3518, 417, 337, 854, 4801, 2114, 4635, 4845, 3008, 2267, 4349, 799, 3176, 3654, 2527, 221, 3715, 362, 4125, 4883, 432, 4567, 3934, 3932, 133, 32, 1173, 225, 2895, 297, 2294, 4559, 4773, 3955, 4589, 3386, 1972, 4775, 3857, 401, 1778, 1916, 30, 2507, 831, 1247, 2151, 489, 2012, 139, 1017, 846, 4825, 3581, 1798, 3664, 4053, 47, 692, 4614, 2928, 3393, 3596, 2075, 388, 1870, 4070, 1198, 3132, 3483, 1166, 4080, 1156, 2598, 4329, 1290, 3856, 4952, 3889, 1676, 2184, 4194, 2464, 3492, 3575, 1199, 2918, 896, 1589, 2846, 293, 56, 2728, 1427, 3069, 4643, 1312, 1681, 4442, 3499, 3678, 1512, 4763, 2702, 3611, 2758, 3708, 1612, 729, 2255, 4034, 3624, 3481, 101, 1524, 1196, 4581, 968, 1079, 3215, 2695, 4631, 3815, 2182, 1108, 3527, 2614, 672, 886, 3480, 543, 3131, 1096, 2526, 4851, 3077, 4494, 3101, 1278, 3711, 78, 1685, 2509, 4848, 1275, 3193, 2686, 3991, 3156, 872, 3880, 4708, 10, 4891, 675, 3150, 1946, 3928, 1144, 3704, 4261, 2159, 3427, 730, 1766, 3657, 2885, 2414, 364, 551, 1161, 2015, 1409, 773, 4167, 4265, 3381, 3734, 3010, 4934, 4496, 3710, 847, 1190, 84, 96, 1632, 183, 2741, 2202, 2461, 4889, 2971, 1137, 268, 1091, 4493, 4283, 3582, 1031, 4764, 3277, 1779, 1025, 1650, 3115, 2395, 3083, 3240, 52, 3790, 3321, 716, 1834, 963, 1953, 2499, 3204, 2892, 3396, 893, 955, 97, 830, 4432, 1194, 4898, 2052, 1200, 3055, 1729, 1075, 2977, 3091, 2273, 196, 1999, 492, 4812, 2425, 1662, 4433, 1069, 1770, 2890, 4314, 2311, 2922, 2806, 3712, 391, 4989, 86, 3292, 3615, 4959, 2481, 3075, 4861, 3044, 320, 1322, 3846, 2398, 1168, 2544, 2919, 146, 4093, 1452, 4031, 2819, 4793, 1019, 857, 1859, 1713, 1369, 1608, 3538, 1830, 483, 3925, 4197, 481, 4766, 1987, 3507, 266, 2469, 2886, 430, 3744, 2768, 1039, 639, 1197, 1223, 2215, 4650, 2492, 833, 4909, 4912, 3979, 1594, 2690, 3931, 3175, 3036, 1600, 3721, 2084, 3742, 36, 1552, 2218, 793, 3237, 2872, 2992, 2322, 143, 757, 722, 4818, 2014, 2035, 939, 2306, 4913, 3757, 587, 4981, 283, 1123, 65, 1070, 4690, 305, 1726, 3099, 3982, 3239, 7, 104, 1675, 960, 533, 309, 3252, 1865, 1744, 3558, 1807, 3225, 1860, 1355, 2245, 3017, 4715, 8, 1985, 2650, 2331, 1465, 1782, 700, 2833, 2832, 3392, 4800, 914, 3705, 2007, 1058, 1652, 1088, 4156, 1381, 1057, 92, 758, 167, 1065, 977, 4374, 4700, 1678, 1089, 1139, 2375, 1181, 2134, 1367, 2248, 4914, 4926, 3792, 3598, 4770, 4904, 2997, 2149, 2192, 51, 3347, 3343, 2121, 3487, 1314, 769, 1919, 4660, 582, 2471, 2419, 2320, 518, 2575, 3078, 162, 2673, 2332, 2638, 2828, 4403, 1357, 809, 3755, 2382, 4613, 2399, 619, 4412, 3661, 2353, 453, 3665, 2391, 1657, 1534, 1107, 2268, 1187, 1742, 4786, 2945, 1795, 813, 1404, 1475, 4915, 3308, 1140, 2449, 1980, 3989, 1433, 1335, 4046, 1293, 1389, 3933, 3826, 838, 2510, 3515, 4112, 4027, 392, 2756, 2634, 2597, 3228, 2089, 3787, 3863, 108, 3829, 1982, 602, 1539, 4579, 3722, 4642, 1049, 933, 3173, 601, 1839, 2292, 2376, 469, 2596, 4153, 3258, 2024, 3638, 3363, 3909, 853, 4226, 4900, 4995, 264, 1812, 3865, 642, 2621, 445, 699, 3087, 4992, 43, 1548, 197, 2772, 3803, 4291, 4029, 3324, 150, 1347, 2194, 3842, 4071, 2938, 514, 35, 2556, 3247, 80, 4073, 2112, 4115, 3250, 2674, 915, 1775, 428, 2946, 826, 2830, 3195, 232, 1001, 774, 930, 4231, 352, 684, 1151, 3226, 683, 875, 1971, 1570, 1155, 4524, 2321, 4271, 2357, 3102, 2605, 3357, 4514, 456, 2857, 2257, 2129, 379, 189, 4933, 3906, 657, 4166, 3119, 869, 2313, 2900, 407, 1753, 2221, 1344, 4220, 3051, 2028, 3129, 900, 1466, 2338, 2423, 5010, 4805, 2884, 640, 3169, 4209, 4994, 2766, 2815, 4550, 3323, 2515, 4454, 2244, 860, 4714, 585, 402, 1850, 98, 1458, 2574, 288, 676, 2165, 2783, 4604, 2097, 1550, 4702, 419, 1491, 4301, 4990, 3337, 1938, 4101, 4048, 621, 1368, 2477, 1743, 3452, 4171, 494, 4949, 608, 2132, 95, 3911, 3431, 1930, 4944, 2528, 4473, 3060, 4134, 2838, 1041, 4810, 20, 2264, 4466, 1718, 2475, 1904, 2327, 4699, 4388, 2073, 776, 666, 3220, 3733, 4193, 3079, 2733, 3768, 384, 559, 3939, 998, 902, 504, 4573, 2298, 170, 4722, 750, 2722, 3599, 49, 2743, 2668, 110, 1969, 4928, 358, 69, 3677, 2592, 4256, 3524, 3723, 4184, 3146, 4521, 1011, 4224, 1232, 3364, 2396, 1291, 3188, 1061, 1889, 3902, 3746, 243, 4651, 3332, 4317, 144, 2814, 4918, 1502, 3255, 4212, 2958, 2822, 1250, 943, 1513, 2031, 1668, 1525, 4546, 3640, 3569, 1243, 4394, 3360, 1611, 1105, 4124, 3019, 2315, 1459, 2081, 705, 1653, 3662, 3269, 3259, 4382, 3140, 4026, 228, 365, 805, 2923, 2853, 910, 4321, 1078, 2021, 1115, 3290, 4348, 4753, 498, 983, 2436, 1029, 3667, 3957, 1767, 1272, 1697, 4476, 1514, 2196, 206, 3139, 2985, 3183, 3341, 2616, 2739, 2843, 1134, 2747, 4386, 4331, 4260, 2909, 412, 1128, 1665, 1648, 40, 1759, 1074, 3557, 184, 3047, 1004, 1334, 3788, 1008, 4899, 1722, 2291, 2472, 2835, 2050, 2261, 3408, 2462, 726, 4024, 449, 114, 545, 1688, 1429, 3382, 4203, 3650, 732, 4499, 119, 4498, 2676, 2771, 1825, 4984, 1952, 553, 3696, 4204, 1269, 704, 1698, 4724, 1700, 3310, 39, 3148, 1076, 4085, 2013, 2847, 4896, 3000, 3033, 3918, 1915, 3192, 1021, 3875, 503, 2164, 2823, 4880, 4221, 2580, 2283, 1780, 724, 1881, 1802, 1461, 1318, 3070, 4177, 887, 4236, 4371, 4593, 3824, 250, 1877, 4281, 263, 908, 1787, 3299, 44, 1206, 3471, 2896, 1214, 4105, 3121, 1383, 3917, 126, 1951, 3762, 1430, 1771, 3616, 4606, 1655, 2137, 1138, 4731, 3791, 4078, 356, 3594, 4872, 4279, 4844, 1501, 728, 1277, 1493, 2839, 4802, 2794, 82, 4525, 2540, 2372, 2763, 2064, 4431, 3320, 4682, 425, 2693, 3053, 3883, 4310, 4824, 4110, 4451, 1944, 3015, 2130, 4060, 1986, 415, 2594, 478, 4727, 3218, 651, 668, 1580, 2873, 534, 1754, 2439, 4685, 4406, 2961, 1162, 2495, 807, 273, 2524, 1597, 2550, 3065, 4592, 2473, 845, 1064, 2761, 2379, 2204, 2106, 1578, 122, 4010, 859, 487, 4069, 3973, 212, 629, 4242, 4758, 1349, 2869, 2582, 1109, 3811, 2914, 5005, 4429, 1852, 4791, 2093, 2091, 1010, 3214, 754, 4396, 1966, 4384, 4107, 2213, 3379, 3956, 1479, 3623, 1649, 3899, 1664, 636, 578, 952, 1937, 821, 2118, 1808, 457, 3457, 2413, 1028, 1988, 3287, 4178, 1487, 2888, 4972, 852, 4554, 3046, 2409, 858, 1132, 72, 4318, 3689, 4967, 2829, 4636, 1868, 3799, 4946, 4674, 2610, 3317, 3388, 3316, 2363, 2936, 522, 3400, 2505, 4608, 3476, 4940, 113, 4175, 2140, 2356, 2981, 2019, 1892, 3093, 1635, 130, 2899, 557, 2941, 686, 3709, 3095, 3961, 4298, 4902, 1709, 530, 1059, 2227, 3371, 2455, 2416, 23, 4504, 3528, 4999, 4632, 4357, 4145, 3807, 4377, 928, 2703, 4839, 176, 4205, 3536, 4088, 3170, 2330, 135, 2773, 3416, 1299, 3013, 800, 669, 4707, 3429, 4665, 1047, 2251, 3216, 3009, 3951, 3391, 1957, 2826, 1098, 2935, 1642, 2865, 2242, 850, 2812, 2911, 3231, 1911, 1976, 123, 81, 441, 3202, 28, 4121, 4470, 34, 3001, 2451, 2960, 4734, 4198, 861, 1164, 4973, 2226, 648, 2018, 24, 1081, 2229, 1897, 4892, 1023, 4249, 3336, 790, 4710, 490, 111, 3349, 3136, 4769, 386, 1043, 4263, 1060, 2969, 3304, 4676, 54, 1854, 4228, 1244, 3450, 2296, 4475, 2742, 2549, 1320, 4520, 507, 3529, 1704, 2855, 3584, 3442, 3643, 2408, 286, 205, 3680, 4162, 2214, 3300, 4838, 843, 310, 4509, 2072, 3448, 1994, 4146, 3508, 1622, 4754, 3430, 4055, 140, 292, 1813, 2303, 272, 783, 4044, 4964, 752, 3726, 3628, 1643, 182, 784, 3635, 145, 978, 3486, 1421, 4368, 3571, 1532, 3648, 16, 4616, 911, 1392, 1063, 463, 1215, 1774, 2658, 3088, 437, 2, 194, 1604, 4890, 1342, 2138, 4742, 3254, 4583, 3613, 2217, 811, 3423, 2133, 3305, 1551, 761, 3380, 3707, 3224, 1340, 191, 2657, 4721, 2637, 3020, 270, 2287, 3213, 4225, 1, 2392, 2314, 2150, 2440, 2699, 2654, 3428, 4359, 2717, 99, 1397, 3415, 4444, 2233, 4233, 2566, 2559, 691, 4811, 661, 3315, 4694, 4484, 3404, 3685, 814, 3062, 2705, 733, 2561, 1360, 1300, 4907, 1496, 1899, 2929, 5, 1191, 4911, 999, 4038, 755, 1422, 4957, 4654, 1055, 3012, 4491, 2420, 3207, 4657, 768, 4339, 2583, 2984, 2797, 3777, 291, 1708, 3022, 1384, 2232, 4920, 659, 1456, 3356, 511, 271, 3817, 259, 2400, 567, 2671, 4633, 4161, 4109, 2996, 1856, 2474, 2427, 1658, 3720, 3085, 1776, 662, 2374, 1103, 1756, 2458, 1090, 2096, 3147, 2094, 1615, 1835, 1607, 1929, 4138, 339, 4531, 4877, 1249, 1820, 2664, 4954, 1204, 2990, 4506, 2349, 4064, 2190, 899, 3048, 1832, 3897, 2774, 4122, 1740, 4277, 4378, 4843, 2991, 3262, 870, 2535, 3484, 3467, 3502, 4545, 3417, 1549, 4706, 751, 4438, 1587, 3947, 4553, 3930, 2384, 4672, 4879, 3293, 3120, 3633, 3153, 3283, 936, 1598, 344, 1707, 1101, 460, 2536, 3718, 3117, 451, 4781, 2522, 63, 1157, 4196, 2026, 4012, 4106, 1454, 1846, 1007, 2579, 1932, 2333, 1337, 1035, 1150, 2155, 1669, 399, 3344, 2608, 19, 735, 2624, 3244, 2274, 3498, 383, 287, 2272, 4062, 341, 1136, 354, 2581, 1727, 4680, 1301, 2189, 380, 4050, 785, 4039, 2727, 4648, 2438, 4806, 4777, 289, 1420, 4765, 2177, 239, 4275, 4217, 241, 3552, 3605, 1376, 2954, 3764, 1258, 4874, 1639, 4313, 3205, 4645, 3794, 1264, 4014, 1731, 4250, 5003, 4669, 1202, 4195, 4471, 795, 4482, 641, 1308, 1080, 4230, 3118, 537, 4293, 3663, 2222, 2562, 749, 3288, 1898, 1637, 208, 2683, 128, 3673, 3424, 3335, 3513, 570, 3756, 2490, 3029, 4165, 796, 879, 4402, 920, 404, 1257, 3327, 59, 3838, 4390, 1890, 4599, 2054, 377, 4733, 829, 1941, 4853, 946, 4513, 1146, 677, 2968, 3760, 2307, 1056, 4556, 4302, 2080, 2989, 4691, 2381, 1327, 2060, 1739, 2623, 4346, 1455, 3418, 91, 4240, 3525, 1974, 3554, 1426, 3251, 3606, 4735, 975, 4128, 989, 3996, 2434, 1606, 2618, 702, 638, 2870, 290, 1495, 4337, 4443, 175, 851, 2609, 3276, 4306, 2104, 4192, 4104, 3211, 4479, 1205, 3530, 3127, 948, 801, 766, 3278, 452, 4391, 1408, 2734, 4568, 1062, 3045, 3987, 3082, 3447, 201, 4075, 216, 1992, 2849, 679, 4863, 818, 1324, 4132, 2818, 1153, 90, 1254, 2456, 340, 727, 4970, 1346, 3410, 3319, 223, 2128, 2713, 2304, 2891, 5009, 4456, 1833, 1436, 736, 1619, 75, 1113, 4180, 2103, 2951, 3412, 3168, 3697, 2415, 820, 1192, 3387, 1224, 3124, 3014, 2328, 4834, 866, 1464, 4771, 4756, 2933, 495, 4500, 2074, 3522, 2908, 1378, 2578, 4065, 4867, 3489, 2845, 4324, 3547, 3494, 3873, 2750, 2428, 429, 3437, 3620, 4678, 4343, 55, 3653, 2107, 3510, 2716, 2480, 1212, 409, 3166, 2223, 3328, 4186, 2811, 3749, 3853, 574, 2421, 667, 4787, 1601, 1179, 1005, 1296, 1981, 4744, 4013, 4953, 2665, 4001, 4578, 643, 981, 2534, 2879, 4860, 3674, 3144, 3260, 3445, 4116, 1333, 3478, 1686, 1670, 3206, 685, 901, 2820, 2061, 1646, 2241, 3007, 1723, 680, 810, 3422, 4576, 623, 3870, 2387, 4399, 2942, 2069, 1216, 2738, 2726, 1476, 505, 2949, 4316, 4856, 157, 1602, 4380, 3591, 862, 4716, 890, 4640, 2142, 2250, 3706, 3366, 4649, 550, 284, 3256, 779, 1403, 3245, 1067, 2998, 4411, 2003, 4575, 1853, 4421, 2092, 2995, 346, 1588, 4191, 1195, 50, 4305, 3505, 3601, 1449, 367, 2861, 3301, 4668, 3384, 2697, 4865, 4103, 4924, 418, 1104, 1474, 3773, 2380, 3322, 374, 3958, 4037, 4054, 1595, 4202, 2663, 2796, 1097, 4379, 3785, 3362, 3886, 351, 4056, 760, 3600, 1537, 1457, 4269, 4564, 3438, 2993, 121, 1336, 3028, 1395, 311, 950, 1790, 400, 219, 1416, 4200, 321, 3941, 1572, 1358, 1593, 984, 3159, 2930, 315, 3005, 3632, 3735, 4840, 1288, 3096, 3778, 4778, 3929, 190, 2539, 1947, 3460, 941, 174, 4566, 4181, 4035, 3504, 3098, 4372, 3985, 4018, 4809, 2678, 2688, 4252, 2802, 4214, 3659, 2293, 925, 3405, 1880, 2767, 4206, 2260, 1625, 1094, 2800, 1033, 3219, 2275, 609, 2485, 394, 438, 734, 2975, 4094, 70, 1174, 3378, 3236, 4455, 1684, 4749, 3771, 4345, 3959, 247, 2670, 1867, 2071, 4784, 3541, 4237, 278, 4582, 3564, 1689, 4875, 198, 304, 138, 3289, 240, 3560, 2000, 2790, 693, 4362, 689, 1887, 2009, 4692, 1054, 4361, 2045, 1006, 2554, 3443, 1351, 803, 624, 3950, 3421, 2453, 3795, 597, 3822, 187, 1695, 1281, 1518, 2262, 1311, 4072, 4033, 3196, 4783, 2100, 1473, 2443, 2719, 1471, 4041, 1909, 4797, 1159, 4179, 2532, 4385, 4878, 3728, 3669, 2432, 4276, 867, 4081, 1241, 1363, 855, 1963, 4588, 770, 3627, 4736, 1544, 1099, 4199, 780, 4925, 4644, 2920, 2921, 2547, 2523, 832, 3610, 4150, 4294, 4517, 2170, 3751, 1690, 3034, 2339, 591, 1231, 4971, 4719, 3003, 2203, 753, 1541, 3636, 611, 3812, 1705, 3532, 2148, 4127, 1410, 192, 4986, 4980, 1749, 2503, 3942, 3670, 1127, 1230, 1693, 905, 4779, 3526, 4569, 4258, 1313, 1306, 3353, 4129, 3683, 4570, 231, 3775, 3210, 959, 4447, 4835, 3030, 1002, 2708, 5007, 137, 413, 3016, 844, 3180, 3702, 1717, 131, 462, 4796, 1129, 635, 4315, 1886, 4092, 4223, 1553, 2178, 3968, 4365, 4888, 3097, 93, 14, 5011, 521, 4679, 2644, 2200, 731, 485, 1424, 544, 2208, 4063, 1538, 1546, 3107, 4000, 3609, 4547, 554, 4006, 3281, 4032, 3922, 1933, 282, 246, 4047, 261, 4572, 5002, 3813, 299, 2368, 892, 12, 4558, 3948, 3949, 1262, 1893, 4370, 4149, 3358, 2653, 3891, 3141, 540, 2765, 227, 1824, 3390, 4285, 2564, 4342, 841, 476, 3975, 4426, 2463, 3878, 613, 3914, 4102, 828, 3753, 2243, 252, 459, 2310, 319, 1481, 2163, 2450, 4901, 335, 1013, 894, 3801, 1928, 4485, 3617, 1530, 4292, 627, 1958, 547, 3923, 904, 4587, 4381, 4757, 4675, 2801, 2466, 2729, 1256, 1906, 484, 4407, 3376, 1472, 4658, 46, 2346, 4234, 1236, 4876, 1093, 1829, 964, 4557, 1068, 27, 4182, 1325, 2514, 575, 4664, 390, 3860, 4968, 4168, 3076, 4759, 1660, 718, 791, 2325, 3974, 1380, 3848, 2905, 1053, 1879, 2645, 1354, 461, 789, 3459, 1917, 2680, 549, 2862, 2613, 4480, 1861, 4219, 1372, 4264, 4960, 2326, 3348, 37, 153, 3420, 4143, 2029, 3435, 148, 720, 1711, 3209, 4358, 2191, 3992, 3579, 366, 242, 2893, 2970, 1176, 816, 4712, 3553, 1666, 646, 2254, 4419, 2915, 4540, 1934, 4439, 1692, 4737, 3500, 953, 3861, 4117, 3858, 993, 980, 2403, 3781, 447, 4895, 1112, 2302, 1110, 2206, 4951, 4803, 1962, 1050, 4373, 1418, 3607, 4452, 4823, 2875, 3818, 3479, 4235, 4745, 3004, 3425, 3359, 3576, 1120, 2295, 1428, 1965, 2841, 2002, 2589, 1183, 1219, 921, 4262, 4833, 1142, 1009, 787, 2759, 3823, 1082, 1451, 2237, 1402, 2183, 3988, 4353, 1285, 3200, 4910, 4486, 3647, 3550, 849, 2124, 565, 255, 3830, 2504, 3882, 1680, 4977, 1691, 620, 1715, 1284, 4449, 3103, 781, 4948, 2345, 129, 992, 605, 3446, 4448, 4170, 2147, 698, 1967, 2836, 3577, 932, 2365, 2369, 1087, 1943, 3080, 3737, 4921, 1260, 4136, 3474, 448, 1356, 1508, 68, 1945, 1227, 2305, 5008, 2180, 2087, 2077, 1377, 4492, 3971, 4639, 1609, 4244, 4618, 4309, 4366, 3810, 3006, 3936, 610, 3965, 2127, 1940, 3394, 4728, 4858, 3037, 164, 2048, 1644, 2085, 2974, 3962, 4079, 1497, 4352, 2986, 1528, 1613, 3439, 527, 486, 2502, 4067, 3779, 2877, 1844, 2065, 4605, 3660, 1182, 4662, 1484, 797, 2301, 2335, 209, 3402, 4397, 3540, 1651, 4414, 3847, 548, 1765, 440, 3944, 4356, 4916, 1927, 2679, 2447, 2027, 1152, 2880, 199, 3841, 2389, 3514, 3997, 3084, 3641, 1610, 368, 4894, 2982, 4663, 2042, 4227, 2108, 2040, 652, 3759, 3840, 2366, 4596, 654, 1016, 1712, 3106, 1573, 1188, 3747, 396, 577, 1015, 4535, 3537, 501, 1910, 4387, 2752, 434, 2901, 2109, 3776, 4268, 3618, 1414, 4983, 3612, 3072, 2484, 1393, 835, 1902, 1983, 4247, 1800, 3977, 3592, 298, 4683, 1218, 2944, 1003, 1978, 3814, 2629, 1186, 2972, 2568, 2039, 124, 3086, 3203, 3064, 2343, 863, 2667, 1084, 2362, 2627, 3468, 332, 116, 3313, 4673, 1382, 2924, 4495, 3676, 4332, 3199, 2378, 2424, 690, 2625, 2058, 3233, 4097, 2516, 3340, 2560, 512, 2950, 2256, 2601, 804, 4311, 4774, 3265, 2125, 1417, 923, 1321, 3161, 674, 1245, 3369, 2122, 508, 4652, 3164, 4300, 3637, 745, 2407, 2620, 3629, 525, 1411, 2119, 2785, 1908, 1843, 2887, 2631, 1298, 513, 4807, 3953, 1677, 3189, 2153, 4395, 2912, 4629, 4906, 353, 2894, 531, 2259, 1914, 1511, 2344, 3935, 1478, 3542, 3361, 969, 3111, 4295, 497, 3869, 2083, 1480, 3701, 3990, 2868, 4850, 1228, 1189, 2220, 2252, 265, 4241, 2735, 1309, 2793, 359, 1823, 707, 3732, 3333, 3800, 502, 4701, 4354, 4025, 1169, 2506, 878, 1827, 4030, 1066, 4709, 4816, 3885, 1585, 2120, 4634, 4323, 106, 464, 4965, 3556, 2660, 1287, 3761, 4776, 2168, 4255, 4548, 3631, 2957, 1523, 3797, 4320, 2781, 628, 4725, 132, 2323, 1568, 1483, 1500, 2642, 966, 275, 517, 142, 1794, 3475, 2488, 4218, 4325, 655, 2782, 2101, 4767, 1663, 2611, 2749, 307, 234, 3750, 3945, 1781, 3068, 1444, 3912, 1831, 1439, 1818, 2825, 682, 1745, 2939, 5004, 4607, 1255, 1210, 3713, 3454, 327, 3167, 1315, 1203, 2234, 739, 3370, 3449, 4003, 1738, 338, 4842, 2126, 3767, 3451, 2834, 4620, 4533, 3827, 3266, 4183, 1638, 1789, 1507, 509, 3534, 4091, 2685, 3862, 3042, 1171, 782, 2649, 149, 1037, 4628, 2224, 4529, 2111, 3253, 4689, 171, 3160, 179, 825, 4017, 151, 1874, 2932, 3820, 4814, 746, 2587, 4148, 2848, 1903, 2804, 4004, 3544, 4068, 2236, 3924, 181, 546, 1361, 3011, 2383, 4577, 3910, 2691, 3002, 987, 1412, 276, 4591, 66, 1038, 1529, 1584, 2603, 4459, 1925, 1599, 4335, 2269, 3212, 4798, 2228, 3695, 277, 1477, 2144, 350, 2737, 4622, 4998, 1706, 2337, 985, 1445, 2518, 6, 3426, 1193, 588, 4052, 990, 2355, 2110, 2588, 67, 2195, 4869, 2047, 3477, 4813, 1872, 2551, 4151, 4251, 976, 195, 4740, 1233, 973, 1180, 455, 4423, 2161, 2493, 4987, 4286, 612, 883, 3373, 77, 324, 4120, 2520, 1837, 3174, 2889, 4935, 4601, 2851, 1935, 4748, 156, 3217, 598, 1373, 4862, 2927, 4164, 1305, 4288, 919, 4369, 3802, 1581, 965, 1531, 2056, 1623, 218, 4936, 109, 3383, 1847, 909, 3517, 4997, 603, 3687, 4142, 1117, 4730, 3125, 343, 3182, 4462, 2113, 2913, 4272, 4489, 1520, 4328, 2626, 213, 4299, 4847, 4289, 817, 663, 1714, 2160, 4308, 1564, 1154, 3151, 3331, 4822, 4750, 3267, 1506, 4163, 1921, 1267, 4303, 4155, 958, 3355, 3066, 3081, 1167, 4609, 1819, 972, 2947, 3859, 1492, 3585, 178, 4511, 3583, 678, 3350, 3158, 4066, 740, 664, 631, 1543, 3645, 536, 3440, 3887, 1379, 3490, 4330, 3234, 3025, 2210, 3157, 2079, 4768, 4133, 316, 4483, 4693, 927, 786, 1991, 3138, 1399, 877, 3535, 4794, 4611, 74, 4257, 2082, 1734, 3796, 4561, 382, 2156, 2807, 3314, 4571, 3587, 523, 3937, 3672, 1542, 2289, 4746, 3501, 2903, 2437, 528, 1605, 4595, 2860, 2687, 3995, 1432, 579, 2718, 2386, 333, 1365, 615, 2468, 4077, 2964, 1032, 1163, 1583, 2778, 2285, 387, 4831, 3655, 1370, 2651, 4304, 3232, 3699, 2448, 2780, 1385, 3172, 1674, 1586, 1158, 1527, 1490, 650, 2497, 2141, 4400, 4932, 1793, 2898, 1014, 4458, 706, 2646, 4416, 345, 2925, 1406, 423, 1845, 253, 4772, 2053, 1948, 4686, 1316, 771, 2669, 2341, 1842, 168, 4213, 3243, 3736, 2377, 1895, 1574, 2076, 2433, 3984, 3851, 4201, 314, 1175, 4273, 233, 154, 3684, 1891, 1271, 4096, 4188, 1441, 230, 4560, 1125, 4019, 1240, 2586, 1811, 4481, 279, 1801, 618, 1294, 4585, 4656, 2558, 3819, 2633, 3881, 2789, 1838, 4327, 4974, 2309, 3497, 1900, 2023, 2572, 3165, 725, 3864, 2684, 3109, 4950, 626, 2467, 710, 808, 1048, 599, 421, 4040, 4751, 3338, 2643, 3354, 2979, 1438, 1431, 3179, 4795, 2088, 3644, 467, 4176, 2090, 312, 688, 3191, 1364, 3682, 4538, 3765, 1270, 3946, 322, 2146, 1640, 2412, 1784, 996, 433, 1788, 360, 3895, 2824, 3407, 1884, 2531, 4267, 703, 1330, 85, 1659, 3866, 3455, 1710, 2622, 4340, 1656, 2348, 1758, 2358, 4830, 318, 1883, 4704, 1786, 173, 3039, 1626, 4922, 4626, 3302, 2736, 3413, 2288, 2049, 3927, 4312, 2181, 4580, 529, 3890, 4978, 4832, 2037, 3463, 491, 4790, 1185, 1046, 3130, 4687, 3549, 865, 1234, 4976, 31, 224, 2454, 1618, 903, 186, 1263, 4659, 2158, 4319, 2429, 4785, 2777, 1160, 454, 1760, 1413, 466, 2098, 3092, 882, 4126, 1696, 1810, 3926, 4383, 1736, 2711, 1814, 1114, 334, 2760, 4624, 2530, 4111, 4376, 637, 215, 347, 2442, 4536, 3181, 4326, 2955, 1276, 1515, 644, 1396, 3642, 1719, 1540, 881, 4696, 3056, 4703, 125, 3651, 4436, 4817, 285, 1641, 2692, 2319, 3993, 743, 25, 1437, 1391, 3904, 3122, 2878, 1119, 738, 4022, 1095, 2840, 3397, 2694, 2494, 107, 2280, 2632, 3832, 2513, 2546, 3352, 3915, 2297, 4087, 1757, 3306, 926, 2351, 1141, 4873, 470, 1505, 3593, 3964, 1434, 3833, 3668, 918, 1975, 4603, 633, 4870, 2078, 2367, 424, 4510, 1279, 2459, 3980, 4389, 2953, 3555, 1732, 3073, 2725, 3578, 1448, 1682, 4917, 2744, 1350, 3590, 2300, 2402, 245, 4190, 4229, 3325, 1526, 3026, 3049, 395, 1768, 381, 349, 4463, 373, 274, 1942, 2813, 625, 1533, 4641, 2724, 3614, 1748, 1280, 389, 4947, 1703, 4405, 1857, 3389, 1388, 4461, 935, 1849, 1761, 880, 3545, 3916, 2258, 3149, 3595, 2666, 4076, 3493, 4738, 2874, 3248, 3769, 1762, 2479, 2070, 3798, 4528, 2173, 4453, 3837, 4627, 2916, 2246, 2397, 2593, 2557, 1126, 516, 269, 607, 4210, 1319, 330, 301, 3774, 2917, 2008, 442, 2810, 4698, 1645, 3470, 4854, 249, 127, 2635, 4929, 4474, 2135, 2698, 4367, 3208, 2978, 177, 4215, 907, 2063, 1851, 4284, 2431, 2102, 1482, 3186, 4574, 1950, 5000, 3054, 4732, 4084, 1724, 3038, 519, 214, 1086, 713, 4684, 1326, 2411, 974, 3229, 3235, 1467, 60, 539, 105, 3548, 2707, 450, 714, 3270, 3919, 3793, 532, 592, 3516, 1135, 11, 2441, 3688, 3903, 604, 3171, 4061, 954, 4090, 3330, 884, 4961, 4681, 2270, 3134, 4468, 281, 4280, 2312, 3223, 515, 997, 1366, 3222, 207, 87, 4118, 4478, 2628, 868, 2714, 4956, 3844, 3375, 3473, 500, 777, 1130, 1630, 248, 4290, 2641, 3693, 4208, 1504, 4671, 406, 1485, 4653, 967, 1821, 1816, 889, 1517, 4114, 134, 1145, 1671, 2521, 2004, 3414, 405, 2682, 1699, 4253, 4074, 2976, 3040, 2405, 4943, 4360, 2852, 3043, 2746, 1519, 1858, 1591, 555, 2201, 4051, 4043, 398, 475, 2017, 4098, 152, 185, 572, 477, 1672, 2659, 3963, 2871, 4422, 1901, 3374, 2602, 258, 2652, 906, 2498, 3748, 1343, 2299, 4868, 2537, 102, 1931, 957, 4420, 4789, 1521, 2615, 3368, 1560, 2394, 4565, 4741, 3067, 4347, 1509, 1862, 506, 2197, 4804, 57, 1616, 3238, 3896, 3884, 4602, 3983, 3464, 1627, 4059, 2966, 2770, 3523, 924, 3089, 576, 308, 1918, 471, 3294, 1387, 3565, 3057, 3894, 3303, 4852, 3867, 4625, 3743, 1052, 673, 3401, 1116, 4130, 2055, 4590, 1498, 4152, 2209, 3221, 541, 822, 719, 1701, 3061, 4937, 4515, 4172, 369, 538, 2999, 1494, 2373, 136, 4988, 3230, 1265, 161, 2647, 2570, 4023, 397, 4713, 560, 3433, 4338, 2700, 3143, 2952, 3094, 2776, 1026, 2543, 778, 2607, 897, 1443, 3284, 2486, 942, 3272, 3177, 1172, 4829, 3703, 3411, 2034, 3052, 2483, 4159, 4527, 3050, 3187, 2809, 3543, 4009, 33, 1124, 1621, 2696, 4123, 89, 1209, 4610, 1211, 2354, 1470, 2342, 1562, 971, 436, 1979, 4497, 4958, 815, 1237, 4982, 4137, 1024, 4113, 4341, 3940, 2617, 1225, 2406, 1415, 3871, 1809, 302, 647, 2238, 2775, 1178, 3691, 4996, 3295, 3681, 38, 2731, 4437, 1882, 1040, 2612, 4307, 951, 586, 2388, 3738, 2038, 986, 2798, 2538, 1566, 3566, 2496, 323, 717, 696, 737, 3246, 2792, 4296, 1687, 4711, 1000, 4523, 1764, 1535, 1468, 1201, 229, 3261, 1328, 188, 118, 3700, 1226, 2842, 681, 1817, 4919, 4584, 2883, 2595, 3114, 403, 4007, 1777, 370, 3058, 931, 1177, 3772, 2803, 2585, 961, 280, 2529, 147, 2216, 3573, 4516, 3110, 3634, 2943, 2567, 4147, 3908, 1556, 4619, 3519, 2762, 3491, 1345, 3488, 328, 2661, 4991, 2837, 2552, 1386, 4695, 4487, 3201, 1923, 306, 632, 3972, 4477, 4846, 1679, 590, 2959, 62, 1871, 3729, 2154, 408, 1803, 3461, 3580, 4427, 3562, 4216, 2417, 3954, 4723, 4717, 303, 4057, 1246, 3162, 3123, 4799, 2606, 357, 3365, 1913, 4472, 1956, 1545, 2636, 4962, 3090, 2723, 2827, 3967, 3658, 1217, 4011, 2445, 2317, 4238, 2188, 2784, 3399, 4336, 571, 1118, 1561, 3142, 1184, 3741, 1996, 3868, 3952, 3403, 2239, 2390, 4082, 1261, 962, 3780, 929, 4530, 763, 3342, 112, 2545, 1297, 1960, 236, 4505, 3970, 4688, 2359, 1073, 4942, 4670, 422, 3621, 4083, 2963, 1536, 444, 3385, 3784, 9, 3311, 949, 2648, 2067, 2576, 721, 3725, 300, 3280, 2361, 4058, 4621, 2701, 1317, 856, 2630, 2730, 4563, 2850, 3850, 747, 4266, 4465, 1486, 4985, 4542, 3854, 1435, 4408, 2207, 169, 4815, 748, 211, 2051, 4355, 3539, 4637, 658, 3586, 4045, 4099, 2230, 3297, 2491, 3839, 1018, 3597, 2364, 4855, 1359, 488, 4941, 1165, 2176, 1220, 1841, 940, 4512, 1620, 4245, 1912, 583, 3719, 2316, 2360, 1896, 267, 2385, 3849, 4254, 3346, 21, 1282, 614, 1423, 1964, 1286, 665, 2167, 4923, 1702, 812, 493, 2501, 3512, 2435, 3626, 3690, 4457, 1848, 3409, 970, 2033, 697, 4549, 1222, 1242, 439, 606, 4028, 3282, 535, 934, 3913, 2068, 420, 1510, 2791, 3608, 3921, 3845, 2329, 100, 840, 4021, 4401, 2393, 446, 372, 2247, 1036, 115, 2476, 4623, 2956, 1863, 2426, 1590, 823, 2318, 4297, 2265, 3966, 1772, 4782, 1516, 979, 4927, 1577, 988, 1888, 4887, 2511, 2219, 3717, 2882, 2709, 3485, 2816, 4539, 3023, 2508, 1993, 4440, 1815, 2410, 762, 141, 3671, 4020, 3900, 1968, 4667, 4718, 1977, 4849, 1565, 4232, 83, 2263, 2290, 2788, 2116, 2906, 1737, 3994, 1489, 2662, 4518, 235, 2755, 4755, 4502, 1997, 3692, 2284, 709, 2011, 2640, 2198, 3568, 3808, 3960, 2212, 4274, 1442, 3567, 1253, 3766, 2967, 3551, 864, 4015, 4460, 103, 2795, 4697, 564, 3035, 4908, 4364, 2844, 336, 4828, 2470, 834, 1989, 622, 1624, 3249, 3074, 2555, 210, 4617, 2548, 1341, 4884, 1374, 3021, 2099, 4544, 1785, 937, 3879, 1034, 1954, 1248, 166, 1633, 4490, 254, 329, 562, 1836, 2866, 426, 158, 3852, 670, 2066, 3453, 2279, 4425, 2764, 2677, 3724, 1733, 2681, 4555, 2131, 4507, 1488, 3782, 1111, 3511, 687, 222, 3227, 220, 4939, 1045, 3828, 4841, 3312, 2753, 3185, 237, 4761, 3570, 2235, 1797, 3999, 1806, 4409, 3432, 3194, 2446, 2881, 48, 3907, 3116, 160, 3190, 3441, 1694, 510, 3406, 3816, 4445, 2710, 3339, 3398, 526, 1752, 1251, 3137, 1894, 1235, 4903, 660, 4424, 2619, 2751, 2225, 1920, 2340, 634, 3603, 2249, 1959, 3969, 4270, 2271, 2025, 2123, 3686, 3888, 1303, 3652, 4551, 3495, 4135, 3625, 2599, 4351, 3806, 1092, 3739, 994, 2041, 411, 3805, 4243, 1440, 1792, 3482, 873, 45, 1208, 1998, 3273, 4049, 3145, 1592, 3395, 3730, 596, 3892, 4938, 378, 200, 1207, 1631, 4826, 2542, 1924, 765, 2745, 4222, 2324, 694, 1390, 3876, 4187, 1750, 3622, 4655, 3809, 2639, 3521, 1721, 2350, 2489, 3, 4488, 3326, 2465, 1425, 3105, 3444, 1571, 1042, 4541, 2422, 3296, 2672, 3345, 2533, 4154, 2460, 1020, 2571, 2704, 573, 3533, 2983, 1614, 4410, 4259, 13, 3572, 363, 2086, 1557, 3155, 947, 4189, 3574, 4739, 2831, 3559, 888, 313, 2174, 1462, 2139, 944, 226, 2712, 4966, 3274, 4173, 2876, 2286, 568, 2282, 2689, 3264, 1582, 193, 1826, 1864, 2500, 2864, 3770, 1634, 1822, 3855, 317, 3133, 885, 1304, 4882, 2032, 3372, 2721, 1469, 203, 3377, 1450, 1331, 1939, 1266, 4762, 4586, 2897, 842, 2044, 1720, 2931, 326, 2987, 2525, 342, 1044, 3520, 1213, 4526, 3024, 295, 4469, 1221, 616, 4857, 79, 4594, 3503, 4519, 3619, 4864, 2145, 1071, 2732, 29, 361, 2266, 4792, 756, 416, 2057, 4859, 4140, 2162, 1667, 2808, 1725, 2308, 2231, 4979, 4886, 876, 945, 4969, 1100, 848, 2452, 4543, 64, 4333, 2478, 4450, 3589, 1292, 2334, 3027, 2001, 563, 2859, 1085, 4752, 4144, 1746, 4404, 600, 819, 4612, 3754, 76, 594, 2590, 3745, 2457, 1072, 3242, 2175, 1955, 2573, 1673, 2136, 2591, 4282, 4630, 3434, 671, 325, 2016, 4666, 982, 3307, 1926, 556, 561, 1522, 656, 4160, 465, 1289, 4350, 4119, 3905, 410, 468, 2569, 499, 4441, 479, 1783, 2430, 1555, 1268, 3986, 4552, 3465, 2336, 3727, 2172, 3198, 17, 3104, 1348, 480, 3285, 1751, 3943, 3978, 3329, 1362, 1936, 764, 120, 4428, 414, 802, 4508, 3630, 617, 2046, 3789, 4808, 1503, 5001, 1148, 3675, 4393, 1558, 1569, 2036, 895, 2779, 3298, 496, 806, 443, 3825, 3367, 3469, 2902, 3531, 794, 772, 1559, 3334, 3874, 1990, 3872, 4720, 4677, 1083, 4836, 1755, 3976, 2281, 1949, 4211, 580, 2706, 257, 552, 1875, 482, 581, 58, 1804, 1885, 4537, 4819, 172, 2769, 524, 1840, 2030, 991, 3698, 4239, 3588, 3059, 3268, 2553, 1647, 4417, 1735, 3466, 4322, 2171, 1352, 4503, 3893, 1051, 1030, 4363, 566, 3981, 701, 4963, 4647, 1973, 474, 1961, 2519, 2973, 916, 159, 4002, 2655, 4, 4344, 4418, 1323, 3063, 4897, 1629, 4871, 2444, 871, 4158, 1463, 4100, 2157, 26, 3716, 1295, 3561, 2404, 913, 2786, 2185, 4398, 938, 94, 4532, 2059, 4334, 3834, 4881, 3126, 3152, 520, 4885, 3458, 1310, 1252, 898, 385, 744, 4042, 4174, 1133, 1796, 2965, 3271, 4780, 3509, 4788, 2152, 4415, 4598, 2020, 5006, 2169, 1170, 3758, 3241, 4089, 1741, 2577, 4646, 1728, 542, 2193, 3286, 2584, 262, 1876, 458, 2656, 1799, 4562, 2211, 3108, 3786, 1338, 2005, 4375, 1401, 584, 2187, 1716, 2910, 3032, 1332, 2988, 3714, 2277, 3113, 827, 1907, 653, 2370, 1866, 375, 569, 1575, 4729, 2962, 2401, 4522, 645, 3100, 1022, 4392, 3763, 355, 2166, 4945, 2817, 2940, 792, 3112, 759, 837, 2757, 1763, 1273, 2276, 2787, 3462, 1307, 53, 2278, 4827, 3679, 1805, 3436, 4464, 1628, 4597, 217, 4931, 4139, 1905, 1791, 767, 244, 260, 3456, 4837, 712, 4467, 839, 1143, 1828, 2006, 2253, 204, 912], "validation": [5262, 5107, 5217, 5152, 5389, 5506, 5074, 5134, 5415, 5243, 5095, 5470, 5357, 5137, 5273, 5449, 5193, 5239, 5476, 5364, 5129, 5353, 5060, 5185, 5154, 5396, 5037, 5053, 5025, 5438, 5422, 5161, 5341, 5139, 5067, 5133, 5012, 5127, 5221, 5471, 5349, 5303, 5228, 5368, 5158, 5390, 5051, 5042, 5171, 5155, 5372, 5420, 5118, 5367, 5441, 5165, 5454, 5146, 5018, 5329, 5393, 5036, 5461, 5369, 5237, 5321, 5251, 5404, 5197, 5478, 5101, 5114, 5040, 5099, 5319, 5366, 5226, 5315, 5170, 5380, 5206, 5314, 5088, 5474, 5225, 5148, 5499, 5403, 5425, 5030, 5109, 5473, 5352, 5453, 5029, 5131, 5400, 5337, 5182, 5333, 5043, 5373, 5267, 5200, 5147, 5087, 5282, 5325, 5122, 5277, 5201, 5026, 5175, 5230, 5244, 5401, 5041, 5096, 5346, 5116, 5324, 5089, 5058, 5457, 5348, 5256, 5259, 5253, 5347, 5308, 5241, 5342, 5477, 5204, 5013, 5318, 5184, 5100, 5102, 5252, 5191, 5216, 5159, 5076, 5418, 5249, 5205, 5335, 5108, 5258, 5289, 5475, 5232, 5202, 5082, 5145, 5500, 5313, 5178, 5023, 5378, 5222, 5323, 5307, 5377, 5064, 5186, 5183, 5312, 5246, 5028, 5071, 5409, 5068, 5022, 5264, 5447, 5301, 5458, 5192, 5494, 5057, 5052, 5265, 5509, 5435, 5507, 5187, 5169, 5073, 5268, 5430, 5215, 5472, 5331, 5359, 5142, 5298, 5266, 5456, 5086, 5245, 5462, 5436, 5069, 5112, 5105, 5050, 5492, 5179, 5339, 5150, 5151, 5078, 5395, 5021, 5429, 5338, 5208, 5115, 5398, 5300, 5376, 5019, 5279, 5310, 5402, 5452, 5284, 5280, 5387, 5417, 5238, 5199, 5079, 5381, 5149, 5424, 5434, 5316, 5480, 5083, 5227, 5442, 5167, 5502, 5466, 5445, 5271, 5495, 5483, 5508, 5383, 5126, 5130, 5257, 5294, 5080, 5463, 5153, 5235, 5360, 5138, 5172, 5496, 5098, 5231, 5370, 5198, 5392, 5320, 5180, 5485, 5260, 5484, 5309, 5047, 5157, 5063, 5132, 5077, 5194, 5388, 5426, 5113, 5062, 5455, 5479, 5450, 5358, 5304, 5248, 5498, 5384, 5302, 5104, 5437, 5111, 5141, 5469, 5481, 5094, 5444, 5274, 5223, 5120, 5188, 5486, 5168, 5055, 5292, 5399, 5213, 5501, 5490, 5511, 5033, 5075, 5084, 5296, 5295, 5017, 5428, 5164, 5085, 5336, 5421, 5291, 5212, 5081, 5482, 5365, 5092, 5254, 5027, 5214, 5439, 5236, 5432, 5386, 5322, 5493, 5160, 5066, 5269, 5488, 5385, 5035, 5311, 5044, 5459, 5446, 5410, 5411, 5072, 5356, 5211, 5020, 5173, 5233, 5219, 5405, 5242, 5210, 5220, 5144, 5128, 5306, 5394, 5297, 5140, 5103, 5039, 5503, 5056, 5031, 5468, 5379, 5247, 5431, 5504, 5363, 5048, 5343, 5234, 5391, 5371, 5412, 5117, 5065, 5229, 5362, 5467, 5334, 5355, 5119, 5045, 5497, 5015, 5285, 5287, 5407, 5097, 5061, 5054, 5427, 5440, 5110, 5345, 5163, 5034, 5261, 5448, 5272, 5423, 5275, 5203, 5406, 5276, 5024, 5156, 5382, 5288, 5143, 5209, 5451, 5419, 5207, 5176, 5361, 5505, 5136, 5330, 5326, 5340, 5135, 5240, 5189, 5354, 5070, 5016, 5344, 5460, 5433, 5299, 5255, 5283, 5328, 5397, 5123, 5162, 5196, 5465, 5510, 5351, 5121, 5464, 5125, 5190, 5059, 5218, 5286, 5290, 5224, 5093, 5270, 5414, 5046, 5278, 5106, 5124, 5281, 5174, 5374, 5413, 5049, 5195, 5293, 5305, 5416, 5263, 5166, 5350, 5443, 5327, 5332, 5489, 5408, 5038, 5317, 5177, 5487, 5090, 5032, 5091, 5491, 5375, 5250, 5181, 5014], "test": [5631, 5878, 5784, 5513, 5749, 5637, 5851, 5813, 5555, 5735, 5606, 5792, 5733, 5964, 5975, 5842, 5650, 5852, 5665, 5821, 5871, 5590, 5575, 5722, 5651, 5907, 5677, 5729, 5682, 5956, 5917, 5986, 5890, 5695, 5521, 6000, 5742, 5809, 5950, 5540, 5999, 5835, 5947, 5618, 5534, 5754, 5681, 5560, 5892, 5680, 5775, 5638, 5874, 5844, 5791, 5671, 5709, 5804, 5523, 5674, 5663, 5675, 5574, 5903, 5992, 5993, 5833, 5664, 5989, 5706, 5866, 5661, 5610, 5683, 5628, 5910, 5757, 5548, 5670, 5915, 5551, 5561, 5710, 5925, 5984, 5724, 5608, 5612, 5745, 5676, 5891, 5518, 5967, 5916, 5520, 5991, 5525, 5725, 5829, 5944, 5522, 5700, 5974, 5762, 5931, 5596, 5911, 6001, 5965, 5830, 5955, 5535, 5788, 5564, 5569, 5679, 5587, 5731, 5726, 5883, 5865, 5568, 5879, 5703, 5741, 5943, 5589, 5743, 5936, 5593, 5698, 6005, 5960, 5770, 5656, 5962, 5982, 5843, 5896, 5554, 5942, 5987, 5585, 5895, 5532, 5626, 6004, 5642, 5752, 5602, 5584, 5968, 5973, 5666, 5897, 5712, 5667, 5768, 5543, 5799, 5909, 5600, 5959, 5919, 5814, 5678, 5751, 5748, 5597, 5727, 5536, 5926, 5939, 5824, 5567, 5527, 5553, 5760, 5924, 5613, 5524, 5945, 5779, 5576, 5588, 5840, 5766, 5863, 5648, 5782, 5785, 5831, 5737, 5937, 5932, 5841, 5633, 5815, 5928, 5884, 5562, 5836, 5622, 5654, 6008, 5581, 5893, 5935, 5858, 5570, 5816, 5934, 5545, 5905, 5875, 5867, 5583, 5803, 5739, 5607, 6011, 5691, 5619, 5780, 5620, 5977, 5941, 5781, 5599, 5750, 5802, 5580, 5963, 5579, 5783, 5902, 5578, 5732, 5668, 5604, 5595, 5537, 5660, 5808, 5565, 5684, 5789, 5861, 5972, 5889, 5855, 5832, 5958, 5542, 5761, 5838, 5859, 5713, 5899, 5853, 5528, 5756, 5994, 5539, 5577, 5929, 5969, 5655, 5901, 5846, 5763, 5652, 5790, 5827, 5702, 5834, 5533, 5515, 5764, 5927, 5573, 5856, 6002, 5694, 5820, 5952, 5828, 6009, 5647, 5635, 5632, 5880, 5767, 5609, 5605, 5649, 5805, 5623, 5693, 5672, 5755, 5980, 5795, 5550, 5687, 5886, 5559, 5659, 5639, 5930, 5516, 5995, 5900, 5885, 5985, 6006, 5961, 5624, 5707, 5546, 5644, 5850, 5758, 5526, 5881, 5747, 5746, 5951, 5636, 5976, 5800, 5705, 5738, 5640, 5946, 5787, 5643, 5774, 5571, 5686, 5662, 5769, 5549, 5877, 6003, 5771, 5720, 5983, 5997, 5940, 5806, 5718, 5744, 5904, 5822, 5627, 5887, 5689, 5690, 5810, 5978, 5657, 5966, 5938, 5547, 5872, 5519, 5906, 5614, 5812, 5719, 5714, 5854, 5692, 5716, 5990, 5556, 5592, 5512, 5920, 5634, 5696, 5530, 5857, 5598, 5921, 5630, 5869, 5776, 5773, 5818, 5529, 5611, 5918, 5777, 5848, 5615, 5517, 5736, 5765, 5839, 5601, 5971, 5541, 5817, 5876, 5658, 5778, 6010, 5948, 5953, 5708, 5882, 5531, 5823, 5552, 5979, 5625, 5646, 5544, 5557, 5734, 5819, 5753, 5586, 5933, 5558, 6007, 5514, 5837, 5711, 5957, 5538, 5970, 5862, 5868, 5826, 5793, 5922, 5685, 5914, 5717, 5772, 5621, 5563, 5594, 5870, 5912, 5740, 5629, 5894, 5572, 5849, 5954, 5996, 5582, 5888, 5988, 5873, 5908, 5688, 5807, 5669, 5998, 5981, 5796, 5798, 5701, 5603, 5759, 5797, 5913, 5794, 5723, 5616, 5898, 5715, 5860, 5653, 5825, 5699, 5801, 5728, 5786, 5949, 5645, 5641, 5811, 5845, 5730, 5591, 5566, 5697, 5721, 5673, 5923, 5864, 5617, 5847, 5704]}, {"train": [2507, 1898, 2802, 742, 2115, 3838, 3182, 1854, 2749, 2436, 573, 3640, 5005, 1762, 3729, 3334, 1740, 1526, 3765, 4758, 3852, 3740, 4555, 3165, 1977, 4338, 24, 863, 4189, 2112, 2751, 2314, 4100, 4732, 731, 1771, 4707, 2595, 1015, 3308, 4601, 2042, 4168, 687, 4890, 1948, 313, 2817, 729, 3262, 1803, 2160, 3970, 969, 4075, 1435, 1074, 2416, 940, 2326, 201, 4366, 4030, 3051, 2963, 881, 382, 1430, 2945, 1892, 4977, 3846, 4487, 1649, 1546, 1622, 3894, 2803, 2333, 795, 2851, 3556, 4476, 386, 1570, 4821, 2725, 4293, 3291, 3426, 31, 4309, 4, 3325, 3914, 1135, 1319, 1753, 4494, 3872, 982, 1572, 3552, 722, 2598, 5004, 2398, 4580, 2805, 4468, 1965, 4087, 3980, 4713, 4694, 2611, 1960, 2113, 1226, 3870, 1766, 398, 1395, 406, 1274, 2297, 2493, 127, 844, 453, 4650, 2787, 2513, 4862, 4459, 3152, 3638, 4565, 4921, 2433, 3999, 3996, 233, 116, 3536, 4242, 3162, 2131, 3090, 3823, 3715, 3354, 3922, 1945, 2278, 511, 672, 2485, 1511, 1201, 260, 770, 4596, 4533, 637, 3956, 932, 2939, 2003, 4466, 2767, 1061, 2862, 4203, 3602, 1853, 2177, 4181, 1862, 1420, 923, 3312, 1445, 3559, 3470, 1591, 3604, 4838, 539, 2602, 45, 918, 417, 1798, 3126, 4390, 352, 2496, 1893, 3032, 946, 3924, 523, 804, 750, 3809, 3414, 4210, 1548, 1334, 1610, 1046, 3759, 4247, 719, 2413, 2001, 3772, 1791, 1304, 3634, 2066, 3854, 1364, 4789, 2561, 4504, 3338, 4922, 3271, 1665, 2478, 2584, 2636, 4161, 3690, 3483, 1643, 4722, 4892, 1321, 1906, 106, 231, 2597, 1065, 4014, 501, 541, 4607, 228, 4757, 2748, 4551, 1, 4365, 1963, 4118, 3347, 628, 3016, 2774, 611, 4324, 4521, 4526, 508, 3561, 2006, 2283, 4525, 2605, 3306, 2356, 2153, 1235, 2861, 551, 1485, 1510, 4545, 2539, 1808, 2351, 4196, 1328, 26, 4275, 69, 4968, 4127, 3033, 3789, 3626, 2759, 724, 4216, 2718, 1092, 2545, 1676, 4152, 2492, 1495, 1063, 109, 1848, 2385, 1073, 4536, 3780, 3108, 4086, 1865, 4126, 3173, 3861, 4911, 458, 4534, 774, 133, 4496, 4057, 3993, 985, 4140, 3836, 4486, 3902, 2073, 3959, 1662, 782, 4578, 1272, 187, 3629, 4389, 200, 242, 4979, 2194, 211, 2806, 966, 4362, 924, 3055, 80, 2099, 446, 66, 759, 3021, 3845, 2845, 2479, 3296, 4160, 1680, 189, 62, 1425, 307, 3184, 888, 286, 2434, 4883, 2008, 970, 2138, 2050, 3616, 769, 3327, 2736, 516, 2238, 3705, 4092, 43, 4412, 1094, 2080, 2203, 3463, 1619, 4628, 880, 3415, 3255, 3590, 176, 3169, 3631, 3337, 1508, 2553, 2100, 3412, 225, 1357, 2855, 2459, 1555, 4942, 3461, 2601, 4019, 397, 3743, 1812, 206, 4164, 3609, 3921, 1454, 4830, 659, 2098, 2638, 258, 3775, 3056, 3141, 2072, 1651, 2705, 86, 3953, 277, 2724, 2566, 3968, 1271, 1397, 279, 3488, 3862, 2303, 542, 2109, 1181, 1428, 1105, 3103, 4202, 1457, 3044, 718, 4620, 383, 1880, 2323, 3702, 494, 1566, 4798, 4528, 1658, 666, 2567, 4085, 2408, 2439, 2892, 1605, 1645, 1561, 2094, 842, 1626, 2866, 151, 439, 4932, 3815, 1743, 1233, 2941, 2723, 2834, 1497, 3404, 3490, 2429, 680, 3250, 426, 4281, 2223, 3676, 2643, 2974, 1415, 3204, 4253, 858, 1055, 4446, 2684, 3802, 2773, 3275, 2243, 915, 1136, 2476, 1050, 67, 2712, 643, 2254, 3040, 2380, 376, 3687, 2619, 2499, 1124, 317, 4855, 1324, 1918, 1251, 1346, 2063, 2620, 4423, 4260, 585, 4083, 3818, 2863, 4071, 4418, 1810, 3364, 4194, 3682, 2878, 2757, 4906, 2655, 229, 4471, 4197, 4499, 4360, 2979, 4706, 793, 2679, 3727, 4726, 2170, 4673, 2929, 4676, 3458, 1029, 1522, 692, 199, 1935, 185, 3091, 3456, 4896, 4480, 3181, 1264, 1302, 4170, 4901, 1199, 4516, 3167, 4593, 751, 2948, 2935, 3258, 4219, 2007, 3540, 3381, 1012, 3613, 4388, 3610, 1873, 4051, 1836, 4690, 4018, 2612, 1024, 2959, 3745, 599, 678, 2780, 3909, 4005, 2899, 1647, 1819, 3396, 704, 2084, 385, 416, 2387, 1335, 4138, 1725, 2533, 4364, 3171, 2828, 4383, 2161, 2004, 422, 91, 2270, 424, 4805, 4206, 2527, 403, 1147, 1465, 3783, 2895, 4336, 498, 2017, 2512, 4384, 3697, 3323, 1407, 4822, 2152, 2879, 414, 1103, 113, 2466, 360, 771, 356, 3969, 4000, 2618, 945, 4305, 4512, 369, 1909, 57, 1563, 3243, 3858, 1225, 3987, 2514, 3657, 814, 1838, 1406, 1190, 4910, 4692, 2819, 1097, 3718, 4988, 259, 2694, 2762, 1113, 139, 395, 3151, 2389, 4004, 1988, 4326, 3730, 587, 4394, 3903, 1167, 3958, 3268, 2641, 4038, 3717, 3195, 2856, 339, 4373, 3441, 4770, 3928, 4495, 3477, 671, 2269, 2262, 3202, 4766, 3480, 2852, 1421, 2592, 70, 3075, 2381, 776, 1818, 1275, 3673, 4440, 1628, 1652, 4403, 3114, 2552, 3699, 2265, 4330, 146, 2055, 1796, 191, 2952, 591, 2464, 4532, 1583, 1171, 3448, 3827, 4963, 1978, 2406, 822, 1432, 3967, 1926, 1790, 2515, 4002, 4035, 2606, 3672, 545, 2807, 327, 1270, 3026, 2744, 2955, 1987, 3724, 2266, 3303, 2312, 1538, 4556, 3764, 4223, 4969, 1601, 1931, 3800, 2615, 4709, 2457, 2494, 1228, 4564, 2105, 3136, 2936, 3585, 2435, 2034, 3579, 2135, 1418, 688, 4993, 2778, 4342, 370, 4740, 1639, 2590, 2556, 1675, 2367, 4792, 4283, 641, 2720, 3847, 996, 161, 4827, 3696, 884, 122, 4743, 958, 1193, 3850, 3131, 1581, 21, 549, 1464, 3014, 4544, 2934, 4184, 935, 803, 1642, 2976, 1014, 3964, 912, 444, 2635, 2733, 712, 1949, 4104, 4320, 515, 4941, 4097, 2329, 1296, 3778, 1220, 1980, 1470, 4243, 1679, 3719, 4627, 2727, 681, 3706, 4435, 1902, 4419, 2572, 4095, 219, 3819, 3266, 910, 3460, 557, 3805, 1801, 2166, 4704, 3506, 1711, 4871, 763, 1369, 3095, 4985, 3213, 421, 336, 2075, 267, 2489, 1131, 4234, 3977, 3565, 535, 101, 2009, 4227, 4404, 2764, 4654, 3264, 4617, 3018, 15, 1411, 4246, 71, 3430, 569, 1736, 601, 4739, 218, 4034, 1210, 2678, 4417, 2191, 4481, 2196, 3670, 907, 2714, 3279, 2300, 1426, 2355, 1027, 3216, 1122, 533, 3147, 3251, 4815, 1900, 2020, 764, 2361, 1772, 2171, 2932, 1378, 2242, 96, 1365, 3283, 1707, 662, 412, 2902, 4379, 3226, 4715, 4634, 5000, 3600, 4319, 1132, 2907, 2360, 2944, 4734, 2289, 1602, 4633, 2302, 1657, 4648, 903, 2206, 4137, 4908, 3878, 2347, 3024, 988, 4172, 396, 3684, 1823, 638, 2522, 288, 4880, 2187, 2526, 249, 2743, 3351, 1851, 19, 3881, 1844, 2607, 507, 2140, 3302, 4759, 4315, 1224, 3041, 2076, 4352, 1882, 997, 1589, 173, 1578, 4735, 787, 4199, 1370, 1162, 1325, 1917, 208, 2101, 1305, 4061, 1678, 2480, 3601, 2091, 1383, 3007, 2661, 1100, 4016, 167, 2897, 3405, 3511, 4861, 2994, 1990, 1690, 4866, 1164, 3603, 1687, 3082, 2296, 3027, 2505, 860, 440, 4531, 2603, 4895, 2392, 4970, 1232, 1350, 4823, 3219, 2424, 378, 2781, 2123, 4236, 2241, 1168, 409, 3767, 3175, 3841, 2124, 1957, 4340, 2267, 3025, 1624, 4848, 1312, 3539, 3737, 1646, 3542, 1155, 663, 3821, 1221, 3081, 2322, 4147, 1839, 2193, 4898, 1031, 1327, 3168, 389, 2448, 4779, 1705, 2554, 1927, 2916, 2784, 1434, 714, 1320, 448, 2407, 1775, 953, 971, 4458, 3656, 1348, 1654, 5003, 3679, 2364, 3808, 1184, 4902, 2654, 4763, 3197, 613, 1189, 3523, 2362, 3580, 4960, 374, 707, 1689, 3454, 391, 2139, 4098, 2445, 2201, 4245, 1638, 3010, 4507, 2288, 4105, 490, 2216, 1393, 4159, 3830, 2867, 4888, 2293, 1441, 794, 2975, 2394, 2542, 3842, 4662, 210, 2334, 3758, 479, 768, 305, 562, 3901, 2148, 1950, 4110, 271, 1669, 673, 4961, 410, 3281, 845, 4144, 1763, 90, 34, 1414, 4090, 2002, 4750, 329, 885, 4169, 44, 851, 2920, 1363, 3394, 3385, 3019, 153, 93, 253, 4479, 1920, 2155, 2820, 3263, 3588, 3619, 1500, 631, 2411, 3820, 492, 2159, 4905, 3170, 1401, 3011, 4641, 1695, 3516, 1755, 836, 4860, 3531, 2188, 741, 3797, 2782, 2940, 3703, 4020, 2825, 5, 1579, 986, 63, 1916, 375, 4788, 3301, 27, 4893, 3735, 1019, 2410, 4914, 4049, 2889, 2559, 1630, 909, 470, 4561, 2923, 4128, 1761, 1340, 3196, 1745, 2969, 2698, 3978, 3015, 4444, 364, 1600, 2318, 1196, 1698, 3346, 2067, 261, 4677, 4387, 13, 2497, 1077, 546, 2586, 4808, 2229, 2428, 3762, 3529, 3397, 594, 2372, 2701, 2918, 222, 1552, 2264, 3589, 292, 1244, 1971, 2036, 2901, 2699, 1577, 1861, 2716, 4660, 4825, 4916, 1455, 358, 2616, 4441, 3311, 4285, 4212, 2016, 4929, 3939, 54, 92, 717, 536, 4550, 916, 1237, 230, 3179, 256, 4157, 3140, 4631, 3439, 2648, 3807, 4013, 99, 807, 2033, 401, 737, 4292, 4647, 2664, 4992, 4800, 3865, 3915, 2915, 2827, 4826, 832, 2540, 1299, 1776, 4025, 3191, 4924, 1554, 2629, 3417, 1586, 2517, 4012, 3440, 1330, 1749, 1952, 3272, 2343, 3725, 2315, 1306, 2843, 1719, 2069, 1741, 2328, 3060, 2280, 3940, 3957, 2962, 2386, 3424, 4782, 1338, 951, 4294, 2310, 2174, 3843, 497, 1811, 3387, 3654, 818, 4819, 2858, 175, 1481, 2783, 2667, 1239, 4573, 3879, 3407, 4351, 4787, 518, 2207, 76, 4703, 4054, 3955, 123, 658, 2319, 367, 3017, 1541, 2993, 2204, 1958, 1925, 1351, 4056, 2594, 1729, 4015, 797, 141, 2763, 4155, 394, 274, 4999, 1408, 3518, 2634, 4094, 3689, 204, 445, 4445, 2498, 4484, 646, 2832, 694, 1713, 4592, 4935, 1174, 4009, 3137, 3039, 1187, 2463, 132, 4068, 1759, 3134, 3348, 1116, 3038, 4558, 3123, 177, 4268, 3000, 3738, 2276, 2331, 2850, 877, 1308, 1140, 451, 368, 2246, 335, 4658, 1780, 3766, 3198, 4699, 3782, 3442, 2662, 1708, 3886, 2872, 3160, 3647, 1663, 802, 3392, 2579, 3643, 648, 2129, 4084, 4568, 4842, 100, 380, 3548, 3694, 4682, 1760, 4344, 3013, 1253, 1151, 2371, 1146, 2801, 2257, 2375, 1901, 42, 2120, 2472, 784, 748, 4510, 20, 593, 1180, 1387, 1488, 2668, 2195, 2145, 4301, 4907, 3105, 4478, 3116, 1469, 895, 3009, 2578, 1490, 640, 2384, 1968, 430, 3833, 1681, 2739, 1142, 1095, 1482, 3576, 738, 1204, 2528, 2359, 1282, 297, 4541, 2292, 1242, 4067, 156, 4927, 890, 1099, 2930, 3059, 160, 2102, 2141, 3586, 2365, 2844, 2599, 3280, 1721, 281, 4032, 251, 2137, 1447, 3693, 4107, 873, 4141, 2325, 4237, 3431, 882, 2234, 4778, 2588, 3384, 1008, 3513, 325, 1039, 4877, 2730, 1238, 1544, 3276, 3236, 1859, 3760, 1714, 1537, 2317, 3321, 2118, 290, 1938, 3358, 1298, 1117, 720, 2252, 4080, 2253, 108, 1786, 1317, 1970, 2604, 4745, 4230, 4785, 2833, 2021, 3984, 3526, 2487, 2853, 3453, 3057, 3150, 1533, 4001, 1611, 4508, 1703, 3034, 59, 3406, 2015, 1487, 77, 4122, 2649, 4064, 1891, 3305, 1169, 81, 2766, 1523, 140, 2795, 4584, 3876, 3188, 4088, 752, 1143, 1534, 2719, 4406, 1178, 4331, 3238, 1930, 1475, 3096, 78, 653, 4917, 1758, 3510, 2132, 575, 3220, 2790, 1709, 4972, 3290, 1218, 2373, 2815, 3371, 639, 431, 4282, 1006, 4449, 4313, 4185, 1825, 721, 4044, 955, 3023, 944, 400, 3379, 3747, 2957, 766, 705, 3357, 3269, 3649, 1153, 2737, 976, 3353, 1417, 2209, 633, 1843, 621, 459, 1216, 163, 3135, 2382, 3877, 4231, 2005, 1036, 4802, 3952, 3704, 1437, 2873, 3855, 595, 1574, 1459, 1826, 4876, 2271, 347, 3352, 829, 2197, 1503, 4918, 3372, 4991, 4059, 2630, 1793, 1326, 913, 4833, 3362, 3063, 2656, 1069, 23, 831, 322, 4847, 4265, 708, 465, 1096, 2742, 1241, 1101, 328, 190, 864, 3932, 524, 4434, 2663, 1332, 1941, 2531, 3900, 4718, 2228, 925, 821, 4405, 4736, 4089, 1653, 3627, 300, 2440, 2483, 4945, 5006, 4612, 3904, 2110, 1717, 4372, 1842, 4028, 1885, 1114, 1747, 3148, 1227, 871, 540, 4102, 1815, 2914, 4549, 165, 2614, 2665, 50, 3930, 3986, 2891, 3100, 2632, 318, 1720, 519, 1360, 2298, 306, 3553, 3887, 592, 4091, 2054, 781, 1603, 3094, 527, 289, 2695, 1090, 311, 4256, 669, 270, 4248, 819, 2503, 1119, 4923, 3073, 3138, 3228, 1781, 4058, 2734, 3754, 800, 1037, 2256, 2423, 2165, 2286, 723, 2917, 4288, 1614, 1877, 1126, 1203, 2564, 4702, 3884, 2481, 1518, 3985, 4832, 3052, 3866, 846, 1985, 1513, 4590, 2534, 1683, 2511, 1291, 1483, 3898, 3650, 2646, 3799, 1261, 2346, 388, 783, 4710, 2227, 4729, 3001, 3723, 1710, 3047, 473, 3318, 4011, 4042, 4171, 886, 3899, 3545, 3058, 1875, 733, 3860, 1458, 4422, 4575, 1197, 4986, 2946, 4691, 2327, 3313, 2163, 1947, 597, 2633, 4623, 1473, 1539, 4407, 1644, 4249, 2104, 758, 1026, 1549, 4455, 3314, 3671, 4796, 2591, 381, 181, 1692, 245, 1059, 2301, 379, 4109, 74, 772, 1728, 4854, 3098, 1855, 689, 2683, 291, 2070, 4868, 2388, 3161, 2675, 600, 577, 1354, 3070, 957, 4003, 765, 1832, 2121, 355, 3709, 3459, 4450, 749, 4934, 3339, 1660, 899, 2811, 3389, 3061, 4583, 3873, 1794, 158, 4343, 2888, 1691, 41, 652, 632, 922, 2330, 1064, 3781, 4363, 3, 504, 4546, 926, 974, 2624, 579, 4552, 4804, 1213, 644, 4680, 357, 4452, 2249, 3770, 3087, 683, 2904, 3503, 2680, 3954, 2306, 709, 3078, 1289, 3370, 3377, 3667, 3194, 4989, 4131, 1254, 4772, 2432, 1255, 979, 4460, 1730, 3478, 2659, 616, 534, 1047, 1531, 4731, 2708, 4698, 4971, 4254, 2353, 1362, 1277, 407, 543, 1704, 949, 2804, 4965, 2627, 2082, 4116, 83, 2953, 195, 2142, 2285, 404, 820, 2419, 4183, 495, 1696, 2898, 1969, 4447, 3225, 1045, 1845, 4956, 405, 3828, 2610, 4040, 4646, 72, 2771, 1380, 4872, 3118, 3571, 2746, 4006, 4031, 1995, 3298, 1585, 321, 4211, 2465, 4179, 2032, 566, 3124, 2454, 4856, 4103, 4154, 1268, 2183, 166, 3190, 701, 1607, 4429, 4762, 1104, 4386, 2482, 2316, 3508, 2074, 1737, 1866, 3174, 1922, 3101, 2051, 3557, 241, 3207, 676, 4295, 3614, 363, 506, 48, 4442, 1568, 3361, 983, 2324, 3669, 3235, 4060, 4397, 7, 2536, 1034, 571, 150, 2571, 4101, 5010, 2502, 3660, 450, 734, 3053, 4724, 2938, 2313, 1575, 3402, 2529, 948, 4437, 2906, 5008, 2052, 3645, 276, 3514, 345, 4670, 4482, 4563, 4975, 1183, 3382, 4524, 4424, 3624, 3599, 798, 3646, 4416, 3685, 2096, 2376, 3509, 5002, 1504, 2321, 68, 147, 2044, 449, 710, 3465, 4306, 2981, 521, 3248, 3632, 1767, 725, 4764, 620, 4129, 4938, 4139, 4843, 1863, 3979, 2060, 3242, 634, 2500, 1543, 4377, 2438, 2214, 2085, 3551, 4594, 4385, 1144, 1507, 349, 777, 4598, 4354, 2383, 887, 2821, 3633, 4562, 4733, 2149, 4714, 2379, 243, 2950, 1349, 40, 2458, 1398, 4463, 3294, 2548, 240, 2311, 2700, 457, 560, 3962, 1525, 1477, 2340, 891, 2491, 761, 4235, 1774, 4571, 4835, 4321, 1979, 2876, 4966, 4485, 868, 2233, 2786, 816, 2083, 2909, 460, 1087, 1924, 4426, 4151, 2530, 1070, 4663, 3383, 905, 4111, 135, 3066, 1110, 2822, 4569, 284, 607, 786, 3158, 4795, 4191, 2682, 115, 143, 4954, 4581, 3562, 936, 1804, 2106, 4967, 293, 4817, 3563, 1686, 4774, 967, 1323, 2896, 2291, 3125, 2057, 1802, 3598, 1666, 4312, 4765, 1584, 3295, 1333, 919, 3287, 805, 4036, 1895, 8, 4625, 466, 1556, 584, 184, 4244, 3320, 1724, 1160, 2900, 4349, 4175, 4355, 408, 2966, 2412, 3569, 4401, 2128, 1604, 1468, 2688, 715, 2666, 3960, 480, 2013, 1192, 4604, 3004, 699, 3005, 1975, 1049, 1303, 3172, 4187, 1527, 3467, 4652, 2809, 4474, 3475, 128, 212, 4958, 1864, 3776, 3450, 1182, 4250, 1912, 1564, 3784, 3934, 3164, 441, 3077, 2180, 2593, 3495, 5001, 3713, 2761, 2715, 4204, 3142, 1266, 693, 3664, 779, 4933, 295, 3088, 4069, 3755, 861, 4668, 3256, 1976, 1545, 2467, 1723, 2430, 4679, 3961, 1856, 1847, 791, 1234, 88, 252, 4771, 73, 3310, 4777, 4987, 2760, 239, 340, 812, 1783, 1309, 874, 1107, 3208, 1386, 603, 3002, 4238, 2964, 4630, 1889, 4348, 1876, 1636, 237, 2040, 1800, 3222, 2947, 1656, 3297, 1081, 1878, 1824, 4520, 4251, 4793, 4266, 1175, 4760, 2652, 1165, 2344, 1706, 3062, 3455, 3444, 4498, 3085, 4501, 262, 4700, 4258, 4367, 4756, 5011, 3950, 4976, 3265, 499, 1091, 1246, 2671, 4602, 3966, 3663, 2237, 2575, 4791, 3636, 2756, 1613, 348, 3642, 897, 3315, 1868, 3433, 4657, 4079, 537, 1283, 711, 3728, 2777, 3234, 1837, 1429, 1474, 4850, 2988, 4685, 3906, 992, 2949, 3395, 2928, 1685, 942, 914, 928, 3769, 373, 2931, 3340, 3230, 2077, 1983, 675, 4747, 2215, 3677, 4267, 2284, 4870, 2922, 2202, 452, 3252, 3917, 121, 626, 3947, 3990, 4353, 3072, 4277, 1461, 2789, 1048, 3121, 4421, 2570, 1011, 1599, 4469, 2729, 456, 2660, 1020, 437, 3994, 3665, 730, 894, 285, 1929, 283, 332, 4643, 4252, 4599, 929, 3734, 2726, 682, 2692, 1551, 2212, 3908, 4029, 2336, 1489, 1598, 2125, 4022, 171, 3133, 2791, 3447, 1501, 2704, 893, 3591, 1785, 3492, 808, 3812, 3449, 1412, 3366, 3411, 2706, 1000, 2573, 3292, 4636, 52, 4173, 2860, 3941, 1491, 4619, 4158, 4849, 3367, 1632, 4853, 1359, 2157, 811, 2785, 3331, 4655, 4708, 4725, 1054, 107, 3791, 3247, 3837, 3378, 2854, 3374, 10, 1173, 3907, 3773, 4284, 2893, 1195, 2401, 3300, 1784, 2925, 3471, 3239, 744, 3655, 1914, 3149, 2903, 3753, 3129, 3076, 1750, 2509, 1460, 517, 4125, 1385, 1134, 2273, 3067, 3163, 3485, 1416, 331, 2741, 4637, 4398, 3578, 1343, 1617, 1870, 1650, 4874, 1998, 1830, 1582, 320, 636, 338, 1536, 857, 3355, 2977, 2541, 4296, 2792, 4392, 1240, 2537, 3521, 436, 1215, 852, 2549, 2421, 3814, 481, 3874, 4614, 2456, 2281, 1834, 4644, 3541, 2800, 1176, 3122, 3089, 4810, 2697, 1057, 2913, 2885, 1222, 2226, 1125, 1071, 1478, 1138, 3929, 674, 824, 987, 4162, 892, 138, 3711, 463, 4221, 4201, 1936, 4232, 4467, 4613, 2501, 4851, 2024, 124, 3177, 4200, 4043, 3577, 1684, 2111, 1494, 3835, 1813, 2345, 4050, 16, 1777, 2370, 2471, 878, 762, 3186, 2555, 104, 371, 4370, 582, 2030, 3639, 1674, 1509, 2444, 1310, 2710, 1009, 4899, 647, 1002, 1179, 1262, 911, 1068, 432, 157, 2144, 196, 3472, 4026, 3595, 4229, 3573, 1209, 2693, 3786, 3839, 4959, 308, 129, 3080, 53, 11, 1088, 393, 3666, 2176, 3741, 4980, 862, 3888, 476, 3811, 649, 2546, 3751, 1944, 1422, 4023, 3102, 767, 3574, 806, 3003, 3933, 3344, 1905, 1388, 4781, 3731, 471, 2874, 3801, 4723, 2911, 442, 3437, 561, 4209, 1123, 3499, 1093, 828, 4869, 399, 1376, 2320, 4837, 2989, 3244, 1558, 3675, 991, 1964, 1974, 3885, 3284, 1043, 2279, 4108, 4218, 1384, 1797, 4669, 4325, 4506, 3648, 2168, 224, 1336, 461, 3974, 4608, 4399, 2239, 4659, 552, 207, 3618, 2972, 3525, 1273, 4409, 4814, 3261, 555, 1718, 1052, 2765, 1001, 1805, 4264, 1419, 136, 3527, 1840, 4603, 4134, 4117, 2535, 2179, 3519, 1831, 4925, 2037, 3237, 1506, 3749, 3083, 1939, 2960, 4302, 3030, 309, 1712, 4303, 1515, 3927, 1597, 502, 4897, 3429, 2669, 4515, 1229, 4374, 3093, 2849, 3982, 2092, 1208, 2095, 1083, 1109, 4875, 1919, 2453, 1606, 2808, 1744, 3349, 522, 3840, 2696, 2167, 1313, 2028, 4226, 2521, 906, 2968, 1051, 1757, 1535, 4597, 1194, 3375, 3343, 1058, 654, 1908, 3798, 2681, 469, 82, 2775, 4341, 51, 2403, 809, 2486, 1493, 4509, 1807, 1375, 3883, 3462, 3674, 902, 2462, 1404, 3623, 4464, 25, 2299, 2405, 3491, 855, 3432, 531, 2366, 4587, 3212, 4145, 2211, 1484, 4671, 4572, 4066, 883, 4689, 174, 3146, 294, 4801, 3324, 2721, 1405, 1377, 2623, 4039, 4228, 2750, 3507, 1281, 4332, 1702, 1754, 840, 2927, 3504, 2221, 2650, 550, 3558, 4519, 3512, 263, 162, 268, 4761, 4123, 3829, 2427, 1290, 3611, 1992, 3803, 221, 1280, 835, 4903, 1962, 4192, 1580, 4582, 1640, 4784, 464, 2933, 3712, 2973, 2404, 1634, 817, 1060, 2672, 839, 4615, 2813, 4217, 2230, 4865, 1076, 843, 1635, 3768, 4335, 2244, 265, 1498, 477, 4186, 3241, 716, 2341, 943, 1170, 3742, 2576, 4323, 1230, 1716, 697, 192, 937, 3826, 2217, 2709, 4930, 4213, 4768, 3254, 4491, 226, 2871, 2, 3592, 1358, 1259, 4017, 4488, 4951, 3889, 2516, 2884, 2864, 4072, 1345, 4518, 1021, 1130, 4645, 2755, 2569, 247, 848, 745, 2797, 1972, 4262, 1973, 4334, 359, 102, 1219, 2287, 4995, 3549, 3517, 36, 4456, 2156, 4207, 227, 333, 2178, 4132, 3681, 3487, 1966, 3537, 667, 956, 125, 610, 4560, 2550, 4794, 2886, 2290, 3153, 3042, 2887, 2068, 1872, 1410, 896, 1243, 2250, 3398, 2348, 4586, 574, 1835, 2670, 2543, 1701, 418, 3716, 2224, 4500, 2926, 1307, 354, 180, 485, 4978, 3583, 2475, 529, 570, 2703, 4753, 1078, 1874, 4195, 3401, 472, 126, 2713, 3494, 1018, 2477, 1413, 4809, 1846, 1512, 110, 4150, 2225, 4864, 2798, 2173, 4752, 3905, 273, 4473, 1616, 1396, 1795, 1727, 4863, 64, 1779, 2690, 4606, 4430, 1559, 1177, 3804, 4093, 1911, 657, 79, 182, 3550, 3825, 3092, 216, 1594, 2717, 3356, 3787, 1207, 4333, 1368, 1726, 4402, 1366, 3144, 1148, 3522, 2012, 2824, 3436, 4514, 1932, 4148, 1502, 2788, 3913, 2468, 235, 3867, 4928, 4632, 898, 350, 4271, 2189, 4357, 1996, 547, 3418, 4527, 1295, 4775, 4711, 1593, 1816, 1576, 4693, 1367, 3515, 4361, 4270, 1361, 4167, 656, 3530, 2520, 3500, 2865, 3139, 474, 4454, 2794, 194, 280, 788, 1042, 1621, 4904, 4027, 1569, 1954, 2560, 1817, 4077, 3240, 428, 2731, 496, 2417, 4356, 2589, 1374, 1436, 330, 2796, 3816, 3249, 1392, 3035, 1278, 4347, 1200, 4073, 4651, 3532, 1547, 1301, 4475, 2869, 4114, 921, 4408, 3992, 346, 2488, 1667, 3110, 3593, 525, 1620, 1293, 2031, 1007, 4443, 491, 2823, 2369, 342, 1486, 475, 4947, 3653, 4308, 3501, 4686, 2350, 3498, 728, 4381, 1913, 2506, 3566, 602, 695, 3154, 4952, 901, 3187, 2134, 972, 244, 366, 4436, 220, 1946, 3698, 630, 4149, 3662, 377, 2673, 856, 3020, 353, 3661, 1997, 3074, 4818, 4177, 3946, 3895, 3231, 4099, 904, 2689, 2056, 2097, 528, 686, 4829, 2532, 4547, 3233, 4433, 2470, 3484, 468, 2848, 1956, 28, 3794, 3119, 706, 454, 2838, 118, 2651, 1256, 1403, 963, 3555, 4585, 4656, 3008, 4427, 4786, 3209, 3045, 1467, 4595, 4996, 1664, 3597, 2711, 236, 4943, 483, 4728, 2868, 2081, 917, 980, 2908, 1121, 1185, 810, 2014, 825, 1618, 30, 4224, 4859, 4577, 1355, 1887, 3086, 679, 4214, 1573, 2877, 2053, 4688, 1075, 4672, 2205, 1444, 4611, 4470, 2937, 4198, 155, 837, 2870, 314, 85, 304, 1673, 4885, 1732, 1682, 2437, 2114, 567, 3422, 1250, 3923, 234, 2277, 564, 2143, 2992, 1347, 179, 1890, 565, 4841, 3206, 3691, 1245, 3246, 84, 3688, 563, 859, 35, 3844, 4339, 2565, 2339, 392, 869, 4076, 2130, 4997, 4748, 2818, 2181, 3856, 4973, 4624, 3289, 4742, 1079, 3006, 849, 1499, 4378, 1852, 2924, 2826, 3635, 2000, 3567, 1770, 4953, 3350, 2558, 2122, 1907, 995, 2577, 3795, 4589, 4483, 4273, 664, 2078, 3130, 1248, 920, 3328, 2942, 4299, 3570, 3159, 3419, 4063, 1040, 3416, 3869, 700, 1700, 2474, 4530, 1394, 38, 1943, 486, 1860, 1067, 3938, 3774, 1942, 1514, 2126, 3097, 3386, 1247, 1353, 1161, 2835, 2642, 4886, 847, 2305, 4712, 95, 2562, 4462, 3822, 4857, 4889, 505, 4639, 3863, 1756, 193, 2857, 1476, 4559, 677, 2089, 553, 154, 1722, 4638, 3951, 2400, 4113, 4307, 3572, 3466, 1017, 740, 1314, 3788, 530, 4328, 1520, 2393, 2119, 1731, 4272, 1787, 1292, 2836, 47, 4522, 4882, 4836, 2747, 853, 2912, 3277, 4900, 4428, 5007, 420, 965, 962, 3686, 3538, 2647, 1276, 3678, 1252, 4816, 4314, 2147, 4182, 3452, 2622, 209, 4675, 22, 2182, 1112, 4820, 4287, 2674, 3183, 4844, 1315, 4944, 3680, 3554, 2943, 1409, 1625, 4502, 4106, 3793, 32, 433, 1567, 2484, 2799, 4081, 4318, 2776, 4300, 2039, 4121, 4566, 232, 500, 2677, 4983, 323, 4124, 2518, 4683, 4720, 727, 3720, 3390, 2065, 1648, 257, 650, 39, 1792, 2812, 1881, 4696, 1528, 148, 3871, 1828, 1994, 3625, 1894, 3193, 589, 2378, 3771, 3317, 3064, 4957, 702, 1163, 4824, 2022, 696, 3668, 4345, 4208, 732, 429, 3700, 4701, 2745, 1111, 4425, 2525, 2047, 3695, 4681, 792, 2107, 3048, 1833, 4420, 2352, 97, 4327, 2185, 756, 1172, 4705, 4263, 3890, 2136, 282, 402, 2399, 4448, 520, 4513, 4919, 2958, 4537, 2626, 1337, 4813, 4649, 1883, 89, 3864, 484, 3099, 1799, 790, 2600, 1391, 438, 4304, 3893, 3445, 590, 3701, 1379, 1688, 4887, 1316, 4755, 3606, 3944, 1782, 3544, 1530, 604, 1389, 2905, 1025, 3910, 3257, 1004, 1540, 250, 1655, 1934, 665, 3341, 1214, 1297, 3926, 3393, 3322, 2919, 4543, 1044, 4431, 1400, 747, 2587, 168, 651, 2335, 4413, 2644, 2391, 4047, 3547, 1451, 4274, 4024, 4570, 316, 1118, 3214, 6, 3286, 1671, 3189, 2010, 619, 4828, 2495, 3464, 310, 4806, 4665, 4529, 1424, 866, 105, 1955, 801, 4834, 1149, 1492, 3560, 111, 713, 0, 2970, 2409, 2368, 387, 624, 3882, 2058, 4322, 2859, 2282, 2691, 1285, 3489, 365, 2255, 2029, 4453, 3659, 1198, 4840, 75, 1951, 178, 990, 3651, 2275, 2685, 1789, 3365, 4329, 3637, 4744, 12, 3790, 3224, 3333, 981, 462, 775, 2740, 2133, 554, 4878, 3043, 2026, 2508, 1449, 4548, 3739, 3534, 186, 4411, 188, 2186, 1841, 4021, 4082, 1217, 4055, 2342, 4337, 2198, 2169, 2954, 760, 3278, 4579, 3763, 3205, 572, 1879, 1629, 3796, 301, 1062, 1440, 2274, 423, 1356, 3036, 1427, 4912, 1257, 4414, 4695, 1849, 3912, 65, 3546, 1940, 4667, 1442, 1223, 1322, 2752, 1279, 4310, 4156, 4062, 3413, 4368, 1699, 4790, 2984, 4719, 2772, 931, 3620, 3989, 3084, 98, 4797, 4165, 755, 3120, 3012, 1923, 4940, 2295, 1615, 1013, 2220, 3469, 4621, 1339, 3832, 4078, 3104, 4493, 145, 2023, 3942, 2735, 2563, 3071, 4982, 4539, 4574, 2657, 1423, 2961, 1814, 1452, 2158, 488, 4269, 4358, 169, 3891, 3481, 4769, 2881, 655, 2490, 2390, 576, 248, 4396, 1084, 4280, 3995, 1553, 3973, 2363, 94, 1382, 900, 930, 975, 246, 3608, 2996, 4727, 1867, 1205, 2245, 3399, 960, 612, 2240, 2377, 4948, 973, 1592, 3948, 3621, 2218, 618, 4949, 968, 61, 2447, 4492, 2910, 1752, 2779, 2309, 4289, 1472, 4188, 512, 2971, 1829, 4773, 3293, 3143, 952, 3345, 510, 3109, 2686, 1231, 3528, 2304, 3376, 627, 1953, 685, 2358, 544, 4255, 3868, 2236, 4070, 1910, 3641, 2449, 4317, 4807, 4041, 1023, 3535, 3180, 796, 1463, 2079, 2222, 1928, 513, 4096, 3630, 2831, 362, 4730, 4610, 3476, 2061, 1212, 2088, 1158, 1517, 1053, 939, 2986, 172, 3851, 1466, 1128, 2259, 959, 622, 134, 778, 4241, 384, 4780, 3925, 1446, 586, 2707, 164, 1152, 3423, 3157, 1738, 315, 425, 989, 5009, 3983, 1672, 4045, 3330, 2830, 3368, 642, 3145, 3963, 1139, 1820, 2108, 556, 3434, 2557, 3628, 2210, 3892, 4915, 183, 4142, 2574, 3307, 1993, 117, 726, 3427, 3493, 2568, 2875, 4913, 3336, 623, 413, 4279, 1311, 605, 1480, 1035, 4220, 1453, 49, 4276, 4946, 1286, 1159, 103, 1748, 698, 4517, 2200, 3777, 2991, 3474, 2882, 3359, 2754, 596, 2338, 1108, 3748, 4375, 4046, 1612, 3607, 3831, 3524, 1267, 4290, 2151, 1157, 1269, 684, 2064, 4605, 4803, 4497, 3468, 1191, 1735, 303, 2354, 2519, 112, 4738, 264, 2769, 1746, 2337, 3733, 3584, 2639, 1637, 789, 4684, 1038, 4678, 3201, 1433, 2090, 4783, 645, 1082, 4737, 2396, 4233, 4511, 2251, 4052, 3203, 337, 1693, 4010, 2469, 1827, 4894, 287, 3981, 2460, 4557, 1085, 372, 4074, 1989, 1821, 4178, 33, 2580, 3857, 3221, 1443, 1003, 2880, 3817, 205, 1869, 4746, 3199, 1016, 170, 4955, 3965, 4505, 4664, 4540, 1751, 351, 3683, 3502, 1661, 1258, 203, 1188, 3215, 4135, 4538, 2608, 3332, 1557, 3496, 144, 3920, 1456, 3949, 3834, 3486, 4007, 4846, 3409, 2837, 2261, 1066, 941, 255, 617, 2208, 815, 2116, 3185, 830, 568, 785, 4297, 4176, 548, 4588, 3714, 3918, 1627, 3975, 1608, 215, 833, 324, 746, 1609, 964, 2621, 1778, 4259, 217, 4754, 1129, 4298, 272, 703, 608, 3156, 838, 3997, 46, 3176, 415, 489, 3935, 3582, 2473, 2637, 3991, 735, 753, 4576, 4119, 4926, 1857, 3497, 3919, 538, 3335, 55, 2585, 2199, 2154, 4477, 2046, 1773, 4037, 1329, 1694, 2732, 3746, 4616, 1933, 269, 447, 4439, 4626, 2414, 1633, 4451, 2455, 629, 434, 1897, 3708, 598, 3373, 2019, 4542, 343, 3482, 1742, 739, 159, 2722, 2596, 1115, 4205, 2127, 827, 3112, 2441, 4438, 3270, 4393, 3309, 3054, 326, 3644, 1991, 2164, 978, 998, 2422, 933, 1086, 1519, 2839, 3849, 2172, 319, 865, 3391, 1145, 14, 37, 615, 1768, 2653, 1352, 29, 583, 4163, 3132, 3937, 4490, 493, 2738, 3069, 3223, 214, 3911, 1631, 3596, 1516, 1341, 4257, 2841, 3217, 2846, 3736, 2442, 4380, 87, 2048, 3326, 3049, 4146, 266, 3117, 4879, 4240, 1431, 3421, 435, 1294, 799, 3211, 4920, 4839, 3824, 4130, 3976, 4503, 3380, 4998, 4741, 58, 2847, 1265, 3792, 4767, 202, 1959, 736, 3410, 198, 2043, 4721, 119, 1550, 4811, 1010, 867, 3329, 2613, 4950, 312, 3446, 1390, 478, 1284, 3707, 2814, 2967, 4909, 1715, 3388, 1986, 2987, 503, 2956, 993, 908, 4553, 3227, 1206, 1150, 2676, 2753, 2978, 1765, 1120, 334, 2213, 1005, 1659, 4391, 606, 1623, 2510, 4371, 1072, 690, 2093, 2609, 1127, 4661, 3267, 984, 3615, 2921, 4472, 4867, 2645, 1287, 2793, 2631, 1462, 2829, 1399, 482, 3106, 1450, 2263, 4640, 1156, 2418, 2628, 4799, 2349, 3428, 4112, 2011, 2617, 3200, 2357, 9, 1373, 223, 1154, 841, 558, 4048, 3945, 947, 581, 3744, 238, 152, 3710, 4033, 870, 131, 2425, 2018, 3438, 2702, 2816, 2551, 1371, 3594, 1562, 3652, 1288, 4239, 660, 344, 3605, 254, 2581, 2982, 1769, 2770, 2431, 3435, 1904, 4311, 4008, 3107, 2524, 4415, 1137, 4193, 813, 4554, 4622, 3897, 2415, 1524, 625, 3218, 4461, 3111, 2451, 4143, 1984, 3369, 1030, 999, 2894, 509, 1263, 467, 3079, 1133, 17, 1641, 278, 4981, 3260, 4316, 1590, 3274, 2374, 3587, 4881, 1249, 3155, 3050, 3192, 114, 1505, 4962, 1032, 2268, 2260, 4831, 2025, 3880, 2999, 2332, 3612, 4984, 3065, 1734, 299, 2840, 4346, 3722, 120, 2397, 927, 4261, 1479, 580, 3658, 3779, 487, 1331, 1733, 1967, 3575, 823, 3420, 2461, 4936, 2117, 3127, 3360, 1884, 4873, 4629, 4535, 3115, 1089, 3113, 3581, 2175, 514, 2538, 3304, 1211, 3285, 1381, 3029, 361, 3232, 2308, 3363, 4350, 950, 2951, 2995, 1961, 4749, 3299, 3046, 4776, 4653, 4053, 4291, 3457, 2452, 2231, 1571, 4465, 4222, 2547, 1596, 1402, 1080, 1739, 390, 2062, 3848, 3875, 614, 2980, 635, 2294, 296, 2687, 2810, 3229, 3750, 3785, 411, 588, 1202, 526, 1822, 1981, 3810, 743, 4751, 1529, 3037, 2041, 3988, 3732, 2258, 4697, 3316, 298, 3403, 4400, 994, 1532, 1098, 4994, 4376, 3473, 275, 4666, 1850, 2146, 2983, 2059, 3726, 2162, 4174, 2219, 2232, 1300, 213, 2640, 3319, 4166, 826, 4457, 3288, 977, 1372, 757, 2184, 1565, 2071, 4432, 1903, 1788, 3533, 1439, 2890, 1521, 2997, 2842, 341, 3031, 2035, 1056, 149, 872, 3028, 1888, 4523, 4609, 876, 4845, 875, 2420, 302, 419, 1587, 854, 4964, 3408, 2758, 2768, 2247, 4115, 4931, 2087, 4974, 3971, 137, 1937, 3273, 3972, 2426, 1899, 1921, 670, 2544, 4136, 3166, 3451, 4278, 4225, 4120, 3342, 1106, 532, 3253, 4939, 609, 3617, 4215, 2965, 3692, 3721, 130, 3622, 780, 1560, 3936, 4395, 2446, 4359, 3520, 1496, 1764, 2235, 2450, 2086, 1448, 1809, 1166, 3505, 1915, 142, 455, 427, 3022, 4190, 1677, 4410, 2504, 4180, 2038, 2985, 4990, 4884, 578, 3761, 1999, 4812, 1186, 4153, 4717, 4674, 3245, 1982, 1022, 1028, 938, 4937, 3943, 56, 18, 1670, 1542, 1668, 443, 2523, 850, 1102, 4858, 1141, 4065, 559, 879, 773, 3068, 2990, 1236, 3543, 1595, 4382, 3282, 3806, 2728, 3400, 4618, 3425, 3859, 4852, 2272, 934, 197, 889, 2190, 4567, 4716, 3916, 4489, 2998, 3259, 3128, 4369, 1588, 2583, 2658, 3896, 1806, 3757, 4133, 3564, 2395, 668, 2192, 2027, 4635, 1858, 4591, 1871, 3479, 2049, 2625, 3756, 961, 4600, 2582, 4642, 2103, 3752, 2402, 1697, 3443, 3210, 1344, 2150, 3178, 834, 4891, 754, 3998, 1260, 691, 2883, 1471, 60, 954, 1041, 3813, 1318, 1342, 3931, 661, 1438, 2045, 3568, 1886, 2443, 1896, 3853, 4687, 2248, 1033, 2307, 4286], "validation": [5084, 5025, 5308, 5147, 5270, 5118, 5148, 5074, 5423, 5062, 5434, 5471, 5433, 5404, 5228, 5445, 5259, 5483, 5287, 5411, 5455, 5131, 5371, 5132, 5362, 5124, 5280, 5460, 5134, 5364, 5198, 5288, 5078, 5410, 5171, 5392, 5391, 5210, 5012, 5418, 5360, 5018, 5363, 5204, 5109, 5294, 5379, 5067, 5063, 5296, 5206, 5456, 5393, 5336, 5330, 5083, 5087, 5317, 5310, 5125, 5311, 5380, 5450, 5243, 5113, 5174, 5482, 5424, 5079, 5242, 5053, 5323, 5493, 5050, 5282, 5347, 5453, 5357, 5194, 5015, 5030, 5447, 5399, 5179, 5320, 5299, 5272, 5388, 5454, 5197, 5153, 5436, 5251, 5041, 5291, 5145, 5281, 5225, 5437, 5503, 5056, 5451, 5496, 5177, 5195, 5095, 5405, 5504, 5091, 5365, 5235, 5129, 5252, 5238, 5135, 5395, 5013, 5487, 5014, 5267, 5229, 5340, 5218, 5345, 5425, 5159, 5511, 5254, 5492, 5387, 5313, 5102, 5500, 5480, 5138, 5265, 5042, 5100, 5431, 5474, 5396, 5032, 5226, 5187, 5398, 5182, 5488, 5414, 5064, 5196, 5151, 5205, 5117, 5506, 5017, 5241, 5301, 5439, 5049, 5255, 5494, 5339, 5029, 5354, 5248, 5101, 5223, 5268, 5346, 5060, 5312, 5071, 5048, 5402, 5428, 5232, 5140, 5256, 5373, 5036, 5502, 5261, 5136, 5200, 5163, 5283, 5221, 5026, 5358, 5240, 5419, 5459, 5505, 5143, 5295, 5089, 5098, 5039, 5211, 5250, 5021, 5277, 5075, 5208, 5245, 5068, 5293, 5279, 5435, 5352, 5217, 5440, 5490, 5372, 5430, 5189, 5165, 5367, 5127, 5426, 5144, 5394, 5111, 5133, 5335, 5497, 5072, 5096, 5315, 5057, 5338, 5326, 5105, 5300, 5164, 5290, 5485, 5356, 5214, 5154, 5383, 5157, 5119, 5192, 5413, 5149, 5027, 5407, 5040, 5481, 5318, 5476, 5120, 5043, 5092, 5045, 5224, 5188, 5190, 5465, 5470, 5314, 5202, 5086, 5495, 5473, 5037, 5401, 5325, 5170, 5306, 5422, 5106, 5258, 5126, 5201, 5038, 5191, 5385, 5080, 5178, 5094, 5239, 5304, 5249, 5366, 5070, 5257, 5382, 5475, 5319, 5510, 5381, 5309, 5443, 5462, 5219, 5230, 5343, 5237, 5507, 5307, 5429, 5266, 5337, 5484, 5444, 5350, 5023, 5262, 5104, 5331, 5378, 5285, 5209, 5384, 5160, 5275, 5274, 5022, 5233, 5146, 5110, 5108, 5297, 5477, 5203, 5061, 5019, 5479, 5212, 5359, 5328, 5351, 5081, 5139, 5486, 5302, 5244, 5028, 5051, 5508, 5034, 5260, 5478, 5464, 5491, 5066, 5355, 5123, 5342, 5175, 5361, 5142, 5183, 5088, 5273, 5376, 5441, 5390, 5467, 5334, 5403, 5374, 5348, 5457, 5047, 5386, 5466, 5329, 5284, 5499, 5316, 5077, 5489, 5082, 5158, 5130, 5321, 5193, 5463, 5128, 5498, 5046, 5099, 5220, 5333, 5097, 5181, 5298, 5031, 5344, 5176, 5156, 5024, 5059, 5458, 5303, 5501, 5247, 5397, 5093, 5292, 5116, 5452, 5035, 5186, 5468, 5185, 5162, 5085, 5415, 5377, 5421, 5016, 5400, 5432, 5150, 5172, 5442, 5236, 5446, 5033, 5417, 5169, 5253, 5406, 5263, 5173, 5076, 5180, 5461, 5278, 5137, 5114, 5207, 5103, 5112, 5020, 5199, 5107, 5327, 5448, 5246, 5052, 5438, 5213, 5375, 5069, 5305, 5509, 5058, 5276, 5222, 5090, 5121, 5044, 5073, 5286, 5115, 5055, 5409, 5412, 5227, 5408, 5166, 5332, 5231, 5168, 5215, 5416, 5427, 5370, 5122, 5155, 5353, 5054, 5141, 5322, 5152, 5234, 5184, 5161, 5389, 5420, 5065, 5269, 5271, 5167, 5324, 5469, 5449, 5264, 5341, 5216, 5369, 5472, 5368, 5289, 5349], "test": [5947, 5928, 5788, 5566, 5833, 5593, 5754, 5640, 5547, 5897, 5986, 5570, 5721, 5965, 5596, 5874, 5921, 5898, 5654, 5554, 5629, 5851, 5927, 5815, 5549, 5701, 5569, 5895, 5957, 5912, 5764, 5521, 6007, 5798, 5929, 5835, 5726, 5660, 5574, 5919, 5946, 5975, 5522, 5828, 5807, 5642, 5952, 5730, 5623, 5520, 5905, 5544, 5742, 5537, 5707, 5792, 5712, 5852, 5854, 5762, 5652, 5997, 5746, 5737, 5760, 5926, 5862, 5577, 5841, 5686, 5717, 5558, 5776, 5578, 5944, 5693, 5939, 6000, 5757, 5685, 5755, 5658, 5985, 5525, 5675, 5555, 5970, 5703, 5589, 5883, 5802, 5753, 5812, 5890, 5518, 5998, 6005, 5704, 5778, 5567, 5821, 5911, 5930, 5602, 5588, 5770, 6003, 5839, 5765, 5796, 5594, 5579, 5799, 5822, 5891, 5990, 5942, 5876, 5925, 5564, 5587, 5910, 5723, 5789, 5585, 5988, 5551, 5811, 5894, 5575, 5829, 5844, 5996, 5972, 5546, 5846, 5653, 5708, 5780, 5885, 5786, 5748, 5586, 5724, 5678, 5512, 5838, 5941, 5955, 5907, 5530, 5865, 5582, 5758, 5611, 5609, 5631, 5756, 5688, 5733, 5561, 5739, 6001, 5519, 5635, 5953, 5515, 5682, 5548, 5840, 5814, 5886, 5781, 5842, 5668, 5785, 5719, 5899, 5674, 5934, 5734, 5639, 5656, 5931, 5573, 5918, 5722, 5740, 5768, 5738, 5655, 5809, 5680, 5866, 5735, 5974, 5695, 5917, 5523, 5857, 5801, 5600, 5968, 5923, 5646, 5774, 5817, 5599, 5649, 5978, 5773, 5562, 5914, 5568, 5645, 5963, 5659, 5536, 5977, 5644, 5853, 5691, 5800, 5837, 6009, 5580, 5715, 5698, 5805, 5615, 5543, 5559, 5808, 5529, 5823, 5741, 5610, 5628, 5617, 5592, 5820, 5648, 5625, 5877, 5887, 5545, 5904, 5964, 5614, 5527, 5633, 5797, 5795, 5745, 5552, 5888, 5606, 5662, 5856, 5664, 5825, 5859, 6008, 5751, 5981, 5697, 5864, 5834, 5940, 5759, 5517, 5913, 5647, 5650, 5727, 5867, 5672, 5938, 5871, 5626, 5630, 5709, 5900, 5576, 5850, 5619, 5624, 5661, 5710, 5954, 5793, 5671, 5843, 5702, 5922, 5711, 5766, 5744, 5627, 5622, 5595, 5514, 5936, 5873, 5636, 5705, 5818, 5816, 5670, 5787, 5849, 5603, 5700, 5832, 6011, 5638, 5669, 5769, 5581, 5991, 5613, 5784, 5791, 5826, 5973, 5597, 5902, 5869, 5699, 5779, 5948, 5889, 5961, 5732, 5665, 5870, 5976, 5819, 5535, 5836, 5880, 5526, 5878, 5830, 5962, 5584, 5861, 5666, 5752, 5663, 5533, 5563, 5591, 5824, 5804, 5831, 5728, 5932, 5542, 5583, 5677, 5524, 5848, 5989, 5863, 5950, 5676, 5679, 5901, 5771, 5608, 5750, 5516, 5590, 5772, 5847, 5673, 5571, 5943, 5860, 5827, 5810, 5971, 5949, 5513, 5696, 5725, 6006, 5813, 5935, 5903, 5689, 5794, 5601, 5806, 5539, 5761, 5684, 5706, 5716, 5528, 5720, 5803, 5637, 5958, 5694, 5538, 5634, 5618, 5956, 5605, 5984, 5560, 5879, 5692, 5540, 5999, 5881, 5681, 5909, 5777, 5858, 5933, 5534, 5747, 5667, 5731, 5979, 5906, 5969, 5607, 5987, 6004, 5713, 5604, 6010, 5945, 5620, 5916, 5783, 5657, 5875, 5641, 5616, 5743, 5736, 5966, 5782, 5951, 5714, 5729, 5982, 5556, 5632, 5937, 5683, 5992, 5643, 5915, 5872, 5994, 5718, 5749, 5960, 5993, 5550, 5565, 5893, 5868, 5572, 5687, 5845, 5790, 5541, 5983, 5920, 5882, 5553, 5967, 5884, 5598, 5959, 5612, 5651, 5908, 5775, 5995, 5896, 6002, 5532, 5621, 5767, 5690, 5924, 5763, 5892, 5855, 5531, 5557, 5980]}, {"train": [287, 2265, 1561, 875, 4354, 1171, 2444, 233, 2752, 4063, 2253, 506, 4075, 2284, 1279, 588, 155, 1756, 1302, 2018, 4434, 1481, 4807, 933, 3303, 3601, 3177, 1993, 624, 2101, 2114, 2274, 1569, 3012, 1635, 2600, 4947, 4888, 2633, 3323, 1544, 2531, 2870, 3805, 4039, 1687, 1321, 1823, 3889, 1283, 2039, 294, 2432, 558, 3484, 3483, 3419, 4496, 2375, 1074, 2509, 329, 2238, 1342, 3765, 1876, 3309, 2000, 3142, 4903, 2680, 15, 1479, 4987, 4244, 400, 3148, 581, 3688, 4877, 1956, 1443, 3992, 4871, 4933, 2254, 248, 3337, 1547, 4189, 3202, 721, 1013, 3599, 3838, 3939, 649, 2729, 2993, 2373, 3497, 4762, 1188, 4236, 4064, 3709, 1453, 153, 1142, 638, 2924, 918, 1282, 215, 1147, 3869, 1254, 2006, 4197, 843, 4813, 4873, 3739, 2888, 2334, 4360, 4442, 2827, 4962, 290, 4022, 2669, 3585, 681, 2129, 2096, 2523, 235, 2404, 4376, 2459, 1212, 4738, 2999, 4729, 1034, 4468, 3393, 2342, 3075, 2370, 4659, 2377, 3842, 1755, 99, 1565, 4436, 2366, 2045, 262, 724, 4011, 3644, 1161, 4546, 1827, 1986, 4226, 4072, 3490, 4712, 466, 69, 1398, 425, 2801, 4325, 1426, 4510, 1141, 2378, 2699, 739, 3528, 4024, 1389, 2105, 1882, 3906, 2854, 4936, 1958, 1177, 1256, 4283, 2578, 3629, 3071, 2447, 4045, 4920, 3467, 4525, 1829, 2704, 486, 4600, 4653, 1914, 1334, 2782, 1599, 602, 274, 993, 1050, 11, 1466, 4557, 4017, 3726, 1295, 778, 1072, 144, 1899, 4450, 3053, 4581, 3626, 949, 3197, 2665, 1429, 2038, 504, 4342, 3568, 4865, 2317, 1353, 3228, 2228, 1451, 761, 324, 857, 4232, 4014, 4687, 2170, 1176, 913, 820, 186, 1068, 4764, 549, 4262, 3356, 630, 1123, 3452, 4943, 4280, 3809, 3049, 1331, 3344, 4793, 2961, 1619, 224, 3003, 2224, 3881, 3615, 1853, 1301, 2208, 2618, 515, 2119, 372, 309, 3893, 4681, 4152, 4963, 3350, 3446, 3224, 876, 1781, 54, 697, 3708, 1170, 691, 2477, 3319, 2775, 3424, 1467, 3508, 2788, 769, 956, 4485, 3405, 554, 3870, 1904, 3305, 2457, 2929, 3018, 2023, 883, 2137, 952, 2990, 4996, 293, 1489, 1688, 699, 2236, 3656, 872, 1622, 1052, 4181, 1060, 1971, 1150, 4117, 2106, 572, 745, 4577, 4252, 4069, 4841, 2215, 2323, 4116, 4850, 2281, 2985, 4629, 3895, 1918, 499, 3423, 4413, 4536, 4452, 4130, 2412, 2590, 979, 1403, 489, 4820, 4101, 4052, 4939, 4425, 2328, 1397, 3021, 4677, 3646, 3410, 3094, 2844, 4714, 612, 1704, 1883, 2755, 268, 3030, 2995, 3117, 3097, 605, 3397, 4311, 1249, 3771, 1046, 4440, 4592, 65, 2984, 4959, 3815, 1804, 4609, 459, 3794, 2392, 2473, 4520, 3791, 2971, 2561, 912, 1847, 1598, 3887, 4634, 4371, 3500, 4199, 3284, 795, 4545, 2297, 3145, 3330, 3437, 618, 1268, 310, 3033, 2795, 121, 677, 4627, 708, 3800, 4861, 2555, 4860, 3941, 2558, 569, 392, 4382, 2614, 4148, 4051, 1757, 4419, 330, 2121, 4134, 4328, 4929, 4966, 2318, 4905, 1575, 4093, 1136, 3827, 1888, 1689, 332, 3689, 4589, 440, 726, 291, 2718, 1769, 1263, 2749, 948, 3883, 4158, 3457, 190, 2294, 451, 1036, 4749, 1630, 4737, 4326, 3189, 3152, 2978, 4137, 3320, 288, 2312, 2205, 1761, 4088, 865, 3192, 4735, 3034, 4145, 2160, 2564, 4147, 3764, 2198, 3731, 3066, 3743, 3149, 2016, 3194, 28, 1703, 4309, 3639, 2086, 2845, 1748, 337, 3398, 384, 3545, 1476, 40, 4337, 860, 2528, 2543, 1680, 203, 1529, 3947, 3353, 1797, 4866, 4023, 3297, 1081, 2915, 1533, 4824, 1421, 163, 3513, 1333, 4587, 4769, 3048, 4524, 2688, 4409, 3072, 2482, 3431, 3551, 4529, 3352, 4716, 3261, 1613, 3361, 1974, 3558, 937, 3306, 4267, 314, 161, 4336, 2975, 2340, 18, 4278, 4426, 1138, 2958, 1678, 5000, 1018, 3695, 3617, 2732, 4870, 2734, 615, 1837, 4658, 1155, 1728, 704, 2951, 3770, 944, 1062, 540, 112, 3570, 3807, 4513, 538, 2012, 3020, 1381, 3089, 2887, 2180, 4444, 947, 109, 2549, 1828, 4554, 2800, 4071, 987, 2821, 396, 3667, 2741, 796, 2054, 2200, 3185, 182, 2581, 4928, 2186, 5003, 2040, 1220, 4416, 748, 1878, 4484, 4453, 4796, 1574, 4062, 3621, 4923, 3972, 1606, 2727, 103, 659, 1257, 3248, 1647, 826, 2376, 320, 1011, 1347, 4567, 2746, 4112, 3679, 345, 1743, 4294, 1166, 145, 3076, 2969, 2296, 1480, 1328, 1915, 1174, 4703, 793, 2389, 667, 2517, 925, 2234, 2163, 3927, 284, 1952, 3390, 395, 3690, 3131, 1990, 1991, 4826, 1809, 3572, 3138, 1660, 994, 253, 2739, 1851, 4301, 1405, 3070, 3485, 3032, 1332, 2900, 321, 413, 13, 3952, 4666, 3343, 3716, 1394, 3068, 1531, 4924, 4908, 34, 1811, 472, 2395, 453, 4726, 387, 3859, 3986, 4552, 2963, 3400, 2489, 4233, 2673, 2768, 1527, 4837, 1792, 568, 3605, 3600, 379, 3331, 966, 2783, 4763, 3176, 2848, 682, 4843, 1751, 4708, 4085, 1315, 666, 4164, 2501, 4006, 417, 160, 4473, 4235, 1979, 2905, 2719, 1735, 3956, 491, 2919, 414, 3495, 1663, 4803, 1000, 3262, 1300, 1946, 131, 4719, 2299, 1873, 3090, 3493, 4497, 303, 1272, 2456, 2524, 4221, 4388, 783, 3518, 3507, 3577, 4885, 4310, 187, 3231, 893, 839, 735, 3106, 2478, 3285, 1041, 3046, 4527, 4663, 686, 1753, 4814, 3659, 2498, 2315, 3455, 3363, 1346, 1252, 2183, 946, 4448, 4571, 2474, 1487, 3168, 2548, 1021, 658, 2940, 684, 2460, 4652, 1463, 4670, 2833, 1242, 4635, 801, 2466, 3208, 2659, 1388, 3821, 4499, 2904, 151, 727, 2611, 2664, 4186, 1850, 4346, 2479, 3334, 1593, 1944, 2484, 3811, 3438, 2813, 1746, 3038, 4706, 4679, 3921, 373, 493, 41, 4682, 3826, 2089, 2503, 2179, 1749, 794, 3783, 4541, 3447, 43, 3831, 1477, 537, 1548, 3155, 867, 1634, 1659, 2885, 1202, 129, 4849, 3420, 1431, 4258, 511, 4739, 1862, 2177, 2705, 282, 2670, 4108, 2134, 1985, 854, 4948, 3681, 606, 76, 2831, 3396, 685, 1903, 4570, 853, 2214, 3504, 3077, 2168, 1172, 1668, 3546, 1259, 3792, 219, 46, 4478, 3959, 2191, 369, 3866, 3468, 597, 4655, 283, 4810, 1802, 4982, 2175, 3750, 2852, 1144, 2629, 1008, 1518, 1181, 834, 1459, 2057, 4277, 5011, 4665, 1711, 1492, 2019, 4595, 521, 1854, 1740, 3317, 275, 1667, 2774, 4139, 1692, 503, 1486, 4651, 3054, 4390, 2462, 2268, 1699, 4113, 2533, 1947, 1055, 1410, 4930, 4768, 2069, 932, 73, 9, 2083, 1117, 4119, 2754, 1824, 1984, 2095, 1642, 1930, 2826, 388, 4641, 1913, 992, 1457, 1439, 4123, 3311, 4121, 2213, 2603, 3672, 2452, 4900, 1722, 664, 2546, 892, 2859, 3044, 4934, 4488, 3111, 2420, 1733, 2710, 1504, 4583, 4794, 1132, 399, 551, 1325, 824, 3944, 1075, 3933, 752, 689, 3413, 2514, 3996, 2269, 3016, 2851, 2204, 3414, 2382, 2219, 1143, 2872, 3884, 4402, 1275, 1047, 3370, 405, 3702, 1744, 3580, 1801, 2500, 3502, 1496, 3277, 4157, 4231, 2640, 3065, 3294, 421, 3216, 3120, 2138, 2046, 4910, 3108, 4122, 3432, 1789, 463, 3936, 4621, 3678, 777, 4970, 123, 1665, 617, 2278, 807, 1363, 962, 1134, 652, 2707, 1288, 101, 4732, 4110, 2261, 406, 2575, 767, 3876, 4998, 4809, 1311, 4114, 2364, 3163, 476, 651, 3022, 1020, 957, 1006, 3916, 4657, 4000, 2736, 3635, 218, 1874, 2422, 806, 4275, 4890, 1515, 4773, 4083, 1545, 3549, 56, 923, 547, 2769, 2344, 1093, 2081, 3023, 4115, 2568, 4585, 3925, 2451, 4227, 2245, 368, 3244, 3269, 3241, 2663, 398, 2354, 4465, 2656, 4771, 2256, 3746, 920, 642, 3825, 4486, 475, 3922, 3784, 3281, 2824, 2631, 3971, 4611, 415, 1937, 3858, 4709, 457, 945, 1308, 269, 4344, 1237, 1806, 4927, 4912, 2369, 989, 4056, 1289, 2864, 4566, 114, 3640, 676, 402, 3612, 3849, 4805, 3670, 72, 2060, 3102, 51, 4213, 1895, 348, 1695, 1810, 855, 906, 1980, 4393, 1607, 2226, 4032, 2977, 446, 2282, 1337, 868, 2715, 3059, 1326, 1795, 1183, 1840, 817, 3119, 356, 2621, 1995, 3478, 1271, 3366, 4314, 2030, 1230, 4925, 1671, 3892, 3623, 432, 3213, 3421, 583, 4594, 4622, 2289, 3625, 1870, 3382, 4759, 2050, 738, 1116, 3848, 4602, 4584, 1129, 4615, 4949, 1031, 646, 385, 2487, 1350, 3351, 4684, 3466, 3645, 773, 480, 4447, 621, 4746, 1402, 4842, 3143, 3820, 965, 1286, 3898, 823, 171, 4758, 1490, 2960, 1045, 3703, 4163, 2425, 2352, 3503, 1830, 136, 825, 1411, 1223, 3684, 225, 4374, 1875, 1229, 2400, 1361, 4598, 1907, 5007, 2820, 4111, 1929, 2646, 4582, 3654, 2465, 4549, 4060, 4628, 17, 2882, 4995, 4906, 3132, 2982, 2345, 4815, 3865, 4411, 4238, 3591, 217, 915, 2047, 38, 3183, 849, 1069, 2660, 2399, 4098, 4957, 866, 792, 1406, 523, 1726, 2677, 3223, 4268, 1423, 2797, 3748, 1154, 4212, 4613, 1253, 1318, 2298, 1357, 401, 1345, 2830, 3456, 4678, 1712, 2722, 4307, 3379, 1408, 1817, 2112, 4223, 19, 4160, 3547, 1512, 4302, 4605, 32, 1538, 3584, 2212, 2255, 2585, 3619, 3406, 1674, 2856, 3238, 3246, 2064, 4839, 2781, 3082, 1583, 2053, 3790, 2446, 496, 1777, 93, 2724, 941, 2439, 3574, 4700, 886, 2612, 2559, 2401, 3058, 1506, 3868, 1856, 2657, 3878, 4428, 3755, 305, 813, 2270, 4690, 591, 1834, 3270, 955, 3633, 3280, 3088, 1650, 26, 1649, 1017, 1522, 562, 226, 1483, 4692, 3199, 1779, 3509, 4038, 322, 4185, 1589, 3573, 1754, 1737, 3512, 1030, 1786, 3682, 779, 1951, 2624, 525, 2301, 2540, 2409, 1978, 2108, 3930, 366, 1822, 4693, 3759, 1, 2363, 2738, 2033, 3589, 566, 3227, 4532, 3073, 1137, 700, 1042, 3004, 4404, 4091, 2691, 3389, 3663, 1970, 1392, 4166, 4230, 4872, 856, 4297, 4775, 3981, 1942, 3767, 4644, 3387, 1371, 1534, 526, 116, 2609, 4917, 3367, 3954, 4467, 1246, 2642, 3732, 1701, 4451, 4318, 4639, 2637, 3301, 4787, 4572, 3409, 4207, 4950, 4926, 2874, 788, 306, 2605, 4305, 2577, 3123, 1448, 3537, 426, 3442, 188, 2653, 2189, 2860, 4531, 2475, 1590, 473, 2082, 2350, 3833, 2295, 3067, 2316, 2302, 594, 3079, 242, 1399, 1115, 3326, 313, 2058, 3548, 1501, 3276, 1175, 2897, 4387, 1066, 2351, 4798, 261, 2326, 2616, 1615, 1102, 156, 1683, 831, 3172, 3618, 943, 1376, 616, 1025, 159, 3662, 4751, 670, 3182, 1520, 4743, 3471, 3749, 3245, 1494, 2947, 1204, 4542, 3779, 990, 1368, 1475, 2554, 567, 502, 1058, 1939, 1885, 3087, 4368, 2001, 2772, 2464, 1447, 4424, 4965, 4831, 2430, 1428, 4818, 636, 2520, 2544, 4454, 780, 325, 781, 1377, 3473, 3610, 16, 3144, 772, 3692, 4254, 660, 2748, 3910, 4674, 2881, 1992, 1787, 3428, 3582, 1693, 4200, 209, 1266, 2817, 2311, 603, 2077, 4662, 2592, 530, 4979, 4978, 3529, 4534, 4574, 4319, 1203, 3279, 1329, 198, 2702, 2828, 1090, 2322, 3240, 2550, 3867, 1717, 3712, 2116, 2901, 260, 673, 1586, 2486, 21, 4855, 3496, 393, 3055, 3005, 2490, 3209, 3824, 1910, 3283, 3780, 3863, 1505, 771, 931, 3717, 840, 2536, 2347, 1418, 4632, 1838, 3531, 3418, 2875, 2154, 4698, 1119, 2842, 3643, 1012, 3295, 4650, 4620, 318, 2339, 3776, 3817, 383, 1893, 4363, 4603, 1317, 2059, 1707, 229, 1023, 3029, 82, 4097, 3286, 1612, 4462, 79, 556, 1314, 756, 818, 3218, 983, 1567, 3325, 2386, 1100, 1935, 3705, 1039, 367, 3804, 1310, 4772, 1038, 3777, 4648, 845, 4777, 95, 3221, 238, 5006, 746, 2407, 1232, 264, 2227, 758, 1669, 3255, 1191, 4722, 3945, 2209, 1057, 2968, 522, 2402, 33, 3105, 2173, 4973, 2078, 3751, 4031, 928, 1813, 1955, 770, 427, 4895, 991, 4339, 4406, 3812, 3974, 4457, 1676, 645, 2511, 1225, 3598, 1627, 2110, 2148, 1624, 4178, 4691, 3215, 4449, 858, 1698, 3017, 2361, 3441, 1029, 3095, 254, 2981, 4216, 802, 3347, 1798, 846, 2103, 1588, 4761, 2911, 1901, 2469, 961, 380, 4403, 1400, 2188, 4487, 2414, 1819, 57, 4034, 1771, 3907, 967, 1284, 3985, 4471, 1654, 729, 3220, 2599, 1499, 3998, 2840, 715, 2035, 1040, 3449, 3161, 519, 4174, 939, 736, 3983, 1438, 541, 1048, 4901, 582, 4685, 1982, 4042, 3627, 2579, 4317, 3085, 4053, 2855, 1373, 4741, 2048, 4004, 3729, 2385, 4533, 2248, 4405, 2494, 2292, 4898, 3156, 888, 2674, 4922, 353, 2087, 3704, 139, 4410, 3045, 1278, 4154, 4016, 1502, 2917, 1696, 1458, 27, 4205, 1905, 20, 2570, 910, 3129, 2583, 3101, 2061, 1672, 3369, 245, 1294, 419, 1484, 2518, 460, 4878, 4829, 1595, 1159, 3902, 4338, 361, 2649, 3014, 4984, 832, 1773, 1846, 1616, 1858, 3896, 513, 619, 4272, 1153, 2123, 104, 4868, 4578, 514, 4429, 3818, 3205, 4880, 3290, 4222, 2435, 1922, 2935, 1073, 3564, 2434, 3383, 1228, 4394, 45, 4234, 1725, 277, 31, 4349, 1107, 1808, 1084, 4874, 10, 3052, 1897, 1528, 2825, 3581, 742, 3371, 1125, 1211, 2952, 4656, 3856, 2884, 559, 909, 50, 1954, 2136, 3926, 784, 2280, 3042, 4495, 102, 2448, 1675, 3797, 3266, 300, 4568, 2310, 3665, 862, 2890, 690, 4237, 2097, 2846, 2443, 6, 2510, 3474, 1396, 1670, 35, 2930, 4224, 4009, 143, 2957, 1133, 1442, 1657, 2125, 4515, 2634, 2367, 4047, 2383, 1602, 790, 339, 461, 3620, 4954, 2144, 2588, 3855, 4193, 2207, 3912, 3187, 3252, 810, 535, 3609, 23, 2065, 1805, 4183, 424, 4138, 2996, 3092, 3571, 4626, 3191, 607, 1618, 4389, 641, 1160, 3962, 3668, 316, 5008, 3634, 1306, 585, 4480, 4269, 63, 4988, 3657, 2814, 3127, 448, 3372, 4176, 2717, 2907, 637, 2785, 77, 3339, 252, 2260, 775, 3302, 2816, 3450, 3733, 2481, 3641, 2090, 545, 4422, 3257, 1745, 2850, 2374, 4019, 2357, 474, 786, 194, 1103, 2273, 1702, 1651, 2809, 4469, 1860, 2668, 4050, 4575, 1912, 4109, 447, 552, 2709, 2794, 4986, 3744, 378, 1896, 4856, 970, 714, 1010, 1157, 3501, 4750, 1729, 2341, 561, 3388, 1087, 3715, 3958, 3233, 2028, 3253, 2909, 1386, 516, 4519, 331, 3166, 2421, 1818, 4175, 4902, 2266, 3661, 4044, 2036, 420, 490, 891, 2320, 1461, 1791, 3365, 2757, 4430, 364, 787, 1662, 4182, 1919, 4046, 4168, 4306, 1868, 3519, 3997, 759, 3448, 2008, 439, 189, 3723, 4539, 512, 3407, 2723, 4548, 4201, 2586, 609, 4770, 575, 2898, 744, 3891, 3982, 4129, 4373, 874, 4891, 3288, 942, 2936, 508, 4757, 3505, 4765, 940, 2925, 3694, 2275, 2551, 3725, 2945, 3628, 4913, 2553, 1112, 604, 2250, 4776, 3250, 4420, 200, 4909, 776, 91, 442, 3345, 672, 3056, 196, 3477, 4680, 4288, 1644, 1274, 4161, 4456, 3920, 4753, 3057, 1307, 4960, 4565, 2654, 969, 83, 3603, 2257, 4276, 1299, 2497, 2966, 1718, 734, 4699, 2726, 4740, 687, 2552, 1469, 2652, 2615, 2166, 423, 1351, 733, 100, 722, 1173, 394, 120, 255, 1290, 4853, 706, 2043, 4219, 1638, 5001, 4704, 1462, 4563, 574, 2324, 2229, 3134, 1557, 362, 657, 1298, 2526, 1233, 1277, 3943, 1536, 4470, 1503, 3949, 3328, 4711, 3550, 458, 3439, 1898, 74, 1190, 3540, 390, 3113, 2896, 811, 573, 3151, 3719, 3335, 3834, 2013, 2365, 3669, 3687, 914, 1999, 3737, 3835, 3830, 4323, 2098, 578, 693, 4544, 1543, 315, 3027, 1427, 2122, 590, 4991, 3062, 668, 3136, 2416, 1541, 2300, 4569, 445, 2356, 2017, 3313, 228, 3463, 3532, 4590, 2025, 4082, 3069, 2955, 3403, 3961, 2153, 4875, 665, 3596, 3084, 1455, 2574, 3653, 150, 4041, 2679, 3417, 1409, 1594, 1549, 4347, 75, 593, 3964, 483, 4981, 3167, 4576, 1576, 1387, 4208, 3235, 1348, 2411, 4736, 2941, 1260, 563, 428, 1738, 296, 3181, 3781, 1339, 2942, 2516, 2271, 1201, 3239, 4099, 4792, 3718, 4836, 3875, 3391, 4530, 4671, 3756, 1194, 4918, 4142, 828, 816, 166, 978, 1550, 230, 3515, 1637, 2353, 4562, 974, 3247, 2986, 1778, 4555, 302, 2037, 908, 3823, 2779, 3843, 2418, 4780, 3511, 1367, 4007, 3226, 2841, 7, 381, 2766, 4516, 4579, 1640, 4858, 1213, 1516, 4884, 4498, 880, 2194, 3381, 2343, 4828, 1105, 1250, 755, 2218, 4078, 2488, 3526, 4341, 1303, 2093, 3754, 1287, 3154, 2145, 720, 1730, 3307, 973, 1082, 4427, 3685, 3901, 2811, 4744, 2682, 1178, 1736, 4092, 4847, 4640, 4472, 2104, 4604, 1886, 3461, 2934, 2133, 1189, 2525, 1135, 2032, 3115, 236, 343, 2437, 2135, 4953, 1921, 623, 3171, 250, 2217, 201, 1120, 3747, 2419, 1923, 3648, 924, 178, 4418, 2015, 2146, 2243, 3808, 482, 4857, 1378, 4149, 4788, 2167, 4384, 879, 2519, 354, 2804, 4830, 257, 4476, 2131, 2740, 4327, 2158, 4985, 1825, 2912, 3116, 4512, 2396, 4894, 2604, 449, 3697, 4257, 2044, 1248, 1684, 975, 3948, 628, 4261, 462, 3686, 4883, 3768, 2725, 2987, 2921, 4367, 2620, 4167, 3433, 3378, 4543, 4104, 710, 487, 4355, 4246, 2790, 527, 1196, 3604, 175, 728, 1691, 2784, 1364, 2124, 3332, 2617, 833, 1537, 2815, 4637, 3025, 4631, 936, 1375, 2132, 2880, 3828, 1927, 3761, 1226, 107, 431, 4266, 579, 4198, 1324, 3453, 3104, 2408, 4146, 4171, 117, 2866, 827, 301, 87, 1603, 1265, 1763, 4528, 4705, 917, 4395, 629, 2304, 1664, 237, 2118, 2692, 2232, 553, 4479, 837, 1579, 146, 4482, 4669, 98, 1281, 1587, 2593, 4029, 4067, 1474, 1869, 2535, 835, 1628, 1470, 2417, 905, 2011, 4443, 4718, 1762, 4190, 4358, 1425, 4438, 996, 3364, 1078, 4845, 1231, 1794, 564, 1473, 2495, 3211, 2658, 1620, 3664, 3880, 173, 4100, 3435, 698, 2468, 437, 1540, 2991, 4916, 1568, 397, 2335, 2405, 207, 4074, 3124, 2506, 1032, 4727, 3957, 654, 2512, 3915, 1063, 4400, 4057, 3039, 4932, 1732, 4676, 4180, 4904, 371, 1700, 869, 298, 3282, 478, 1972, 2380, 227, 1731, 4217, 1962, 2542, 935, 42, 5004, 3701, 863, 2778, 725, 1820, 4596, 3222, 2576, 1988, 197, 1774, 3991, 39, 1199, 149, 3597, 2566, 1217, 1168, 1044, 3730, 2678, 1007, 2330, 4834, 2796, 3404, 586, 1950, 797, 4942, 2264, 2928, 2666, 4507, 241, 1096, 1562, 4730, 2696, 2538, 4441, 1130, 2894, 3260, 2630, 4720, 3937, 2169, 4356, 3752, 2496, 3879, 709, 1894, 4379, 1881, 1393, 861, 2771, 1205, 4195, 92, 2, 2165, 3333, 2471, 2150, 4887, 340, 2314, 4383, 211, 3118, 1056, 148, 3622, 2683, 1715, 4089, 819, 3908, 3510, 4334, 1759, 2492, 1879, 2303, 3769, 192, 88, 1338, 951, 2429, 1560, 2589, 2965, 1891, 317, 1472, 1437, 2244, 276, 701, 789, 2545, 471, 4494, 747, 1677, 376, 4408, 4623, 2786, 3542, 633, 2643, 1456, 3544, 1782, 1609, 2720, 3133, 3377, 4672, 2899, 495, 2776, 4586, 782, 2980, 222, 4351, 2049, 3938, 3950, 199, 3929, 2761, 878, 4439, 4352, 1767, 643, 4625, 2932, 1843, 4256, 2242, 4756, 625, 1343, 3886, 126, 2305, 4795, 455, 4140, 3035, 1709, 1360, 249, 4209, 842, 292, 4702, 1365, 1708, 507, 1776, 4315, 4359, 4675, 119, 1320, 4660, 4102, 1633, 2571, 3882, 4455, 3349, 4475, 2258, 1415, 4938, 1417, 4463, 2211, 1468, 4560, 2291, 1358, 3861, 1003, 1750, 890, 1807, 2130, 3359, 3934, 2359, 763, 2319, 3594, 4646, 2415, 2372, 4095, 2756, 2868, 2598, 509, 2557, 4263, 1445, 1146, 3969, 326, 4300, 960, 4407, 3009, 1416, 901, 3940, 1780, 2671, 4800, 1572, 2007, 4755, 4059, 4725, 3789, 703, 3341, 1180, 4308, 2582, 2976, 2206, 608, 1585, 2567, 2450, 4688, 4911, 1498, 570, 2944, 4251, 3559, 3840, 3242, 4286, 647, 382, 2449, 4214, 3093, 2747, 3251, 2808, 2172, 1061, 4825, 128, 1352, 1845, 1088, 3967, 2485, 1998, 2651, 4715, 804, 234, 1983, 3745, 2992, 1551, 533, 1625, 1835, 409, 1118, 3425, 4330, 871, 611, 2703, 1872, 3315, 4136, 985, 4274, 4279, 1758, 3977, 528, 4173, 4003, 1140, 3354, 1424, 4707, 1747, 4977, 22, 847, 3114, 3081, 536, 4614, 805, 3978, 2476, 1803, 1207, 4398, 2661, 3489, 1434, 2662, 2970, 4863, 1511, 108, 1724, 2174, 1401, 4385, 3494, 3193, 3523, 3786, 2293, 319, 4194, 3905, 1197, 1099, 1563, 2117, 4391, 799, 3923, 408, 4876, 1471, 2321, 753, 1009, 4597, 3384, 2876, 1697, 2636, 2750, 3491, 2031, 1316, 626, 4509, 3112, 3980, 336, 791, 3539, 4782, 3636, 2398, 2763, 2438, 1033, 4415, 3874, 719, 798, 1241, 3814, 1926, 3787, 3200, 4481, 3312, 2632, 251, 3586, 2803, 4008, 1721, 3642, 2216, 2597, 4816, 580, 3008, 4668, 3535, 971, 4921, 3322, 1148, 344, 2286, 517, 1227, 4218, 1236, 576, 751, 809, 2195, 2684, 1871, 4321, 221, 1269, 97, 1857, 3458, 3911, 133, 4159, 632, 3990, 1001, 1218, 2594, 2066, 2072, 2142, 4377, 3086, 4285, 1705, 2938, 3583, 2151, 2155, 4271, 1760, 1019, 4897, 55, 365, 3000, 4228, 1106, 4989, 2337, 411, 702, 4086, 4126, 3355, 4446, 2792, 2675, 2853, 3265, 4260, 1920, 4322, 3486, 1706, 4094, 1552, 4754, 213, 1796, 3408, 2619, 2713, 4799, 3864, 2091, 2521, 4366, 4867, 919, 3465, 1716, 2431, 4313, 312, 785, 3256, 3327, 2902, 3024, 1643, 2440, 4054, 713, 266, 1151, 1685, 921, 635, 1626, 2823, 4517, 2793, 14, 4265, 2712, 2989, 1095, 1258, 882, 4247, 749, 4835, 1546, 2650, 3080, 1209, 297, 2237, 1436, 1639, 2798, 418, 1981, 4504, 4293, 3147, 272, 4049, 2070, 4289, 484, 4125, 907, 2610, 3128, 4242, 3141, 2767, 2681, 520, 679, 600, 2491, 3890, 4033, 1292, 1800, 705, 4968, 351, 3536, 333, 2972, 897, 4701, 454, 246, 4633, 4365, 2026, 4951, 1596, 2220, 2950, 3822, 2565, 231, 4037, 1821, 2027, 1957, 1521, 497, 3380, 4316, 3578, 1646, 4249, 3164, 3995, 3567, 2149, 2584, 342, 1611, 3762, 3683, 4079, 1866, 2743, 2390, 5002, 2397, 3469, 4822, 1485, 84, 3525, 1890, 1799, 4483, 4417, 4, 4012, 634, 4760, 1933, 96, 4731, 2834, 3481, 741, 2074, 4696, 2499, 596, 8, 4127, 430, 456, 323, 1842, 2857, 2865, 3099, 1110, 4747, 177, 4304, 543, 2622, 184, 214, 4969, 1255, 4437, 1054, 25, 2837, 2693, 4035, 1139, 2759, 2379, 3287, 653, 4500, 2368, 3740, 2799, 1200, 3638, 4493, 176, 1936, 3810, 3872, 1564, 4919, 4564, 4333, 3249, 2906, 3593, 1623, 1909, 1280, 4298, 2714, 4961, 29, 4610, 2249, 3368, 370, 122, 4151, 2128, 4636, 2655, 1482, 4248, 3232, 1005, 4550, 760, 1185, 2336, 1091, 895, 2287, 2914, 4721, 477, 1945, 1605, 2810, 711, 4551, 977, 3135, 4081, 281, 4840, 671, 3953, 2455, 350, 4689, 3816, 2348, 662, 4645, 2648, 3098, 4695, 4971, 3968, 1071, 3169, 412, 113, 174, 1766, 3470, 1989, 140, 4618, 3230, 4522, 4155, 1864, 995, 4013, 870, 4423, 3575, 2613, 436, 485, 4956, 1831, 1192, 2055, 1215, 4976, 1026, 1681, 1247, 4329, 2147, 1510, 2252, 2181, 2537, 359, 2690, 4852, 1264, 433, 2092, 1291, 4717, 4967, 4638, 2143, 3293, 4364, 338, 1641, 3714, 2454, 3236, 5010, 1419, 2010, 1902, 4940, 407, 2331, 2879, 2595, 2445, 1513, 2829, 2946, 2088, 2185, 3524, 1720, 2922, 2541, 1385, 4284, 2433, 2410, 3793, 1948, 1571, 3299, 2223, 4832, 3178, 4694, 4005, 622, 3110, 3649, 3318, 3488, 1661, 1043, 873, 1768, 374, 3795, 614, 2306, 4240, 927, 3711, 4215, 851, 765, 510, 958, 3774, 1016, 3803, 4211, 1849, 627, 223, 2933, 3498, 1285, 1790, 3291, 3411, 1024, 2259, 2773, 2931, 1240, 3006, 4028, 1382, 3234, 972, 3841, 243, 4225, 4859, 4734, 695, 2608, 2127, 172, 5, 4253, 1694, 1859, 3100, 1969, 2838, 982, 1356, 3775, 1014, 1370, 3631, 4295, 2974, 4503, 422, 2507, 1235, 1645, 3850, 3989, 1158, 2997, 850, 24, 1313, 546, 1994, 3707, 838, 565, 1198, 4370, 1210, 999, 2735, 4882, 4540, 3479, 4386, 444, 4399, 1379, 2453, 162, 3298, 762, 3713, 3829, 3204, 4607, 4273, 4955, 4781, 3459, 3554, 3278, 244, 1508, 1309, 2843, 2358, 1460, 4433, 953, 4789, 4362, 3854, 980, 4974, 12, 3273, 3736, 1374, 3001, 1977, 2024, 81, 3040, 1938, 1444, 4491, 1304, 3588, 4559, 3125, 3329, 3917, 1124, 137, 1524, 4958, 1844, 49, 3735, 3395, 4170, 3757, 62, 2005, 1966, 1532, 2822, 4713, 2346, 2029, 964, 1145, 3201, 2463, 2327, 2276, 1597, 2176, 2285, 2701, 4869, 165, 4724, 1127, 2210, 2483, 963, 4220, 4752, 3851, 2406, 1449, 2426, 4643, 2587, 3520, 2099, 2923, 981, 4477, 2233, 3357, 3727, 1383, 884, 125, 308, 4896, 4187, 4027, 3304, 4889, 105, 1653, 3563, 3928, 1077, 4348, 2504, 2267, 4844, 3412, 3871, 841, 2240, 2325, 4783, 1002, 2869, 1305, 4068, 2591, 2787, 2009, 3385, 2073, 4204, 864, 2556, 3146, 3258, 4458, 1884, 4915, 1465, 4191, 2886, 1478, 4946, 1079, 4606, 479, 154, 204, 692, 4811, 94, 2802, 2391, 481, 3036, 2251, 3852, 1340, 1553, 4396, 3275, 2760, 1997, 3758, 1131, 2939, 723, 3051, 4937, 4361, 4521, 3031, 1064, 2889, 3243, 1222, 135, 4421, 44, 4397, 4241, 4026, 3860, 4259, 3741, 1934, 2708, 2164, 1953, 2002, 4697, 3436, 4514, 4312, 1877, 898, 930, 3802, 2777, 3555, 3955, 2239, 3614, 4474, 2003, 1931, 4239, 391, 1053, 2467, 3760, 3440, 1098, 4999, 1262, 1111, 1839, 4766, 2313, 544, 270, 1833, 4983, 887, 1179, 1517, 193, 613, 542, 4619, 3210, 1391, 2034, 3184, 464, 4320, 1608, 4492, 1558, 3611, 1049, 1224, 2493, 534, 3918, 4156, 852, 4523, 4076, 4972, 3909, 1570, 1632, 4464, 183, 2192, 1775, 1658, 3873, 3002, 2247, 650, 3130, 334, 2913, 2926, 3174, 2182, 61, 3083, 375, 4935, 2689, 1435, 2534, 648, 1164, 922, 3806, 4823, 4733, 4445, 2623, 3203, 3630, 2962, 4332, 1621, 470, 2601, 1094, 3050, 2959, 202, 1113, 2953, 2502, 1900, 4786, 3394, 157, 2639, 3846, 4381, 1591, 4202, 3314, 1162, 2835, 926, 2115, 441, 181, 984, 1867, 3308, 1245, 2569, 911, 3268, 3173, 740, 663, 3673, 3336, 1741, 3362, 1059, 674, 3970, 1940, 4593, 1126, 4124, 1959, 754, 259, 4196, 4229, 1244, 1167, 655, 1422, 3064, 3206, 4128, 263, 1855, 4353, 1336, 2022, 195, 4002, 2203, 5005, 3324, 3832, 2964, 1446, 1414, 2573, 3482, 4812, 830, 2641, 4505, 4131, 2916, 4723, 3951, 377, 2721, 4654, 206, 3358, 3321, 2272, 1083, 4506, 4642, 2393, 130, 3963, 1812, 3624, 2283, 4172, 1327, 3960, 3607, 4106, 1035, 2094, 2764, 1880, 1440, 37, 1784, 750, 3595, 2547, 1941, 2051, 3078, 3966, 3514, 3454, 1783, 2381, 1169, 4556, 631, 3579, 2873, 1297, 4066, 2141, 3931, 3924, 2100, 2120, 1184, 661, 3772, 1719, 4040, 2742, 2627, 2021, 1335, 1372, 328, 132, 2806, 1525, 1655, 1497, 2193, 694, 2839, 2235, 3541, 4854, 58, 389, 3026, 3799, 2943, 1430, 2187, 718, 2973, 279, 505, 1261, 678, 1454, 2903, 1975, 1943, 2508, 3015, 3888, 814, 2645, 3010, 2539, 1987, 59, 1413, 4184, 4103, 452, 2883, 800, 1814, 815, 902, 4612, 3175, 2572, 1600, 696, 3121, 730, 2737, 3472, 1679, 1452, 1968, 532, 3464, 278, 4580, 829, 3676, 4188, 2387, 498, 170, 158, 3857, 2819, 3310, 2529, 822, 1022, 4790, 3374, 115, 3973, 2954, 355, 2076, 3680, 3946, 3011, 3342, 152, 2522, 4461, 3533, 1239, 2231, 4518, 3521, 3402, 4881, 4791, 1089, 71, 66, 3207, 86, 4601, 2221, 2847, 70, 2667, 1037, 1514, 1195, 3706, 4245, 4821, 3415, 2695, 4892, 2733, 2580, 2607, 968, 2349, 1889, 2596, 2080, 2910, 2758, 360, 656, 3801, 1785, 2700, 1165, 2470, 1601, 1156, 2625, 4862, 4994, 774, 1495, 4077, 4848, 4036, 3606, 1015, 3162, 584, 3602, 3671, 2178, 1617, 518, 589, 208, 988, 1887, 240, 4412, 3180, 1752, 1555, 4080, 4785, 1450, 1412, 1067, 1573, 2084, 1924, 1949, 821, 347, 1519, 1243, 2878, 3899, 3837, 168, 4833, 4392, 4350, 3292, 2333, 3041, 4599, 2920, 4784, 2184, 465, 1836, 3919, 1208, 1488, 4742, 2893, 335, 1772, 1065, 1996, 2628, 295, 3566, 3225, 2338, 2861, 3060, 500, 1251, 1076, 4264, 4508, 3289, 4132, 1369, 52, 1507, 304, 60, 3159, 488, 639, 1330, 2789, 4065, 1559, 3562, 4745, 1925, 1362, 4141, 4270, 4290, 3386, 3109, 3214, 1976, 3885, 3785, 3150, 341, 1349, 1086, 1690, 2770, 1114, 2201, 1582, 1906, 3900, 2171, 531, 524, 3587, 4375, 4010, 539, 106, 4107, 2140, 1380, 4664, 3373, 2042, 4165, 3499, 4980, 1788, 4118, 3738, 386, 1636, 2246, 2162, 1648, 894, 716, 1121, 3877, 1764, 2563, 599, 904, 1420, 3608, 1581, 4864, 766, 1908, 4120, 403, 2606, 3782, 571, 3360, 4661, 3506, 997, 4649, 3430, 4941, 2602, 90, 1097, 560, 258, 48, 3592, 4591, 4778, 1765, 2891, 1354, 4907, 1109, 1163, 4340, 4210, 2156, 299, 3338, 3196, 3914, 4553, 1610, 3443, 4535, 3096, 3773, 358, 363, 1727, 4990, 5009, 2994, 4827, 4710, 3217, 2716, 4511, 2706, 2731, 141, 110, 3219, 4624, 2638, 1911, 3693, 273, 3061, 4378, 4573, 124, 808, 1085, 4335, 2698, 2071, 127, 1961, 2532, 731, 899, 4547, 2728, 2052, 2998, 1584, 3427, 289, 265, 4030, 3522, 3074, 1214, 4608, 4061, 1149, 1714, 764, 1973, 429, 3553, 950, 30, 3475, 3561, 1152, 2647, 4133, 3376, 2329, 220, 4459, 3699, 1296, 3480, 3847, 1666, 2562, 1556, 1219, 2126, 1341, 737, 1187, 2672, 4997, 2626, 2832, 1027, 2111, 4135, 179, 889, 555, 1273, 4952, 3451, 2107, 2222, 2480, 3658, 2988, 4501, 2513, 1404, 2427, 2113, 2102, 1216, 3675, 3932, 3229, 3272, 3976, 2014, 2308, 4945, 1092, 4993, 232, 167, 3720, 2751, 205, 3660, 346, 1122, 2967, 4096, 2159, 4537, 1916, 1960, 2791, 3655, 885, 4291, 2753, 3556, 3401, 3237, 4886, 916, 598, 3043, 1322, 4728, 3028, 3984, 4025, 812, 53, 4748, 2515, 3862, 2371, 3557, 3063, 4846, 2908, 2812, 4630, 3987, 1080, 2458, 185, 443, 4931, 1344, 404, 1863, 2937, 4838, 595, 757, 2263, 2685, 1852, 3798, 67, 247, 1963, 1359, 1793, 3019, 111, 954, 435, 1865, 732, 2530, 2020, 3534, 2197, 3942, 3666, 3492, 2063, 4296, 280, 4001, 4299, 3913, 2075, 3766, 3195, 3565, 4431, 434, 2062, 2644, 2279, 1270, 1530, 529, 592, 4018, 1816, 1861, 3742, 1182, 4435, 2067, 881, 3652, 4804, 2892, 80, 450, 4683, 3434, 4177, 2694, 2230, 467, 1267, 4801, 3788, 2424, 4686, 3139, 3569, 3674, 3993, 169, 1917, 1441, 2836, 2307, 3651, 743, 3445, 877, 2948, 1652, 938, 3462, 3158, 4673, 3140, 1234, 2858, 285, 3700, 3724, 2079, 4964, 4767, 3677, 4489, 1395, 1967, 4070, 3274, 4561, 416, 976, 2388, 180, 3836, 1770, 1051, 4250, 4899, 1312, 4502, 3839, 4538, 1433, 1128, 3179, 2394, 1101, 1710, 327, 1407, 1004, 4914, 256, 212, 1848, 4021, 36, 3126, 142, 134, 1464, 3157, 1293, 3122, 688, 2241, 4303, 2225, 929, 1238, 3476, 4357, 1323, 3429, 1892, 3340, 2807, 3734, 3590, 3616, 4797, 3897, 286, 4143, 3722, 934, 620, 1832, 4526, 1523, 2676, 640, 675, 3988, 2687, 2863, 557, 4490, 4292, 4774, 577, 3271, 4043, 1493, 2895, 4243, 267, 3979, 2730, 1682, 4192, 3103, 1554, 848, 4073, 3994, 4055, 4806, 2527, 3710, 3013, 64, 3904, 1566, 3530, 1826, 3007, 4432, 3965, 1432, 349, 2918, 3037, 1614, 3728, 959, 2461, 1104, 3267, 4345, 85, 1932, 3426, 717, 164, 4015, 4324, 4281, 2428, 601, 1028, 2805, 3538, 3186, 3698, 0, 1656, 3975, 2867, 3560, 4084, 4020, 118, 2332, 3552, 4206, 644, 2152, 210, 3212, 1500, 501, 1577, 1841, 2780, 3813, 3819, 2109, 1580, 3422, 2413, 3254, 410, 2360, 707, 3543, 1186, 4802, 4343, 1355, 1206, 4090, 438, 1723, 4105, 3375, 1384, 680, 4153, 4779, 3637, 4048, 2403, 3999, 4331, 3165, 469, 1539, 2262, 4372, 3721, 4588, 2877, 2041, 3259, 2290, 3650, 1742, 2199, 1592, 4616, 998, 4255, 1390, 2711, 3487, 683, 2139, 4162, 2056, 4667, 1673, 2309, 1491, 2765, 2068, 3392, 2505, 3300, 191, 4169, 1070, 3844, 1739, 2355, 2979, 3763, 4944, 4150, 3170, 2157, 3263, 836, 1526, 2004, 4975, 550, 1928, 2085, 3778, 1964, 1604, 3444, 986, 2277, 3160, 4851, 3935, 2818, 4558, 4893, 3845, 587, 1535, 903, 138, 2161, 2849, 4817, 803, 669, 4617, 4819, 3399, 2196, 3153, 4058, 4401, 3527, 492, 896, 859, 2423, 352, 311, 2436, 768, 468, 1108, 2635, 239, 4879, 712, 307, 4203, 1578, 4380, 68, 2442, 494, 3, 3346, 1629, 2697, 2949, 4992, 1276, 3107, 4414, 2288, 3188, 3903, 1509, 1631, 3853, 3894, 1965, 1542, 271, 147, 2762, 4460, 1221, 3296, 4087, 3264, 4466, 610, 2871, 3696, 2190, 1319, 2362, 548, 1193, 2560, 2745, 357, 3691, 3613, 4144, 3091, 2384, 216, 3348, 4647, 2202, 2862, 1686, 3198, 900, 1734, 3576, 1713, 2472, 89, 3647, 2686, 2744, 3796, 3516, 2927, 3416, 3047, 4287, 4808, 1366, 3316, 78, 3632, 3753, 1815, 47, 4282, 2956, 4369, 3190, 3517, 3460, 2983, 2441, 844, 4179, 3137], "validation": [5085, 5141, 5101, 5313, 5393, 5108, 5346, 5343, 5110, 5436, 5320, 5106, 5200, 5265, 5144, 5444, 5350, 5053, 5409, 5063, 5249, 5418, 5158, 5348, 5470, 5431, 5330, 5112, 5109, 5179, 5122, 5087, 5318, 5303, 5070, 5104, 5046, 5375, 5278, 5127, 5125, 5081, 5425, 5261, 5446, 5230, 5026, 5043, 5396, 5069, 5307, 5421, 5468, 5155, 5244, 5478, 5385, 5437, 5387, 5508, 5419, 5219, 5165, 5202, 5311, 5094, 5407, 5254, 5075, 5268, 5114, 5057, 5130, 5084, 5347, 5215, 5132, 5014, 5092, 5462, 5408, 5445, 5314, 5232, 5016, 5138, 5310, 5266, 5150, 5022, 5100, 5443, 5412, 5145, 5262, 5035, 5500, 5471, 5316, 5167, 5271, 5430, 5067, 5332, 5181, 5288, 5354, 5034, 5256, 5259, 5226, 5224, 5228, 5174, 5119, 5209, 5185, 5401, 5065, 5413, 5491, 5253, 5162, 5301, 5186, 5391, 5450, 5390, 5337, 5297, 5490, 5388, 5157, 5091, 5312, 5505, 5117, 5427, 5045, 5062, 5489, 5220, 5073, 5456, 5426, 5037, 5373, 5435, 5308, 5086, 5442, 5214, 5299, 5455, 5369, 5242, 5420, 5222, 5235, 5293, 5424, 5168, 5247, 5404, 5469, 5447, 5351, 5148, 5480, 5164, 5334, 5378, 5355, 5041, 5400, 5027, 5205, 5276, 5236, 5511, 5017, 5153, 5250, 5325, 5282, 5467, 5103, 5206, 5208, 5090, 5040, 5056, 5305, 5453, 5255, 5111, 5294, 5483, 5213, 5183, 5432, 5309, 5182, 5246, 5304, 5260, 5386, 5137, 5457, 5509, 5252, 5237, 5066, 5465, 5151, 5118, 5429, 5201, 5218, 5507, 5463, 5061, 5269, 5327, 5143, 5362, 5238, 5370, 5093, 5116, 5243, 5331, 5367, 5082, 5323, 5286, 5229, 5170, 5032, 5120, 5234, 5195, 5049, 5300, 5042, 5198, 5464, 5018, 5012, 5020, 5128, 5360, 5187, 5475, 5124, 5241, 5064, 5374, 5402, 5298, 5315, 5458, 5283, 5038, 5274, 5275, 5257, 5177, 5136, 5494, 5379, 5060, 5460, 5270, 5441, 5204, 5417, 5296, 5059, 5359, 5339, 5481, 5482, 5121, 5154, 5263, 5361, 5357, 5451, 5193, 5015, 5024, 5474, 5454, 5397, 5333, 5097, 5199, 5322, 5251, 5376, 5495, 5098, 5030, 5239, 5139, 5019, 5044, 5280, 5440, 5273, 5051, 5099, 5368, 5349, 5493, 5277, 5365, 5439, 5459, 5033, 5173, 5302, 5448, 5172, 5052, 5166, 5225, 5129, 5207, 5095, 5422, 5079, 5245, 5223, 5048, 5221, 5356, 5403, 5096, 5326, 5142, 5372, 5189, 5279, 5405, 5502, 5233, 5113, 5068, 5013, 5191, 5217, 5290, 5353, 5047, 5506, 5394, 5344, 5358, 5180, 5102, 5498, 5115, 5203, 5488, 5415, 5501, 5163, 5126, 5077, 5058, 5289, 5105, 5477, 5140, 5055, 5285, 5476, 5123, 5340, 5338, 5292, 5389, 5175, 5398, 5039, 5212, 5381, 5192, 5328, 5434, 5210, 5080, 5452, 5287, 5438, 5324, 5135, 5380, 5319, 5416, 5264, 5341, 5382, 5025, 5089, 5147, 5267, 5169, 5336, 5076, 5216, 5161, 5248, 5284, 5272, 5466, 5160, 5423, 5258, 5473, 5352, 5410, 5342, 5472, 5364, 5231, 5178, 5152, 5399, 5366, 5414, 5306, 5023, 5159, 5461, 5392, 5497, 5433, 5184, 5329, 5486, 5071, 5503, 5146, 5449, 5395, 5078, 5190, 5196, 5227, 5295, 5021, 5134, 5479, 5496, 5485, 5036, 5197, 5492, 5107, 5428, 5194, 5074, 5510, 5371, 5029, 5499, 5504, 5054, 5281, 5176, 5188, 5028, 5131, 5156, 5484, 5072, 5050, 5321, 5335, 5240, 5384, 5133, 5345, 5171, 5377, 5088, 5031, 5363, 5291, 5211, 5406, 5411, 5487, 5317, 5083, 5149, 5383], "test": [5807, 5788, 5536, 5744, 5742, 5673, 5924, 5828, 5605, 5787, 5550, 5945, 5654, 5643, 6006, 5674, 5795, 5923, 5782, 5869, 5530, 5660, 5760, 5602, 5759, 5641, 5838, 5686, 5890, 5915, 6008, 5730, 5732, 5796, 5717, 5844, 5739, 5802, 5672, 5821, 5611, 5938, 5861, 5794, 5937, 5549, 5957, 5554, 5685, 5907, 5981, 5982, 5799, 5933, 5774, 5947, 5968, 5632, 5556, 6004, 5846, 5871, 5577, 5990, 5719, 5590, 5565, 5925, 6009, 5870, 5769, 5737, 5831, 5789, 5790, 5675, 5649, 5627, 5822, 5765, 5600, 5585, 5882, 5934, 5881, 5943, 5879, 5814, 5952, 5575, 5784, 5895, 5999, 5987, 5655, 5902, 5996, 5548, 5523, 5858, 5886, 5520, 5900, 5785, 5916, 5960, 5638, 5772, 5606, 5555, 5984, 5690, 5920, 5566, 5962, 5884, 5664, 5679, 5905, 5584, 5639, 5599, 5701, 5708, 5539, 5607, 5534, 5941, 5716, 5688, 5998, 5841, 5715, 5959, 5773, 5706, 5949, 5522, 5692, 5904, 6011, 5883, 5705, 5988, 5618, 5992, 5948, 5544, 5872, 5961, 5581, 5542, 5777, 5615, 5892, 5853, 5813, 5620, 5989, 5580, 5845, 5894, 5562, 5977, 5762, 5712, 5636, 5604, 5936, 5852, 5647, 5613, 5592, 5560, 5714, 5847, 5843, 5854, 5750, 5912, 5650, 5995, 5537, 5927, 5889, 5919, 5899, 5823, 5864, 6003, 5896, 5874, 5622, 5922, 5816, 5663, 5621, 5893, 5770, 5629, 5970, 5753, 5812, 5837, 5815, 5652, 5576, 5860, 5630, 5569, 5574, 5835, 5898, 5921, 5619, 5532, 5819, 5612, 5763, 5867, 5829, 5986, 5557, 5866, 5624, 5741, 5689, 5720, 5667, 5735, 5966, 5631, 5803, 5642, 5825, 5985, 5538, 5979, 5935, 5951, 5751, 5836, 5573, 5914, 5834, 5726, 5906, 5723, 5699, 5662, 5958, 5965, 5793, 5833, 5531, 5761, 5878, 5525, 5512, 5808, 5909, 5976, 5529, 5786, 5792, 5766, 5698, 5755, 5875, 5583, 5597, 5749, 5768, 5946, 5963, 5596, 5736, 6002, 5601, 5588, 5972, 5582, 5593, 5659, 5680, 5665, 5771, 5856, 5940, 5830, 5521, 5865, 5997, 6005, 5930, 5727, 5779, 5614, 5908, 5885, 5944, 5661, 5964, 5656, 5804, 5928, 5891, 5728, 5848, 5528, 5722, 5578, 5748, 5778, 5857, 5651, 5684, 5653, 5670, 5994, 5696, 5517, 5975, 5526, 5991, 5693, 5695, 5731, 5859, 5563, 5677, 5820, 5710, 5983, 5811, 5910, 5700, 5775, 5876, 5702, 5971, 5635, 5950, 5567, 5840, 5711, 5901, 5850, 5724, 5877, 5973, 5541, 5564, 5676, 5682, 5839, 5551, 5725, 5863, 5805, 5897, 6007, 5625, 5954, 5967, 5926, 5591, 5809, 5668, 5678, 5913, 5862, 5818, 5746, 5832, 5644, 5669, 5533, 5733, 5791, 5842, 5616, 5657, 5756, 5754, 5797, 5568, 5594, 5745, 5697, 5587, 5718, 5955, 5603, 5887, 5524, 5953, 5929, 5707, 5993, 5764, 5880, 5956, 5598, 6010, 5691, 5553, 5608, 5767, 5527, 5687, 5855, 5586, 5709, 5609, 5658, 5747, 5552, 5570, 5911, 5781, 5780, 5783, 5648, 5801, 5547, 5623, 5579, 5518, 5873, 6000, 5589, 5535, 5681, 5694, 5800, 5640, 5740, 5646, 5939, 5826, 5974, 6001, 5637, 5558, 5721, 5645, 5917, 5513, 5571, 5776, 5757, 5559, 5683, 5743, 5628, 5817, 5713, 5634, 5515, 5610, 5729, 5617, 5978, 5734, 5824, 5851, 5703, 5595, 5942, 5704, 5810, 5626, 5561, 5633, 5903, 5758, 5888, 5738, 5545, 5540, 5827, 5931, 5543, 5980, 5969, 5516, 5546, 5671, 5798, 5666, 5849, 5752, 5932, 5918, 5868, 5572, 5806, 5514, 5519]}, {"train": [2531, 1666, 4162, 1476, 4762, 2347, 4461, 941, 3932, 4171, 3124, 3181, 4822, 2540, 4145, 4357, 3004, 612, 1923, 2747, 3883, 572, 2446, 630, 729, 2750, 1204, 1528, 827, 3077, 2214, 1924, 3698, 2762, 916, 156, 2671, 4248, 3758, 2185, 442, 3985, 4643, 130, 4801, 1787, 1829, 860, 3541, 4947, 1456, 191, 2506, 117, 3166, 2664, 4611, 4013, 1525, 1872, 4782, 3470, 3926, 2108, 3589, 3478, 2488, 1414, 3445, 3227, 2760, 1585, 1284, 625, 648, 3996, 4348, 2939, 4269, 4230, 4300, 1249, 2102, 3009, 4729, 4615, 4749, 6, 4000, 943, 3830, 4119, 3490, 2694, 239, 2418, 4080, 2567, 2421, 2846, 2243, 2766, 2144, 594, 3974, 4814, 1485, 3784, 3516, 4659, 2655, 3068, 1691, 2726, 3392, 3694, 4191, 3864, 2701, 3074, 516, 4780, 3006, 2625, 2666, 4276, 229, 4596, 420, 2921, 4389, 4874, 4925, 24, 3912, 1026, 2954, 608, 866, 236, 3648, 523, 1831, 2768, 3449, 879, 1334, 1175, 353, 2018, 1282, 4957, 4787, 4018, 1933, 1413, 4688, 3164, 326, 3278, 3215, 2462, 4622, 2752, 417, 170, 3225, 2428, 3859, 2881, 2657, 4540, 2686, 3143, 2942, 3488, 3804, 4369, 3306, 4121, 2695, 847, 4359, 363, 526, 338, 1226, 547, 4949, 3535, 3975, 933, 2720, 1917, 4969, 4395, 3908, 2566, 4826, 2804, 993, 2783, 2216, 3199, 182, 1214, 2728, 4310, 4534, 2035, 1401, 3208, 3594, 2306, 4197, 597, 2309, 1292, 730, 1820, 2352, 325, 2556, 73, 619, 2092, 2979, 4399, 1742, 3133, 3138, 3942, 3277, 3393, 2192, 975, 106, 2796, 143, 3790, 2926, 3051, 1227, 3473, 4457, 2220, 4353, 1512, 2885, 4823, 4964, 1540, 3482, 1069, 2872, 40, 1789, 3944, 1081, 479, 4125, 658, 3300, 1888, 3186, 4859, 2167, 1526, 4680, 1590, 668, 2932, 4385, 188, 246, 2775, 131, 1575, 4231, 4115, 93, 4945, 4512, 139, 3222, 809, 4533, 4890, 1714, 4648, 4531, 2400, 2326, 3426, 1255, 2602, 2364, 5011, 1233, 1552, 774, 515, 4290, 2084, 331, 2635, 3921, 3831, 925, 1801, 4351, 1107, 3098, 3103, 4153, 1352, 2276, 1910, 1760, 2886, 1521, 4982, 3656, 4261, 3113, 2992, 445, 3067, 4400, 2067, 4375, 583, 1302, 4896, 1036, 4829, 3986, 1491, 1100, 1891, 3767, 3265, 1301, 1213, 4968, 4066, 818, 449, 4309, 4499, 4345, 1412, 1723, 4442, 3561, 1674, 1230, 443, 861, 109, 4379, 2641, 2968, 1435, 1397, 386, 4847, 2, 372, 2993, 2480, 4803, 707, 1849, 1724, 1527, 3123, 932, 2063, 4709, 4560, 2149, 26, 1053, 4802, 1702, 3266, 336, 4990, 2211, 1369, 527, 528, 2135, 3349, 2162, 3688, 4722, 610, 2902, 3323, 865, 3735, 1771, 119, 4865, 4955, 2809, 8, 1763, 3666, 4022, 3937, 4527, 3279, 3439, 4415, 4751, 1740, 2794, 4339, 2093, 3264, 4216, 2619, 4294, 292, 1961, 2402, 4637, 1877, 4189, 4298, 79, 1105, 3894, 3280, 4913, 4578, 4449, 3406, 3303, 2524, 4785, 4410, 3334, 85, 3832, 3884, 4325, 276, 3289, 4583, 4106, 1880, 731, 4713, 3453, 4071, 4109, 1095, 4747, 1661, 2420, 1164, 3723, 3022, 3960, 4371, 4467, 2630, 2931, 4970, 4971, 3786, 4078, 779, 3053, 596, 2543, 743, 3737, 3169, 1433, 190, 1773, 2908, 3239, 4328, 1846, 3847, 1547, 2568, 2937, 3307, 651, 3893, 3274, 2733, 1776, 1488, 1206, 1378, 1025, 2456, 2042, 4273, 4437, 4438, 3504, 875, 4686, 2123, 484, 2601, 534, 775, 3865, 3655, 2186, 1453, 3560, 1556, 4002, 4996, 2269, 2212, 4630, 2273, 2986, 542, 3609, 3539, 1296, 1874, 3355, 3957, 2176, 475, 2981, 570, 4537, 3348, 2716, 305, 3120, 81, 308, 448, 3647, 2127, 3395, 1349, 2504, 1278, 4301, 102, 692, 2138, 755, 695, 4044, 1570, 431, 1513, 2457, 2948, 2101, 4669, 1647, 3050, 3214, 2451, 4833, 1059, 2023, 2550, 1758, 2301, 3327, 4154, 2343, 2389, 1318, 620, 2411, 3880, 3977, 3030, 4818, 4723, 3889, 739, 3292, 1916, 4349, 4690, 3669, 956, 1558, 3272, 213, 233, 3389, 605, 2339, 986, 826, 2213, 497, 824, 1188, 3653, 1733, 2542, 659, 1852, 2399, 766, 3296, 4113, 3596, 3197, 3662, 1375, 1077, 3399, 1553, 4474, 1128, 4679, 1197, 3011, 2351, 288, 1376, 3147, 1468, 3247, 1873, 313, 116, 1029, 3000, 3184, 1858, 4255, 4956, 501, 2746, 2485, 140, 1989, 1589, 654, 873, 1219, 68, 3570, 3522, 4484, 2240, 680, 4627, 1160, 2064, 2548, 586, 3794, 2890, 3357, 1060, 361, 900, 4952, 3330, 780, 35, 1443, 1345, 2465, 2646, 4030, 2078, 3741, 2587, 4529, 3065, 2299, 2434, 2187, 1155, 3616, 2549, 1314, 2225, 203, 1444, 3192, 1819, 2613, 3236, 2780, 4901, 3760, 2547, 194, 930, 1591, 3027, 4642, 1969, 3246, 356, 2827, 4317, 3777, 3149, 3443, 1464, 207, 4211, 3956, 7, 4572, 4773, 2285, 37, 3818, 693, 3641, 3365, 3901, 4555, 1235, 3481, 4573, 2172, 3911, 1886, 333, 3540, 2966, 1030, 1701, 2591, 1631, 1862, 1478, 2091, 2816, 682, 2965, 2379, 3757, 3233, 4995, 921, 2617, 4923, 1055, 304, 4373, 3746, 4702, 1410, 4824, 3713, 1623, 2681, 1902, 4553, 1864, 2870, 1490, 343, 3639, 1822, 1047, 477, 4089, 2705, 4450, 4965, 1868, 1721, 717, 3928, 4897, 433, 16, 3340, 1441, 3223, 3377, 834, 1616, 1034, 4592, 590, 39, 4332, 2011, 1802, 3213, 4215, 99, 3086, 1656, 4299, 2987, 399, 4547, 3899, 1427, 4296, 2813, 275, 3771, 1114, 4245, 4963, 1028, 3435, 2303, 2054, 444, 1056, 1516, 1389, 2930, 4745, 3533, 960, 1515, 3024, 2239, 3409, 3025, 4302, 196, 1264, 3176, 2598, 2271, 4278, 2711, 1361, 3761, 4915, 2014, 2160, 2378, 3371, 772, 4102, 3064, 3848, 2593, 4880, 3869, 511, 2338, 4746, 3699, 4509, 1366, 646, 1731, 4264, 1821, 2588, 4039, 4808, 300, 3487, 3825, 1407, 2270, 4393, 3373, 3391, 1741, 2030, 3747, 553, 3465, 1138, 708, 2843, 3538, 1646, 3664, 3881, 3358, 4790, 4096, 531, 4891, 3812, 1843, 4218, 983, 3375, 1373, 1598, 679, 845, 4838, 3382, 2758, 164, 36, 1698, 4172, 1593, 1965, 3131, 1737, 3333, 3152, 199, 4263, 4136, 4293, 2369, 299, 3530, 2583, 2278, 403, 1448, 1950, 1234, 4134, 1931, 2941, 4916, 1221, 4594, 2341, 2453, 670, 332, 2907, 4454, 2247, 3162, 5009, 2889, 744, 1870, 1941, 1313, 2032, 3983, 4282, 1180, 2390, 3872, 2002, 495, 514, 1243, 984, 1606, 2700, 3384, 4202, 1571, 4807, 1211, 478, 1147, 529, 2393, 2191, 4868, 538, 1041, 1840, 889, 3608, 848, 4658, 4789, 340, 1185, 2812, 3512, 4506, 4884, 3168, 1919, 1613, 1692, 1832, 366, 1514, 3178, 656, 2632, 4934, 4037, 1665, 624, 2595, 2491, 3876, 4011, 41, 3567, 1106, 2738, 2300, 4867, 3257, 128, 107, 2157, 441, 3948, 265, 2774, 4190, 1280, 1371, 1202, 1304, 1421, 235, 1262, 1367, 3174, 1800, 2624, 173, 218, 145, 4413, 249, 134, 435, 980, 1524, 3497, 2697, 1439, 4382, 1618, 4525, 2964, 2611, 1058, 4711, 4221, 2165, 4760, 4948, 3191, 4743, 2608, 705, 1507, 2914, 1132, 1812, 1535, 2337, 2745, 4862, 3895, 3501, 1276, 760, 2815, 2444, 3431, 4376, 715, 3529, 716, 3148, 2721, 1673, 2898, 3260, 776, 4236, 4781, 4035, 294, 814, 4062, 1648, 4851, 1944, 1193, 1450, 2006, 498, 2633, 2814, 4166, 2388, 4411, 1297, 2963, 1087, 3416, 2155, 2498, 115, 3545, 990, 2137, 1817, 3649, 2706, 1294, 2696, 324, 3676, 2779, 3782, 4950, 3976, 467, 4220, 2673, 4324, 2261, 1826, 3618, 2177, 2530, 2039, 2439, 2665, 3010, 3146, 2971, 410, 2770, 136, 2708, 4034, 3780, 2729, 2972, 1685, 2128, 539, 1113, 2884, 4653, 4655, 1471, 4944, 1275, 1804, 295, 3251, 4185, 2289, 3726, 2238, 4286, 4451, 490, 1348, 225, 421, 3532, 4275, 3092, 1790, 1511, 522, 3759, 226, 314, 2474, 4478, 45, 95, 3087, 3404, 3484, 1894, 2918, 3525, 2021, 1409, 4873, 368, 1065, 3796, 4430, 4739, 389, 3511, 2911, 4315, 1824, 2260, 1534, 1390, 4541, 1201, 491, 1530, 1881, 1903, 1735, 1604, 3196, 2207, 2041, 168, 3281, 4899, 1638, 3140, 1770, 186, 2702, 3962, 636, 1049, 367, 4009, 423, 3019, 4112, 510, 3469, 4110, 1247, 1341, 4875, 291, 1850, 1687, 4466, 2333, 4268, 2181, 3434, 2974, 1159, 745, 3220, 4974, 2710, 1588, 4783, 2099, 2609, 1273, 2848, 298, 2503, 613, 1930, 4152, 3255, 934, 1700, 2869, 3483, 3800, 1372, 2113, 839, 840, 3430, 830, 3566, 379, 833, 4718, 4267, 2408, 3228, 1696, 2718, 2248, 5005, 880, 3256, 2471, 4073, 4954, 3576, 4562, 4423, 3170, 428, 1098, 2359, 1258, 474, 3509, 4204, 1300, 1728, 2709, 940, 4210, 4703, 212, 3013, 1643, 662, 2649, 3503, 3455, 4504, 1605, 2427, 585, 4471, 607, 1018, 4819, 4988, 3432, 2741, 4094, 2034, 1993, 3163, 3325, 645, 1309, 1537, 354, 4927, 1458, 400, 4297, 1720, 2473, 3194, 3210, 1040, 3909, 4008, 1668, 1876, 4842, 3779, 2254, 2748, 231, 274, 278, 2001, 571, 2414, 1517, 2050, 899, 638, 309, 2508, 1667, 890, 1650, 1353, 1679, 4156, 504, 3405, 4485, 4462, 805, 4448, 4104, 4180, 1129, 3810, 4650, 83, 4016, 4675, 3054, 747, 545, 2253, 75, 1350, 1254, 4440, 3580, 4036, 4530, 4107, 1835, 3548, 2988, 3368, 3002, 4800, 1442, 4095, 2118, 2494, 740, 118, 4936, 4344, 1396, 1901, 1531, 2523, 2057, 1215, 2280, 4056, 3474, 2928, 4542, 1921, 881, 796, 2340, 2600, 3887, 2998, 2756, 20, 1311, 1119, 1118, 4714, 1792, 1784, 32, 148, 2529, 677, 3868, 472, 1985, 3506, 2249, 1324, 870, 3317, 2496, 3114, 2189, 2245, 60, 675, 3101, 3028, 2995, 1629, 2912, 698, 2967, 1963, 1274, 4367, 409, 3508, 3578, 3637, 3582, 4544, 4464, 2110, 2692, 2622, 318, 4699, 3626, 1970, 1842, 162, 2020, 3219, 4754, 3104, 720, 215, 712, 1523, 606, 4536, 4118, 4568, 14, 3212, 180, 3625, 3546, 4425, 1898, 1003, 255, 1354, 2336, 2295, 1157, 1130, 3690, 127, 1544, 2991, 540, 23, 872, 330, 4883, 3946, 2077, 2372, 2819, 4479, 4306, 4057, 4633, 3339, 4445, 2470, 604, 166, 1729, 4727, 1786, 1807, 1327, 1328, 964, 507, 1781, 1033, 4662, 1799, 2200, 2009, 3965, 2810, 3135, 1502, 4811, 991, 3834, 3188, 4186, 4656, 2070, 4356, 3762, 3640, 3018, 3661, 919, 1935, 1074, 3850, 2143, 911, 2824, 1434, 1244, 4091, 4570, 4520, 4854, 3592, 2629, 290, 2867, 4701, 4958, 857, 4864, 3663, 1625, 2449, 2024, 637, 1897, 1246, 4088, 2698, 3385, 370, 1682, 2151, 4991, 1798, 1744, 3268, 352, 3446, 4407, 4750, 1237, 1592, 3493, 3615, 1913, 4538, 1223, 4486, 4843, 4446, 4237, 2433, 2043, 2784, 2395, 2328, 3963, 1002, 2899, 3801, 1438, 3598, 1839, 2194, 2980, 380, 374, 2383, 1283, 3343, 4770, 3740, 80, 678, 53, 4394, 2915, 4414, 192, 1317, 3528, 2690, 931, 4719, 2955, 3686, 89, 4928, 825, 1279, 38, 1655, 1076, 4476, 2068, 4597, 4005, 2858, 2874, 200, 1717, 2422, 3930, 499, 1576, 4168, 4419, 4728, 2883, 334, 3828, 4043, 1861, 358, 2605, 2512, 2501, 57, 3907, 2350, 1580, 2499, 2668, 2012, 1392, 3719, 3692, 4628, 46, 2179, 2318, 938, 1564, 176, 2546, 4404, 3898, 4028, 2044, 4654, 3342, 2158, 2742, 2734, 3185, 4870, 2704, 2559, 2394, 1044, 1449, 2131, 2820, 4111, 4392, 2518, 799, 4494, 2959, 2296, 2242, 1562, 1212, 1813, 1174, 530, 2866, 2805, 3254, 2853, 3814, 3189, 2342, 3457, 3590, 578, 4077, 4906, 3674, 1220, 4574, 2090, 1640, 4835, 3718, 5006, 1853, 4234, 1207, 3347, 1649, 296, 560, 3953, 773, 2581, 3452, 713, 4495, 2951, 821, 3595, 2029, 3144, 4219, 2156, 2545, 165, 3685, 1654, 4489, 1907, 2736, 4691, 4386, 2104, 3139, 3232, 232, 929, 4488, 2119, 3964, 91, 4586, 3557, 685, 63, 3959, 1501, 541, 4879, 1143, 1184, 2244, 770, 3995, 1927, 3438, 3412, 1966, 2308, 2901, 398, 284, 1383, 4181, 684, 3813, 3182, 1323, 3346, 807, 4844, 1780, 1630, 4744, 1101, 1356, 1752, 1732, 1285, 3344, 4999, 4639, 1855, 2888, 223, 1122, 4046, 1795, 548, 4014, 4161, 3707, 259, 1601, 4439, 1686, 1885, 4143, 1426, 1992, 1494, 2945, 998, 722, 4672, 427, 1779, 4848, 4772, 174, 171, 4333, 439, 4720, 1382, 2946, 1195, 2592, 3997, 3047, 123, 1310, 3774, 2759, 3206, 592, 2558, 1400, 4707, 1725, 3423, 3954, 3742, 4557, 4634, 2628, 1380, 4087, 1983, 4810, 629, 1169, 1343, 2970, 810, 121, 2307, 556, 3345, 581, 3428, 1330, 4326, 1293, 1990, 525, 2059, 3882, 1305, 649, 3517, 1120, 1671, 665, 1614, 4105, 4603, 1269, 2377, 676, 2154, 2500, 0, 552, 3095, 761, 3703, 2022, 2040, 2984, 4370, 3827, 3297, 1216, 4724, 869, 1475, 2603, 2314, 4559, 1031, 1569, 3823, 4150, 238, 3724, 1533, 2046, 1722, 3183, 2432, 2237, 452, 58, 565, 483, 376, 2204, 3991, 4973, 4093, 4599, 2272, 3755, 1908, 201, 3555, 784, 1871, 562, 228, 1139, 1337, 4435, 90, 1710, 4065, 996, 381, 13, 1610, 888, 489, 3242, 3902, 2403, 253, 831, 2687, 1909, 1793, 1198, 1424, 3125, 3644, 3007, 4507, 394, 2943, 2782, 4358, 4912, 4550, 256, 4320, 4083, 4959, 2553, 2445, 4905, 1896, 2662, 4663, 1032, 1088, 1038, 1067, 2577, 1803, 1506, 1619, 277, 2522, 3066, 2882, 3293, 214, 2487, 4099, 3643, 1532, 4681, 4609, 4601, 1217, 4277, 3611, 777, 733, 4677, 2086, 1972, 2823, 2584, 351, 2082, 2017, 4015, 3635, 1218, 458, 3572, 1703, 1637, 2290, 4217, 1996, 3172, 1395, 4872, 1339, 2560, 1599, 177, 2437, 2919, 1639, 1561, 3394, 1565, 2062, 871, 2982, 3947, 4985, 424, 1151, 4178, 3950, 2526, 1620, 3187, 1713, 3356, 883, 1683, 2344, 4042, 4398, 438, 3687, 3229, 4894, 2840, 1329, 4755, 3851, 3073, 3410, 2056, 4468, 1379, 3744, 885, 577, 2304, 4841, 836, 4469, 798, 3366, 2606, 335, 4092, 3413, 4483, 234, 1167, 3379, 1755, 3745, 2376, 1131, 4027, 1222, 757, 476, 2786, 3156, 1393, 1340, 2975, 4327, 34, 4576, 396, 2495, 101, 2795, 3057, 1430, 2509, 146, 2873, 137, 2571, 1253, 322, 3792, 922, 2900, 2215, 2013, 4458, 4343, 2642, 1342, 2682, 3388, 3938, 1699, 2100, 759, 3491, 391, 2436, 4137, 1090, 3768, 461, 3134, 3766, 1008, 496, 843, 3283, 4188, 454, 3706, 1263, 3311, 1385, 2331, 3315, 2667, 1765, 1374, 413, 4195, 462, 2450, 4501, 1111, 1315, 4120, 3943, 3076, 4496, 1322, 1007, 4610, 723, 641, 1672, 4303, 3226, 1573, 3524, 3798, 125, 3933, 2410, 1982, 3973, 4825, 4836, 1104, 928, 2405, 2801, 584, 487, 4674, 2005, 4481, 546, 994, 2541, 3360, 3645, 3119, 4561, 623, 2799, 2626, 4735, 2927, 1173, 535, 832, 4010, 3654, 1912, 2455, 3705, 3536, 4241, 1615, 2265, 2442, 952, 3500, 2798, 1863, 64, 419, 293, 4565, 3554, 482, 1203, 3476, 1772, 4607, 4907, 1709, 626, 4820, 521, 4741, 4635, 2183, 3046, 2224, 617, 4715, 2320, 1582, 4460, 1363, 224, 1166, 3243, 3749, 3383, 1602, 2790, 1557, 4877, 1387, 2675, 819, 2107, 355, 3341, 4350, 3568, 3917, 3313, 3922, 4403, 3180, 2623, 4155, 2406, 2576, 703, 3203, 486, 704, 1988, 2188, 1911, 4175, 342, 122, 4053, 1295, 4238, 4863, 1607, 1416, 4274, 4581, 3171, 3079, 2961, 2578, 4064, 2585, 2277, 2319, 3190, 4670, 580, 567, 451, 144, 1068, 1762, 852, 3198, 447, 1148, 1015, 3472, 2825, 3787, 4067, 1641, 2440, 2051, 4758, 1859, 1546, 2829, 603, 4612, 3507, 66, 2142, 2562, 709, 1411, 4938, 4591, 3807, 2130, 2407, 2136, 3151, 329, 2658, 4664, 327, 4792, 1066, 3036, 2230, 100, 4396, 208, 4892, 3020, 48, 3966, 4855, 2887, 2652, 1566, 4352, 1676, 1543, 615, 270, 3651, 1834, 4705, 1308, 4778, 2055, 328, 3697, 3672, 1942, 2677, 76, 2909, 2852, 3700, 4187, 4849, 4908, 492, 4360, 3029, 808, 1009, 2817, 92, 1922, 1208, 3838, 1071, 2773, 74, 914, 3712, 2182, 3695, 4858, 1172, 1473, 2614, 3904, 640, 2478, 4569, 1096, 1127, 2620, 4135, 2297, 84, 2722, 2140, 4340, 2416, 3471, 4812, 1609, 4361, 142, 3714, 600, 4052, 2292, 2417, 817, 1115, 3403, 1567, 158, 1250, 2322, 4580, 4816, 404, 4977, 3765, 1751, 3084, 1806, 4805, 4280, 563, 3835, 4256, 2481, 1256, 244, 669, 3537, 795, 1466, 4992, 2316, 2423, 794, 3451, 55, 3361, 1879, 1086, 3822, 4930, 4314, 3878, 4383, 3998, 711, 113, 574, 4584, 4806, 1079, 455, 3102, 3324, 1470, 3826, 4975, 3631, 4777, 3665, 862, 579, 3069, 1624, 4006, 4103, 2850, 2731, 1291, 2033, 886, 1953, 4108, 4726, 4616, 2362, 2345, 4989, 3788, 221, 1368, 2557, 979, 1070, 1266, 2950, 4535, 2466, 3978, 3179, 187, 470, 4983, 2227, 1680, 1014, 1716, 2028, 3906, 1446, 1057, 3910, 2751, 4768, 2483, 3874, 800, 425, 1358, 2925, 2139, 1797, 446, 2938, 3702, 2844, 1750, 3286, 3969, 3619, 3200, 963, 2962, 3319, 4660, 3386, 1082, 3287, 4003, 1836, 3032, 3285, 263, 18, 3970, 1156, 3083, 10, 894, 1224, 4020, 4791, 3871, 2893, 4051, 989, 150, 1497, 2836, 3350, 1178, 4250, 2355, 4342, 4432, 3088, 250, 4207, 4493, 537, 1508, 3454, 383, 5, 4498, 1937, 962, 70, 1603, 3290, 4832, 509, 4900, 558, 891, 4828, 4337, 2166, 1136, 3776, 4243, 1681, 1194, 4813, 47, 1746, 2905, 3805, 1635, 1791, 1973, 3005, 316, 3160, 4311, 1051, 4212, 2802, 4972, 178, 4412, 3305, 2159, 751, 262, 850, 849, 2535, 4490, 347, 4258, 4689, 1346, 936, 3646, 2235, 2684, 946, 3924, 465, 2580, 3299, 4338, 3258, 3729, 3754, 1734, 4939, 3284, 2771, 4026, 344, 1597, 3109, 816, 2640, 4182, 1938, 2288, 3038, 595, 2904, 44, 2839, 2038, 4061, 3670, 3842, 4431, 1176, 2217, 1595, 2507, 2112, 2565, 3890, 3715, 273, 418, 4049, 4441, 1080, 3468, 1013, 77, 2723, 3417, 4575, 1981, 2256, 1838, 4447, 2807, 882, 1420, 3939, 3862, 2977, 1663, 647, 627, 306, 4259, 3240, 3362, 1360, 51, 2891, 4693, 4229, 4424, 2880, 1695, 4831, 1149, 910, 3987, 3016, 3015, 1268, 1548, 1706, 4887, 4926, 4023, 2989, 4433, 4068, 1510, 898, 3209, 480, 3829, 797, 4270, 4590, 1818, 1332, 1316, 2396, 4978, 3617, 1331, 2894, 4796, 1457, 216, 4979, 1123, 1078, 2346, 3816, 181, 735, 4856, 4551, 4626, 3905, 350, 4233, 1869, 3216, 710, 3678, 4434, 189, 362, 2016, 3918, 2382, 2544, 4147, 3444, 4033, 4232, 4943, 3993, 3795, 787, 4725, 4167, 1578, 2572, 1825, 2917, 3949, 4149, 4427, 555, 2095, 3811, 393, 11, 1153, 4543, 3096, 706, 4882, 242, 2997, 3485, 3128, 2472, 3789, 1165, 4305, 3591, 3750, 4667, 1091, 377, 2842, 2268, 2198, 3316, 4625, 673, 416, 3407, 323, 970, 4372, 2262, 1054, 87, 3633, 954, 185, 4621, 42, 653, 4165, 4341, 2976, 4240, 4984, 4717, 3733, 2645, 3896, 3498, 3684, 69, 628, 1306, 2461, 387, 3667, 1904, 2266, 2634, 2532, 1808, 4909, 1948, 2327, 3763, 2573, 3603, 4850, 4567, 1559, 2241, 72, 3999, 4176, 2749, 2387, 261, 1830, 1978, 987, 3709, 4472, 995, 1429, 550, 3364, 184, 254, 459, 559, 1272, 1563, 2106, 1052, 3756, 1265, 2163, 2047, 4251, 2334, 258, 2643, 1024, 918, 2251, 2096, 3001, 2053, 4585, 2683, 532, 2031, 217, 2236, 2497, 4198, 2115, 3224, 961, 147, 1775, 1061, 4101, 301, 4756, 2026, 4426, 727, 2596, 1191, 2195, 4837, 1945, 3602, 19, 3936, 4910, 3458, 3628, 1043, 2724, 4252, 2010, 2808, 4641, 3424, 3920, 2525, 4422, 4600, 2283, 337, 3271, 4776, 3331, 561, 1050, 3551, 4646, 1577, 3211, 3638, 1125, 1190, 3689, 3237, 4795, 1459, 4017, 384, 222, 948, 3376, 3021, 460, 4765, 1399, 1326, 4329, 4606, 3992, 2703, 3155, 2133, 4122, 4598, 4465, 967, 2419, 4666, 3173, 519, 903, 12, 227, 3982, 4516, 4040, 999, 4678, 4671, 3778, 1016, 1205, 2229, 2476, 1386, 2868, 2575, 395, 3886, 3408, 2511, 1925, 3817, 2152, 691, 1749, 3248, 4116, 1626, 1005, 88, 453, 4129, 2999, 2302, 321, 2274, 4387, 3564, 1347, 1124, 789, 748, 1955, 1472, 3892, 544, 2637, 3821, 3559, 1465, 3837, 1126, 3072, 3116, 3866, 1245, 935, 2076, 27, 2715, 1239, 3363, 1001, 683, 3600, 635, 4024, 1747, 503, 103, 3952, 977, 1542, 3730, 3337, 2025, 3604, 4889, 536, 2924, 1900, 179, 1298, 3627, 2398, 2713, 4038, 289, 3734, 279, 3244, 2679, 2994, 2903, 3372, 1403, 1943, 4307, 3, 1431, 2957, 1865, 67, 4786, 3586, 841, 1991, 672, 4227, 2141, 31, 1141, 429, 3322, 3900, 2610, 3675, 4881, 2184, 268, 2978, 924, 1947, 2233, 2755, 4706, 736, 2353, 4029, 920, 1761, 1505, 978, 1823, 3502, 2735, 3722, 1493, 4206, 867, 1766, 3396, 2861, 1572, 4761, 687, 3622, 3121, 2071, 851, 4757, 4698, 3014, 151, 3553, 1418, 1492, 3558, 4323, 3145, 3727, 844, 2000, 3335, 3751, 2175, 3620, 4638, 3891, 2826, 2647, 3374, 2467, 3238, 2105, 4518, 664, 1753, 3433, 3205, 1182, 4503, 4532, 1999, 2098, 973, 311, 198, 2124, 1487, 3459, 3728, 968, 2514, 3221, 4321, 3806, 1121, 3425, 4148, 1000, 893, 909, 4636, 2401, 3062, 756, 4704, 4492, 3888, 4377, 1154, 4025, 4554, 2923, 3448, 4183, 175, 4164, 512, 3877, 4857, 657, 3436, 2803, 2452, 3738, 3971, 388, 3217, 2122, 4766, 4871, 3318, 3091, 823, 1010, 3060, 2563, 2392, 4623, 4048, 3588, 426, 2206, 4114, 2121, 1062, 98, 412, 3100, 2134, 3245, 690, 905, 2279, 4313, 838, 3048, 247, 2831, 3856, 1743, 1006, 2430, 2537, 3023, 471, 1404, 4397, 219, 1027, 738, 2109, 4004, 4142, 3175, 4539, 1236, 2793, 1422, 2693, 1705, 1232, 1884, 4931, 141, 5000, 1289, 4929, 2837, 2659, 3849, 1461, 3045, 753, 2983, 3988, 2170, 3693, 1645, 4779, 4075, 674, 4124, 450, 56, 3785, 1133, 4123, 5004, 149, 1210, 2822, 3273, 2479, 3329, 4146, 4652, 481, 4487, 2527, 183, 3958, 3282, 4798, 3259, 1406, 3479, 3731, 1145, 4845, 3840, 3089, 3845, 4997, 4346, 3058, 3941, 1012, 3419, 2761, 4966, 901, 493, 2468, 2913, 2574, 1689, 2922, 282, 3308, 4254, 3769, 54, 1608, 2854, 319, 1075, 4239, 1161, 3359, 3606, 3534, 241, 2676, 94, 837, 4937, 1727, 3773, 2528, 3544, 2219, 2196, 205, 4721, 2486, 1769, 1171, 3336, 4604, 1267, 405, 4364, 4475, 28, 3158, 3660, 957, 4566, 3167, 434, 2037, 2116, 3157, 591, 3492, 4804, 2906, 4885, 4368, 1168, 4893, 1658, 3105, 2228, 3263, 734, 339, 3262, 3543, 804, 718, 4347, 1545, 2036, 1715, 1481, 3824, 9, 111, 3510, 3234, 1109, 1851, 3398, 4463, 3967, 2670, 3857, 3142, 2818, 4631, 618, 4140, 1541, 1794, 2719, 1660, 4649, 1971, 1504, 4242, 2896, 4141, 1011, 3043, 3652, 3477, 21, 4775, 3415, 513, 1398, 1084, 4860, 155, 3515, 3117, 4331, 3429, 602, 3489, 1020, 877, 2627, 1704, 3218, 4661, 3860, 791, 3867, 1847, 1995, 4676, 4993, 2621, 2257, 633, 593, 1469, 2415, 96, 2073, 4001, 599, 1351, 4921, 4522, 4817, 3925, 4920, 2019, 2990, 1596, 3781, 2358, 1594, 666, 3743, 4549, 3499, 2072, 652, 3397, 3819, 1355, 3061, 2454, 1303, 2811, 4914, 937, 202, 763, 1097, 2384, 269, 153, 3599, 3055, 3623, 3081, 1402, 2332, 769, 4917, 4523, 3460, 2875, 4967, 3574, 642, 5003, 2616, 1867, 4647, 3929, 1974, 4556, 2777, 1072, 2555, 2223, 1975, 2226, 746, 1200, 3725, 2425, 3044, 1929, 917, 671, 732, 681, 634, 3378, 4742, 2612, 2286, 3207, 2878, 5001, 1621, 1073, 4716, 4007, 2294, 4159, 811, 4519, 4579, 2120, 3575, 904, 2281, 3696, 1768, 2246, 1892, 2648, 1697, 2267, 815, 564, 2475, 197, 2680, 2363, 3463, 2744, 3721, 3085, 3075, 4712, 4571, 788, 1587, 915, 1617, 2087, 2534, 1536, 4193, 2431, 3039, 4070, 283, 2293, 3037, 643, 4876, 110, 4086, 2448, 4528, 1684, 303, 1809, 3783, 2828, 152, 4203, 1860, 2743, 3437, 4895, 3312, 822, 2464, 1500, 4700, 3739, 3682, 3650, 251, 3581, 3106, 508, 4408, 4980, 4902, 345, 1856, 4513, 1519, 4262, 1918, 1960, 907, 4249, 4257, 4244, 382, 752, 969, 3090, 3683, 1939, 4552, 4084, 3094, 3080, 566, 4577, 4058, 3241, 3195, 169, 598, 2564, 1103, 4878, 4287, 2521, 2178, 133, 3668, 4624, 1583, 582, 3304, 4589, 3605, 4657, 3642, 1035, 4131, 4799, 1959, 2754, 2284, 1940, 3034, 4253, 3621, 3870, 2208, 4797, 2552, 1782, 4288, 965, 587, 97, 2360, 3981, 2513, 4335, 1388, 264, 981, 2969, 1893, 4157, 195, 906, 4932, 2739, 3294, 1551, 2876, 3353, 3791, 1384, 2952, 892, 1425, 1357, 1841, 33, 2429, 2375, 1021, 2607, 2654, 2489, 3903, 4214, 390, 589, 4651, 2263, 4158, 4366, 2864, 4853, 3122, 3288, 1477, 4736, 1805, 1914, 725, 1287, 2222, 3351, 749, 2232, 3990, 2767, 4330, 2259, 4480, 3309, 3844, 1299, 3839, 430, 3505, 4076, 3380, 2027, 554, 2171, 2173, 4063, 1693, 1344, 3955, 2973, 853, 1827, 1518, 4548, 696, 4732, 660, 688, 3235, 950, 2203, 1181, 4304, 2366, 3579, 437, 1977, 1452, 1288, 4502, 3984, 1186, 281, 1229, 1997, 4665, 2539, 4953, 1187, 2787, 2231, 285, 1023, 1158, 1482, 1228, 4764, 272, 1196, 974, 2490, 4987, 346, 132, 2349, 4898, 1949, 2250, 3107, 1718, 3159, 3691, 2385, 976, 61, 533, 3583, 1225, 3659, 4418, 3562, 2015, 302, 2008, 3854, 4196, 4201, 3549, 3979, 3193, 1140, 3793, 371, 3118, 4160, 4444, 2089, 1463, 3634, 4564, 365, 3836, 3033, 506, 3082, 473, 4384, 4951, 4428, 3934, 856, 2132, 4888, 1019, 4436, 82, 4763, 2582, 2833, 803, 3269, 3252, 341, 4050, 1984, 2958, 3298, 466, 947, 2255, 1370, 2443, 1883, 4363, 689, 1636, 3464, 3913, 1240, 2114, 160, 661, 3411, 315, 3112, 864, 3314, 1875, 663, 1039, 1415, 4194, 3748, 3099, 4587, 939, 2660, 614, 4128, 266, 209, 4904, 3369, 2996, 741, 3495, 1447, 2851, 1319, 4940, 3230, 4962, 4045, 4793, 955, 783, 3026, 2727, 3367, 2404, 3915, 2935, 1844, 373, 2291, 1833, 4759, 1652, 2651, 1290, 4730, 1600, 4834, 1628, 4697, 3764, 4748, 1627, 1954, 2007, 1117, 4208, 3132, 701, 3547, 4271, 2631, 2477, 2458, 2800, 3632, 3161, 392, 2180, 4737, 3775, 1550, 2317, 4774, 2370, 1574, 1708, 1828, 1905, 1642, 4682, 1423, 1987, 3820, 2252, 3141, 2413, 4312, 1866, 4097, 4319, 456, 1633, 2335, 4032, 2221, 1745, 4733, 2691, 1889, 4459, 1359, 2871, 1915, 1774, 1777, 5010, 3610, 2201, 3858, 1677, 4380, 286, 1581, 1085, 4986, 126, 4322, 1474, 1257, 3108, 1419, 2934, 3129, 3809, 4336, 3885, 4177, 2287, 876, 1815, 518, 2519, 357, 2841, 4809, 569, 1657, 4919, 3914, 62, 3070, 297, 3577, 3275, 2368, 287, 4788, 724, 1335, 4117, 2765, 2045, 2561, 1712, 210, 4491, 3927, 4546, 4452, 4082, 3597, 1137, 2305, 4961, 280, 2589, 2150, 3301, 436, 1022, 2644, 4998, 3049, 1, 2493, 1756, 4821, 4602, 4378, 3931, 4169, 432, 4903, 2234, 2517, 2460, 1952, 2438, 4673, 4060, 2538, 3456, 3710, 3961, 104, 3110, 4012, 3945, 1788, 631, 2916, 4429, 4830, 2533, 2661, 2678, 4687, 3873, 4151, 621, 765, 4731, 4225, 726, 728, 1967, 3673, 792, 2310, 1480, 2218, 3701, 3968, 4365, 3593, 3093, 3521, 2315, 4684, 1934, 2329, 3017, 1499, 1183, 43, 378, 2863, 2789, 1890, 59, 1730, 1063, 1307, 655, 3291, 2699, 2579, 2125, 1503, 2960, 1568, 1962, 714, 887, 2190, 1560, 204, 3679, 1144, 1719, 1092, 2313, 3153, 163, 754, 2586, 2788, 2797, 3513, 1764, 2386, 2638, 2879, 468, 4840, 502, 2348, 2772, 406, 4526, 3935, 3753, 2505, 846, 2650, 120, 2066, 414, 781, 457, 114, 942, 1994, 2380, 1926, 3466, 3338, 3402, 697, 1555, 4132, 2199, 422, 3111, 2685, 4173, 2356, 397, 782, 2391, 2330, 4079, 4170, 4055, 4144, 858, 2080, 3467, 812, 3250, 1810, 488, 3202, 4223, 86, 1248, 2554, 2778, 1320, 3352, 926, 1622, 1094, 3671, 3861, 230, 3972, 2933, 411, 1362, 3035, 4861, 2321, 908, 1242, 3677, 4588, 4545, 895, 463, 4421, 771, 2174, 4514, 3614, 897, 3951, 252, 3989, 2048, 2129, 945, 4224, 494, 1736, 4771, 2463, 2282, 243, 3137, 2669, 4200, 206, 3332, 4283, 3923, 3526, 3980, 1857, 2956, 1711, 3381, 4279, 1381, 3295, 1170, 3414, 1814, 3310, 1538, 2857, 3150, 3657, 4, 1417, 1251, 3853, 524, 5002, 3527, 1440, 3852, 1455, 2161, 806, 2441, 2004, 2323, 3994, 2065, 3681, 1754, 3563, 4047, 4521, 3328, 2202, 4558, 2707, 4163, 1179, 65, 3584, 3916, 4247, 440, 138, 2737, 2832, 988, 4960, 2791, 859, 15, 2075, 1554, 385, 1042, 22, 3031, 4794, 700, 1509, 601, 2482, 1878, 1428, 2459, 4222, 2856, 2515, 464, 1549, 1951, 310, 4235, 245, 2895, 52, 4605, 2725, 25, 3704, 2860, 2892, 4734, 415, 4918, 1271, 4477, 786, 3267, 320, 5008, 912, 1979, 4582, 966, 3427, 1046, 1998, 576, 3708, 2520, 4941, 4784, 3441, 2781, 5007, 3846, 3253, 1932, 1112, 2674, 878, 2357, 982, 568, 4852, 3736, 349, 3078, 1986, 517, 4059, 3542, 1796, 4510, 2426, 1529, 1670, 3071, 4619, 2597, 2570, 1738, 4138, 4740, 4417, 4455, 1364, 2148, 1928, 2397, 4839, 1241, 4453, 3612, 1099, 4632, 762, 1238, 1037, 4184, 4072, 835, 1338, 1690, 1694, 2381, 2536, 4456, 30, 4291, 971, 4285, 3421, 3204, 1460, 543, 667, 2940, 4292, 758, 4192, 702, 972, 2740, 3052, 3601, 1634, 3550, 3613, 1522, 1408, 154, 3797, 2469, 1004, 790, 4388, 1325, 3552, 4420, 4085, 4815, 4738, 1152, 3496, 632, 813, 802, 4266, 1882, 1116, 2435, 2258, 874, 2103, 4205, 50, 4694, 3136, 4563, 3879, 2168, 1956, 2312, 4767, 3717, 2147, 3518, 1312, 2551, 4827, 3720, 1659, 3126, 1678, 1150, 1757, 4911, 2859, 4644, 2656, 2083, 4246, 4126, 2838, 3630, 3420, 2094, 2088, 4199, 609, 644, 4618, 4994, 1462, 3556, 1653, 2193, 4019, 2897, 4769, 3177, 4265, 3401, 3636, 551, 2785, 3770, 1108, 1980, 4334, 124, 4946, 3056, 1336, 4054, 2367, 1739, 505, 4318, 1539, 3531, 4021, 2516, 1486, 1017, 4508, 1064, 3863, 1281, 3063, 3480, 1498, 1102, 1778, 4281, 2849, 359, 3519, 1277, 1936, 3919, 2264, 1520, 2146, 1767, 159, 742, 1231, 4355, 4405, 240, 4981, 1644, 2275, 3127, 2845, 2920, 1163, 951, 2764, 4708, 2169, 3165, 2769, 1816, 4228, 1083, 1495, 1391, 257, 4074, 4710, 1135, 1759, 2164, 78, 2712, 2324, 4614, 699, 1895, 1405, 4683, 1333, 4081, 913, 3803, 2821, 1261, 4041, 639, 469, 4374, 4696, 2910, 2835, 3370, 4174, 3201, 785, 408, 2947, 1899, 1209, 1579, 4629, 3711, 3320, 2074, 2757, 1048, 2944, 29, 3855, 768, 2079, 112, 1887, 2953, 3261, 4213, 360, 4402, 855, 2732, 3475, 4846, 1920, 3270, 1252, 2209, 4381, 157, 4935, 3276, 1669, 2569, 2510, 1664, 923, 1189, 1479, 997, 575, 1906, 369, 2373, 1785, 2205, 3494, 721, 312, 767, 2145, 868, 260, 1484, 1489, 1270, 4595, 2714, 3442, 4524, 2502, 1662, 1432, 896, 3658, 3040, 1377, 3833, 1848, 3716, 4098, 3629, 2085, 1093, 2097, 2847, 1957, 375, 1146, 364, 1811, 4179, 2688, 1192, 2197, 172, 4470, 3003, 3130, 4645, 2862, 2126, 3354, 3587, 4226, 622, 611, 3799, 4295, 1177, 2424, 1134, 2663, 401, 801, 1854, 4362, 4922, 3772, 211, 2599, 3732, 4640, 4130, 2763, 2409, 2792, 1321, 3571, 4505, 4593, 2311, 4416, 1286, 854, 407, 863, 1260, 1688, 1726, 828, 4752, 1483, 3514, 3041, 4390, 3422, 271, 3815, 4511, 3486, 3462, 2615, 135, 3387, 829, 161, 953, 1089, 949, 1496, 4933, 4497, 4500, 4406, 2210, 2371, 2604, 4443, 2618, 193, 3042, 3752, 3008, 105, 1946, 4515, 3390, 4473, 737, 2717, 2689, 4924, 3321, 2929, 4668, 778, 750, 4209, 4090, 4976, 686, 2052, 2594, 650, 1651, 1964, 1365, 237, 902, 549, 1783, 307, 2639, 2877, 4284, 3115, 1707, 2636, 4260, 317, 3154, 248, 1611, 1445, 485, 2069, 3520, 3450, 4409, 1837, 2653, 4620, 992, 3843, 820, 3585, 1958, 616, 3231, 4133, 1612, 3461, 2834, 1675, 3808, 944, 402, 694, 129, 3680, 4354, 2058, 1045, 884, 4685, 3447, 1110, 842, 2730, 2361, 4517, 1976, 1436, 167, 4391, 4100, 4139, 3607, 4942, 2855, 2830, 2753, 2374, 3012, 557, 2484, 500, 4069, 2672, 1968, 1259, 4695, 793, 4289, 3418, 2354, 3897, 17, 4608, 2492, 4401, 2060, 71, 1437, 2985, 2298, 1584, 4869, 2003, 3940, 1199, 2081, 3302, 2949, 1394, 927, 4308, 2865, 220, 2806, 3059, 3875, 2049, 3440, 3523, 4316, 959, 4692, 4127, 2776, 4482, 348, 764, 4886, 108, 2117, 4613, 2111, 1632, 2325, 985, 4617, 3400, 1845, 2153, 3097, 2061, 3624, 3565, 3841, 3569, 2590, 588, 49, 4272, 3326, 4753, 2447, 2936, 1586, 1162, 2365, 573, 1748, 2412, 267, 1142, 3573, 3249, 3802, 958, 1454, 1467, 1451, 719, 520, 4866, 4031], "validation": [5315, 5413, 5094, 5162, 5020, 5147, 5048, 5061, 5277, 5193, 5044, 5053, 5144, 5255, 5066, 5474, 5086, 5351, 5469, 5226, 5181, 5091, 5349, 5401, 5457, 5188, 5387, 5089, 5323, 5132, 5033, 5198, 5251, 5285, 5412, 5419, 5149, 5430, 5050, 5308, 5222, 5161, 5335, 5373, 5421, 5347, 5034, 5483, 5209, 5453, 5139, 5460, 5202, 5465, 5208, 5032, 5437, 5300, 5369, 5264, 5352, 5072, 5420, 5088, 5494, 5344, 5288, 5127, 5097, 5451, 5042, 5037, 5128, 5107, 5059, 5133, 5079, 5476, 5219, 5385, 5316, 5224, 5497, 5123, 5194, 5490, 5153, 5307, 5342, 5406, 5510, 5466, 5432, 5165, 5427, 5435, 5429, 5391, 5069, 5082, 5108, 5365, 5491, 5390, 5433, 5043, 5087, 5500, 5471, 5047, 5381, 5186, 5163, 5119, 5253, 5211, 5241, 5023, 5041, 5250, 5173, 5368, 5405, 5317, 5077, 5031, 5495, 5302, 5054, 5506, 5298, 5415, 5190, 5022, 5057, 5090, 5192, 5475, 5416, 5199, 5389, 5084, 5271, 5260, 5480, 5216, 5422, 5367, 5280, 5481, 5292, 5486, 5327, 5499, 5396, 5252, 5098, 5175, 5106, 5052, 5447, 5319, 5322, 5213, 5182, 5458, 5407, 5068, 5013, 5150, 5291, 5244, 5310, 5109, 5445, 5249, 5200, 5354, 5485, 5258, 5444, 5080, 5187, 5441, 5326, 5360, 5070, 5076, 5305, 5212, 5395, 5071, 5105, 5236, 5075, 5408, 5423, 5262, 5235, 5325, 5078, 5346, 5417, 5261, 5496, 5184, 5201, 5265, 5203, 5400, 5440, 5353, 5215, 5273, 5016, 5439, 5393, 5503, 5172, 5379, 5058, 5297, 5025, 5065, 5238, 5418, 5166, 5233, 5275, 5330, 5482, 5493, 5197, 5036, 5462, 5324, 5340, 5450, 5320, 5279, 5399, 5294, 5487, 5138, 5019, 5045, 5110, 5270, 5358, 5207, 5064, 5434, 5426, 5468, 5263, 5246, 5103, 5125, 5039, 5083, 5051, 5142, 5309, 5196, 5220, 5370, 5093, 5289, 5383, 5164, 5470, 5114, 5096, 5299, 5111, 5102, 5247, 5492, 5242, 5152, 5366, 5382, 5455, 5337, 5449, 5339, 5167, 5116, 5507, 5063, 5384, 5359, 5062, 5137, 5210, 5060, 5118, 5014, 5355, 5336, 5136, 5160, 5143, 5282, 5225, 5214, 5189, 5104, 5272, 5174, 5386, 5191, 5017, 5092, 5281, 5334, 5157, 5409, 5488, 5403, 5425, 5204, 5248, 5332, 5049, 5502, 5038, 5459, 5321, 5356, 5328, 5195, 5159, 5452, 5239, 5240, 5205, 5372, 5259, 5290, 5154, 5134, 5237, 5120, 5177, 5015, 5428, 5095, 5171, 5156, 5176, 5438, 5293, 5454, 5363, 5338, 5223, 5388, 5018, 5392, 5343, 5484, 5424, 5030, 5463, 5046, 5229, 5350, 5055, 5067, 5267, 5024, 5329, 5362, 5185, 5467, 5158, 5081, 5218, 5035, 5168, 5446, 5221, 5478, 5170, 5443, 5178, 5479, 5131, 5021, 5508, 5169, 5135, 5398, 5394, 5129, 5357, 5511, 5122, 5303, 5464, 5313, 5056, 5287, 5296, 5183, 5375, 5151, 5461, 5026, 5301, 5331, 5501, 5312, 5217, 5314, 5145, 5274, 5245, 5257, 5028, 5230, 5179, 5101, 5456, 5341, 5377, 5148, 5099, 5361, 5232, 5374, 5100, 5234, 5306, 5073, 5268, 5410, 5256, 5266, 5286, 5411, 5489, 5509, 5027, 5180, 5228, 5304, 5448, 5318, 5431, 5140, 5436, 5397, 5115, 5085, 5472, 5124, 5345, 5146, 5074, 5371, 5130, 5333, 5414, 5141, 5311, 5402, 5380, 5269, 5404, 5348, 5231, 5364, 5012, 5040, 5442, 5113, 5276, 5121, 5473, 5155, 5254, 5504, 5117, 5206, 5477, 5126, 5505, 5295, 5029, 5227, 5498, 5284, 5112, 5243, 5278, 5376, 5283, 5378], "test": [5643, 5954, 5668, 5951, 5731, 5965, 5919, 5771, 5573, 5894, 5584, 5734, 5899, 5750, 5775, 5853, 5891, 5997, 5702, 5577, 5529, 5611, 5740, 5575, 5607, 5546, 5804, 5856, 5645, 5798, 5811, 5813, 5935, 5736, 5864, 5629, 5906, 5714, 5544, 5737, 5658, 5910, 5597, 5515, 5631, 5623, 5615, 5881, 5514, 5880, 5699, 5722, 5732, 5869, 5576, 5933, 5657, 5790, 5683, 5566, 5942, 5854, 6002, 5600, 5766, 5724, 5762, 5888, 5795, 5791, 5838, 5763, 5605, 5659, 5728, 6011, 5707, 5620, 5941, 6001, 5520, 5616, 5592, 5636, 5701, 5559, 5761, 5936, 5977, 5521, 5713, 5591, 5689, 5900, 5893, 5959, 5715, 5684, 5718, 5939, 5569, 5639, 5773, 5970, 5710, 5779, 5522, 5793, 5904, 5669, 5670, 6000, 5776, 5711, 5551, 5833, 5596, 5797, 5911, 5571, 5558, 5817, 5815, 5547, 5594, 5730, 5890, 5846, 5810, 5735, 5986, 5665, 5586, 5917, 5976, 5988, 5635, 5667, 5752, 5564, 5548, 5903, 5695, 5871, 5562, 5916, 5512, 5534, 5949, 5835, 5542, 5583, 5964, 5768, 6009, 5743, 5867, 5873, 5844, 5960, 5918, 5816, 5973, 5675, 5678, 5535, 5690, 5805, 5808, 5794, 5983, 5930, 5664, 5861, 5781, 5748, 5849, 5770, 5764, 5993, 5978, 5803, 5603, 5756, 5892, 5822, 5847, 5747, 5588, 5644, 5885, 5787, 5604, 5944, 5598, 5818, 5789, 5872, 5627, 5674, 5723, 5780, 5860, 5841, 5948, 5545, 5974, 5628, 5912, 6004, 5523, 5550, 5725, 5859, 5749, 5612, 5824, 5914, 5655, 5994, 5785, 5987, 6006, 5716, 5783, 5842, 5772, 5563, 5982, 5524, 5652, 5878, 5992, 5672, 5801, 5758, 5929, 5989, 5923, 5950, 5574, 5692, 5721, 5778, 5691, 5637, 5828, 5806, 5836, 5693, 5947, 5706, 5971, 5874, 5961, 5585, 5530, 5922, 5777, 5980, 5802, 5898, 5526, 5579, 5945, 5739, 5908, 5962, 5840, 5837, 5537, 5985, 5543, 5741, 5671, 5518, 5915, 5788, 6007, 5610, 5991, 5630, 5759, 5931, 5921, 5554, 5955, 5832, 5660, 5858, 5745, 5539, 5799, 5800, 5927, 5531, 5624, 5565, 5653, 5868, 5649, 5877, 5809, 5638, 5782, 5677, 5924, 5694, 5673, 5886, 5848, 5727, 5889, 5532, 5796, 5968, 5609, 5925, 5587, 5517, 5662, 5975, 5705, 5519, 5688, 6008, 5640, 5932, 5679, 5696, 5590, 5697, 5581, 5857, 5555, 5593, 5784, 5901, 5681, 5934, 5834, 5552, 5769, 5786, 5907, 5700, 5686, 5617, 5852, 5883, 5682, 5952, 5967, 5614, 5909, 5765, 5709, 5622, 5972, 5527, 5984, 5528, 5570, 5946, 5726, 5599, 5560, 5641, 5625, 5760, 5792, 5738, 5663, 5953, 5733, 5897, 5729, 5751, 5676, 5561, 5865, 5845, 5661, 5746, 5938, 5827, 5648, 5647, 5896, 5963, 5606, 5866, 5525, 5632, 5774, 5557, 5685, 5969, 5823, 5541, 5536, 5913, 5958, 6005, 5755, 5990, 5956, 5875, 5650, 5717, 5595, 5876, 5862, 5578, 5608, 5820, 5981, 5879, 5863, 5829, 5996, 5999, 5666, 5582, 5754, 5926, 5884, 5882, 5843, 5687, 5957, 5556, 5572, 5807, 5719, 5928, 5540, 5654, 5979, 5538, 5998, 5821, 5704, 5698, 5995, 6003, 5825, 5831, 5839, 5742, 5767, 5895, 5757, 5855, 5567, 5619, 5708, 5589, 5905, 5602, 5812, 5943, 5580, 5819, 5887, 5703, 5533, 5656, 5814, 5937, 5902, 5826, 5830, 5646, 5712, 5966, 5601, 6010, 5680, 5940, 5513, 5568, 5516, 5642, 5851, 5626, 5613, 5850, 5634, 5618, 5744, 5633, 5651, 5870, 5553, 5920, 5720, 5753, 5621, 5549]}, {"train": [1420, 1119, 2661, 3485, 4220, 4289, 4316, 4431, 1694, 2902, 2243, 2348, 2590, 1034, 1518, 2444, 3486, 2860, 4803, 2069, 4341, 1594, 989, 4915, 4504, 1073, 2551, 3988, 2087, 5010, 710, 2411, 4262, 3833, 3629, 4951, 2575, 1790, 703, 1230, 577, 4406, 170, 2988, 3850, 2021, 918, 278, 4055, 1992, 906, 1809, 1907, 4237, 3726, 1141, 377, 4794, 1584, 2089, 4537, 1284, 55, 4589, 1455, 1092, 4143, 3422, 522, 975, 4984, 2576, 4394, 1613, 1668, 3826, 1046, 386, 3399, 1966, 387, 626, 2334, 4287, 2809, 991, 1556, 2374, 4180, 4315, 651, 1244, 2425, 1430, 856, 4503, 2997, 1349, 47, 3568, 1846, 1486, 1372, 4351, 802, 3827, 200, 3140, 1312, 1310, 2483, 3516, 156, 1618, 2804, 3109, 1692, 2337, 1354, 670, 3195, 1590, 1431, 3353, 4019, 3959, 3759, 982, 3433, 2872, 3059, 206, 2923, 3663, 4221, 308, 4490, 3118, 1957, 1494, 1863, 3204, 628, 1072, 525, 2861, 3458, 4597, 2184, 2565, 509, 1531, 4528, 2817, 2410, 2245, 2828, 2973, 2017, 4269, 4660, 3233, 997, 4777, 2499, 1068, 4737, 1707, 1764, 841, 4854, 3610, 3974, 1937, 2028, 463, 1887, 3436, 1241, 4889, 2709, 1013, 94, 2645, 552, 2810, 1157, 2662, 4312, 2124, 196, 3728, 2134, 4657, 874, 4050, 678, 3431, 4962, 1416, 4659, 3907, 1397, 1570, 3487, 4749, 1168, 4253, 1355, 379, 1044, 2568, 4372, 3443, 2229, 1409, 3791, 812, 3065, 4089, 3936, 4199, 318, 3046, 187, 559, 2366, 3559, 3527, 2822, 3579, 413, 4603, 3993, 3193, 3667, 2002, 1904, 35, 749, 1696, 1553, 4134, 1175, 1296, 3569, 632, 2226, 4225, 2725, 1514, 78, 3449, 1930, 1347, 4599, 4035, 1463, 1905, 2146, 4165, 852, 1852, 2482, 2746, 4020, 4031, 815, 2370, 1017, 265, 1370, 1925, 2238, 4311, 818, 478, 2428, 4999, 4286, 1300, 2492, 2548, 4416, 4494, 3346, 2733, 1873, 446, 162, 290, 4435, 2532, 2620, 2059, 1772, 1888, 4877, 2415, 2730, 4085, 1193, 2197, 229, 4234, 1984, 3957, 1417, 556, 3240, 1169, 1495, 3830, 635, 4547, 2398, 2049, 4711, 3199, 2391, 2744, 3406, 2241, 1910, 4522, 3119, 2363, 1340, 4826, 2213, 624, 3054, 4874, 1945, 3750, 4168, 4676, 1554, 949, 3365, 263, 4409, 2074, 2735, 1869, 3146, 2305, 659, 955, 467, 4181, 3597, 1551, 3132, 1710, 2009, 1796, 2835, 466, 2037, 3812, 3535, 3902, 2640, 471, 4397, 1388, 1369, 1035, 4028, 2776, 2509, 236, 2266, 342, 1936, 4519, 606, 3452, 4931, 99, 4549, 642, 3111, 4122, 4258, 254, 675, 2901, 3564, 2695, 4715, 1052, 300, 2918, 2681, 4622, 1425, 1248, 1731, 813, 4655, 1794, 4507, 2208, 422, 226, 3899, 1740, 2486, 4481, 2351, 12, 2400, 4472, 3336, 3114, 1295, 3575, 1996, 2192, 4274, 2047, 2911, 3174, 1867, 2414, 2365, 2655, 1257, 2294, 2395, 518, 1582, 481, 1100, 2083, 1368, 3563, 2487, 2663, 557, 2925, 412, 4188, 214, 2338, 972, 4939, 1182, 875, 1135, 4641, 3345, 2355, 4972, 2780, 3596, 2409, 2417, 4689, 316, 1348, 1704, 4844, 2846, 3092, 956, 4924, 2460, 4602, 2880, 1429, 1569, 2886, 4850, 345, 3192, 2164, 4763, 560, 225, 4010, 2890, 3031, 3020, 366, 324, 1578, 376, 1602, 1999, 3678, 922, 4304, 92, 2904, 3701, 2158, 1652, 930, 305, 180, 4197, 476, 1787, 2137, 3548, 2050, 4444, 1833, 1477, 970, 76, 690, 1197, 233, 1743, 1121, 31, 3856, 4768, 1371, 995, 3918, 1926, 4573, 1630, 2659, 4283, 319, 984, 3302, 4713, 1475, 3496, 4156, 2651, 2199, 645, 2535, 1186, 4746, 2086, 470, 172, 1152, 2957, 240, 756, 4990, 4814, 117, 570, 4912, 4191, 3477, 4899, 2263, 3141, 2748, 4774, 780, 1339, 1636, 883, 3053, 248, 3209, 1564, 4317, 3779, 321, 2202, 415, 3147, 2716, 3186, 4466, 1240, 3781, 2088, 3483, 798, 4064, 4707, 3891, 2715, 2249, 1895, 2868, 3107, 3601, 1131, 4720, 3428, 405, 769, 4870, 2020, 4155, 1745, 3184, 3854, 1233, 2381, 1473, 4252, 4728, 2377, 3033, 3593, 3245, 3038, 486, 3100, 143, 2717, 2240, 3671, 4082, 878, 4479, 880, 2752, 4840, 3839, 543, 2466, 847, 1454, 1916, 3908, 2336, 3595, 4568, 48, 158, 3317, 3926, 2097, 3489, 3771, 4447, 4875, 853, 145, 2138, 3410, 2583, 1059, 1341, 4003, 3180, 2840, 3016, 558, 752, 3388, 4529, 2859, 4375, 2368, 3958, 729, 4473, 4539, 2139, 3938, 3258, 4183, 1107, 2035, 4173, 2742, 821, 2792, 114, 178, 4150, 2658, 3251, 327, 4116, 816, 687, 4045, 840, 343, 4996, 4550, 1128, 3013, 279, 296, 1779, 1862, 3991, 3951, 2863, 483, 1002, 3980, 2625, 3169, 4327, 2819, 2172, 4456, 276, 4208, 734, 2705, 1535, 217, 1540, 2781, 142, 2344, 1058, 4161, 1069, 4685, 4506, 4248, 444, 3409, 2447, 2677, 838, 3704, 3997, 3375, 5005, 2023, 2494, 208, 347, 3985, 1301, 3747, 1350, 2095, 4634, 1795, 2650, 2253, 3532, 2292, 1780, 2151, 2104, 3896, 183, 1083, 4623, 4565, 943, 1461, 3634, 1221, 3123, 2359, 4651, 2703, 2526, 2941, 3072, 364, 4600, 4887, 2207, 4294, 4754, 2315, 1098, 864, 262, 2708, 966, 954, 1816, 1402, 301, 1825, 744, 4099, 1931, 2132, 3022, 2286, 2254, 4907, 150, 1735, 1523, 1303, 4114, 3098, 482, 1519, 2670, 2125, 2683, 4390, 2426, 2092, 2612, 1184, 3684, 3466, 231, 605, 4592, 2342, 2952, 2929, 2121, 4934, 2580, 4855, 3642, 1218, 3040, 4810, 958, 4851, 4241, 4918, 2903, 2961, 4564, 832, 4178, 220, 3420, 3773, 1048, 4410, 67, 761, 2812, 1897, 940, 128, 274, 1750, 4687, 108, 3953, 2081, 2940, 4439, 4169, 148, 3179, 3150, 774, 1828, 2881, 3494, 477, 3034, 2098, 4177, 1191, 1548, 2598, 614, 1624, 4029, 3503, 3817, 3521, 4049, 1199, 3465, 2843, 4438, 1671, 763, 1183, 1316, 188, 212, 3331, 1980, 2734, 3426, 435, 1205, 547, 2827, 1180, 3804, 4063, 4652, 4679, 2119, 2427, 1946, 1812, 3407, 3505, 1797, 2627, 1768, 980, 454, 3412, 1747, 1893, 582, 4295, 3427, 4218, 862, 1011, 1506, 3247, 526, 1552, 2392, 2552, 2788, 1012, 4942, 4016, 4943, 3001, 1469, 1592, 2910, 4328, 3861, 2316, 3425, 4973, 3362, 4864, 230, 4359, 3226, 3814, 4137, 4916, 3074, 4903, 3586, 3097, 4512, 819, 2061, 4848, 1727, 381, 2838, 3921, 1236, 1408, 1299, 2044, 1838, 4268, 399, 4941, 4452, 4764, 4544, 3517, 851, 2982, 4571, 3461, 1492, 1719, 2434, 1742, 4355, 3045, 4489, 3866, 1411, 3366, 800, 1149, 4926, 3157, 3676, 242, 14, 4896, 4347, 1685, 836, 3822, 495, 4340, 3668, 942, 73, 1714, 3323, 4404, 3230, 2373, 1875, 4977, 1823, 3869, 3644, 2402, 728, 4138, 2959, 2726, 1172, 751, 4232, 820, 2697, 2090, 2357, 905, 4636, 130, 4698, 3463, 1675, 2935, 1137, 1351, 5007, 1718, 123, 871, 4938, 1435, 3356, 4356, 2284, 4325, 2686, 1279, 540, 3786, 2541, 2864, 3910, 4868, 2954, 3840, 2496, 4524, 1886, 3784, 4429, 2176, 2531, 2019, 4836, 2775, 3886, 796, 2665, 3301, 2644, 3440, 630, 1479, 3030, 2811, 3429, 1061, 2268, 4008, 1700, 1204, 2128, 2111, 1057, 4656, 4094, 1357, 1045, 4166, 2930, 3956, 814, 348, 4042, 2537, 2265, 3047, 4586, 4066, 2011, 2193, 4702, 4075, 4048, 2900, 2966, 493, 4650, 1187, 3807, 3578, 1754, 2604, 727, 1516, 3095, 538, 4790, 28, 4765, 1663, 4024, 508, 1807, 4759, 1262, 2906, 2283, 2080, 4388, 881, 4809, 2729, 4027, 203, 858, 425, 4538, 4535, 184, 3373, 664, 4405, 3989, 2048, 4450, 1640, 1237, 2478, 2031, 2636, 2990, 3296, 4303, 4337, 2761, 2449, 3979, 3500, 1433, 4605, 1202, 232, 2272, 4680, 2719, 4543, 3159, 2032, 3906, 704, 2944, 1908, 2454, 54, 2766, 4157, 1733, 758, 4946, 4795, 1194, 3607, 3892, 267, 3722, 1091, 1345, 2931, 4572, 3783, 4209, 3405, 4670, 3852, 127, 4684, 1858, 1274, 437, 3113, 2850, 4350, 4331, 835, 4770, 2905, 2386, 5006, 4761, 2755, 1777, 1394, 2555, 134, 1508, 1989, 2246, 804, 1814, 3300, 4245, 3749, 578, 340, 1365, 211, 1413, 4172, 2987, 1729, 762, 4140, 506, 3534, 3403, 2318, 396, 1290, 4118, 1502, 3744, 587, 4588, 2323, 3580, 1647, 3212, 3249, 3011, 3999, 987, 1933, 4585, 1028, 3257, 138, 924, 3205, 1775, 1566, 795, 3588, 2367, 4313, 2296, 3739, 3382, 1898, 4876, 750, 1108, 2504, 4645, 1081, 3692, 4970, 3623, 4653, 1577, 2016, 498, 2237, 551, 1424, 4077, 1036, 3271, 3949, 1238, 1586, 585, 1378, 1054, 574, 665, 2129, 3780, 3413, 998, 488, 3950, 652, 4480, 361, 3867, 2882, 2420, 3721, 3277, 3677, 2685, 2713, 4824, 1352, 4738, 4175, 234, 1501, 4545, 2773, 2404, 1649, 3515, 3234, 2831, 4982, 3151, 3267, 3626, 3838, 2647, 2155, 542, 1118, 2854, 2109, 808, 3653, 3352, 3278, 2637, 2522, 3813, 4757, 3419, 1471, 3153, 4170, 892, 4119, 2871, 4801, 4278, 1332, 2205, 3848, 1319, 2858, 3538, 0, 2234, 1666, 823, 946, 3583, 2791, 1062, 3922, 3348, 1022, 90, 4339, 3818, 3923, 2516, 2495, 1067, 2278, 4992, 1443, 3042, 2500, 299, 1094, 4338, 2643, 419, 3200, 1510, 2969, 2567, 3361, 3246, 2115, 3223, 4465, 421, 4374, 2894, 4816, 2692, 41, 1014, 2247, 4125, 3080, 281, 1879, 4587, 2556, 4486, 2153, 894, 4069, 3777, 4128, 4065, 1374, 1154, 4367, 4997, 4928, 914, 4335, 1251, 4067, 3154, 1138, 1896, 2423, 3736, 3067, 4508, 51, 3931, 2330, 4553, 4646, 4975, 3079, 3041, 2110, 544, 2443, 3729, 4706, 2413, 1414, 2631, 3528, 3236, 1161, 3318, 1739, 1114, 417, 2524, 4361, 264, 2559, 1334, 1331, 3955, 3699, 1289, 1995, 2300, 1064, 1427, 545, 932, 898, 4395, 962, 2688, 4510, 22, 4306, 4782, 3290, 4846, 3796, 2619, 3004, 45, 120, 3599, 3650, 3714, 2488, 3964, 3618, 4940, 4786, 4994, 3647, 1543, 1574, 961, 2938, 3845, 3656, 3389, 2024, 3851, 1366, 730, 2491, 784, 111, 4039, 2256, 3297, 2839, 1870, 2634, 565, 344, 1386, 58, 1256, 109, 915, 3963, 4288, 2993, 2328, 4536, 1950, 189, 633, 4985, 4421, 1655, 1385, 3185, 4047, 3307, 1198, 677, 2463, 1360, 152, 2992, 4919, 2547, 1314, 783, 2774, 5009, 2919, 1932, 3660, 2789, 198, 1481, 339, 3820, 3491, 2133, 3679, 3805, 4548, 1460, 3253, 3605, 3617, 996, 1338, 4540, 2977, 3920, 1145, 908, 4793, 2364, 1913, 760, 2118, 3378, 4642, 4132, 2842, 3737, 2617, 4830, 4001, 3864, 2641, 1167, 75, 149, 4412, 3895, 3093, 927, 3689, 2832, 1951, 2356, 2586, 1541, 1147, 1009, 4100, 310, 3705, 85, 98, 3772, 709, 139, 1788, 4102, 529, 2430, 4052, 2638, 1440, 4726, 3264, 277, 375, 1641, 1255, 4216, 1480, 2978, 650, 3967, 572, 434, 484, 1407, 3770, 1737, 1813, 4366, 2312, 2039, 2221, 4618, 4290, 2303, 3498, 4246, 3086, 1358, 689, 3026, 3105, 4442, 4398, 4151, 4432, 2759, 19, 1562, 1634, 1985, 1160, 2371, 2849, 461, 567, 3961, 1082, 1170, 3415, 3841, 503, 2691, 140, 1746, 993, 4785, 2179, 2120, 846, 2652, 2630, 4662, 1643, 4236, 4658, 4805, 3965, 3837, 2769, 1337, 1084, 3137, 3683, 4190, 4276, 2228, 1181, 2664, 3572, 1373, 4058, 2233, 3996, 4648, 3872, 3849, 1625, 1287, 1608, 4126, 2116, 1544, 4904, 285, 828, 599, 748, 4153, 3758, 1721, 4453, 3723, 2570, 3549, 3372, 4492, 4485, 919, 2694, 4616, 657, 489, 3685, 4012, 886, 3734, 218, 2157, 126, 4321, 4373, 1881, 432, 2148, 271, 4752, 4952, 2219, 667, 4688, 4434, 2706, 548, 831, 911, 4057, 68, 2187, 1250, 2042, 3973, 873, 129, 4319, 3282, 4446, 3913, 1076, 4285, 3984, 247, 1760, 3459, 1293, 511, 459, 147, 3640, 1983, 2501, 355, 586, 904, 933, 338, 3995, 1192, 779, 4935, 2569, 4458, 4558, 622, 1560, 1956, 2194, 1678, 1056, 2126, 3659, 3377, 3202, 3224, 1000, 4620, 4006, 3968, 2393, 3764, 4026, 4860, 2777, 515, 3155, 617, 1222, 3919, 1924, 4730, 4787, 4133, 2468, 367, 1821, 4307, 4343, 1162, 3883, 2282, 3289, 384, 235, 1558, 3743, 4631, 3319, 1245, 1380, 280, 1651, 4584, 2520, 2538, 2680, 1912, 32, 4888, 3590, 1489, 3842, 122, 1015, 2273, 4091, 4905, 2741, 3930, 2464, 2591, 4575, 3337, 4601, 3862, 575, 1662, 726, 1189, 2169, 824, 4046, 1918, 2013, 106, 2295, 1691, 4626, 2476, 4531, 4222, 1786, 978, 2800, 2558, 4725, 306, 4986, 1227, 1909, 2869, 3358, 1026, 155, 2514, 2707, 679, 1459, 1616, 1051, 21, 3262, 4682, 3609, 1753, 3189, 4211, 4878, 4965, 3039, 2026, 3019, 1484, 3408, 2939, 4857, 39, 2885, 1949, 1302, 2149, 4299, 3396, 3280, 2764, 4520, 2684, 2616, 312, 3082, 1353, 2533, 4695, 449, 3291, 3589, 3198, 3328, 1313, 207, 3615, 3603, 3782, 409, 1970, 270, 2214, 2696, 1922, 3740, 2851, 4305, 4900, 1885, 1292, 600, 4034, 3070, 770, 4418, 2102, 2943, 2820, 4072, 1225, 4872, 2579, 2010, 573, 2525, 153, 1699, 717, 3115, 1824, 2385, 2896, 4932, 4675, 103, 2469, 4483, 2639, 2704, 1001, 2848, 4991, 2699, 1437, 1396, 2261, 1333, 3613, 3171, 1844, 1031, 3558, 256, 3514, 3929, 4441, 402, 3376, 4856, 537, 2442, 1606, 2727, 3857, 4862, 4838, 3075, 1010, 4354, 3148, 3877, 3524, 2857, 3384, 2076, 266, 4176, 1415, 4129, 1155, 4710, 1706, 994, 3448, 2093, 3164, 496, 2248, 74, 4497, 1900, 4691, 900, 1112, 2350, 3983, 474, 2574, 3254, 3309, 3432, 3525, 2949, 4788, 4342, 4699, 1536, 2189, 1468, 1845, 1089, 3885, 3898, 4105, 2679, 289, 4521, 1142, 133, 781, 168, 2025, 3166, 4925, 3778, 411, 2607, 2216, 4967, 1877, 1942, 209, 641, 1389, 3190, 3418, 2867, 3947, 4502, 1270, 733, 3175, 612, 3457, 2056, 4534, 2690, 1773, 2960, 1106, 797, 3165, 410, 1802, 3049, 788, 1962, 1611, 258, 2856, 3937, 3735, 2928, 2834, 1507, 3014, 4005, 2418, 4898, 4396, 4859, 3901, 1505, 1637, 3145, 1129, 2072, 4783, 4282, 773, 4360, 157, 3122, 213, 2669, 4332, 272, 974, 4273, 4917, 1275, 1826, 4284, 1974, 944, 1224, 2912, 2826, 3738, 2624, 1654, 3799, 3136, 3990, 2489, 4201, 2106, 4182, 1878, 580, 2180, 2593, 4517, 1973, 1243, 3803, 4690, 4112, 3242, 2290, 43, 1329, 401, 1539, 2018, 3470, 1842, 268, 4353, 2675, 2067, 950, 3823, 3511, 3844, 2837, 3315, 1549, 684, 261, 4324, 3539, 252, 373, 1784, 4647, 2198, 3493, 383, 2754, 174, 2433, 2217, 4557, 2354, 4059, 1258, 3809, 4514, 4120, 1522, 1038, 3321, 4937, 20, 2909, 705, 497, 1803, 3255, 4302, 3635, 3414, 827, 1571, 899, 4379, 4578, 594, 1434, 636, 1320, 4590, 3551, 2723, 37, 1452, 1736, 1074, 3566, 1971, 3062, 1993, 1384, 2581, 2823, 860, 2621, 4532, 4820, 5008, 2123, 4828, 86, 3860, 1140, 1453, 4963, 1464, 2, 3935, 3561, 3090, 1947, 2873, 3168, 563, 4499, 1156, 1857, 334, 884, 1271, 2979, 2052, 3003, 3248, 1633, 2921, 161, 1422, 4275, 625, 3584, 2589, 7, 1837, 2131, 939, 2798, 3971, 571, 644, 1684, 336, 3716, 3542, 1534, 4796, 1585, 2946, 118, 5000, 4038, 3430, 2668, 433, 2833, 1448, 25, 2674, 4733, 4023, 4677, 4533, 4217, 3531, 1190, 608, 2550, 849, 2252, 896, 1939, 2888, 4841, 4551, 1513, 2852, 1620, 1527, 1117, 3050, 3717, 2892, 2457, 3554, 3790, 4644, 1436, 2140, 1383, 3081, 4476, 1294, 1210, 1280, 1328, 4910, 1889, 1791, 3094, 46, 4097, 1041, 1979, 960, 3104, 4364, 350, 2170, 1490, 640, 2542, 4152, 4758, 3327, 3172, 1876, 1423, 2991, 4460, 517, 3509, 1268, 3627, 4632, 3286, 1037, 4381, 3688, 2765, 3952, 2534, 531, 2530, 4092, 492, 4869, 3225, 10, 3385, 2165, 228, 1124, 1615, 3367, 3048, 4983, 4104, 2066, 82, 2698, 1827, 4087, 2407, 1419, 2539, 1998, 4256, 3643, 4669, 3915, 3662, 2585, 4736, 2063, 3402, 837, 3124, 3308, 4436, 3480, 151, 1915, 3326, 457, 3502, 3379, 4885, 1272, 3574, 4101, 2917, 4853, 4103, 1016, 3351, 4477, 3216, 737, 1626, 3846, 2458, 3664, 3874, 4461, 3811, 3905, 2878, 1669, 1093, 3693, 2646, 2920, 3632, 455, 986, 3713, 2156, 4895, 2353, 57, 3630, 3870, 3131, 743, 3421, 1565, 2448, 2656, 4417, 1466, 4894, 2895, 3622, 2971, 4798, 4740, 307, 480, 3052, 2054, 1403, 1321, 721, 2728, 3306, 4009, 3398, 4815, 3355, 2578, 4873, 1032, 52, 2825, 3884, 388, 2968, 1557, 3329, 1596, 286, 414, 4743, 3774, 237, 1406, 1342, 1703, 1568, 2926, 4186, 4326, 3116, 2186, 1705, 1987, 3347, 2737, 201, 3497, 938, 1322, 794, 490, 917, 2877, 713, 243, 707, 907, 1672, 4210, 4583, 4625, 3894, 79, 3691, 320, 2143, 2573, 4799, 1104, 3359, 3637, 1929, 2963, 1209, 2421, 3875, 4124, 4205, 1080, 1050, 1861, 4370, 1874, 3152, 216, 4954, 3424, 2242, 2847, 2378, 3499, 2345, 1781, 3577, 3241, 621, 2544, 115, 2802, 4566, 2343, 1491, 4411, 2818, 1892, 3606, 3368, 4981, 2108, 2980, 4560, 2038, 3043, 2965, 2981, 389, 3680, 4136, 739, 2352, 1576, 4037, 2710, 1150, 3381, 2036, 3724, 93, 3536, 4665, 3055, 1642, 1848, 11, 3298, 4229, 4562, 4413, 2144, 1547, 1055, 1318, 1217, 430, 1801, 863, 4771, 564, 4581, 4513, 2008, 4380, 3797, 1053, 1219, 4901, 722, 523, 3639, 2302, 65, 2033, 553, 1133, 1485, 1456, 1988, 2508, 909, 3863, 4681, 528, 719, 4595, 453, 3682, 452, 1305, 747, 661, 3397, 615, 4561, 1099, 2572, 2622, 953, 3012, 3310, 176, 1555, 5001, 504, 702, 4389, 4179, 4113, 1986, 2225, 901, 1914, 3654, 3207, 4947, 3526, 1214, 4308, 512, 4196, 4219, 2682, 2824, 876, 4897, 879, 3208, 1716, 462, 3303, 4259, 4392, 194, 1658, 3160, 2898, 1123, 3087, 4923, 2375, 1638, 2785, 1521, 2470, 4694, 3520, 2821, 499, 603, 1110, 1418, 766, 1722, 5003, 3700, 4251, 1120, 1664, 3553, 4723, 3889, 1497, 119, 934, 4108, 888, 1680, 3473, 1103, 2171, 1203, 848, 2372, 185, 1589, 3391, 4238, 8, 990, 3712, 1267, 2732, 4300, 2932, 136, 2006, 3173, 3914, 72, 1097, 3028, 3299, 1831, 877, 1612, 1457, 2561, 1597, 1575, 3221, 2462, 895, 2721, 4776, 3120, 2899, 591, 2927, 3746, 239, 1344, 3058, 885, 16, 3732, 4322, 1446, 407, 3537, 916, 1941, 771, 1304, 4013, 1686, 3183, 2075, 697, 3658, 3158, 164, 1379, 3582, 465, 807, 2040, 952, 1264, 3293, 2654, 2167, 2571, 1561, 1071, 3371, 524, 4745, 2387, 3416, 4570, 4298, 3847, 3917, 643, 2712, 2166, 1042, 701, 3244, 793, 2456, 1088, 102, 694, 1573, 2994, 1211, 1726, 4714, 204, 2388, 3731, 2976, 3069, 3010, 1960, 4546, 3027, 500, 144, 4145, 3789, 2787, 3235, 1176, 1132, 2955, 4609, 3513, 363, 1487, 1778, 1866, 4886, 4509, 4174, 3681, 2453, 4792, 672, 4078, 3585, 1769, 2396, 2608, 4051, 4376, 2794, 2564, 61, 1550, 4866, 799, 394, 3981, 764, 2762, 765, 1493, 1659, 2347, 2455, 789, 4424, 631, 219, 3504, 604, 4144, 776, 3335, 3962, 1229, 4496, 1679, 3025, 2836, 1018, 121, 2262, 2751, 1298, 1785, 3103, 3719, 1632, 801, 2475, 4525, 3304, 1847, 3986, 4004, 3934, 4541, 408, 3237, 2747, 660, 2536, 1346, 1174, 1127, 159, 501, 1783, 592, 4192, 3007, 3259, 3897, 2015, 448, 3495, 1326, 579, 3927, 1223, 2924, 3484, 2070, 3279, 855, 4056, 698, 304, 2277, 4358, 4778, 1840, 507, 857, 1650, 1482, 1723, 3876, 1677, 1426, 368, 4791, 1003, 2480, 2951, 1362, 4041, 1667, 2844, 3077, 4638, 1512, 1711, 3370, 3506, 1260, 1665, 3858, 1317, 505, 2596, 3229, 2600, 706, 3108, 4261, 3438, 2451, 4734, 502, 3176, 3611, 4914, 87, 1442, 809, 2689, 2711, 533, 2251, 4443, 2145, 4301, 4823, 1689, 3752, 2942, 3314, 436, 2922, 4527, 3023, 2473, 1263, 2287, 3051, 1546, 3143, 3718, 2724, 2753, 3893, 2756, 2397, 4212, 2127, 1273, 1533, 4922, 2601, 1765, 107, 4643, 782, 1308, 1185, 4148, 1776, 3274, 1581, 2459, 4363, 2424, 822, 4204, 4693, 3541, 3882, 3602, 2505, 125, 2062, 4267, 595, 4054, 4847, 166, 1598, 2014, 2055, 2441, 3674, 3313, 2321, 4073, 4697, 4696, 2130, 2154, 2672, 4189, 2078, 3032, 2440, 3456, 2288, 3467, 4678, 981, 1829, 4591, 2007, 3976, 445, 4797, 2577, 4593, 700, 329, 2218, 803, 4610, 3139, 3600, 1835, 3587, 1090, 2401, 4780, 5, 1232, 337, 4673, 1891, 4384, 322, 4750, 1253, 1252, 1883, 3117, 4244, 4781, 3035, 1040, 2815, 568, 3941, 450, 27, 3194, 1525, 4084, 718, 4672, 4314, 70, 1623, 2175, 3127, 4079, 971, 2084, 2582, 3468, 245, 3998, 3933, 3562, 2209, 3651, 3161, 398, 2326, 2763, 4998, 1401, 4377, 731, 1498, 3673, 3865, 34, 3730, 2862, 609, 3364, 3652, 3210, 4663, 2077, 4852, 4070, 4879, 3270, 1676, 1725, 724, 4459, 4831, 1239, 2950, 4146, 360, 2606, 132, 2177, 4455, 4141, 936, 1075, 714, 4515, 4291, 3417, 4731, 1096, 928, 4911, 4909, 1474, 3357, 4440, 3709, 2382, 1730, 3311, 1952, 2136, 1234, 1871, 1254, 1621, 257, 2667, 2566, 5004, 2380, 1153, 2701, 3672, 66, 4187, 3464, 597, 4107, 3390, 4495, 2231, 1033, 2611, 2562, 244, 4849, 195, 4748, 4880, 3333, 576, 3078, 3338, 2510, 4858, 2429, 3834, 1749, 3177, 3555, 2419, 861, 777, 1935, 2770, 4430, 1902, 3755, 4624, 4906, 56, 4449, 2996, 2332, 4254, 182, 4468, 3806, 2267, 3530, 1125, 4297, 732, 3715, 2743, 3828, 3220, 3128, 4613, 1488, 1030, 2408, 1789, 4147, 3581, 2379, 3232, 2320, 4451, 2185, 3213, 2513, 1146, 549, 992, 3450, 472, 4115, 3383, 3149, 3943, 1151, 2079, 2304, 1670, 1853, 1631, 4580, 2795, 1635, 889, 3793, 555, 4945, 791, 3767, 440, 1579, 843, 1398, 590, 2758, 2666, 3824, 4755, 4320, 3162, 309, 1282, 4556, 4280, 785, 3720, 897, 4668, 1148, 1884, 2461, 1049, 4892, 926, 3239, 2970, 175, 4247, 1530, 1399, 3462, 2767, 3134, 4523, 2041, 2301, 4893, 1629, 4393, 4498, 3387, 2360, 3879, 869, 2814, 1660, 1382, 390, 1958, 4709, 817, 4933, 720, 634, 4226, 3766, 2306, 4462, 154, 4385, 3707, 378, 4769, 1496, 1545, 1720, 2560, 4971, 2490, 3439, 1715, 2163, 4833, 1215, 4233, 1444, 1159, 2891, 4902, 3540, 1860, 2034, 2224, 2649, 135, 3756, 662, 4563, 2168, 3320, 4011, 3021, 535, 4243, 163, 4193, 742, 4574, 3000, 4224, 1330, 3056, 3880, 1500, 3475, 3696, 2314, 4511, 2497, 4231, 3354, 921, 4061, 4002, 3312, 441, 1882, 2540, 3138, 4117, 1171, 2232, 2465, 4415, 3881, 4271, 3638, 638, 1511, 4044, 4829, 4475, 1462, 663, 833, 104, 1111, 2889, 3089, 4239, 1628, 3288, 1537, 3126, 2322, 36, 1020, 666, 3325, 893, 4033, 314, 4021, 2916, 370, 1139, 3710, 4015, 3191, 516, 1759, 4130, 494, 1811, 3305, 1178, 3071, 2700, 2001, 3698, 4383, 4686, 333, 3757, 3641, 1761, 1220, 655, 695, 4093, 1039, 1609, 4378, 740, 589, 1520, 775, 317, 3888, 4154, 3135, 326, 423, 3330, 4727, 4346, 192, 4272, 530, 4800, 882, 1432, 253, 4843, 1158, 2313, 404, 71, 1367, 291, 491, 1928, 4454, 4969, 1025, 842, 4633, 1207, 2554, 3261, 18, 1661, 4596, 887, 4142, 1681, 2546, 4627, 2603, 3706, 4203, 485, 4230, 30, 562, 1865, 1599, 3594, 451, 2678, 2317, 2260, 4242, 2807, 487, 1143, 4953, 1868, 4863, 3369, 2745, 1387, 4722, 1856, 637, 1963, 1815, 4, 1890, 3101, 2416, 741, 3110, 4751, 825, 1078, 1943, 3268, 4426, 4344, 181, 2845, 2588, 15, 1917, 2506, 4883, 4025, 1122, 1188, 1872, 4202, 1748, 4619, 4542, 1959, 1105, 88, 2073, 3492, 3501, 238, 3982, 4135, 3621, 4807, 3476, 2801, 4839, 627, 2865, 1610, 3276, 3859, 3017, 3939, 4716, 1047, 1363, 2051, 4825, 2281, 4420, 4162, 845, 4206, 1961, 4576, 1281, 1800, 4804, 1315, 1087, 2841, 2799, 1109, 648, 4964, 3765, 521, 3106, 2915, 3598, 3215, 3742, 3214, 4257, 3099, 420, 303, 3263, 190, 1447, 2161, 251, 811, 2043, 141, 2879, 4832, 77, 4030, 669, 3217, 442, 3576, 3404, 3260, 3392, 1805, 4265, 2493, 4739, 4292, 4913, 4608, 753, 723, 3966, 4606, 3343, 2271, 33, 1836, 1393, 1854, 9, 4478, 1653, 2307, 1580, 1095, 1247, 513, 3394, 4074, 4930, 4213, 854, 1810, 1231, 2673, 2874, 2484, 447, 3218, 221, 3795, 167, 3341, 1524, 2584, 735, 787, 2099, 4194, 2297, 311, 4635, 4834, 3203, 4000, 3815, 1438, 1070, 2435, 541, 2757, 3944, 2181, 1208, 2085, 1392, 4989, 3675, 193, 2432, 3969, 1450, 1752, 3490, 1086, 937, 1066, 755, 4086, 3281, 4671, 2594, 3322, 2671, 3088, 2962, 1968, 3015, 3550, 1529, 1593, 806, 4095, 2064, 4767, 1261, 3612, 1428, 2362, 2210, 2239, 4014, 4819, 2948, 1702, 4604, 3507, 870, 746, 3129, 1819, 429, 3592, 2450, 3624, 4330, 323, 3690, 3711, 3446, 69, 2399, 460, 2325, 1277, 3445, 1818, 4598, 4228, 1364, 3636, 1311, 1770, 3591, 584, 1259, 4813, 4195, 2311, 3036, 4111, 4365, 397, 4106, 1906, 2437, 1445, 629, 3708, 4163, 1356, 3451, 1269, 3560, 2227, 357, 867, 2738, 294, 2597, 3284, 4470, 4022, 137, 3798, 566, 3269, 2750, 3741, 2887, 1116, 330, 1043, 3386, 536, 2805, 4369, 3646, 2211, 4958, 3066, 13, 1391, 29, 456, 3733, 903, 3900, 346, 3231, 3163, 2204, 3768, 479, 4817, 3211, 1567, 4516, 1343, 1410, 4552, 2998, 1965, 948, 1683, 602, 2405, 2989, 4955, 3945, 1325, 2236, 4293, 2958, 3435, 1029, 3556, 3657, 1981, 1376, 1822, 331, 1476, 3334, 3620, 3434, 1709, 4386, 4867, 4637, 2592, 380, 4773, 3686, 920, 3853, 935, 3785, 2876, 546, 3970, 1617, 3960, 3350, 805, 424, 3265, 3447, 169, 4419, 2403, 4487, 2270, 1515, 3571, 4164, 4433, 2258, 4408, 2335, 1395, 681, 601, 1361, 2117, 224, 4649, 2285, 403, 2358, 4505, 1997, 1657, 639, 1400, 3769, 1060, 165, 4334, 3565, 2383, 3972, 4457, 2477, 3324, 736, 4053, 2467, 2507, 2613, 1953, 674, 618, 2394, 4076, 680, 393, 4741, 1, 1023, 4821, 1622, 293, 514, 4577, 1008, 110, 656, 1297, 868, 1645, 354, 3975, 4526, 3455, 4357, 3349, 4742, 1864, 2830, 623, 1441, 4448, 4827, 4484, 3821, 4215, 3523, 1921, 2446, 40, 3932, 1559, 2299, 2045, 4427, 4040, 4615, 2512, 2615, 2545, 385, 2517, 3393, 464, 64, 4666, 1483, 4296, 42, 951, 3570, 3924, 2162, 2100, 4882, 2142, 2439, 3665, 1708, 4569, 1994, 4908, 2422, 359, 241, 4474, 1855, 1291, 2384, 1216, 611, 4071, 588, 2653, 202, 1381, 913, 197, 4214, 3073, 4987, 3125, 4329, 539, 4747, 469, 426, 708, 2331, 173, 1470, 3794, 2103, 1226, 658, 3096, 3008, 2760, 2091, 3340, 3441, 4555, 4184, 2983, 91, 2203, 2096, 1758, 4806, 3453, 3911, 3670, 1903, 1646, 2986, 2808, 4488, 2438, 2908, 4387, 4323, 2113, 1126, 80, 1200, 4160, 2884, 931, 1144, 1246, 4081, 3130, 2000, 3400, 1538, 3228, 1242, 2632, 2660, 2431, 4260, 60, 3751, 4309, 941, 1648, 3102, 979, 2105, 5002, 778, 3754, 4891, 2160, 3227, 2722, 738, 2150, 4368, 3044, 2190, 2274, 4263, 3695, 2999, 1375, 2523, 2870, 1687, 1324, 1503, 2212, 2718, 259, 1173, 443, 583, 4639, 3825, 2605, 1682, 834, 3816, 325, 2786, 910, 3197, 829, 269, 4345, 3332, 3904, 4936, 4845, 2956, 1693, 2275, 671, 649, 2793, 3083, 4250, 1850, 4968, 3557, 2913, 3005, 2772, 2390, 3472, 3469, 3518, 999, 4399, 1944, 295, 2587, 406, 4362, 3522, 3543, 2768, 2609, 4198, 2797, 4036, 3454, 473, 210, 4131, 872, 1007, 2947, 4721, 3871, 4703, 4753, 96, 2563, 1899, 2897, 287, 382, 3188, 3702, 620, 653, 1499, 3604, 1465, 4729, 1517, 2657, 688, 2005, 4080, 1163, 1063, 4961, 4060, 711, 4957, 3037, 4784, 3029, 2230, 2068, 4407, 4822, 352, 2629, 4881, 3057, 891, 977, 49, 3363, 2369, 2474, 186, 4664, 4837, 3488, 1458, 712, 4861, 2308, 458, 438, 3727, 2201, 2289, 1792, 1901, 3024, 2934, 4096, 2936, 2985, 550, 1639, 2147, 3763, 4110, 616, 2602, 4352, 2361, 1102, 97, 2339, 3295, 3167, 4667, 4724, 1213, 3753, 1021, 2191, 3266, 3760, 3616, 2914, 2813, 4401, 1077, 1583, 964, 4960, 1627, 1532, 1390, 1934, 3687, 1911, 4554, 4640, 4167, 4400, 2893, 839, 4464, 790, 2310, 2346, 1004, 2349, 4139, 3085, 1195, 4756, 2071, 4683, 391, 4976, 646, 2196, 101, 4281, 2519, 341, 3533, 3380, 757, 3544, 1697, 2543, 1955, 292, 328, 1673, 4582, 1309, 965, 1701, 3912, 692, 3903, 2182, 249, 2806, 715, 4674, 105, 2796, 2984, 4530, 3775, 3745, 810, 2057, 313, 4109, 1212, 395, 1166, 3474, 1164, 351, 1286, 3316, 2114, 3573, 3843, 4007, 6, 4017, 3294, 4921, 2595, 1278, 4712, 1196, 3479, 2502, 2784, 2244, 4121, 2628, 1249, 4043, 2029, 2030, 2599, 2112, 2633, 24, 1817, 4348, 676, 1588, 2933, 4062, 4719, 374, 1728, 4266, 969, 3219, 4978, 4391, 1504, 4403, 4988, 4927, 4802, 3992, 786, 3206, 3800, 596, 4493, 4654, 2481, 3831, 959, 759, 1977, 4630, 3787, 4579, 392, 4090, 4974, 3666, 4227, 2623, 1113, 1177, 1526, 3133, 349, 2122, 3873, 957, 2521, 1990, 3084, 4018, 1741, 768, 3273, 3283, 4425, 3697, 1841, 3978, 17, 1972, 246, 2498, 3256, 2527, 3661, 255, 2614, 358, 2135, 691, 1738, 3344, 3292, 2549, 3182, 844, 4871, 2771, 3156, 4811, 3471, 2195, 146, 431, 685, 4255, 3061, 1923, 302, 3829, 418, 3018, 968, 2004, 3802, 124, 647, 2259, 2883, 2341, 3649, 4808, 1563, 4944, 619, 3832, 223, 2058, 2269, 2141, 3878, 3460, 3243, 3835, 1412, 3009, 4959, 2635, 1421, 1920, 371, 4068, 2511, 2676, 2829, 4701, 416, 4223, 1806, 1734, 3112, 1674, 3411, 332, 2280, 1206, 4760, 4700, 4500, 3761, 2471, 2082, 1964, 2094, 3614, 1509, 63, 947, 2964, 1851, 1756, 1938, 527, 1601, 1065, 3002, 4279, 3648, 2436, 4966, 1982, 4772, 1762, 4098, 1688, 2376, 1793, 2553, 2803, 1919, 1079, 3669, 335, 767, 1323, 4705, 3792, 3890, 4812, 4277, 3442, 205, 2188, 3633, 4704, 3987, 4200, 100, 963, 116, 2022, 1115, 1767, 4628, 2528, 369, 1605, 985, 607, 983, 4240, 519, 3508, 3478, 1288, 2152, 4865, 2012, 3776, 1285, 3, 4950, 532, 1843, 4956, 1587, 4732, 1976, 4708, 4735, 1134, 925, 1283, 976, 284, 3287, 4567, 4083, 4692, 4949, 668, 273, 4469, 131, 2257, 673, 1572, 89, 3510, 693, 1266, 598, 4249, 84, 1404, 297, 3725, 2953, 2060, 3631, 2452, 696, 3519, 3142, 2972, 4171, 1763, 4611, 3748, 2319, 2739, 1644, 1607, 2626, 2855, 3547, 4621, 2720, 1808, 2101, 4884, 428, 1724, 95, 902, 1614, 215, 1695, 468, 3482, 2200, 3855, 44, 2610, 682, 2907, 1717, 3788, 3342, 2046, 2327, 4463, 1690, 1019, 4235, 1179, 4979, 725, 3942, 3238, 4159, 3529, 3946, 654, 581, 2298, 4032, 4993, 4842, 826, 3144, 4428, 945, 699, 2003, 4123, 1782, 1405, 859, 2027, 4929, 4482, 2782, 4402, 38, 3272, 2731, 2702, 3481, 3545, 3928, 62, 2389, 3006, 4423, 2223, 3810, 3187, 520, 2779, 4762, 2255, 2937, 23, 2053, 3275, 2740, 2178, 1136, 2853, 683, 2714, 1478, 4629, 2618, 3645, 4920, 1595, 3444, 4614, 288, 4818, 3076, 3925, 1591, 2412, 3395, 3808, 4349, 3201, 177, 4207, 3628, 372, 3252, 2183, 3619, 3655, 3121, 222, 365, 112, 3954, 2279, 1859, 1307, 4467, 1027, 3887, 1766, 1927, 3977, 3068, 3196, 4127, 2264, 1467, 113, 4318, 3423, 3285, 1954, 191, 4559, 3064, 4607, 1774, 3091, 3360, 1880, 988, 4594, 81, 2445, 315, 2687, 4518, 1771, 1005, 2866, 4270, 3546, 1542, 4371, 4779, 2749, 2967, 1130, 613, 2324, 686, 1165, 298, 1713, 1834, 4264, 866, 59, 4471, 1804, 745, 2736, 2518, 1894, 1276, 2642, 283, 850, 4414, 160, 2693, 1755, 1451, 2215, 1975, 1085, 1235, 3625, 3063, 1336, 1101, 1757, 1978, 275, 4766, 830, 3948, 1359, 3694, 1656, 2276, 1991, 53, 3170, 792, 3512, 1712, 3374, 1327, 754, 3940, 4158, 50, 865, 2816, 3060, 2250, 3222, 4717, 2329, 356, 593, 4775, 171, 4948, 2945, 1830, 4744, 3836, 561, 4612, 3181, 2340, 2174, 3762, 439, 2293, 510, 83, 1604, 3703, 4835, 1948, 1472, 1439, 4149, 1619, 1600, 260, 4445, 1820, 2529, 973, 3250, 2503, 1306, 2472, 179, 3801, 1832, 1839, 4890, 3909, 4336, 1969, 2065, 2206, 929, 3567, 923, 4333, 1751, 2309, 4661, 3608, 1798, 2173, 1603, 3339, 2557, 475, 4382, 2995, 2291, 2333, 4617, 5011, 1849, 2778, 2235, 967, 2783, 534, 912, 2648, 2974, 1024, 26, 4437, 2790, 1698, 1528, 1228, 1744, 1799, 2479, 2159, 1006, 3819, 1377, 716, 1940, 554, 3437, 1732, 1201, 362, 282, 3994, 4310, 400, 4088, 1335, 199, 610, 4491, 4501, 2975, 2222, 1967, 2875, 3868, 2107, 4422, 1449, 4718, 250, 2406, 3401, 4995, 4980, 3552, 227, 2485, 4789, 4185, 890, 1265, 3916, 569, 3178, 2515, 2220, 353, 772, 427], "validation": [5272, 5170, 5031, 5149, 5142, 5190, 5304, 5377, 5209, 5359, 5049, 5345, 5424, 5291, 5373, 5241, 5418, 5360, 5452, 5503, 5428, 5275, 5442, 5017, 5470, 5263, 5437, 5488, 5340, 5252, 5238, 5328, 5143, 5215, 5228, 5383, 5475, 5075, 5159, 5343, 5131, 5501, 5412, 5301, 5355, 5106, 5305, 5283, 5494, 5414, 5124, 5366, 5046, 5172, 5054, 5436, 5292, 5362, 5242, 5173, 5012, 5111, 5202, 5286, 5262, 5196, 5257, 5087, 5336, 5484, 5511, 5245, 5446, 5400, 5354, 5296, 5298, 5187, 5329, 5115, 5092, 5074, 5032, 5254, 5122, 5353, 5370, 5321, 5036, 5144, 5497, 5169, 5022, 5246, 5063, 5507, 5356, 5440, 5160, 5458, 5479, 5382, 5310, 5069, 5110, 5264, 5307, 5179, 5315, 5287, 5444, 5113, 5037, 5490, 5103, 5052, 5088, 5098, 5466, 5161, 5313, 5331, 5364, 5309, 5294, 5481, 5217, 5493, 5047, 5434, 5237, 5079, 5186, 5482, 5181, 5140, 5023, 5185, 5407, 5250, 5397, 5425, 5320, 5326, 5367, 5429, 5062, 5028, 5216, 5456, 5219, 5183, 5093, 5420, 5465, 5226, 5295, 5344, 5347, 5395, 5080, 5474, 5472, 5200, 5461, 5342, 5462, 5312, 5293, 5378, 5015, 5234, 5468, 5322, 5167, 5369, 5163, 5351, 5134, 5445, 5455, 5492, 5469, 5259, 5384, 5506, 5165, 5129, 5510, 5139, 5285, 5064, 5410, 5357, 5145, 5308, 5051, 5374, 5021, 5094, 5253, 5061, 5471, 5416, 5385, 5419, 5500, 5403, 5402, 5265, 5118, 5038, 5333, 5231, 5258, 5027, 5255, 5411, 5175, 5495, 5289, 5050, 5427, 5375, 5261, 5278, 5335, 5431, 5081, 5390, 5365, 5120, 5473, 5164, 5393, 5016, 5166, 5128, 5208, 5090, 5487, 5318, 5406, 5082, 5327, 5389, 5441, 5210, 5260, 5319, 5282, 5158, 5198, 5401, 5332, 5178, 5013, 5085, 5341, 5422, 5213, 5443, 5112, 5454, 5415, 5126, 5467, 5426, 5485, 5450, 5097, 5014, 5058, 5239, 5188, 5306, 5505, 5056, 5303, 5290, 5076, 5155, 5386, 5324, 5099, 5477, 5223, 5146, 5053, 5451, 5154, 5107, 5066, 5409, 5033, 5194, 5268, 5180, 5502, 5147, 5464, 5136, 5152, 5206, 5192, 5379, 5421, 5114, 5195, 5349, 5212, 5020, 5025, 5405, 5334, 5417, 5266, 5404, 5346, 5387, 5150, 5151, 5184, 5220, 5135, 5381, 5221, 5330, 5449, 5438, 5071, 5288, 5277, 5396, 5498, 5392, 5423, 5350, 5029, 5041, 5218, 5413, 5105, 5148, 5068, 5302, 5432, 5156, 5311, 5203, 5235, 5358, 5439, 5035, 5060, 5077, 5026, 5137, 5157, 5430, 5274, 5280, 5251, 5284, 5101, 5108, 5459, 5042, 5380, 5478, 5096, 5132, 5117, 5024, 5045, 5394, 5271, 5463, 5070, 5199, 5019, 5138, 5162, 5256, 5399, 5040, 5211, 5447, 5193, 5133, 5224, 5177, 5176, 5197, 5486, 5086, 5201, 5240, 5225, 5300, 5072, 5205, 5435, 5317, 5496, 5123, 5269, 5457, 5116, 5504, 5337, 5084, 5102, 5244, 5453, 5460, 5236, 5325, 5314, 5059, 5171, 5141, 5100, 5222, 5119, 5121, 5248, 5276, 5230, 5267, 5361, 5227, 5483, 5281, 5018, 5189, 5352, 5229, 5095, 5489, 5153, 5214, 5448, 5270, 5339, 5130, 5055, 5039, 5091, 5207, 5297, 5232, 5398, 5508, 5089, 5083, 5125, 5243, 5073, 5057, 5499, 5279, 5174, 5391, 5323, 5491, 5127, 5273, 5348, 5249, 5368, 5067, 5433, 5509, 5043, 5104, 5030, 5065, 5363, 5480, 5408, 5371, 5034, 5109, 5388, 5191, 5316, 5182, 5078, 5476, 5048, 5247, 5299, 5233, 5372, 5044, 5168, 5338, 5376, 5204], "test": [5657, 5674, 6008, 5912, 5618, 5821, 5848, 5882, 5621, 5849, 5732, 5709, 5740, 5903, 5854, 5945, 5537, 5889, 5746, 5952, 5552, 5606, 5813, 5699, 5936, 5689, 5607, 5959, 5532, 5966, 5745, 5664, 5588, 5847, 5609, 5686, 5942, 5720, 5569, 5886, 5575, 6009, 5803, 5594, 5703, 5637, 5683, 5766, 5632, 5881, 5634, 5919, 5973, 5956, 5825, 5911, 5586, 5739, 5786, 5892, 5682, 5902, 5574, 5744, 5597, 5561, 5715, 5548, 5661, 5770, 6003, 5784, 5549, 5567, 5643, 5933, 5719, 5939, 5531, 5787, 5947, 5651, 5557, 5521, 5645, 5516, 5862, 5613, 5910, 5571, 5824, 5764, 5630, 5815, 5653, 5514, 5623, 5858, 5960, 5519, 5658, 5975, 5794, 5857, 5694, 5941, 5633, 5615, 5998, 5931, 5873, 5897, 5666, 5754, 5879, 5921, 5696, 5963, 5776, 5938, 5872, 5871, 5797, 5904, 5726, 5981, 6007, 5538, 5852, 5948, 5951, 5984, 5940, 5783, 5826, 5601, 5790, 5675, 5839, 5913, 5835, 5993, 5692, 5779, 5717, 5576, 5978, 5795, 5707, 5734, 5880, 5678, 6002, 5965, 5908, 5891, 5584, 5928, 5827, 5566, 5646, 5731, 6010, 5676, 5687, 5979, 5805, 5527, 5923, 5568, 6011, 5828, 5518, 5542, 5760, 5883, 5996, 5656, 5756, 5647, 5693, 5608, 5778, 5845, 6004, 5648, 5905, 5702, 5550, 5831, 5624, 5728, 5672, 5988, 5989, 5914, 5572, 5551, 5865, 5755, 5750, 5860, 5541, 5769, 5593, 5558, 5955, 5631, 5878, 5713, 5890, 5982, 5598, 5604, 5967, 5733, 5801, 5917, 5983, 5846, 5701, 5763, 5704, 5735, 5748, 5833, 5513, 5525, 5591, 5565, 5793, 5671, 5585, 5736, 5638, 5957, 5611, 5834, 5751, 5842, 5512, 5626, 5564, 5749, 5579, 5811, 5840, 5639, 5640, 5543, 5581, 5758, 5946, 5844, 5898, 5774, 5547, 5708, 5915, 5667, 5563, 5863, 5690, 5899, 5723, 5876, 5580, 5867, 5895, 5644, 5986, 5773, 5909, 5789, 5546, 5599, 5627, 5864, 5949, 5556, 5798, 5806, 5610, 5660, 5673, 5812, 5555, 5605, 5837, 5530, 5526, 5935, 5706, 5777, 5929, 5757, 5781, 5592, 5977, 5768, 5961, 5843, 5985, 5654, 5718, 5969, 5636, 5901, 5887, 5900, 5714, 5668, 5573, 5810, 5596, 5861, 5832, 5926, 5968, 5808, 5642, 5602, 5741, 5868, 5991, 5520, 5679, 5762, 5722, 5818, 5870, 5809, 5850, 5578, 5533, 5670, 5583, 5920, 5859, 5659, 5641, 5681, 5677, 5729, 5866, 5807, 5820, 5925, 5877, 5540, 5655, 5958, 5765, 5665, 5737, 5924, 5695, 5788, 5927, 5792, 5617, 5799, 5622, 5625, 5943, 5962, 5869, 5698, 5649, 5590, 5836, 5782, 5816, 5635, 5524, 5800, 5562, 5906, 5742, 5523, 5976, 5974, 5688, 5603, 5934, 5802, 5684, 5628, 5528, 5680, 5529, 5771, 5616, 5785, 5995, 5553, 5885, 5838, 5612, 5992, 5994, 5559, 5619, 5747, 5830, 5536, 5829, 6006, 5697, 5700, 5894, 5752, 5874, 5888, 5856, 5582, 5767, 5570, 5990, 5721, 5932, 5577, 5999, 5663, 5515, 5534, 5705, 5791, 5743, 5875, 5922, 5517, 5761, 5918, 5796, 5930, 5589, 5650, 5944, 5817, 5804, 5822, 5554, 5522, 5851, 5753, 5595, 5775, 5691, 5953, 5964, 5545, 5841, 5997, 5972, 5980, 5937, 5855, 5725, 5544, 5710, 6000, 5819, 5950, 5652, 5971, 5724, 5620, 5823, 5738, 5662, 5987, 5896, 5685, 5893, 6001, 5907, 5669, 5730, 5916, 6005, 5780, 5884, 5560, 5954, 5853, 5600, 5711, 5759, 5970, 5629, 5587, 5727, 5614, 5539, 5716, 5535, 5772, 5712, 5814]}, {"train": [4665, 453, 125, 4532, 2306, 4944, 3359, 3435, 3118, 296, 2819, 3759, 2091, 4775, 304, 1051, 1154, 4227, 1750, 2581, 4772, 4628, 2137, 4970, 1360, 4827, 4791, 3708, 2928, 4784, 2096, 1079, 4663, 2273, 1082, 1703, 3937, 3379, 3182, 1275, 2993, 151, 2905, 2710, 245, 4267, 3017, 2539, 4095, 1939, 3180, 4338, 2027, 2656, 3523, 4378, 2944, 130, 4277, 4221, 2392, 1736, 4369, 3014, 4106, 2128, 4546, 3229, 1749, 848, 2552, 4821, 1374, 2674, 2639, 3246, 4769, 1201, 2588, 2015, 2599, 3584, 68, 2379, 3361, 126, 3911, 4638, 3074, 2801, 3106, 236, 2458, 2109, 4506, 159, 2727, 4300, 1322, 4189, 2380, 958, 2213, 73, 4793, 737, 4630, 1080, 2104, 526, 273, 187, 4123, 2579, 2456, 85, 4874, 4445, 2665, 3445, 2092, 3783, 1876, 286, 1538, 2619, 1002, 2726, 3767, 439, 1252, 303, 2357, 1866, 3102, 2988, 2704, 4150, 4192, 1112, 4742, 1845, 1791, 1686, 2219, 4981, 3048, 2223, 3238, 3984, 870, 2969, 4810, 3585, 3382, 3557, 680, 1381, 2528, 114, 879, 500, 1732, 2045, 4955, 1731, 2751, 4306, 2208, 3019, 4559, 2718, 3513, 3712, 1282, 1647, 2870, 3352, 1072, 291, 2487, 328, 1077, 1152, 2079, 2992, 3295, 1332, 2902, 1717, 4174, 4324, 3669, 3516, 70, 2067, 2673, 3701, 4656, 2942, 4263, 4230, 204, 1016, 871, 1672, 1959, 3714, 2025, 409, 4654, 3413, 2013, 1431, 2176, 691, 4995, 1126, 250, 2258, 4099, 4328, 1899, 433, 4431, 292, 1461, 2282, 3881, 2959, 157, 4423, 3142, 4361, 3989, 3789, 1579, 807, 143, 535, 2002, 4256, 4073, 3158, 3302, 2167, 2498, 480, 772, 3883, 1492, 652, 1250, 1985, 1441, 3063, 55, 3832, 1978, 4640, 3038, 847, 3284, 2035, 2715, 4045, 3145, 3788, 2225, 572, 4996, 91, 2394, 2805, 3449, 2709, 217, 3776, 3864, 3570, 2772, 1211, 571, 1288, 2114, 1671, 1626, 1668, 1843, 3830, 565, 336, 3533, 2668, 1859, 1639, 3800, 1463, 2937, 3578, 2906, 4813, 487, 4509, 411, 2215, 3422, 2912, 2130, 2034, 4679, 4427, 4050, 3515, 1107, 2348, 4298, 541, 4257, 3135, 4390, 179, 4155, 313, 3020, 4834, 1610, 4849, 5002, 906, 2197, 4674, 315, 2372, 3342, 922, 1774, 4407, 256, 3477, 1289, 1459, 4777, 3583, 3136, 4641, 4713, 3847, 401, 4229, 4646, 3942, 602, 1778, 2712, 1279, 2977, 455, 2147, 3162, 1847, 710, 4434, 751, 3476, 2464, 3428, 3988, 4124, 694, 171, 4921, 1095, 3279, 4634, 4145, 2615, 3051, 861, 4054, 3281, 3268, 860, 1600, 3769, 4254, 2304, 2292, 185, 1246, 1839, 3391, 3762, 1188, 1825, 781, 2935, 3026, 2976, 4837, 1711, 2435, 2084, 4161, 2414, 730, 789, 314, 378, 4247, 482, 1678, 1134, 4194, 587, 62, 1828, 2480, 3995, 2179, 3927, 2701, 4788, 2194, 1330, 895, 1061, 156, 392, 2584, 4731, 4956, 4933, 2570, 1327, 2901, 1755, 4175, 479, 4997, 4383, 4366, 1233, 854, 841, 429, 3380, 3398, 4108, 2916, 1817, 465, 2652, 1297, 1772, 4472, 4525, 2211, 3186, 3994, 1932, 3657, 4608, 84, 3968, 4712, 641, 3733, 713, 4882, 398, 2302, 2070, 4835, 3331, 585, 363, 1212, 1295, 3998, 997, 2681, 1545, 760, 2199, 3442, 2455, 2144, 2482, 4694, 2699, 425, 3267, 3146, 1504, 1230, 352, 2831, 120, 1488, 733, 1792, 1053, 636, 2980, 2186, 1674, 399, 633, 2017, 4535, 268, 1302, 414, 3540, 2263, 413, 2538, 2102, 3318, 1995, 3130, 673, 926, 131, 2237, 3239, 4403, 1380, 1677, 1695, 696, 4042, 777, 2447, 3790, 1879, 2725, 316, 4547, 4452, 4865, 1811, 2254, 3423, 4704, 2170, 2750, 117, 4819, 4838, 4213, 4606, 1364, 1448, 567, 2506, 1005, 1362, 4449, 1910, 3965, 3552, 1733, 829, 3447, 3384, 2536, 1904, 3123, 3270, 2548, 975, 645, 2119, 1540, 3410, 2400, 4187, 3742, 3713, 2463, 1588, 443, 4264, 3859, 394, 3774, 1873, 69, 4302, 2445, 3967, 4845, 1264, 742, 3448, 1181, 213, 2391, 26, 501, 4347, 3724, 4916, 1209, 1464, 2173, 1192, 2424, 4781, 2582, 1251, 3000, 302, 1421, 82, 2853, 1479, 1583, 2361, 656, 3912, 3228, 1173, 1278, 882, 423, 387, 4895, 704, 3153, 3586, 4402, 4863, 4471, 4653, 1555, 724, 24, 2934, 4460, 1587, 342, 3673, 2914, 1734, 3704, 3392, 2587, 2317, 4920, 4918, 1665, 2406, 2163, 2875, 4544, 4244, 1767, 3343, 2039, 4063, 2854, 2089, 1982, 1236, 2051, 4239, 2139, 1216, 3464, 2274, 386, 2326, 1984, 4127, 3040, 4468, 3248, 4799, 2345, 808, 1224, 3848, 1799, 3666, 963, 3461, 3681, 1661, 4765, 1227, 4884, 2459, 1968, 3062, 4833, 625, 2724, 1735, 1191, 2549, 2594, 1204, 3204, 2074, 690, 4991, 222, 4203, 4831, 2054, 1679, 1358, 105, 3116, 4368, 3548, 4514, 4624, 2078, 2466, 846, 4007, 404, 1878, 3150, 1537, 3745, 4224, 4783, 2373, 936, 211, 725, 111, 2745, 4166, 732, 2689, 4249, 1830, 1522, 2821, 2417, 1546, 1054, 2889, 651, 2395, 4212, 4275, 676, 3559, 2983, 3723, 3741, 2288, 3842, 3748, 1145, 220, 515, 457, 2075, 2907, 2932, 2058, 3030, 3326, 1536, 4447, 4139, 446, 3868, 948, 17, 384, 640, 621, 688, 2291, 3403, 2501, 4226, 4622, 4199, 3039, 3144, 3027, 2181, 3097, 466, 4946, 786, 81, 509, 1627, 3354, 4751, 2169, 697, 4013, 529, 4800, 3931, 4081, 2000, 3100, 2755, 3446, 1172, 1393, 1318, 1572, 3561, 1298, 3517, 1556, 4552, 2783, 249, 1190, 3918, 2295, 4412, 3408, 2500, 629, 2033, 1383, 686, 2429, 2171, 3605, 1228, 2368, 591, 4599, 251, 2708, 3377, 4875, 3949, 3178, 3878, 3816, 2281, 4754, 4386, 5009, 3563, 4773, 3181, 2216, 3255, 2007, 3951, 4566, 2762, 814, 3574, 1, 4232, 2110, 1738, 3397, 1705, 1650, 1162, 1202, 468, 49, 644, 367, 562, 2342, 94, 1771, 3656, 838, 4579, 1439, 918, 155, 267, 4032, 1662, 3999, 3256, 2312, 4576, 4110, 1808, 3316, 2228, 4869, 3737, 262, 1457, 4494, 4339, 4168, 4978, 4524, 4969, 1757, 3372, 1456, 1914, 4066, 1449, 2775, 1491, 2191, 1573, 4739, 4308, 2041, 2826, 1484, 4157, 408, 928, 2883, 4271, 5, 4610, 969, 1576, 708, 723, 3732, 1420, 1945, 2650, 1207, 2655, 4373, 3293, 2694, 4655, 959, 3827, 2409, 4052, 3416, 2613, 1007, 3869, 1530, 2986, 701, 3071, 2809, 888, 2094, 2142, 815, 2953, 2221, 4240, 3224, 4510, 2982, 1405, 3581, 3219, 4508, 752, 4888, 3521, 2434, 3562, 2155, 3068, 950, 1718, 1144, 3134, 1131, 836, 739, 3928, 3271, 573, 768, 2865, 1664, 603, 2201, 4396, 3809, 2072, 2251, 4332, 4435, 2492, 627, 924, 643, 2446, 3217, 1339, 1494, 3879, 1138, 3691, 3836, 3560, 3620, 703, 3501, 1168, 3496, 780, 3176, 982, 663, 2895, 4632, 4676, 1990, 1993, 4708, 4313, 1257, 3258, 3781, 4177, 803, 2707, 2437, 914, 1898, 1730, 665, 734, 22, 1862, 3575, 3661, 1552, 170, 2625, 2469, 4661, 2265, 3351, 1833, 3286, 3807, 3893, 2077, 1205, 4356, 4585, 4159, 1180, 3640, 3389, 3396, 2920, 1398, 4497, 3948, 2880, 2019, 19, 12, 4923, 67, 568, 3257, 4868, 371, 1624, 1885, 2232, 3184, 88, 2010, 671, 3462, 2432, 4112, 1052, 2729, 1029, 3599, 4190, 1509, 2160, 3699, 307, 1414, 1903, 2598, 90, 4717, 3076, 677, 1842, 2835, 4394, 373, 3710, 2957, 2275, 3023, 15, 1740, 3650, 525, 2245, 3022, 4162, 1628, 3481, 1335, 4473, 2798, 1166, 4365, 4469, 1966, 167, 2122, 3622, 3990, 4367, 1058, 699, 4096, 2871, 3173, 3486, 4071, 3785, 537, 2352, 1266, 3456, 13, 1157, 1008, 1897, 1513, 1019, 662, 3802, 305, 3305, 205, 4589, 118, 4906, 1854, 2140, 4235, 756, 4720, 649, 4612, 4790, 3795, 5010, 3070, 2097, 896, 1424, 580, 4197, 940, 469, 4602, 4015, 1865, 4387, 866, 1561, 3127, 3940, 4307, 989, 1745, 1744, 494, 2226, 4999, 4279, 4691, 4567, 4438, 4701, 4557, 4577, 2453, 1026, 1832, 1165, 1447, 1465, 196, 4272, 2235, 3287, 4787, 2012, 339, 4337, 2362, 79, 2913, 819, 1329, 3757, 3903, 3839, 3735, 718, 2807, 1128, 4250, 2580, 1967, 4268, 119, 320, 4979, 1655, 3775, 3630, 3923, 2164, 1436, 1680, 1760, 4669, 235, 2659, 5005, 1475, 544, 1018, 3192, 3546, 727, 1549, 1046, 1342, 2143, 2847, 2329, 1369, 3851, 18, 3987, 4490, 4450, 294, 4290, 2157, 628, 4688, 594, 3791, 1422, 3718, 1943, 2398, 3703, 4985, 102, 833, 635, 4767, 1262, 4331, 2442, 560, 1806, 2863, 2869, 546, 3191, 1075, 3468, 2542, 3997, 3844, 3750, 1375, 3782, 165, 4696, 2233, 1247, 3826, 1593, 1533, 1103, 4957, 1566, 1838, 1220, 2375, 632, 4940, 4292, 3165, 4344, 764, 2600, 2927, 1285, 3756, 326, 3312, 905, 2521, 1124, 1396, 164, 3218, 4500, 61, 2055, 410, 246, 3405, 3665, 4987, 2956, 3922, 1645, 1013, 2861, 3356, 375, 4018, 3196, 761, 209, 2752, 1365, 4930, 1724, 2136, 471, 1506, 1462, 207, 3096, 340, 1869, 3067, 2486, 57, 4657, 2515, 3368, 993, 3161, 4425, 593, 2630, 1980, 4914, 1355, 620, 2257, 2504, 3291, 874, 979, 2677, 3787, 4430, 1512, 4024, 2107, 4377, 4109, 3230, 4911, 478, 624, 370, 1692, 4296, 498, 3247, 4974, 2892, 798, 3889, 1171, 2465, 1471, 325, 4697, 2926, 1011, 2852, 3761, 4989, 4584, 3653, 2525, 1858, 4542, 3225, 1088, 93, 4086, 2918, 3886, 1347, 1411, 4141, 1786, 3840, 221, 2309, 4091, 3095, 4411, 2134, 1226, 2214, 2558, 4752, 3110, 3614, 1829, 4346, 837, 2780, 4747, 4021, 1177, 4107, 3676, 195, 2404, 653, 4270, 1814, 945, 3589, 2776, 4297, 4334, 4733, 3986, 1574, 1481, 2896, 2618, 3877, 1041, 1547, 3855, 1754, 2888, 3222, 1851, 1569, 2700, 4809, 3488, 1788, 3850, 5000, 4030, 2246, 150, 4475, 3032, 1889, 3489, 4604, 766, 1682, 442, 223, 685, 981, 2227, 909, 3793, 506, 3579, 4844, 3553, 834, 5003, 3765, 4635, 3752, 3700, 173, 1438, 2038, 1834, 1442, 5004, 3778, 2366, 2671, 4252, 2814, 1117, 4789, 3675, 1451, 1363, 3812, 3232, 1519, 257, 4348, 4012, 2759, 3348, 1085, 3243, 3419, 2556, 214, 350, 3639, 876, 4503, 4186, 4993, 4005, 2702, 28, 4488, 4436, 2573, 2269, 3917, 3093, 1136, 1809, 4939, 4457, 4932, 4881, 3235, 1203, 4716, 3065, 3603, 4818, 726, 1010, 4748, 32, 1544, 2711, 2679, 451, 2643, 4595, 2608, 202, 2731, 2502, 4295, 4301, 1337, 4660, 2697, 3465, 4943, 2975, 3474, 3667, 2517, 2545, 3371, 3684, 3904, 290, 4664, 4588, 3494, 1795, 968, 3137, 4479, 4545, 2206, 4149, 3009, 2771, 505, 1525, 101, 1856, 238, 3411, 1357, 1034, 867, 351, 4028, 3443, 1918, 1758, 2537, 4778, 4808, 2879, 3532, 228, 4990, 4304, 3426, 1222, 2316, 3117, 865, 4351, 2586, 1658, 1972, 9, 4583, 1956, 4364, 3008, 278, 430, 1276, 3636, 3157, 2843, 2753, 3418, 4206, 4758, 1789, 2836, 391, 3542, 4980, 4158, 966, 192, 1286, 1941, 4975, 2829, 1887, 4465, 2401, 2385, 4605, 4048, 2332, 4756, 4205, 3340, 2812, 2310, 403, 1242, 4225, 2, 194, 4992, 953, 4090, 1651, 1912, 4607, 1741, 3289, 416, 1156, 2230, 2890, 3467, 2764, 2305, 1807, 4305, 705, 4563, 3907, 1770, 4234, 2101, 2195, 1311, 1911, 3353, 4908, 3885, 464, 1272, 23, 4976, 1198, 4890, 121, 2758, 513, 692, 946, 2834, 3996, 137, 2145, 1244, 3431, 3843, 2604, 3357, 4796, 1999, 2682, 1991, 264, 4702, 912, 216, 1307, 2705, 4381, 230, 3882, 1953, 133, 3711, 516, 2638, 1532, 639, 4601, 2648, 3694, 4922, 2832, 2647, 1353, 2997, 20, 827, 4942, 1142, 1137, 2138, 584, 793, 1618, 3202, 144, 4163, 4441, 1981, 3490, 3205, 530, 728, 4762, 1652, 2644, 4432, 597, 4294, 397, 149, 899, 1404, 2640, 2741, 3231, 802, 531, 77, 3550, 2606, 1419, 247, 521, 1343, 4994, 3276, 1646, 1386, 4245, 153, 276, 3439, 2773, 994, 4152, 3002, 2168, 4816, 2384, 4554, 1300, 2494, 2874, 3806, 3394, 2632, 927, 382, 3814, 3344, 2796, 4415, 4529, 2308, 52, 4370, 3582, 312, 4388, 4937, 4498, 2744, 3199, 3207, 1524, 1848, 4101, 2837, 169, 1310, 4288, 1071, 795, 3080, 162, 1199, 1965, 2174, 1934, 3947, 2924, 1726, 1012, 206, 1634, 1779, 3768, 4164, 3098, 765, 3744, 3921, 2908, 2855, 3113, 2609, 1499, 3420, 1864, 558, 4098, 2513, 2567, 1175, 1909, 1490, 4527, 891, 1389, 2561, 2621, 281, 770, 1924, 1937, 3101, 418, 4689, 3674, 3588, 474, 1115, 4444, 3245, 1979, 3336, 3236, 1835, 1410, 2028, 2720, 1425, 4283, 3507, 1382, 1309, 2667, 3526, 3637, 178, 1701, 4451, 1539, 3360, 3874, 275, 381, 1098, 4649, 588, 908, 3747, 3037, 1370, 937, 2770, 4089, 366, 2018, 709, 10, 858, 2457, 999, 873, 272, 4672, 2377, 923, 1882, 140, 4104, 2856, 4877, 2624, 1110, 1118, 4913, 3055, 2810, 2149, 2262, 2530, 1675, 2754, 3399, 2930, 664, 492, 3956, 4693, 4745, 2601, 2743, 3693, 1317, 2436, 1292, 2797, 1970, 2999, 2403, 3200, 1321, 1446, 4518, 4867, 2443, 1510, 1691, 4035, 4236, 3664, 2532, 1187, 1777, 2175, 1681, 4782, 821, 2675, 4617, 840, 2419, 4593, 2533, 1304, 4536, 3890, 1531, 3740, 563, 1476, 2967, 166, 1529, 1125, 4491, 4406, 504, 188, 1625, 951, 3168, 3482, 2614, 900, 1819, 4053, 3692, 241, 2087, 3706, 4677, 2627, 4352, 1445, 2210, 2040, 2739, 1700, 4478, 428, 1584, 2917, 97, 1089, 2448, 284, 2484, 3503, 2964, 4409, 4363, 904, 507, 25, 2962, 2963, 4047, 1604, 4480, 2009, 87, 1976, 2166, 4636, 3155, 2666, 1563, 447, 3872, 2663, 4651, 270, 2339, 329, 3433, 3091, 1917, 551, 2165, 3387, 3011, 4097, 3406, 4179, 3838, 4512, 3054, 3322, 4285, 3860, 3531, 2247, 4666, 604, 2553, 3263, 4218, 3566, 2629, 4740, 2355, 4486, 2151, 1280, 2654, 3033, 4094, 2550, 4565, 2311, 43, 3220, 4852, 1066, 4950, 1708, 3197, 3596, 2351, 2241, 4463, 1319, 2857, 499, 2061, 1045, 427, 3670, 4020, 1660, 1581, 3753, 1033, 50, 2060, 1818, 4587, 2440, 3799, 757, 2973, 1739, 1487, 4022, 4894, 1636, 3139, 2146, 11, 242, 30, 3651, 389, 1253, 1765, 3075, 2979, 1558, 1949, 1619, 1032, 4327, 181, 3214, 3846, 4198, 2220, 2792, 4682, 3498, 2966, 3451, 485, 1601, 1296, 3677, 400, 1060, 3662, 2872, 1094, 2057, 261, 3534, 1351, 3804, 4785, 1341, 564, 4670, 1270, 2255, 1906, 2529, 1929, 1927, 3358, 4511, 1916, 2260, 4077, 773, 1729, 3705, 355, 212, 2093, 186, 472, 1086, 3652, 3425, 804, 3303, 4505, 1575, 4592, 154, 3766, 337, 1258, 1669, 3805, 3296, 1595, 2462, 4325, 4652, 1950, 3506, 3212, 3924, 4326, 190, 83, 4474, 4627, 2864, 4060, 884, 1040, 1009, 1933, 4538, 2103, 3875, 3278, 4117, 1690, 4113, 2279, 2974, 2844, 3311, 2503, 218, 219, 2172, 3043, 4899, 626, 2670, 2349, 1141, 3610, 343, 3424, 1455, 4040, 2098, 2585, 2315, 3472, 3958, 1607, 1648, 2382, 4092, 856, 483, 881, 1155, 1132, 4541, 2867, 4144, 863, 2497, 3183, 4293, 1787, 1613, 929, 243, 522, 4611, 42, 21, 51, 3524, 1986, 1654, 1269, 2922, 3429, 2891, 3299, 1241, 4846, 1263, 907, 2422, 1323, 3124, 4167, 4185, 4314, 3539, 3493, 99, 3079, 4216, 4706, 393, 2733, 2207, 4499, 4621, 4278, 1167, 972, 4678, 4815, 3852, 1888, 3195, 539, 2493, 684, 613, 3971, 3897, 4609, 1836, 805, 2133, 1900, 3234, 1983, 1901, 4965, 1699, 3213, 2692, 3151, 536, 601, 654, 4841, 3867, 14, 3274, 822, 648, 3641, 1947, 3072, 3013, 542, 3264, 4181, 650, 3319, 4681, 4476, 2021, 3537, 2418, 3720, 855, 3459, 2148, 890, 1059, 3169, 3086, 1963, 553, 1480, 2037, 3978, 2300, 4111, 3350, 1293, 4764, 947, 3654, 1320, 3511, 3508, 484, 3060, 2248, 2485, 4070, 2732, 702, 3171, 3504, 1867, 395, 1334, 1746, 2696, 4848, 489, 4883, 3739, 1477, 1827, 1684, 675, 1676, 4210, 3166, 1656, 2961, 3983, 4433, 3629, 1714, 614, 1140, 3811, 2184, 4214, 1193, 1955, 4067, 2444, 4531, 100, 722, 2626, 1954, 3680, 1047, 845, 4825, 4774, 3522, 1921, 3211, 1486, 4291, 3991, 2430, 4692, 2212, 1101, 1881, 2116, 877, 3606, 2099, 2183, 107, 4253, 208, 2544, 2793, 3120, 2746, 1603, 2278, 441, 365, 4208, 2645, 4231, 1108, 346, 3828, 1974, 4730, 2052, 3695, 2044, 3519, 4461, 1243, 616, 741, 957, 4603, 3061, 1542, 253, 4936, 2156, 1644, 3518, 4945, 1352, 528, 231, 3993, 3727, 2303, 2460, 80, 1944, 4169, 579, 1520, 33, 3484, 3480, 1551, 4376, 1528, 1314, 3249, 935, 1570, 4100, 2823, 569, 2778, 1863, 3233, 720, 4947, 4573, 3309, 1003, 3612, 754, 1062, 2909, 1868, 1081, 3746, 3122, 2958, 2622, 3734, 3073, 3325, 3164, 1936, 2081, 1964, 4961, 3366, 3577, 2641, 322, 4967, 2806, 1913, 5007, 2287, 913, 3328, 4398, 1210, 4680, 557, 2526, 976, 4397, 2076, 2314, 857, 3018, 4502, 362, 2162, 2209, 2113, 2131, 3853, 1472, 349, 903, 3007, 683, 3094, 4779, 2135, 738, 4814, 3663, 2277, 2117, 2575, 2799, 4261, 3064, 3128, 4889, 300, 2787, 892, 4493, 4924, 3619, 3541, 3818, 956, 4749, 919, 1621, 2820, 2985, 45, 2374, 4454, 3726, 2242, 3668, 2531, 3932, 4709, 658, 973, 2605, 745, 2259, 4281, 2683, 1390, 3820, 3438, 2449, 4555, 2562, 2024, 2877, 2576, 1092, 348, 4795, 4811, 1299, 4201, 2774, 4743, 2825, 4737, 1508, 4526, 1189, 4442, 1737, 2330, 586, 237, 1543, 152, 2747, 638, 4136, 1615, 2534, 3035, 2706, 872, 4122, 735, 2029, 1001, 3736, 298, 3103, 3933, 4209, 1394, 4907, 4349, 450, 1240, 1614, 3687, 2020, 4523, 1139, 1840, 3962, 2642, 4220, 4753, 2356, 4925, 1214, 3115, 1926, 1849, 415, 2127, 3898, 92, 2399, 682, 4520, 40, 3731, 4116, 2779, 862, 106, 4513, 331, 3943, 2507, 849, 4515, 2972, 954, 2714, 3083, 358, 1742, 2083, 3611, 762, 1256, 4228, 1197, 4824, 27, 2438, 109, 3025, 3346, 436, 540, 1580, 2572, 1748, 420, 4319, 71, 3835, 3375, 2276, 853, 2996, 4459, 4859, 4615, 2551, 3763, 502, 4419, 2728, 2182, 3975, 1782, 271, 356, 1803, 1659, 3455, 581, 2946, 2516, 4675, 4299, 4879, 2088, 2597, 4823, 1260, 4003, 1997, 3938, 1683, 3216, 4392, 4384, 4138, 163, 3647, 2402, 53, 2790, 486, 1000, 3187, 657, 3571, 3402, 1151, 1935, 3863, 3955, 2541, 3460, 3307, 3825, 3386, 1070, 3109, 234, 4854, 1822, 1713, 2664, 2592, 2952, 3401, 4043, 3112, 4960, 1331, 2126, 4642, 3082, 4910, 4644, 554, 4734, 2152, 4282, 4321, 4968, 1505, 3845, 1766, 642, 3808, 988, 2848, 3773, 1804, 1534, 3250, 2611, 1160, 2204, 1617, 1616, 2336, 4528, 1557, 3332, 4597, 2268, 4963, 1489, 3514, 875, 2218, 1229, 3253, 361, 1637, 4151, 2335, 1992, 4380, 3499, 3974, 4408, 1208, 1942, 4266, 2713, 4222, 4645, 4418, 1629, 1305, 4898, 2236, 619, 4897, 1642, 3754, 1526, 2205, 4049, 3244, 1169, 4393, 3457, 1923, 461, 4121, 852, 4896, 4958, 746, 1928, 1514, 4286, 1723, 3786, 869, 3458, 4391, 2324, 2690, 1725, 2734, 1336, 1571, 2408, 1133, 1632, 3010, 1428, 4128, 2851, 1090, 310, 1274, 811, 412, 4711, 3485, 4196, 259, 1385, 1183, 608, 1273, 2634, 1958, 2369, 4014, 4600, 2846, 3976, 332, 3554, 2068, 1721, 58, 4023, 2267, 380, 3265, 933, 244, 4873, 354, 1496, 2416, 4215, 4951, 1460, 4687, 1871, 3597, 3900, 3525, 2849, 2325, 3635, 1960, 3251, 4534, 1430, 4548, 1423, 589, 440, 886, 1599, 1880, 3085, 3551, 4938, 3210, 1121, 4075, 168, 437, 4360, 885, 810, 1800, 2483, 4082, 3053, 818, 1348, 3313, 3642, 667, 4917, 3147, 3294, 1359, 2981, 3841, 2612, 277, 4055, 2022, 2062, 4618, 4856, 1440, 4056, 4036, 488, 4650, 631, 4051, 2307, 2822, 1176, 920, 4443, 3682, 3404, 2286, 3822, 3492, 59, 902, 1850, 2014, 3111, 240, 4353, 1568, 4389, 3226, 4178, 3568, 2800, 4372, 462, 2396, 4684, 974, 345, 1975, 2748, 3252, 3050, 1667, 1940, 3777, 758, 1073, 2794, 809, 1267, 590, 4464, 458, 2514, 4710, 2850, 3824, 3854, 2359, 3648, 4807, 3160, 622, 1687, 1135, 4115, 1020, 1450, 248, 3871, 4570, 3272, 3638, 2196, 2393, 825, 4315, 2347, 2685, 1622, 4903, 2477, 2023, 3707, 3237, 285, 4631, 4591, 4000, 4616, 318, 3066, 357, 1182, 1022, 4647, 3920, 2522, 4153, 3717, 3567, 1113, 4276, 4915, 1378, 2439, 3241, 510, 2945, 4495, 3678, 3203, 3719, 4668, 3627, 2048, 4487, 4721, 1591, 3308, 1712, 1802, 4362, 3470, 3301, 2660, 2684, 3972, 2842, 4861, 1238, 4130, 3221, 4812, 2527, 820, 4880, 4065, 3780, 4062, 1780, 3376, 1752, 4165, 3297, 3407, 4817, 1076, 183, 4350, 1063, 791, 549, 3266, 2631, 75, 4019, 2340, 2080, 1794, 1728, 1764, 3779, 2489, 477, 2894, 3555, 2358, 1006, 3683, 293, 2571, 4851, 3450, 2470, 1855, 2911, 4429, 1763, 4530, 3631, 1225, 31, 148, 1987, 3604, 3491, 4521, 2802, 1631, 1391, 2795, 3925, 796, 3891, 108, 2730, 4078, 4424, 495, 4728, 4792, 2252, 3321, 700, 3081, 1875, 2950, 3042, 3810, 4428, 2321, 1354, 2499, 4902, 3036, 1577, 2578, 880, 54, 263, 2735, 3254, 3505, 3572, 2491, 258, 1099, 2032, 669, 1129, 4413, 3660, 3545, 4569, 48, 1998, 2559, 497, 476, 1031, 524, 605, 1500, 4806, 4613, 3801, 1048, 831, 1698, 2943, 4560, 2617, 47, 2777, 2222, 44, 115, 4072, 1147, 3092, 239, 4202, 4977, 2535, 297, 3569, 4501, 368, 1948, 229, 1120, 215, 3469, 2123, 4034, 1501, 1670, 3015, 1710, 1633, 3016, 95, 3390, 36, 970, 4311, 3056, 2915, 4233, 4695, 1521, 3089, 4069, 4025, 3223, 4057, 660, 4492, 911, 1158, 122, 72, 4826, 2365, 2564, 1605, 3616, 2125, 2413, 4750, 4400, 917, 3031, 1776, 1283, 1907, 3792, 711, 787, 2473, 3764, 4572, 1715, 146, 3906, 3275, 3730, 842, 3690, 2026, 3280, 2818, 901, 3378, 1515, 1408, 3602, 4329, 435, 889, 706, 2153, 2948, 2005, 2454, 1119, 3001, 4064, 175, 1050, 3952, 2680, 113, 2243, 4876, 740, 1161, 1291, 4148, 1507, 89, 266, 4866, 4303, 407, 698, 1326, 4919, 421, 4242, 4870, 2547, 3899, 3282, 4255, 4667, 3385, 160, 1759, 4180, 661, 35, 4741, 1930, 3052, 659, 3929, 4044, 3088, 2788, 1104, 4421, 1548, 4340, 1988, 4489, 3133, 1100, 1416, 3966, 1612, 2695, 2030, 921, 3671, 943, 1021, 545, 1720, 341, 1143, 4088, 552, 2189, 4033, 3643, 4137, 646, 1316, 308, 4797, 417, 4462, 2546, 1768, 1255, 2989, 255, 1922, 295, 1403, 3188, 3961, 1069, 3502, 4926, 2387, 1961, 3794, 3454, 3364, 839, 2203, 503, 1796, 1502, 850, 4156, 4574, 3209, 4683, 3771, 2555, 3613, 226, 145, 617, 3156, 743, 3894, 2388, 3896, 98, 338, 1994, 321, 4243, 2334, 1237, 4312, 4016, 4001, 2180, 4317, 2461, 1038, 1219, 1649, 949, 1761, 1747, 2095, 898, 127, 2610, 3058, 1025, 3715, 534, 4732, 1102, 980, 2653, 2188, 3045, 78, 1478, 2839, 4173, 3954, 729, 1284, 2789, 2327, 4217, 4722, 1109, 3084, 2364, 2337, 4760, 1178, 3979, 978, 2519, 419, 383, 4757, 2925, 1565, 1407, 3644, 1014, 2858, 566, 3345, 2367, 233, 3304, 575, 3400, 4759, 3591, 1401, 2069, 1611, 1815, 4029, 1535, 4414, 1043, 595, 755, 3970, 1344, 3500, 3315, 1790, 2970, 460, 2161, 3985, 4596, 2949, 4829, 132, 4986, 2757, 3395, 134, 434, 695, 3347, 3339, 3388, 938, 707, 3770, 4041, 4872, 1303, 2990, 2633, 4287, 3298, 1946, 2376, 2738, 224, 2082, 2063, 1413, 4416, 4038, 934, 599, 582, 4549, 1373, 2468, 1895, 2882, 1813, 1195, 177, 4133, 915, 4927, 1891, 1277, 527, 3, 306, 1306, 2234, 4357, 4962, 2451, 2238, 4171, 4537, 2042, 2723, 4639, 388, 4273, 543, 962, 578, 4714, 4659, 1801, 6, 129, 4798, 2105, 2523, 2661, 2756, 4832, 2703, 4405, 3646, 1971, 583, 34, 1769, 138, 46, 4103, 2328, 2231, 4129, 1653, 110, 1271, 1666, 4893, 3277, 377, 3172, 4207, 3547, 674, 4971, 3866, 4698, 1821, 4358, 3858, 2421, 1418, 1444, 406, 1248, 769, 990, 1235, 4146, 353, 3716, 4878, 1592, 4776, 1372, 3934, 1485, 1585, 2574, 868, 2471, 3412, 4132, 3121, 3427, 1406, 2760, 2899, 1773, 2512, 955, 142, 2565, 2873, 1630, 3751, 2884, 3314, 3125, 3862, 992, 4662, 4988, 2616, 3170, 4008, 4594, 4715, 4084, 797, 1623, 576, 1743, 3471, 2845, 2791, 2737, 1324, 4401, 1685, 4801, 1174, 1387, 1194, 1861, 1996, 4140, 753, 1349, 2415, 3888, 4984, 3414, 1371, 670, 800, 2106, 4211, 3363, 2353, 1707, 939, 3857, 2688, 3544, 1567, 759, 225, 1345, 4458, 56, 330, 2224, 2651, 550, 4345, 2508, 2763, 2298, 3960, 3919, 1234, 3721, 1523, 1586, 4455, 4727, 135, 431, 4238, 4335, 2452, 1429, 776, 3114, 1153, 2043, 189, 3861, 2886, 2390, 4009, 4905, 2511, 161, 4705, 2596, 4269, 606, 1663, 4289, 3059, 3300, 1163, 681, 3355, 1427, 2132, 1973, 158, 3829, 2637, 4310, 3373, 2354, 4237, 2719, 2557, 1517, 2781, 2389, 1473, 532, 4707, 4864, 2202, 2859, 3520, 182, 1078, 3041, 2158, 1325, 1482, 4105, 60, 124, 2691, 3198, 2411, 2716, 4059, 4191, 1844, 4481, 3941, 555, 538, 1597, 1553, 3870, 4379, 2120, 2769, 1399, 2876, 1518, 1594, 4323, 2192, 309, 4440, 1130, 788, 799, 2510, 1068, 3440, 347, 2603, 4485, 3310, 2003, 2919, 634, 1793, 1893, 1111, 1483, 2283, 323, 2824, 3261, 2386, 3365, 463, 2270, 2563, 611, 4262, 1872, 4540, 4836, 200, 1366, 4948, 3004, 4219, 3194, 172, 1562, 1951, 3926, 2628, 4160, 3370, 4771, 2768, 1673, 4154, 405, 1820, 1989, 2607, 2782, 4786, 2938, 4439, 689, 2239, 623, 3163, 2984, 1015, 1919, 4374, 533, 3478, 2108, 4885, 481, 3283, 3959, 1458, 1797, 986, 3760, 1294, 3190, 1433, 4891, 3273, 4952, 4395, 3969, 806, 4517, 1826, 4027, 3497, 3335, 335, 3167, 3044, 2939, 1785, 1116, 4031, 1004, 3876, 2595, 232, 3964, 2036, 1564, 1694, 197, 4265, 778, 1185, 2350, 3625, 3479, 2344, 1503, 816, 299, 894, 2301, 1435, 86, 1170, 4437, 610, 767, 4251, 1023, 3573, 1497, 1164, 4805, 1044, 0, 2827, 1067, 287, 1554, 4571, 3444, 1495, 4482, 4135, 4543, 1896, 324, 878, 65, 2090, 279, 2766, 1268, 3672, 930, 198, 3087, 1024, 3441, 490, 1290, 4342, 1217, 1702, 2817, 2363, 1693, 3177, 2921, 4355, 2620, 4467, 4735, 2187, 4142, 4629, 3679, 3148, 2256, 2960, 4093, 600, 2736, 3349, 3817, 830, 1356, 1239, 1417, 672, 2887, 1437, 1392, 3141, 3950, 4422, 4037, 317, 4725, 1784, 3126, 4746, 1578, 4519, 2931, 3535, 1716, 4580, 2589, 4284, 3798, 2830, 4375, 2860, 637, 1688, 965, 3645, 3580, 4912, 4068, 390, 333, 4195, 4744, 952, 3367, 813, 4333, 66, 570, 3982, 832, 2065, 74, 1902, 3132, 4804, 1493, 4843, 422, 717, 438, 3626, 1340, 1696, 996, 37, 4330, 260, 2929, 4448, 3024, 1395, 180, 775, 1467, 4726, 2412, 3758, 1602, 2016, 3738, 721, 1638, 3227, 3901, 4246, 942, 3722, 112, 2602, 668, 4949, 265, 2543, 716, 2540, 3138, 2318, 2049, 1179, 4954, 3564, 3487, 2658, 4551, 991, 3383, 3953, 2253, 289, 2381, 556, 459, 426, 4, 3483, 2050, 444, 3823, 2266, 4533, 1218, 5008, 3702, 4822, 104, 424, 3913, 1452, 3005, 3262, 3930, 3689, 4539, 2428, 1074, 1706, 1853, 1469, 191, 1550, 2115, 763, 1952, 2881, 1017, 454, 4026, 1123, 4558, 280, 4017, 379, 3341, 4385, 712, 282, 2698, 4998, 3320, 2784, 2427, 2840, 910, 4182, 983, 4446, 2425, 4417, 184, 3592, 2293, 1338, 1643, 4892, 2994, 4477, 4909, 4119, 3892, 1837, 615, 784, 3530, 1087, 2813, 1213, 1030, 2635, 4404, 2111, 4941, 3849, 3090, 2200, 2031, 3831, 618, 4686, 2749, 2481, 3833, 823, 2479, 4010, 4200, 2811, 1709, 1287, 2808, 2124, 4862, 749, 4248, 4598, 4102, 3029, 3338, 3512, 327, 2431, 4724, 2475, 1931, 3880, 3908, 3615, 4516, 4853, 5006, 3333, 2903, 2649, 2229, 1415, 4633, 4260, 2898, 2560, 960, 432, 4590, 3012, 2520, 139, 1511, 2064, 1036, 4183, 3686, 916, 274, 3905, 396, 2331, 1096, 2995, 2765, 2296, 596, 771, 2686, 2360, 2740, 16, 4857, 4562, 2178, 4858, 4578, 1281, 4382, 2420, 1697, 4280, 3432, 1056, 4643, 3556, 252, 3034, 3006, 4061, 4002, 2904, 210, 116, 1308, 574, 1254, 385, 174, 3201, 3473, 4204, 1841, 731, 2524, 2006, 561, 897, 1127, 1753, 3945, 1114, 607, 3003, 2261, 3334, 3944, 4619, 38, 1783, 2338, 2046, 2433, 4830, 2554, 887, 96, 2290, 4928, 577, 1367, 4964, 2878, 2646, 1146, 4309, 3963, 520, 147, 123, 1589, 1620, 1215, 1831, 3175, 2803, 998, 3973, 2838, 1883, 4223, 2488, 3108, 2423, 824, 3916, 4241, 3743, 828, 201, 1249, 1977, 4648, 3285, 2240, 4931, 301, 559, 2478, 1232, 1598, 254, 3729, 4039, 630, 1027, 2008, 1905, 2264, 693, 2053, 1516, 1886, 1065, 2936, 3837, 4699, 655, 2250, 1816, 2828, 2272, 3797, 1962, 2717, 2965, 128, 2816, 3649, 817, 4320, 3046, 1775, 1037, 744, 1402, 2991, 1443, 203, 512, 2410, 1097, 3028, 3594, 2150, 344, 4188, 1541, 1870, 518, 1925, 4556, 1846, 3772, 932, 3803, 4847, 2868, 1606, 1206, 2313, 1908, 4803, 2767, 1582, 4766, 1722, 3784, 2971, 4336, 4901, 2893, 1049, 2509, 4114, 4671, 4568, 4484, 1590, 2940, 2518, 1860, 2112, 2467, 3369, 2476, 1468, 984, 1083, 5001, 4550, 1084, 2378, 3980, 3558, 3565, 1938, 4934, 376, 2047, 2722, 2672, 4582, 3174, 3895, 2657, 864, 1609, 3725, 4184, 4420, 402, 1328, 719, 2804, 3688, 4143, 2193, 1762, 227, 3536, 3655, 2474, 2071, 519, 944, 4973, 1221, 3495, 3749, 1641, 4087, 4076, 2154, 687, 4673, 1857, 2951, 1877, 1186, 39, 103, 456, 1466, 4316, 4626, 4719, 1196, 3796, 2761, 736, 2185, 1412, 3049, 517, 1434, 288, 3856, 1042, 136, 1064, 1265, 4561, 2341, 511, 1409, 1035, 3632, 3634, 2593, 445, 3621, 3821, 961, 2129, 1346, 3914, 4079, 4842, 470, 2569, 2815, 2954, 2100, 4176, 1498, 4959, 1812, 2294, 3600, 2636, 794, 1890, 1426, 3755, 2968, 1261, 76, 3981, 4354, 3685, 4170, 1377, 4736, 1453, 1149, 4755, 334, 1596, 2987, 3208, 3107, 2297, 4637, 364, 2923, 1223, 3393, 893, 4860, 1756, 3698, 1105, 4904, 3601, 4522, 3292, 4120, 4006, 4586, 3910, 2900, 1852, 3154, 7, 1824, 3834, 2343, 1640, 2590, 3317, 4426, 3021, 3475, 4780, 1823, 3129, 1798, 3587, 3696, 783, 1969, 1727, 3185, 609, 4134, 4886, 3529, 3819, 3193, 3105, 4483, 3509, 1884, 374, 3463, 2496, 2568, 3327, 4074, 283, 3415, 925, 1106, 4763, 360, 1657, 4820, 859, 2159, 3466, 3104, 2249, 2280, 3510, 785, 4887, 4729, 1454, 3538, 4982, 3140, 2721, 4504, 1315, 4768, 1039, 4359, 2217, 2299, 2742, 1527, 4871, 592, 3946, 3069, 3323, 3528, 1474, 2056, 2998, 2862, 4983, 3131, 4058, 3269, 678, 3709, 4953, 496, 3992, 612, 2955, 2472, 883, 2441, 2577, 1350, 3421, 4553, 2591, 4575, 3813, 3329, 2397, 369, 4723, 3409, 598, 508, 3240, 4126, 985, 4318, 2866, 3179, 2885, 4839, 4761, 4738, 3815, 1689, 1200, 4004, 3437, 1608, 269, 931, 1560, 1400, 4581, 493, 779, 2678, 1751, 3436, 1093, 1432, 2001, 3609, 1397, 3915, 4929, 1376, 1159, 4371, 2066, 2897, 3330, 3259, 2198, 3909, 3189, 3159, 1368, 3206, 3260, 1231, 2073, 3887, 3957, 3593, 2244, 3078, 176, 2910, 3099, 2933, 3149, 2978, 4507, 452, 3337, 987, 4410, 4274, 514, 4625, 995, 3152, 4840, 774, 748, 4802, 4399, 3143, 1874, 1388, 4690, 3430, 4828, 977, 4343, 1361, 475, 2941, 2284, 448, 2370, 3939, 1055, 467, 1379, 2785, 967, 4855, 4935, 41, 4564, 1313, 2841, 1559, 1312, 3576, 801, 4322, 3617, 3590, 2289, 4011, 4131, 3624, 4259, 359, 5011, 3902, 1150, 1333, 3452, 2085, 3884, 1122, 2947, 3935, 1245, 679, 63, 714, 3607, 4966, 1894, 4770, 3324, 3659, 1057, 2687, 193, 4258, 851, 4085, 3417, 1635, 4794, 64, 812, 2011, 473, 2426, 2177, 964, 2623, 2319, 750, 2121, 2004, 647, 4685, 8, 3527, 3362, 3290, 547, 2320, 199, 4900, 4118, 2583, 4147, 2285, 1957, 3623, 2662, 2346, 4658, 2490, 3598, 4703, 2676, 3242, 2786, 3608, 1920, 1148, 2495, 2566, 3543, 782, 3306, 372, 3633, 4850, 1470, 523, 319, 3977, 4496, 2086, 4080, 1259, 3549, 311, 3119, 4718, 844, 3595, 790, 792, 2371, 2333, 1892, 3453, 666, 141, 4466, 2669, 747, 3628, 1805, 4470, 2833, 4700, 4972, 2271, 3697, 1384, 3381, 491, 2059, 1704, 826, 2405, 4046, 4125, 2323, 3936, 4614, 2190, 1810, 1719, 3434, 4620, 1091, 2450, 3047, 449, 1028, 2322, 3728, 3374, 2141, 4193, 1781, 4083, 3658, 3077, 971, 835, 1915, 4172, 941, 3873, 29, 1301, 715, 4623, 2505, 4456, 548, 2693, 3618, 843, 3865, 4341, 2118, 3215, 1184, 4453, 3057, 2407, 2383, 3288], "validation": [5211, 5320, 5218, 5117, 5440, 5219, 5046, 5377, 5035, 5307, 5288, 5510, 5090, 5501, 5418, 5282, 5058, 5313, 5024, 5439, 5060, 5349, 5189, 5474, 5094, 5407, 5446, 5304, 5093, 5483, 5494, 5032, 5346, 5213, 5027, 5044, 5255, 5434, 5350, 5101, 5070, 5252, 5364, 5315, 5083, 5443, 5505, 5502, 5459, 5220, 5129, 5419, 5017, 5455, 5127, 5287, 5105, 5296, 5059, 5382, 5048, 5223, 5273, 5281, 5062, 5305, 5373, 5039, 5124, 5141, 5068, 5052, 5020, 5442, 5436, 5413, 5202, 5254, 5476, 5021, 5078, 5465, 5356, 5186, 5398, 5365, 5486, 5156, 5458, 5410, 5066, 5098, 5038, 5126, 5240, 5208, 5429, 5109, 5102, 5149, 5173, 5216, 5097, 5099, 5207, 5508, 5283, 5290, 5286, 5488, 5268, 5257, 5190, 5029, 5432, 5131, 5076, 5274, 5381, 5241, 5372, 5247, 5080, 5319, 5493, 5239, 5433, 5303, 5217, 5285, 5492, 5188, 5194, 5258, 5448, 5192, 5384, 5120, 5132, 5394, 5191, 5065, 5042, 5206, 5158, 5279, 5238, 5404, 5140, 5343, 5235, 5393, 5297, 5116, 5357, 5335, 5104, 5187, 5236, 5480, 5195, 5406, 5435, 5386, 5144, 5339, 5171, 5253, 5075, 5168, 5159, 5092, 5430, 5347, 5358, 5133, 5507, 5454, 5222, 5025, 5072, 5280, 5095, 5016, 5034, 5342, 5251, 5119, 5509, 5071, 5196, 5310, 5284, 5047, 5096, 5460, 5497, 5491, 5228, 5177, 5182, 5180, 5403, 5293, 5426, 5178, 5438, 5087, 5152, 5400, 5487, 5242, 5088, 5197, 5018, 5423, 5100, 5311, 5128, 5424, 5378, 5166, 5043, 5161, 5466, 5183, 5022, 5399, 5229, 5340, 5085, 5298, 5111, 5376, 5069, 5294, 5291, 5390, 5363, 5204, 5051, 5336, 5456, 5333, 5272, 5221, 5201, 5306, 5355, 5367, 5447, 5153, 5289, 5370, 5437, 5299, 5318, 5050, 5490, 5248, 5179, 5181, 5226, 5462, 5506, 5359, 5379, 5452, 5489, 5145, 5057, 5427, 5302, 5368, 5019, 5214, 5468, 5415, 5337, 5391, 5169, 5361, 5136, 5495, 5464, 5108, 5322, 5122, 5344, 5414, 5504, 5425, 5148, 5383, 5233, 5224, 5210, 5164, 5215, 5243, 5366, 5107, 5053, 5457, 5369, 5441, 5345, 5146, 5081, 5170, 5137, 5028, 5380, 5112, 5327, 5416, 5139, 5212, 5498, 5091, 5199, 5246, 5374, 5074, 5106, 5245, 5155, 5200, 5167, 5473, 5332, 5049, 5271, 5477, 5113, 5277, 5461, 5388, 5055, 5321, 5275, 5165, 5013, 5203, 5150, 5431, 5265, 5134, 5249, 5045, 5348, 5054, 5143, 5409, 5387, 5300, 5026, 5232, 5499, 5121, 5184, 5325, 5484, 5401, 5511, 5157, 5269, 5067, 5023, 5040, 5084, 5334, 5428, 5061, 5329, 5147, 5481, 5163, 5209, 5231, 5250, 5482, 5270, 5351, 5172, 5260, 5244, 5227, 5385, 5450, 5064, 5309, 5162, 5261, 5037, 5015, 5371, 5031, 5125, 5420, 5479, 5110, 5323, 5397, 5360, 5422, 5135, 5412, 5174, 5077, 5392, 5030, 5237, 5408, 5467, 5151, 5154, 5470, 5324, 5118, 5308, 5389, 5353, 5114, 5041, 5086, 5295, 5444, 5138, 5123, 5056, 5130, 5352, 5485, 5262, 5354, 5326, 5036, 5115, 5330, 5338, 5082, 5033, 5312, 5073, 5193, 5301, 5503, 5230, 5160, 5205, 5471, 5292, 5276, 5328, 5063, 5314, 5341, 5463, 5445, 5264, 5451, 5263, 5453, 5142, 5478, 5331, 5175, 5421, 5014, 5278, 5500, 5316, 5362, 5089, 5395, 5375, 5449, 5225, 5259, 5176, 5475, 5472, 5317, 5469, 5402, 5405, 5185, 5103, 5234, 5079, 5411, 5198, 5266, 5417, 5496, 5012, 5396, 5256, 5267], "test": [5692, 5587, 5959, 5646, 5915, 5582, 5991, 5876, 5565, 5817, 5845, 5764, 5981, 5839, 5750, 5772, 5899, 5618, 5827, 5804, 5714, 5625, 5766, 5955, 5691, 5548, 5800, 5968, 5596, 5953, 5945, 5540, 5756, 5758, 5825, 5673, 5806, 5631, 5575, 5808, 5731, 5589, 5662, 5880, 5520, 5824, 5940, 5647, 5870, 5936, 5901, 5983, 5964, 5668, 5887, 5844, 5599, 5683, 5807, 5960, 5718, 5952, 5720, 5656, 5918, 5702, 5917, 5615, 5701, 5971, 5799, 5773, 5897, 5822, 5643, 5843, 5982, 5593, 5538, 5926, 5666, 5682, 5640, 5796, 5872, 5519, 5957, 5678, 5833, 5990, 5693, 5778, 5595, 5967, 5801, 5997, 5629, 5562, 5708, 5536, 5890, 6004, 5788, 5938, 5555, 5976, 5831, 5826, 5891, 5795, 5878, 5739, 5585, 5853, 5573, 5592, 5784, 5947, 5534, 5948, 5782, 5857, 5770, 5588, 5572, 5591, 5539, 5928, 5648, 5858, 5688, 5980, 5644, 5805, 5925, 5780, 5672, 5836, 5828, 5621, 5992, 5568, 5934, 5527, 5576, 6007, 5579, 5986, 5996, 5670, 5912, 5603, 5757, 5904, 5619, 5545, 5524, 5877, 5725, 5687, 5749, 5574, 5989, 5820, 5660, 5620, 5939, 5552, 5943, 5713, 5946, 5910, 5802, 5710, 5906, 5944, 5850, 5787, 5797, 5748, 5521, 5902, 5879, 5669, 5694, 5636, 5781, 5578, 5566, 5737, 5743, 5719, 5706, 5551, 5855, 5961, 5634, 5892, 5841, 5512, 5611, 5869, 5638, 5935, 5815, 6005, 5951, 5608, 5835, 5848, 5900, 5792, 5811, 5998, 5532, 5560, 6001, 5679, 5769, 5794, 5974, 5613, 5812, 5686, 5685, 5564, 5829, 5775, 5837, 5721, 5744, 5791, 5920, 5703, 6003, 5834, 6008, 5655, 5762, 5707, 5942, 5559, 5759, 5932, 5530, 5771, 5712, 5514, 5763, 5550, 5889, 5993, 5664, 5903, 5760, 5742, 5786, 5518, 5715, 5789, 5963, 5882, 5543, 5537, 5852, 5798, 5732, 5776, 5659, 5746, 5649, 5954, 5810, 5929, 5907, 5727, 5984, 5558, 5923, 6002, 5865, 5557, 5921, 5813, 5754, 5698, 5547, 5973, 5965, 5689, 5896, 5868, 5723, 5895, 5987, 5717, 5606, 5888, 5774, 5632, 5533, 5604, 5847, 5969, 5916, 5861, 5674, 5553, 5816, 5994, 5751, 5544, 5949, 5577, 5704, 5875, 5733, 5570, 5777, 5699, 5639, 5842, 5734, 5541, 5840, 5563, 5657, 5652, 5711, 5851, 5633, 5864, 5862, 5513, 5653, 5680, 5628, 5745, 5790, 5705, 6000, 5975, 5726, 5753, 5859, 5684, 5893, 5854, 5645, 5661, 5729, 5523, 5783, 6009, 5898, 5616, 5546, 5962, 5908, 5724, 5522, 5525, 5873, 5654, 5665, 5919, 5627, 5818, 5809, 5610, 5941, 5966, 5567, 5542, 5995, 5849, 5863, 5623, 5793, 5914, 5838, 5594, 5950, 5695, 5814, 5909, 5765, 5881, 5535, 5586, 5886, 5978, 5779, 6010, 5958, 5728, 5819, 5931, 5927, 5624, 5663, 5913, 5747, 5741, 5922, 5671, 5676, 5697, 5528, 5860, 5584, 5832, 6011, 5933, 5515, 5526, 5556, 5722, 5830, 5581, 5768, 5911, 5803, 5988, 5874, 5651, 5979, 5761, 5583, 5642, 5972, 5569, 5516, 5894, 6006, 5716, 5561, 5884, 5597, 5970, 5637, 5956, 5977, 5846, 5738, 5867, 5529, 5554, 5667, 5549, 5609, 5700, 5612, 5622, 5601, 5985, 5856, 5626, 5635, 5871, 5598, 5677, 5630, 5883, 5930, 5571, 5690, 5740, 5999, 5755, 5531, 5767, 5605, 5730, 5517, 5735, 5602, 5885, 5905, 5752, 5696, 5590, 5866, 5736, 5823, 5607, 5641, 5821, 5681, 5937, 5650, 5785, 5617, 5658, 5709, 5614, 5580, 5675, 5924, 5600]}, {"train": [93, 4509, 1185, 4411, 4560, 1852, 1054, 4988, 11, 4244, 920, 3410, 1920, 5008, 3627, 1314, 3507, 4663, 4654, 4384, 3456, 362, 2266, 4702, 4036, 1092, 1045, 945, 339, 1064, 4902, 4054, 4056, 2762, 3266, 3004, 206, 3180, 998, 3063, 2426, 3944, 598, 138, 1088, 1693, 2919, 4644, 3587, 1709, 4141, 1646, 4132, 1183, 457, 162, 1258, 3292, 3164, 4492, 3237, 3790, 3619, 4940, 2254, 2540, 957, 1139, 3702, 4139, 97, 2583, 787, 3191, 1529, 3642, 4333, 2629, 3902, 4844, 1147, 1101, 2217, 2097, 4669, 1737, 568, 1718, 3249, 1685, 4611, 1884, 3820, 2189, 3646, 1648, 3609, 674, 4085, 2086, 2364, 4924, 3392, 2916, 3166, 4296, 3391, 3415, 754, 860, 4592, 4752, 240, 3345, 3225, 455, 245, 2799, 364, 4338, 1943, 3201, 4075, 4135, 3528, 1525, 2297, 2849, 1519, 211, 4588, 2901, 3155, 2419, 4090, 237, 4445, 3805, 2756, 4766, 4580, 2842, 4257, 866, 3491, 2836, 786, 1988, 564, 1080, 1338, 4347, 2375, 3539, 821, 2729, 231, 4930, 980, 317, 4552, 4849, 185, 1040, 1752, 2763, 1969, 3282, 530, 82, 1016, 3236, 388, 107, 4292, 906, 766, 2478, 2905, 415, 2160, 3163, 1510, 3626, 44, 1022, 2525, 1155, 4433, 2237, 1804, 1405, 3482, 4569, 1953, 3509, 4822, 918, 3064, 1464, 319, 1994, 910, 1450, 1523, 1346, 3106, 1058, 4657, 2773, 3168, 1767, 2290, 2596, 4974, 271, 1361, 2125, 1995, 4899, 1398, 644, 1862, 722, 4666, 2994, 4327, 2137, 2881, 838, 3735, 1403, 3640, 3904, 3975, 1873, 4442, 3954, 703, 2888, 4374, 548, 2325, 2377, 2115, 2735, 218, 123, 4753, 1171, 2978, 4586, 4396, 117, 826, 2823, 3021, 504, 3364, 400, 1339, 3447, 3133, 3374, 353, 29, 2195, 1814, 2106, 163, 2575, 2084, 3872, 299, 1933, 3071, 2787, 4620, 1197, 4946, 1674, 2746, 1774, 573, 1032, 1509, 3489, 4261, 819, 4630, 1341, 4675, 4574, 4220, 3318, 2500, 2496, 2576, 3892, 4867, 4455, 4439, 3785, 4263, 4386, 4102, 2064, 3703, 3706, 839, 1033, 4033, 4926, 4298, 2781, 1104, 3360, 1301, 3833, 291, 3709, 2320, 3218, 4910, 584, 2207, 3796, 3118, 1037, 3475, 4011, 1942, 147, 1777, 1837, 4593, 3814, 2855, 3889, 1743, 1486, 4928, 1303, 1919, 4911, 3260, 1587, 3487, 3746, 3968, 2213, 3586, 2223, 4971, 2973, 4794, 1225, 4965, 1788, 514, 937, 2463, 2262, 243, 469, 4760, 3277, 3143, 1672, 4443, 1363, 1086, 2868, 868, 1174, 4686, 946, 1184, 2645, 3914, 593, 2040, 734, 4665, 4495, 2620, 566, 1765, 3772, 2427, 1626, 1374, 777, 4332, 2394, 2006, 3397, 4385, 4161, 1937, 4256, 1102, 1985, 1618, 3091, 3824, 3263, 2063, 1849, 3108, 501, 928, 3714, 693, 2691, 3051, 2453, 3926, 4404, 1983, 1481, 953, 4741, 4239, 3930, 559, 2062, 4074, 4126, 2828, 2611, 696, 567, 4008, 2196, 898, 1598, 1636, 2248, 3875, 4530, 3775, 911, 2506, 3343, 3153, 3884, 2651, 4967, 269, 1666, 1325, 4713, 3950, 2073, 2048, 3840, 1198, 2140, 241, 3533, 1678, 4466, 2177, 4730, 572, 1349, 1582, 2807, 589, 4987, 2989, 4643, 3726, 1430, 2226, 136, 2212, 1387, 4005, 4128, 3224, 702, 3510, 1368, 1270, 3179, 4973, 4201, 626, 541, 2271, 1110, 4594, 517, 258, 1854, 4904, 915, 4746, 1392, 485, 22, 3849, 4388, 4290, 3469, 4418, 2760, 4250, 923, 20, 2852, 4581, 4956, 2790, 738, 556, 207, 63, 4268, 1206, 4440, 727, 2984, 4124, 1877, 1987, 4051, 4040, 1961, 4626, 1637, 5003, 4178, 1597, 4591, 2800, 2805, 4395, 3671, 3899, 321, 4801, 1689, 1812, 4954, 2682, 995, 2240, 1013, 192, 4343, 3736, 416, 2099, 405, 3739, 3782, 1017, 2388, 792, 2288, 4951, 149, 5006, 2449, 377, 2856, 4276, 2565, 1899, 1579, 981, 4706, 1112, 2730, 2171, 1861, 3982, 1818, 3428, 307, 4827, 3146, 871, 333, 3453, 1656, 3212, 2615, 2699, 986, 4747, 1796, 882, 540, 1007, 2476, 203, 732, 4607, 361, 172, 2993, 1010, 1031, 2273, 3288, 480, 461, 2358, 1512, 4521, 4181, 48, 2979, 2300, 4284, 1655, 4423, 3948, 1606, 3679, 294, 1948, 2631, 1974, 3141, 318, 4656, 4017, 4532, 4059, 4499, 2482, 4022, 1399, 2609, 2459, 4199, 4862, 4955, 4720, 4680, 1036, 571, 888, 3850, 4852, 4175, 1204, 2652, 4613, 2679, 1035, 1214, 3765, 578, 46, 827, 2089, 4421, 3298, 663, 3126, 4087, 3351, 4599, 720, 3874, 3525, 4109, 1863, 1438, 499, 357, 2508, 1836, 1925, 4684, 2208, 1130, 2716, 4364, 497, 3916, 383, 3399, 2123, 2683, 4796, 1865, 3737, 4223, 1972, 248, 3534, 4358, 3999, 4770, 3896, 453, 1223, 3897, 1811, 2493, 1134, 582, 1573, 66, 3567, 2339, 3170, 768, 3084, 4778, 824, 694, 4383, 4773, 330, 2283, 4998, 2124, 3933, 4757, 4895, 1232, 1952, 549, 2812, 2708, 1451, 3623, 1733, 1766, 112, 492, 3074, 879, 4642, 4777, 1623, 642, 3401, 300, 2389, 1658, 3531, 3558, 2968, 3450, 3524, 4208, 2164, 1900, 4377, 2108, 3578, 4754, 1684, 743, 3215, 2501, 4994, 3150, 4830, 2711, 4449, 1440, 4577, 2291, 3683, 2771, 2034, 3675, 175, 1751, 4921, 713, 1334, 2010, 1593, 4295, 3598, 3647, 2312, 2379, 2378, 1662, 1839, 3038, 2071, 4021, 2912, 1568, 2133, 4537, 4859, 972, 3066, 2981, 1488, 3424, 553, 2638, 4355, 520, 2740, 2447, 4305, 3182, 3998, 4570, 118, 1682, 4252, 4824, 365, 2085, 54, 4693, 942, 1148, 4408, 1748, 4517, 2101, 3520, 3752, 1059, 4723, 3682, 1156, 1434, 4086, 3901, 3774, 636, 2326, 62, 2001, 1028, 760, 603, 2661, 802, 4840, 4533, 250, 903, 1959, 360, 4122, 3931, 532, 2180, 1356, 3094, 139, 4803, 4764, 4932, 2466, 3252, 1299, 4925, 201, 1323, 1259, 3932, 2504, 2678, 1883, 3732, 151, 1550, 3723, 2983, 1166, 639, 5004, 1745, 3596, 119, 30, 3299, 1622, 3987, 1009, 1676, 3123, 2837, 4280, 3728, 3780, 2580, 2539, 2166, 2432, 4048, 2118, 4373, 3111, 2361, 3239, 3202, 1379, 4933, 2751, 4875, 4563, 820, 687, 4094, 2179, 3691, 2092, 1661, 3154, 1518, 4014, 1831, 1043, 2953, 2873, 1824, 2996, 3885, 3368, 1205, 4898, 1851, 4035, 4878, 3549, 467, 3308, 4870, 1456, 1612, 3748, 391, 516, 2494, 3435, 2607, 1427, 1694, 3, 4010, 1916, 3766, 1927, 3139, 2054, 1008, 4917, 3538, 2534, 2121, 456, 2280, 2472, 2906, 4091, 69, 1472, 4536, 3566, 1947, 489, 847, 751, 3193, 3029, 4069, 1590, 1062, 1878, 4869, 2056, 3784, 3373, 4571, 2415, 3152, 4573, 1026, 629, 40, 2541, 4678, 2386, 586, 4279, 4798, 1892, 2287, 3001, 3035, 1093, 154, 2634, 2802, 1515, 4452, 3605, 4436, 2498, 1720, 3222, 3890, 59, 1449, 562, 2479, 1992, 4291, 4273, 116, 3516, 1038, 3480, 3229, 2930, 4009, 3556, 832, 1677, 2935, 4748, 408, 4488, 2579, 34, 4637, 1562, 4000, 711, 204, 4627, 1867, 4245, 3326, 1827, 4118, 780, 4985, 1211, 1657, 4419, 2234, 3451, 1462, 1149, 3200, 4382, 4217, 4711, 276, 1170, 2933, 2513, 2677, 1591, 1634, 3727, 1152, 4467, 615, 4310, 3894, 3221, 1000, 2186, 913, 1572, 2440, 4589, 3548, 2835, 4606, 96, 1810, 10, 563, 1687, 3472, 3908, 1077, 4660, 3485, 789, 4839, 3103, 4315, 1141, 4073, 2132, 3612, 3198, 1659, 648, 1692, 745, 1561, 412, 4850, 2454, 3365, 4496, 2477, 5001, 3413, 2956, 3044, 2009, 4528, 2893, 4743, 3883, 809, 403, 3459, 164, 623, 1167, 2345, 2643, 3386, 2399, 1556, 183, 4771, 2487, 1060, 2070, 1127, 4230, 253, 836, 2915, 1508, 4947, 194, 3878, 3654, 2658, 1604, 262, 673, 3377, 3602, 4903, 4546, 2502, 1029, 3267, 967, 1165, 1210, 4437, 2524, 4346, 3052, 1357, 1513, 2127, 4776, 2342, 2681, 795, 2272, 2065, 4997, 3443, 3003, 4789, 983, 4872, 769, 810, 4670, 1096, 2866, 2075, 551, 196, 4584, 2553, 4016, 153, 370, 881, 1108, 4782, 2142, 1821, 281, 1846, 3990, 1444, 2238, 1284, 2335, 3065, 3008, 4480, 1049, 4369, 741, 4913, 1233, 3279, 3209, 4673, 814, 3778, 3861, 3965, 1402, 1168, 1744, 475, 311, 1067, 2846, 973, 4931, 2206, 4225, 3827, 129, 2545, 1697, 1053, 4287, 3471, 2267, 2998, 762, 37, 2222, 616, 1081, 298, 3230, 3823, 3104, 1179, 1548, 4342, 354, 940, 4003, 3388, 1457, 1734, 2307, 1906, 3497, 3821, 2396, 1710, 2257, 4129, 4949, 1894, 4447, 3382, 2741, 4427, 4113, 936, 1030, 4792, 969, 2571, 2665, 4278, 4659, 2161, 3729, 2531, 4519, 4329, 2944, 3656, 2556, 79, 2954, 4636, 4639, 2436, 4834, 2701, 3787, 2959, 3197, 3958, 2205, 463, 1131, 1730, 3905, 259, 2261, 3232, 1805, 429, 4970, 238, 2201, 670, 2100, 273, 886, 3409, 4481, 784, 2780, 2433, 1274, 2610, 4595, 2952, 4893, 4544, 1188, 841, 4578, 219, 252, 1934, 110, 3614, 4099, 3527, 2832, 3873, 313, 4218, 1815, 2183, 17, 4307, 1957, 3086, 4281, 1949, 800, 197, 21, 3187, 4688, 1975, 2552, 2311, 1278, 5007, 27, 2902, 4392, 613, 892, 3781, 2277, 1011, 3856, 2860, 3597, 2130, 3188, 807, 3440, 1911, 865, 3560, 2789, 3159, 1570, 3481, 3002, 3638, 1372, 274, 2286, 3829, 2416, 3327, 4742, 3219, 4505, 4195, 2165, 3075, 801, 3522, 4361, 1072, 4057, 4615, 2870, 3633, 212, 3506, 3210, 4317, 2199, 637, 3570, 2214, 893, 4482, 266, 4187, 3358, 3822, 491, 2011, 2562, 896, 1144, 4470, 3810, 3819, 3306, 931, 3622, 328, 4671, 1601, 1813, 2333, 4267, 326, 4845, 1713, 927, 8, 4707, 2712, 2922, 47, 4543, 3315, 4641, 1332, 753, 1230, 2559, 1830, 2304, 4874, 302, 2229, 962, 1833, 3419, 527, 3964, 2346, 1480, 3144, 1268, 4508, 3385, 2158, 68, 3995, 1876, 4055, 1880, 3808, 4733, 2947, 1917, 2907, 2403, 2686, 6, 4297, 1453, 654, 4150, 3701, 3743, 3011, 984, 4959, 3404, 242, 2521, 2815, 4914, 4610, 1742, 2668, 4212, 444, 2443, 396, 605, 2925, 2074, 2350, 3994, 2626, 4936, 420, 4887, 2243, 4966, 437, 1757, 3019, 4413, 3081, 3305, 1629, 2961, 1908, 2987, 1531, 2149, 2854, 1006, 844, 4962, 3842, 2867, 2143, 2549, 649, 4504, 3501, 3223, 4020, 4060, 1872, 1539, 3276, 4817, 4871, 1926, 966, 3505, 202, 4154, 4918, 3689, 959, 215, 3477, 1494, 1344, 3417, 3526, 4698, 64, 3843, 4412, 3680, 1495, 308, 1793, 4293, 912, 651, 535, 4529, 4430, 3710, 2588, 770, 4779, 791, 2792, 4331, 4952, 1705, 1528, 758, 4359, 746, 1386, 224, 1203, 4565, 2131, 78, 2757, 2999, 2002, 1411, 9, 4661, 4387, 3269, 3657, 1371, 4809, 542, 1516, 1458, 1345, 2039, 2551, 3173, 3357, 2511, 2742, 3037, 2128, 4120, 1, 1140, 1719, 3117, 812, 3495, 2934, 1879, 1114, 478, 4854, 4231, 4041, 1621, 4628, 521, 659, 2753, 2986, 3699, 4877, 4127, 3705, 525, 3713, 2372, 2059, 2428, 335, 2464, 1574, 1600, 2239, 3733, 4076, 4676, 1439, 2878, 4185, 1790, 3208, 2554, 25, 2503, 53, 3192, 384, 2080, 4028, 3206, 1850, 3250, 3199, 905, 3517, 440, 3411, 1838, 3721, 4701, 2784, 111, 1845, 4160, 1266, 3425, 4238, 2013, 3390, 4002, 484, 4600, 731, 4131, 38, 1044, 289, 4265, 3110, 4808, 4725, 1436, 925, 1534, 932, 1215, 1142, 2112, 2354, 1474, 2767, 3421, 3711, 2537, 2546, 1426, 1660, 3017, 4739, 4397, 379, 3036, 167, 2159, 2774, 4853, 1819, 1589, 3348, 1609, 1025, 759, 3832, 1538, 1764, 4687, 1397, 195, 4224, 1466, 1968, 992, 4064, 3130, 2456, 1348, 4416, 3877, 2685, 4143, 1716, 137, 2950, 3077, 2715, 2488, 781, 3738, 1162, 4683, 1586, 739, 3559, 3432, 4957, 3337, 3484, 2332, 4349, 4696, 4259, 3423, 359, 2898, 641, 3098, 2963, 3338, 686, 4919, 257, 1542, 2228, 4838, 885, 84, 1019, 4493, 1664, 917, 2091, 2374, 1417, 1826, 1631, 2533, 2814, 2793, 2577, 1069, 447, 503, 76, 1795, 3717, 837, 2275, 4575, 166, 280, 3720, 740, 3639, 1039, 2249, 4352, 4805, 601, 3962, 1389, 393, 155, 4294, 406, 4210, 1610, 1860, 2457, 169, 4762, 1432, 3700, 2794, 538, 2941, 597, 1563, 794, 2157, 3340, 3583, 1881, 2145, 3034, 3943, 1412, 2087, 236, 2348, 2274, 2344, 1300, 4562, 1445, 3599, 3431, 1478, 4714, 1090, 1257, 3936, 875, 4972, 1608, 1946, 2461, 1547, 2235, 1627, 3857, 2495, 4976, 3628, 4486, 3361, 45, 2028, 621, 4167, 4107, 1973, 4380, 2687, 1433, 3557, 607, 4884, 493, 4080, 1936, 2122, 2647, 92, 4690, 4088, 3992, 389, 1875, 2418, 3589, 2051, 1780, 1293, 2323, 2484, 3755, 3362, 4325, 2625, 2974, 833, 4203, 1076, 4816, 2481, 3991, 1243, 3563, 3100, 2382, 1100, 1956, 1886, 4394, 627, 3259, 329, 3303, 2369, 2522, 3668, 3644, 4363, 288, 1260, 1971, 560, 3381, 691, 4980, 278, 3767, 4163, 4254, 4198, 2078, 247, 77, 1945, 4781, 198, 4674, 4410, 3056, 4477, 90, 4192, 1138, 338, 4186, 1754, 3665, 3160, 3806, 1585, 1847, 3242, 574, 3670, 2068, 4312, 1217, 1504, 372, 4354, 3313, 2826, 4564, 1707, 2225, 502, 4340, 3749, 422, 3704, 4432, 3227, 358, 3734, 3851, 3666, 4420, 1365, 2148, 4147, 39, 4391, 1691, 2726, 4101, 1477, 425, 3957, 2769, 3088, 322, 3864, 1190, 4206, 3835, 1271, 960, 188, 717, 382, 4689, 3158, 2429, 105, 3971, 1758, 1602, 3751, 352, 1680, 3134, 3685, 2152, 394, 518, 1476, 2604, 4851, 1244, 561, 1921, 2330, 490, 270, 4535, 938, 1841, 3540, 156, 1063, 337, 2627, 148, 1700, 2486, 3801, 3072, 2417, 3906, 592, 4775, 1428, 468, 295, 4446, 3577, 3486, 2982, 3478, 3618, 4228, 4652, 3457, 2352, 4200, 4682, 3220, 1469, 1951, 4651, 2331, 312, 186, 1385, 4302, 3653, 15, 2425, 979, 507, 2395, 1759, 4251, 174, 4842, 2635, 4166, 2624, 2707, 1446, 1350, 3420, 3496, 3985, 3287, 132, 2517, 4901, 1853, 1897, 2966, 4879, 3132, 4389, 2400, 4274, 1187, 3464, 1890, 3678, 620, 3455, 4819, 4207, 2918, 4863, 3462, 3018, 1209, 24, 2045, 2253, 1351, 1393, 263, 2176, 3951, 3107, 2690, 2890, 3818, 3156, 4050, 260, 1914, 4474, 1280, 3770, 4193, 4708, 799, 4348, 1581, 3687, 171, 1212, 2976, 3398, 4019, 3629, 72, 4183, 3568, 4159, 2889, 3255, 2606, 432, 3096, 1437, 2469, 3323, 3463, 2371, 3400, 1735, 3175, 1958, 2185, 1738, 2095, 4515, 3915, 4162, 2910, 4155, 604, 2126, 2083, 718, 3125, 223, 4174, 3630, 2431, 3813, 978, 2154, 3165, 2872, 2581, 2573, 1643, 1893, 4092, 3207, 4977, 4381, 3993, 2694, 1001, 4953, 2058, 2200, 376, 929, 108, 2719, 3571, 1226, 1715, 4612, 1068, 1459, 305, 157, 2859, 2839, 2731, 1583, 1347, 3848, 3661, 3663, 3947, 1285, 264, 4655, 4826, 3674, 3076, 1882, 4393, 2018, 3204, 2172, 4732, 3102, 4514, 2628, 2139, 140, 3310, 1485, 3803, 4534, 4012, 2321, 846, 85, 721, 1782, 1245, 2025, 1055, 2264, 2270, 1686, 4145, 1395, 4018, 658, 579, 3467, 1447, 656, 3105, 3235, 3763, 1085, 2844, 133, 1484, 1313, 5010, 143, 3070, 1955, 3304, 1706, 4299, 1317, 3555, 625, 2589, 3927, 4341, 2788, 1708, 3058, 2748, 2512, 3658, 3412, 2359, 4900, 4846, 4923, 863, 4646, 3847, 2674, 4658, 2749, 3341, 3243, 2155, 4258, 3183, 2170, 1535, 1679, 2980, 647, 1253, 210, 2445, 3376, 1834, 3297, 4038, 1809, 3418, 4140, 3918, 3707, 2351, 104, 2827, 3371, 1355, 681, 1667, 1319, 2215, 3692, 697, 2897, 3716, 124, 2029, 4694, 706, 3093, 2250, 2843, 3370, 345, 1452, 226, 2146, 4583, 4471, 2329, 3632, 423, 2862, 4047, 3845, 2340, 1778, 397, 4995, 2885, 4649, 4745, 1103, 470, 3022, 3891, 4170, 369, 1801, 355, 4182, 2210, 1370, 460, 4907, 2670, 3768, 3600, 2884, 4568, 4007, 2926, 3807, 1021, 994, 4112, 1594, 3241, 23, 1024, 2656, 1242, 2324, 2957, 3552, 1193, 954, 476, 4138, 1704, 4908, 1180, 4691, 1126, 2105, 1633, 771, 1799, 3020, 737, 3753, 2560, 3837, 3097, 3301, 2303, 707, 2279, 612, 2236, 773, 2462, 951, 115, 1986, 1424, 3750, 2779, 3744, 2821, 1157, 135, 1410, 2737, 2355, 742, 2263, 306, 2282, 3513, 121, 222, 283, 2365, 3831, 958, 4795, 3053, 735, 1932, 26, 1136, 1763, 265, 3870, 1463, 4883, 3416, 2434, 466, 343, 4722, 2936, 609, 971, 1216, 1400, 4596, 438, 1175, 2988, 949, 3226, 2216, 3508, 2281, 4025, 3909, 690, 1712, 2892, 3449, 2367, 3895, 3786, 4810, 2044, 4527, 2218, 3636, 159, 4024, 876, 56, 1099, 1441, 3000, 1340, 4275, 4821, 3124, 512, 1588, 2269, 2765, 1276, 1688, 1537, 4750, 2198, 2219, 3128, 2838, 1250, 374, 543, 1541, 1651, 3740, 3624, 3869, 3092, 635, 2387, 1575, 2470, 3049, 1605, 2617, 4942, 2509, 1650, 4184, 3512, 4246, 993, 1989, 3247, 4500, 1117, 2738, 3286, 3492, 3378, 2402, 1613, 3996, 4235, 3347, 3474, 1498, 3314, 752, 1595, 1251, 4765, 704, 495, 1887, 1343, 3783, 246, 692, 4277, 1277, 4916, 610, 3211, 3858, 2135, 775, 4366, 570, 2519, 1856, 4089, 2798, 3754, 4030, 4836, 2473, 2758, 81, 880, 4177, 4724, 4712, 3289, 4306, 3333, 2900, 2718, 3254, 473, 4896, 546, 1041, 2251, 2485, 2451, 3976, 3177, 2047, 4557, 4551, 1304, 4518, 3888, 3879, 131, 2411, 2713, 1014, 4134, 1015, 724, 4176, 2806, 2555, 763, 4807, 2050, 1632, 4321, 748, 1505, 3617, 1034, 3384, 3554, 2031, 2932, 4319, 3161, 2672, 2299, 1895, 346, 1753, 3978, 2256, 446, 3581, 4378, 4906, 1087, 87, 2818, 688, 1192, 3090, 1163, 448, 4576, 4511, 1905, 2614, 3407, 2076, 4555, 4079, 165, 1219, 106, 7, 290, 3637, 178, 1711, 2845, 1468, 4188, 3574, 3545, 2336, 1369, 617, 4058, 4619, 2754, 2090, 1639, 4196, 1272, 1121, 4829, 2775, 2734, 1965, 1075, 4097, 1306, 1123, 3213, 1702, 2830, 4328, 4806, 2985, 2574, 3140, 4668, 3697, 1802, 2241, 1394, 890, 798, 4632, 3389, 1002, 2014, 1616, 1635, 2776, 4539, 3839, 1324, 818, 3795, 4242, 3089, 3050, 4456, 1283, 3414, 2037, 3542, 2023, 1722, 2190, 2871, 1213, 3131, 4116, 4417, 3929, 128, 859, 3757, 1614, 4169, 3350, 417, 4790, 2467, 2917, 4888, 366, 1239, 2825, 1703, 539, 4114, 2072, 4990, 161, 3641, 1798, 889, 2103, 2584, 679, 2550, 4820, 1194, 2224, 2891, 2877, 2370, 1113, 3448, 1526, 324, 3815, 595, 2675, 4802, 1739, 180, 3573, 2942, 1918, 209, 3479, 3536, 293, 2102, 100, 3319, 4172, 2722, 3591, 1699, 2390, 1295, 2570, 4171, 2557, 4478, 4400, 1576, 1406, 395, 2857, 4476, 3355, 2676, 2182, 4042, 1503, 2518, 1652, 2561, 2245, 1998, 1066, 1079, 772, 2702, 987, 3045, 4881, 2295, 3925, 368, 4034, 4323, 3731, 3095, 3138, 114, 1382, 805, 4311, 1996, 4609, 3043, 125, 968, 3715, 4728, 1020, 3405, 1993, 3544, 4618, 2230, 381, 1159, 974, 2439, 2191, 3635, 3811, 4062, 1536, 1151, 479, 2338, 749, 2637, 428, 3900, 2927, 2632, 4756, 4314, 3676, 3922, 4318, 4249, 4774, 3793, 4548, 1915, 2204, 2752, 4316, 1442, 630, 4945, 3745, 2725, 678, 2032, 3205, 83, 1545, 709, 4214, 3458, 2928, 3054, 1326, 4650, 4320, 3584, 3562, 4769, 1116, 3804, 4367, 52, 4407, 3190, 1731, 4758, 1308, 2244, 655, 190, 852, 3621, 2252, 1522, 3231, 1331, 3547, 1698, 410, 191, 1454, 1004, 668, 1784, 4324, 909, 845, 1172, 3938, 498, 1358, 4975, 1089, 2847, 4667, 3631, 3989, 3023, 2558, 4240, 1018, 4749, 4031, 152, 1816, 3027, 3311, 3283, 2055, 1407, 2315, 331, 4704, 1236, 4151, 643, 3470, 2406, 2759, 3067, 1571, 1982, 1506, 733, 3354, 2543, 2414, 1267, 1928, 4516, 569, 2310, 3688, 1178, 2639, 1533, 4357, 1569, 2144, 3762, 4337, 1695, 4336, 2178, 1435, 3062, 2520, 1807, 902, 4264, 1967, 4084, 4491, 1246, 2356, 667, 233, 1105, 4123, 2585, 1869, 1176, 494, 1316, 3278, 4960, 1391, 788, 4283, 944, 3613, 1696, 531, 916, 2858, 1756, 2578, 1855, 1384, 214, 2412, 955, 2129, 600, 3379, 1342, 4156, 1630, 4483, 522, 4943, 4703, 2648, 2721, 399, 4524, 4550, 2903, 3353, 3186, 4461, 286, 1653, 3742, 536, 1404, 4882, 14, 591, 340, 2567, 1083, 4365, 2605, 3649, 1070, 1663, 4093, 4072, 1514, 3759, 2949, 1296, 4523, 3854, 4405, 680, 2069, 2410, 1380, 4213, 1726, 2710, 513, 4566, 4685, 4891, 988, 2795, 4991, 2026, 1182, 4403, 2851, 2265, 3652, 4968, 1475, 1868, 398, 43, 2357, 2197, 685, 3973, 2015, 3010, 141, 2904, 3722, 653, 511, 2612, 3594, 1056, 3430, 91, 4330, 228, 349, 2007, 793, 1208, 1551, 4406, 796, 4023, 3046, 2505, 4705, 581, 665, 2770, 4350, 829, 4787, 2349, 239, 4288, 3309, 3284, 3148, 4507, 873, 2819, 676, 698, 2231, 884, 4237, 2684, 2404, 4922, 2489, 323, 2384, 2820, 3686, 2337, 2033, 2602, 4894, 58, 3109, 999, 3369, 225, 1191, 341, 4451, 2151, 534, 42, 4454, 1898, 434, 4799, 2021, 182, 4645, 3860, 1073, 303, 2895, 2586, 3032, 1647, 452, 4221, 2619, 5009, 251, 4424, 797, 4438, 4541, 3579, 2875, 2194, 1061, 4441, 1835, 450, 4912, 4334, 2173, 1532, 4458, 310, 1808, 4095, 776, 101, 4032, 4823, 3693, 2035, 145, 4629, 3825, 823, 2924, 1871, 3383, 213, 348, 99, 941, 3886, 3006, 2341, 855, 3941, 2700, 3120, 2169, 3234, 1499, 1960, 2298, 75, 588, 1125, 1050, 675, 3122, 344, 1858, 4133, 2641, 1962, 3708, 3725, 3800, 1329, 606, 61, 640, 1181, 1418, 2458, 3454, 3069, 2316, 1057, 4783, 2764, 2695, 3681, 1098, 1248, 4149, 4190, 3593, 1273, 4106, 3912, 4282, 1200, 650, 2896, 2594, 4211, 528, 2564, 2057, 2042, 4078, 1224, 2053, 4614, 2259, 3963, 3881, 4963, 4180, 2284, 2088, 2444, 2318, 3244, 3826, 2630, 401, 2601, 991, 4173, 4344, 599, 3655, 1725, 2680, 4189, 2077, 4601, 1717, 4978, 4219, 4428, 594, 411, 585, 4608, 4512, 3758, 1950, 3294, 939, 4165, 2138, 2929, 4004, 970, 3176, 4487, 537, 3799, 1822, 3923, 4540, 3893, 4362, 4248, 2824, 3741, 3919, 515, 3040, 2600, 2363, 4398, 2052, 3659, 4029, 3588, 3928, 1196, 4818, 3116, 2649, 4735, 4303, 1789, 1638, 3541, 4744, 1443, 3660, 1761, 3761, 2150, 682, 1420, 2822, 1901, 3028, 3776, 4039, 4681, 700, 4157, 2955, 142, 1736, 4473, 899, 177, 3907, 1377, 3696, 1352, 2526, 2233, 3868, 4179, 2705, 950, 158, 3550, 2162, 2848, 363, 1820, 1023, 705, 948, 1555, 3445, 4489, 1491, 4873, 4262, 3007, 2242, 1978, 4301, 1234, 2468, 4015, 421, 4414, 2483, 3214, 1781, 2347, 2285, 1741, 3437, 975, 1005, 1690, 1094, 2894, 431, 2538, 2536, 3519, 1668, 4699, 4938, 2141, 442, 2305, 1502, 3217, 254, 1701, 1997, 4677, 268, 3677, 3257, 2438, 2187, 1517, 2931, 4110, 184, 4800, 926, 3956, 2829, 3429, 89, 1133, 3082, 3402, 4812, 409, 4399, 3151, 3363, 4993, 3025, 74, 261, 2181, 1596, 2642, 645, 3033, 779, 462, 356, 3669, 2306, 4052, 3162, 3270, 922, 4950, 857, 1641, 622, 3779, 4269, 669, 4356, 1263, 3039, 4371, 4070, 2334, 965, 1311, 1567, 3882, 2861, 113, 1620, 2899, 2293, 1794, 2572, 1857, 1161, 2535, 2499, 5002, 1580, 887, 4625, 4964, 4479, 4831, 2081, 4300, 1724, 851, 1097, 2653, 646, 3261, 413, 2202, 4604, 985, 947, 1135, 4351, 1727, 3296, 3320, 730, 2296, 3253, 2568, 4865, 2768, 4006, 3521, 933, 1896, 2409, 2766, 4772, 4617, 2246, 3196, 2450, 424, 3083, 2662, 2066, 3014, 1848, 4825, 3367, 2260, 2184, 1507, 3181, 3157, 1760, 934, 1624, 825, 3812, 3030, 1409, 1787, 2696, 3189, 1564, 3535, 4142, 624, 176, 2405, 3817, 3265, 3442, 414, 3061, 3336, 632, 661, 2616, 2376, 4146, 3601, 1390, 51, 2693, 3553, 1360, 144, 2704, 4152, 3009, 3518, 1321, 2786, 2017, 1500, 249, 3264, 1335, 4191, 4390, 459, 2019, 2921, 576, 3760, 1607, 4664, 672, 4520, 2743, 402, 387, 4961, 1578, 4435, 2507, 3129, 4309, 28, 181, 4243, 1938, 2547, 1554, 842, 3648, 2943, 4013, 3228, 1423, 1592, 2004, 3406, 782, 1367, 1489, 1353, 3024, 5005, 695, 3012, 2813, 2908, 747, 4510, 4098, 3530, 1543, 664, 1874, 4372, 2211, 244, 2975, 1460, 94, 2309, 1286, 2314, 1337, 2745, 1320, 2732, 4501, 3514, 1003, 102, 2441, 1378, 2882, 806, 2492, 5000, 815, 1800, 4232, 4876, 4026, 657, 31, 1292, 3380, 4815, 1207, 4063, 350, 4697, 1642, 1078, 964, 3079, 3611, 2659, 3059, 2972, 109, 2809, 1530, 2294, 13, 1524, 216, 614, 1747, 1980, 2618, 3764, 3834, 1465, 1186, 1903, 3281, 4833, 2727, 3233, 4071, 3984, 1431, 1143, 1307, 3203, 596, 3880, 189, 2874, 487, 2523, 1376, 4788, 3498, 2796, 2969, 347, 1931, 2876, 2000, 1047, 3718, 4401, 3698, 3945, 1970, 2020, 3372, 4759, 4065, 3444, 4415, 1859, 4616, 1154, 908, 1120, 1864, 3866, 2258, 3565, 1111, 3031, 4465, 4125, 2706, 2471, 1939, 275, 2114, 2061, 2437, 2247, 1740, 3983, 50, 1119, 3511, 3876, 956, 2911, 4457, 1241, 1195, 1027, 877, 1425, 3285, 1487, 3543, 2168, 767, 652, 4638, 2322, 3494, 3910, 4304, 2883, 3312, 4468, 2292, 2491, 4205, 2012, 736, 4811, 4605, 441, 1929, 3436, 315, 3238, 1922, 3112, 2147, 2865, 1222, 3300, 2967, 221, 3295, 2497, 4526, 1169, 3788, 2094, 1829, 2174, 1907, 3719, 3694, 963, 4308, 848, 3465, 2817, 4227, 565, 4979, 454, 683, 2005, 1145, 3476, 3940, 2590, 4115, 351, 2650, 1336, 1755, 783, 3620, 2016, 2003, 3349, 1051, 200, 4892, 3114, 4376, 4857, 4502, 2909, 883, 1052, 3080, 4379, 816, 1771, 4066, 187, 1414, 168, 3113, 433, 524, 4717, 3606, 272, 849, 2833, 1490, 4422, 2808, 611, 930, 996, 2595, 3650, 4545, 4984, 3972, 4270, 2413, 3792, 4868, 2991, 2755, 2801, 1288, 2383, 3846, 2621, 3590, 1312, 4513, 2362, 4247, 3119, 2360, 3862, 3616, 2067, 831, 4469, 2514, 2692, 4944, 4453, 4475, 2666, 4582, 1199, 4767, 1422, 296, 785, 2804, 1413, 3980, 3331, 2474, 385, 4460, 2613, 4121, 3499, 3942, 2043, 1471, 3194, 1275, 1281, 2301, 4889, 2422, 2327, 4804, 1150, 1772, 1330, 4886, 1671, 3595, 3446, 3268, 662, 179, 67, 4339, 2992, 86, 4640, 989, 2110, 3604, 853, 2098, 1269, 4992, 4204, 4360, 744, 2328, 3859, 2116, 1683, 3898, 4885, 4939, 4948, 4001, 3844, 631, 1302, 2778, 3015, 4598, 3592, 435, 2480, 1560, 633, 4464, 3643, 3852, 1976, 2082, 496, 830, 3042, 4864, 3921, 3184, 2380, 103, 2608, 4791, 4590, 3564, 4459, 982, 2660, 3828, 2831, 1129, 3452, 4335, 120, 4843, 1746, 4814, 726, 486, 3603, 2289, 1229, 439, 2255, 1084, 2313, 2990, 1768, 4522, 4813, 634, 5, 4559, 3466, 774, 1107, 3245, 2221, 1115, 3395, 924, 4061, 547, 2397, 864, 255, 2864, 1261, 4556, 2582, 1977, 2268, 2920, 3087, 1501, 2163, 3967, 2392, 4653, 3099, 235, 205, 419, 1109, 3317, 4572, 3330, 1416, 1124, 2657, 505, 282, 1540, 1645, 3248, 552, 3684, 4861, 2709, 0, 3149, 4848, 279, 3575, 445, 3488, 2785, 1279, 2761, 1158, 2840, 1904, 4, 2420, 3953, 2368, 1558, 2977, 4353, 2914, 4215, 2671, 1228, 1221, 555, 2527, 2717, 3135, 4841, 1310, 3396, 3422, 2962, 1552, 57, 1082, 4525, 1776, 1840, 4715, 2398, 2946, 4999, 2869, 3920, 1298, 1496, 3939, 390, 1046, 4260, 4044, 3924, 2113, 1930, 3977, 1202, 4144, 2736, 3634, 1012, 935, 2542, 976, 2744, 4111, 4920, 2167, 790, 2455, 3439, 284, 3493, 1941, 2104, 1137, 523, 509, 4647, 2622, 373, 481, 4558, 4104, 2803, 1521, 3645, 1122, 2937, 3913, 55, 1714, 2664, 4835, 70, 2024, 2516, 3169, 2393, 4096, 2302, 1619, 719, 3816, 73, 4736, 3607, 4621, 1940, 4579, 545, 3756, 267, 3167, 3798, 4915, 1318, 430, 870, 2532, 232, 689, 2192, 2797, 2997, 2714, 3979, 1617, 4222, 1842, 4045, 1866, 1832, 2834, 4927, 4837, 2049, 2373, 3503, 4603, 4234, 660, 277, 4049, 1095, 1401, 3393, 1644, 127, 3461, 3408, 2569, 2591, 2008, 2850, 3121, 1255, 4941, 3865, 3339, 3830, 2220, 3335, 4429, 757, 2317, 4905, 4255, 2, 1888, 1557, 4740, 3178, 2965, 4633, 583, 95, 314, 3344, 3789, 4498, 618, 3041, 4981, 2733, 3240, 1923, 1999, 550, 3667, 628, 1235, 1963, 2041, 304, 1806, 3672, 1262, 1294, 1322, 714, 16, 1762, 3959, 3057, 764, 3174, 71, 1238, 1843, 4897, 4634, 850, 869, 404, 4958, 2563, 1074, 2153, 723, 2995, 840, 477, 2655, 3321, 907, 3515, 1770, 529, 1467, 5011, 4119, 1164, 3903, 2644, 482, 4622, 4462, 684, 2816, 710, 3316, 2951, 193, 2381, 2673, 98, 3490, 4716, 1520, 1315, 1153, 4761, 2203, 380, 4726, 1106, 3853, 4108, 4450, 3537, 1289, 557, 2391, 2633, 4345, 3359, 3246, 4648, 2117, 1118, 3500, 3147, 2424, 2598, 407, 3271, 575, 1791, 900, 1291, 1132, 371, 2111, 1559, 474, 755, 1091, 3809, 3580, 1721, 2209, 813, 436, 4494, 2510, 1910, 4285, 3073, 4937, 4266, 1282, 3773, 519, 1546, 2853, 2811, 287, 336, 712, 2667, 3438, 3887, 4368, 458, 3777, 2442, 3673, 443, 80, 3572, 3460, 4692, 3546, 544, 4117, 392, 2529, 854, 1493, 4241, 4136, 608, 2022, 725, 878, 3142, 4226, 1240, 2623, 2940, 3258, 1825, 1670, 1264, 4828, 3988, 4597, 418, 2636, 1817, 3468, 3960, 1544, 3871, 4561, 3352, 3838, 1673, 3504, 1549, 1388, 1383, 3325, 1237, 3101, 1889, 4194, 464, 2193, 1935, 483, 1309, 1785, 500, 4053, 3394, 4081, 2698, 1584, 3426, 2688, 2276, 2308, 2120, 4909, 2134, 3332, 4635, 230, 1231, 856, 1381, 3016, 1979, 3185, 867, 1071, 4077, 4082, 2703, 2970, 1792, 4602, 2887, 36, 4768, 2448, 1177, 3195, 4731, 4587, 919, 872, 904, 3441, 716, 4662, 4567, 1603, 1913, 1364, 901, 2791, 2544, 2841, 3078, 4679, 1252, 4197, 2401, 4229, 3911, 1615, 122, 4672, 1473, 2278, 3356, 1611, 2515, 3324, 1732, 4409, 1723, 1984, 3946, 4929, 943, 4797, 1470, 2175, 3797, 1375, 375, 33, 3934, 4538, 3473, 862, 2640, 4370, 4755, 4785, 1511, 316, 4506, 4793, 3771, 3115, 2960, 4718, 4860, 1964, 1048, 506, 2366, 2452, 708, 3256, 4236, 60, 2913, 1256, 2783, 997, 2093, 1227, 1327, 4444, 35, 4554, 3608, 4490, 3651, 4856, 803, 699, 4130, 146, 3291, 3013, 170, 921, 533, 558, 778, 2408, 3502, 4547, 1201, 3867, 378, 1492, 587, 3961, 2603, 3171, 4313, 4067, 3585, 2423, 1669, 1779, 3769, 2863, 602, 327, 449, 4100, 3690, 2597, 4168, 4786, 1599, 4631, 2136, 4375, 2475, 4289, 2156, 4043, 2879, 1944, 451, 4531, 2407, 4989, 3969, 2886, 134, 858, 3427, 309, 4253, 3986, 2119, 3055, 4484, 1287, 1665, 1160, 2880, 1728, 1247, 160, 4983, 1844, 334, 2430, 173, 4855, 4969, 2319, 4727, 3375, 4105, 977, 471, 4216, 808, 1769, 1991, 1565, 4402, 1990, 4737, 590, 301, 2566, 3949, 1396, 426, 3863, 2938, 3532, 472, 2096, 1773, 1640, 1128, 765, 2530, 914, 4286, 1328, 3791, 2923, 3068, 861, 12, 4982, 3724, 619, 4585, 1461, 715, 3625, 4068, 3970, 4426, 367, 4434, 1359, 3403, 130, 1429, 3561, 4448, 2592, 2646, 1729, 1479, 1497, 3935, 677, 891, 18, 3172, 2728, 2060, 4624, 2435, 2188, 126, 1249, 4202, 728, 1042, 2810, 756, 4037, 3262, 4710, 1527, 526, 4700, 3917, 4847, 3060, 234, 828, 2939, 1297, 297, 666, 1828, 2465, 1455, 3387, 990, 4729, 3275, 325, 811, 4751, 3048, 4709, 1482, 1885, 2030, 3730, 804, 4472, 1362, 1966, 1902, 1783, 3610, 3569, 2750, 4326, 488, 1954, 3251, 1290, 386, 4784, 1566, 4272, 3712, 320, 2587, 3216, 3328, 65, 1254, 3855, 256, 342, 638, 2528, 3937, 4485, 3523, 1218, 2593, 1870, 834, 3137, 3747, 1749, 729, 3293, 208, 4322, 1577, 508, 1366, 4271, 4553, 1786, 3952, 3551, 3529, 2958, 3997, 2663, 1173, 2385, 3047, 4623, 1220, 894, 4858, 4780, 1483, 1775, 2232, 3329, 1333, 4890, 3966, 4695, 3981, 4209, 220, 2772, 4027, 1421, 2964, 2446, 2107, 4046, 843, 1909, 3136, 150, 4431, 2777, 2971, 2421, 2046, 227, 1649, 961, 3273, 3302, 3346, 3841, 88, 3695, 4866, 3434, 2739, 2747, 2027, 3334, 2669, 3366, 1305, 3085, 4425, 2654, 2599, 1415, 4738, 701, 49, 1625, 3955, 1797, 1146, 3274, 1803, 1265, 577, 3322, 3662, 4880, 4996, 3026, 2036, 1681, 554, 3483, 1628, 2689, 2460, 2490, 3802, 1189, 1354, 1065, 4549, 3307, 32, 19, 4734, 229, 1750, 4164, 427, 1823, 4153, 292, 4233, 2079, 2945, 1654, 3433, 1924, 41, 1981, 580, 895, 3664, 2697, 2720, 761, 4158, 822, 874, 817, 1408, 3145, 4503, 4542, 3794, 3290, 4719, 4497, 3836, 332, 1675, 1373, 465, 2724, 2548, 4103, 2038, 4935, 4083, 952, 3582, 285, 3127, 3280, 2343, 4763, 1912, 3342, 1553, 3272, 750, 1891, 2353, 2723, 4832, 217, 2782, 4934, 4148, 671, 3615, 1419, 1448, 2948, 2227, 199, 3576, 510, 4986, 2109, 4137, 3974, 897, 4721, 4463, 3005, 835], "validation": [5042, 5449, 5379, 5456, 5307, 5277, 5092, 5501, 5309, 5154, 5263, 5477, 5286, 5339, 5167, 5433, 5454, 5498, 5436, 5511, 5084, 5074, 5200, 5090, 5034, 5471, 5247, 5405, 5019, 5371, 5300, 5145, 5233, 5481, 5193, 5355, 5076, 5467, 5118, 5365, 5351, 5066, 5212, 5383, 5451, 5310, 5185, 5181, 5158, 5025, 5343, 5157, 5093, 5470, 5232, 5172, 5392, 5050, 5065, 5183, 5330, 5044, 5202, 5290, 5089, 5253, 5221, 5348, 5207, 5366, 5250, 5465, 5144, 5447, 5432, 5400, 5096, 5403, 5458, 5138, 5248, 5324, 5384, 5381, 5085, 5469, 5078, 5416, 5014, 5024, 5302, 5125, 5393, 5354, 5410, 5283, 5260, 5214, 5506, 5255, 5246, 5326, 5062, 5500, 5364, 5047, 5387, 5289, 5137, 5099, 5337, 5476, 5102, 5374, 5382, 5083, 5437, 5264, 5101, 5268, 5335, 5322, 5244, 5197, 5097, 5049, 5080, 5353, 5385, 5217, 5249, 5294, 5063, 5474, 5347, 5033, 5234, 5402, 5483, 5147, 5064, 5298, 5308, 5075, 5295, 5201, 5241, 5225, 5245, 5036, 5280, 5493, 5120, 5224, 5203, 5256, 5128, 5497, 5296, 5057, 5113, 5503, 5479, 5054, 5357, 5320, 5333, 5056, 5130, 5325, 5331, 5168, 5441, 5509, 5480, 5345, 5251, 5412, 5306, 5208, 5323, 5431, 5110, 5459, 5161, 5129, 5398, 5031, 5119, 5460, 5195, 5218, 5363, 5115, 5418, 5445, 5304, 5505, 5131, 5421, 5399, 5116, 5278, 5236, 5112, 5341, 5243, 5258, 5211, 5091, 5411, 5291, 5155, 5423, 5045, 5156, 5419, 5276, 5462, 5199, 5356, 5073, 5389, 5122, 5176, 5314, 5390, 5082, 5237, 5292, 5204, 5266, 5413, 5133, 5487, 5239, 5196, 5303, 5018, 5135, 5429, 5240, 5216, 5032, 5510, 5055, 5319, 5508, 5430, 5180, 5360, 5069, 5297, 5081, 5466, 5226, 5132, 5305, 5187, 5210, 5440, 5043, 5453, 5068, 5361, 5425, 5223, 5504, 5427, 5229, 5373, 5174, 5124, 5377, 5386, 5484, 5284, 5235, 5051, 5281, 5178, 5489, 5052, 5422, 5238, 5182, 5220, 5189, 5149, 5334, 5315, 5177, 5270, 5107, 5329, 5368, 5109, 5406, 5098, 5162, 5022, 5111, 5352, 5140, 5035, 5442, 5104, 5070, 5121, 5067, 5194, 5367, 5060, 5209, 5316, 5391, 5139, 5190, 5026, 5369, 5166, 5293, 5037, 5388, 5376, 5041, 5242, 5313, 5318, 5159, 5053, 5219, 5438, 5428, 5072, 5414, 5206, 5448, 5030, 5496, 5311, 5394, 5372, 5136, 5439, 5188, 5227, 5299, 5380, 5350, 5123, 5103, 5152, 5175, 5285, 5046, 5108, 5450, 5017, 5191, 5163, 5444, 5359, 5478, 5100, 5257, 5407, 5020, 5213, 5443, 5267, 5507, 5282, 5016, 5404, 5485, 5169, 5087, 5148, 5287, 5106, 5230, 5184, 5077, 5288, 5455, 5170, 5499, 5490, 5143, 5461, 5362, 5262, 5146, 5358, 5321, 5279, 5059, 5151, 5254, 5312, 5079, 5396, 5023, 5164, 5375, 5261, 5179, 5482, 5215, 5095, 5171, 5126, 5160, 5173, 5029, 5028, 5327, 5134, 5205, 5420, 5332, 5401, 5013, 5328, 5061, 5165, 5273, 5252, 5415, 5346, 5473, 5475, 5094, 5378, 5495, 5048, 5105, 5486, 5502, 5150, 5153, 5435, 5424, 5259, 5071, 5086, 5457, 5142, 5494, 5274, 5271, 5058, 5040, 5039, 5231, 5397, 5088, 5038, 5463, 5409, 5027, 5127, 5426, 5228, 5117, 5468, 5492, 5340, 5269, 5021, 5491, 5222, 5464, 5452, 5342, 5370, 5336, 5395, 5434, 5265, 5417, 5272, 5488, 5114, 5317, 5446, 5275, 5012, 5192, 5408, 5015, 5338, 5472, 5301, 5344, 5349, 5141, 5198, 5186], "test": [5560, 5801, 6010, 5854, 5541, 5603, 5999, 5624, 5844, 5746, 5672, 5959, 5684, 5911, 5862, 5546, 6003, 5583, 5822, 5995, 5896, 5756, 5787, 5643, 5641, 5885, 5580, 5776, 5980, 5690, 5752, 5803, 5651, 5972, 5944, 5806, 5653, 5596, 5962, 5625, 5994, 5979, 5575, 5737, 5693, 5889, 5611, 5971, 5794, 5606, 5590, 5620, 5845, 5946, 5540, 5770, 5618, 5990, 5796, 5648, 5807, 5593, 5988, 5566, 5515, 5790, 5677, 5820, 5866, 5680, 5914, 5682, 5615, 5749, 5855, 5791, 5852, 5652, 5799, 5927, 5778, 5880, 5711, 5887, 5843, 5722, 5964, 5779, 5666, 5800, 5656, 5765, 5864, 5529, 5978, 5728, 5808, 5810, 5530, 5929, 5772, 5943, 5762, 5740, 5695, 5903, 5548, 5623, 5829, 5574, 5941, 5948, 5893, 5629, 5663, 5528, 5839, 5975, 5567, 5532, 5802, 5825, 5573, 5719, 5733, 6006, 5761, 5576, 5634, 5813, 5520, 5699, 5669, 5786, 5836, 5989, 5544, 5679, 5764, 5945, 5939, 5642, 5902, 5513, 5963, 5860, 5531, 5884, 5984, 5630, 5562, 6001, 5570, 5798, 5588, 5678, 5713, 5658, 5910, 5837, 5915, 5610, 5909, 5797, 5708, 5645, 5664, 5723, 5622, 5775, 5657, 5561, 5812, 5584, 5907, 5731, 5609, 5550, 5918, 5632, 5965, 5987, 5785, 5516, 5692, 5925, 5717, 5931, 5924, 5823, 5730, 5838, 5804, 5691, 5565, 5700, 5865, 5953, 5676, 5608, 5966, 5525, 5937, 5545, 5983, 5755, 5758, 5514, 5522, 5637, 5763, 5982, 5815, 5867, 5552, 5992, 5783, 5614, 5835, 5721, 5599, 5833, 5670, 5832, 5757, 5705, 5942, 5766, 5729, 5675, 5998, 5874, 5709, 5628, 5527, 5869, 5594, 5715, 6007, 5969, 5598, 5857, 5882, 5547, 5886, 5821, 5788, 5591, 6002, 5928, 5958, 5957, 5890, 5850, 5920, 5536, 5848, 5831, 5668, 5842, 5922, 5539, 5631, 5704, 5644, 5743, 5930, 5689, 5710, 5616, 5662, 5517, 5649, 5604, 5706, 5597, 5877, 5724, 5759, 5824, 5549, 5782, 5512, 5667, 5542, 5726, 5563, 5956, 5901, 5748, 5612, 5627, 5745, 5683, 5940, 5973, 5635, 5732, 5906, 5734, 5846, 5858, 5938, 5967, 5870, 5534, 5898, 5923, 5621, 5899, 5935, 5949, 6009, 5827, 5881, 5868, 5974, 5533, 5694, 5904, 5582, 6004, 5768, 5674, 5878, 5851, 5769, 5701, 5564, 5805, 5809, 5712, 5871, 6000, 5811, 5518, 5891, 5856, 5753, 5819, 5727, 5633, 5595, 5952, 5926, 5557, 5535, 5773, 5551, 5863, 5859, 5587, 5991, 6005, 5736, 5900, 5892, 5933, 5853, 5817, 5919, 5646, 5626, 5894, 5559, 5834, 5725, 5655, 5754, 5638, 5913, 5830, 5696, 5586, 5523, 5977, 5985, 5996, 5600, 5685, 5714, 5698, 5780, 5660, 5816, 5947, 5997, 5519, 5640, 5718, 5905, 5826, 5681, 5774, 5986, 5993, 5702, 5537, 5585, 5686, 5703, 5739, 5688, 5751, 5742, 5538, 5650, 5895, 5578, 5960, 5917, 5654, 5968, 5951, 5543, 5932, 5554, 5789, 5659, 5954, 5849, 5735, 5639, 5792, 5818, 5921, 5556, 5617, 5908, 5558, 5665, 5793, 5781, 5760, 5571, 5553, 5970, 5912, 5592, 5619, 5840, 5671, 5976, 5950, 5955, 5814, 5981, 5647, 5750, 5707, 5875, 5569, 5613, 5777, 5872, 5744, 5888, 5747, 5568, 5601, 5572, 5581, 5883, 5784, 5636, 5767, 5661, 5720, 5526, 5936, 5771, 5716, 5577, 5697, 5916, 5673, 5795, 5555, 5607, 5828, 5687, 6008, 5873, 5521, 5524, 5602, 5961, 5861, 5738, 5741, 5847, 5897, 5879, 5579, 5605, 5934, 5589, 5841, 6011, 5876]}, {"train": [4687, 4983, 1853, 3300, 2451, 2864, 2141, 1835, 3680, 2298, 423, 3380, 1774, 1968, 2313, 1777, 2643, 685, 4744, 2699, 1644, 4493, 2664, 1989, 80, 901, 4489, 307, 4161, 4974, 862, 2491, 2170, 1133, 2143, 2552, 593, 3349, 4724, 2513, 2088, 1666, 3326, 3429, 2601, 4341, 968, 547, 2707, 3210, 1124, 2619, 4756, 1321, 2901, 2083, 1204, 949, 1783, 4973, 977, 393, 4878, 4406, 985, 875, 4165, 4209, 2965, 1842, 2990, 4455, 323, 5007, 1162, 1315, 2750, 2417, 4382, 226, 4467, 398, 2001, 3704, 160, 3933, 4646, 853, 2683, 771, 3162, 2286, 4218, 3292, 4709, 4694, 3418, 4589, 1863, 4411, 2855, 3682, 3115, 2045, 3359, 3989, 3797, 3200, 475, 4567, 4543, 4222, 4781, 16, 447, 452, 2970, 4288, 3293, 4511, 3462, 3908, 1248, 2913, 3895, 586, 4227, 521, 256, 2476, 3371, 3314, 3880, 3502, 3936, 4934, 3046, 4793, 2054, 230, 472, 172, 4795, 153, 2496, 4514, 2138, 3964, 4906, 557, 1905, 418, 3536, 234, 520, 2545, 93, 2186, 1632, 4228, 3219, 496, 3138, 884, 217, 250, 1969, 837, 2217, 2078, 140, 4893, 3033, 4997, 4579, 2027, 3607, 1031, 1431, 3734, 1409, 379, 2505, 2262, 2036, 3141, 2881, 4913, 3495, 1445, 1744, 4482, 2940, 2676, 2176, 3087, 537, 2345, 4518, 4734, 4552, 3390, 3713, 4442, 3235, 4861, 1349, 4985, 5009, 4629, 598, 2917, 4735, 2131, 2010, 4739, 4950, 4068, 3504, 1213, 1115, 4424, 1821, 1499, 3826, 3681, 1427, 1705, 4386, 4132, 2910, 3307, 4221, 1264, 2105, 1978, 871, 1273, 3733, 1327, 2934, 3265, 787, 4336, 4038, 46, 846, 4594, 2953, 4938, 4412, 328, 1704, 1675, 4952, 350, 3979, 1169, 4093, 4340, 91, 2332, 3724, 1589, 3885, 4868, 3972, 3705, 2256, 1688, 3255, 3830, 3684, 2832, 2929, 1183, 4471, 3881, 867, 2617, 8, 4126, 451, 3478, 3785, 3103, 2310, 3017, 2981, 272, 1465, 4574, 1719, 3068, 4070, 409, 859, 1370, 4408, 3554, 4363, 2778, 481, 459, 4256, 3765, 1749, 4931, 4941, 28, 1872, 3352, 1683, 2376, 2289, 1937, 638, 2768, 3629, 2258, 3986, 1520, 2295, 4129, 4662, 4235, 3013, 840, 4430, 2960, 4599, 1555, 1246, 510, 1295, 2194, 2107, 377, 4081, 834, 4591, 2032, 27, 3856, 3097, 3178, 4268, 544, 1801, 1832, 2925, 4902, 4155, 1918, 2573, 4577, 693, 4112, 2613, 3457, 4190, 1403, 2622, 2799, 278, 3697, 2096, 1792, 995, 4296, 1486, 1357, 3461, 1290, 2158, 691, 998, 1469, 3276, 3961, 3735, 4361, 4490, 1043, 3331, 3221, 3275, 1270, 2876, 3395, 4057, 4028, 330, 3288, 3806, 206, 2272, 769, 155, 4294, 4794, 3078, 4891, 3573, 673, 4580, 3336, 4313, 1105, 1437, 1584, 3790, 2948, 2330, 260, 1742, 2113, 252, 3493, 3556, 584, 2311, 4778, 1276, 288, 3603, 810, 1365, 4289, 3179, 668, 425, 764, 3538, 1559, 4918, 4809, 449, 3481, 711, 1659, 1908, 2326, 1328, 95, 4245, 4500, 2020, 246, 2432, 4437, 1236, 1693, 4608, 3063, 3271, 1883, 2672, 273, 597, 131, 2433, 3647, 3260, 1062, 3408, 877, 4279, 362, 4711, 438, 1654, 1960, 3456, 1251, 2522, 4393, 3090, 65, 4874, 3387, 4550, 4937, 4929, 2470, 2641, 3208, 3548, 3613, 4056, 2030, 1534, 3572, 103, 266, 309, 1628, 4480, 3565, 4301, 4971, 709, 3237, 443, 1425, 3228, 1674, 97, 2091, 1504, 713, 684, 3837, 1051, 2647, 4807, 4197, 568, 4389, 639, 371, 961, 3487, 3655, 1130, 3863, 2684, 3533, 1456, 468, 1602, 4765, 3743, 1727, 3174, 1287, 704, 4558, 703, 3484, 2401, 485, 3951, 3252, 1668, 3403, 1502, 3439, 4502, 3792, 574, 4182, 1166, 2033, 3844, 3619, 3011, 2355, 1585, 3310, 1084, 2605, 4962, 4803, 4095, 4641, 2883, 1670, 268, 4021, 4330, 450, 728, 2221, 828, 1755, 58, 4529, 3109, 3048, 63, 4888, 778, 2968, 2599, 2459, 3379, 669, 1467, 680, 1916, 2972, 41, 3250, 4352, 2029, 501, 4002, 494, 1771, 1212, 4517, 952, 1106, 3551, 4718, 2576, 3474, 2171, 1201, 4743, 2259, 3769, 997, 3832, 3282, 4508, 3498, 4443, 1753, 4332, 4912, 2841, 1492, 4257, 1856, 1549, 2629, 2081, 57, 3917, 4181, 4665, 2899, 1545, 368, 66, 3409, 2773, 3966, 3685, 683, 2954, 1329, 4628, 3512, 3615, 1732, 3026, 831, 457, 420, 1254, 3171, 3209, 4871, 4660, 651, 1129, 2177, 3043, 2587, 3197, 136, 3505, 364, 2499, 4370, 1087, 1592, 904, 179, 2400, 4131, 2779, 454, 3747, 546, 4307, 4538, 4335, 2328, 789, 1085, 2252, 2517, 640, 1430, 3467, 1153, 882, 2312, 1580, 1478, 3542, 682, 4462, 3014, 1412, 511, 4904, 1582, 1925, 4562, 694, 2293, 50, 2314, 3036, 756, 2811, 3092, 1405, 3808, 4612, 3656, 4759, 439, 4527, 887, 2919, 3003, 2997, 1829, 2180, 2932, 1572, 4670, 4350, 3693, 1402, 1876, 2520, 2652, 1019, 4753, 1072, 886, 2677, 529, 2868, 4924, 4699, 3584, 849, 3508, 1054, 4054, 4116, 4719, 1797, 652, 591, 2922, 3902, 4826, 1040, 912, 735, 2214, 1185, 3005, 1934, 4762, 2890, 198, 2147, 3965, 4657, 1382, 1535, 4139, 3431, 2818, 4128, 3906, 3509, 2041, 2222, 4229, 344, 5, 4193, 117, 1938, 2793, 2824, 2324, 4693, 2044, 2644, 1041, 2924, 4812, 2064, 4818, 1965, 2477, 2946, 2414, 2556, 1954, 2758, 2134, 4792, 811, 2116, 3738, 4160, 1411, 360, 4991, 3348, 74, 3083, 4284, 506, 1595, 746, 4996, 4483, 3635, 1137, 1637, 3067, 1900, 2130, 3901, 2203, 1604, 4776, 132, 174, 2488, 3821, 1887, 4914, 2006, 974, 3202, 4521, 440, 2794, 2463, 1056, 2687, 3458, 1330, 4848, 662, 1307, 3346, 4097, 4968, 2322, 1066, 1734, 785, 2671, 1375, 2751, 601, 431, 1368, 2588, 4804, 314, 3623, 4014, 2593, 4741, 802, 3617, 4033, 228, 2978, 4703, 3116, 1503, 2656, 2191, 4246, 724, 857, 1748, 4419, 518, 4595, 2119, 254, 4982, 4281, 4433, 2146, 3110, 3342, 82, 1649, 512, 3795, 3636, 1320, 2368, 1422, 793, 2803, 2554, 3620, 39, 599, 1181, 3725, 731, 2860, 4404, 3585, 271, 4265, 3850, 2343, 248, 4560, 1750, 352, 3132, 1665, 650, 3897, 719, 4841, 2553, 2444, 4700, 1849, 3479, 2307, 4633, 4947, 2169, 4621, 4416, 2609, 4740, 3262, 2402, 1924, 4449, 3544, 940, 499, 1967, 2285, 1631, 3768, 4958, 435, 4251, 4359, 1651, 2850, 1897, 1794, 4046, 3098, 2173, 1956, 2989, 4943, 4286, 158, 723, 524, 2724, 446, 402, 2350, 4091, 3338, 2952, 1451, 4738, 2450, 1843, 1005, 4031, 367, 231, 4423, 1953, 383, 2816, 101, 2375, 2957, 2914, 2847, 965, 311, 2456, 4303, 620, 4365, 2875, 4121, 2127, 2497, 915, 3625, 4712, 4844, 3782, 4001, 4977, 893, 126, 1167, 3164, 1779, 2759, 678, 4278, 3754, 4675, 3861, 1754, 3820, 265, 3182, 2152, 4685, 1814, 4485, 1491, 4208, 2056, 954, 105, 193, 2237, 1281, 4475, 3771, 1973, 4269, 3970, 3670, 2984, 1609, 4196, 868, 4077, 2481, 286, 2753, 2807, 1173, 2055, 922, 4531, 4086, 1868, 1733, 1839, 2737, 4487, 4463, 2887, 4353, 3506, 3958, 3926, 3865, 1487, 1117, 1990, 2585, 3216, 2236, 2757, 4364, 2419, 4541, 2789, 3144, 2034, 1395, 946, 671, 4833, 3638, 781, 2595, 2895, 4224, 1233, 2066, 1743, 4869, 218, 2879, 4377, 3935, 1237, 1725, 1146, 4676, 491, 1650, 215, 4261, 4464, 1524, 914, 2866, 3815, 3476, 2530, 3884, 1496, 3851, 2620, 2859, 4882, 4664, 1070, 2229, 4572, 4164, 3061, 1761, 3924, 4337, 2320, 4613, 944, 905, 118, 659, 1603, 4405, 1694, 4415, 1314, 3709, 2461, 556, 1804, 55, 1353, 3059, 5000, 4243, 2923, 2244, 3775, 1029, 505, 2544, 4300, 1791, 3316, 2189, 3634, 4157, 3066, 1971, 243, 734, 4309, 614, 388, 4390, 4195, 2928, 3803, 624, 4777, 4494, 15, 4329, 1260, 303, 4624, 3836, 3907, 4034, 188, 238, 866, 3696, 4530, 2242, 1275, 1575, 670, 1363, 389, 1914, 1795, 69, 3218, 3835, 2916, 3663, 752, 4598, 2885, 559, 1994, 1943, 587, 1731, 1032, 4065, 4174, 2747, 2135, 3447, 3993, 312, 666, 1027, 1958, 3888, 151, 4723, 285, 1672, 2136, 1454, 3730, 247, 414, 773, 1221, 4796, 1678, 751, 2227, 3130, 3750, 1741, 3155, 4570, 221, 1860, 1351, 4241, 2837, 1093, 4043, 4318, 3192, 3287, 9, 513, 1208, 1225, 253, 918, 2263, 4484, 4036, 433, 4117, 925, 1655, 2663, 2660, 321, 2512, 759, 3766, 519, 4596, 1647, 4372, 3608, 1184, 3847, 4481, 973, 1333, 2164, 2061, 441, 3099, 1713, 1917, 1423, 4542, 3982, 4067, 3281, 1621, 2388, 2397, 3560, 2373, 4923, 2425, 1461, 4317, 883, 374, 2618, 3366, 4806, 3198, 189, 1111, 428, 181, 163, 2422, 3227, 3833, 1547, 1529, 4698, 1505, 4225, 702, 3176, 3446, 3211, 1259, 4823, 2769, 2765, 2385, 2489, 522, 2058, 1993, 2374, 4963, 4134, 1309, 2080, 274, 625, 430, 833, 2413, 2411, 2363, 679, 3106, 1610, 1834, 4900, 4495, 4654, 2775, 4199, 4203, 4240, 2072, 1935, 150, 3190, 3177, 11, 1022, 1955, 2998, 1904, 4571, 3987, 1576, 3717, 1408, 4678, 3280, 4642, 3911, 1217, 2572, 1768, 4073, 2726, 1102, 1240, 4402, 4764, 4357, 3787, 1974, 129, 4360, 3671, 353, 1286, 4136, 370, 3715, 4260, 812, 503, 4183, 3075, 4894, 1642, 4836, 1419, 1002, 4909, 2852, 1278, 4568, 4843, 3800, 4167, 3365, 3866, 2183, 1453, 3404, 988, 201, 4978, 3740, 4109, 1808, 1815, 3406, 4035, 1763, 2731, 3108, 1921, 2798, 1473, 4425, 4732, 3347, 1833, 1205, 3860, 207, 1226, 1081, 987, 2094, 2449, 2722, 51, 2276, 2833, 3052, 2851, 1342, 2128, 1393, 483, 1889, 788, 1099, 615, 706, 4421, 3521, 2569, 1823, 2124, 308, 3643, 4042, 1951, 445, 2843, 761, 2338, 2844, 1729, 3564, 967, 407, 4896, 5001, 3434, 3838, 296, 804, 476, 415, 3396, 3441, 4478, 1514, 3337, 88, 2420, 1948, 3142, 843, 1751, 2297, 2583, 2390, 375, 794, 2430, 784, 1347, 200, 1008, 4547, 3980, 3910, 37, 4840, 4255, 897, 2503, 2766, 1601, 3107, 4930, 5004, 1758, 2098, 1826, 149, 4787, 3686, 492, 419, 3318, 1892, 4520, 4890, 2025, 993, 3674, 2795, 4013, 2132, 3978, 795, 1459, 122, 2810, 2963, 3491, 427, 1214, 3330, 854, 3123, 281, 3489, 4052, 4770, 1517, 1806, 3480, 4863, 504, 2283, 262, 3997, 3445, 3373, 2359, 4779, 2379, 4062, 4211, 3566, 743, 4824, 4736, 156, 892, 2727, 2085, 92, 4714, 2858, 2482, 1247, 2870, 942, 910, 3272, 482, 4643, 1300, 3883, 3931, 1311, 2657, 4410, 4859, 4512, 4886, 841, 1807, 1439, 4327, 2845, 3402, 642, 2836, 1026, 2996, 1874, 3240, 130, 1608, 2467, 1017, 4507, 3742, 1288, 4470, 1109, 4460, 4015, 4041, 3622, 3776, 1282, 3801, 1625, 2584, 143, 3592, 171, 4232, 2142, 983, 4967, 13, 4058, 1312, 3475, 4834, 932, 2223, 2342, 3633, 3319, 3947, 3930, 532, 3195, 2000, 3229, 86, 2834, 2627, 4150, 1862, 498, 4588, 2531, 2543, 4617, 2442, 3035, 1802, 2688, 3805, 4096, 3516, 4297, 4100, 1344, 856, 473, 595, 1063, 3332, 3649, 4147, 177, 1507, 3126, 950, 1202, 2979, 4127, 3416, 1332, 2507, 2708, 2005, 4722, 3859, 4618, 4845, 3793, 4201, 2109, 38, 4945, 3658, 3711, 4348, 1190, 3591, 4993, 1164, 3135, 4532, 2287, 4202, 2521, 4084, 1049, 4503, 2856, 3973, 2666, 543, 656, 3621, 2407, 2511, 1996, 1142, 4248, 4731, 3960, 1506, 2770, 1706, 2623, 1685, 525, 4273, 3812, 295, 3517, 2937, 3203, 1479, 3058, 4831, 3327, 3497, 1977, 4315, 2668, 2889, 376, 2383, 3804, 324, 1765, 4451, 325, 3752, 1131, 4238, 550, 4557, 1080, 1182, 3234, 2347, 3580, 931, 3969, 2267, 2592, 851, 621, 716, 890, 1234, 2716, 487, 2733, 213, 1458, 70, 3283, 1929, 631, 3245, 2912, 224, 3134, 4, 294, 47, 2761, 4005, 2550, 1873, 3990, 464, 1415, 305, 990, 876, 1949, 958, 3309, 863, 3659, 162, 4244, 3053, 3688, 3360, 2900, 2542, 3718, 2133, 3401, 1721, 548, 562, 1203, 2621, 2264, 4785, 2877, 654, 2808, 855, 78, 758, 579, 3392, 3777, 1509, 2547, 3819, 3322, 705, 239, 292, 1787, 1944, 4094, 4397, 4627, 715, 1256, 1811, 2918, 779, 2335, 3976, 4573, 3749, 203, 1238, 4048, 3323, 1064, 1061, 2539, 612, 1218, 538, 4469, 4810, 1168, 4103, 808, 1952, 1689, 1757, 1745, 4666, 1025, 4254, 898, 284, 2565, 3118, 564, 2675, 1472, 3368, 1950, 3055, 3700, 4169, 3612, 1518, 399, 3614, 3510, 1982, 2179, 4748, 4866, 4747, 3723, 2782, 4892, 467, 4078, 4590, 4556, 1963, 4752, 2755, 4727, 4651, 4447, 4916, 2995, 4680, 1077, 290, 2546, 4233, 880, 458, 641, 2646, 2074, 1827, 4459, 1769, 896, 2260, 1888, 4213, 2416, 2941, 1331, 3111, 3702, 4114, 1661, 935, 125, 1441, 963, 2673, 1068, 528, 4961, 1390, 333, 2292, 2065, 1586, 3555, 3532, 2710, 1010, 3442, 1932, 372, 1471, 4737, 2577, 2557, 4179, 2279, 54, 4089, 1939, 2199, 1138, 1302, 2738, 1297, 2139, 1762, 3079, 3736, 645, 3296, 2700, 4123, 1879, 1103, 1147, 304, 3570, 535, 3212, 2823, 3857, 3825, 2596, 3660, 507, 1262, 4009, 1554, 4563, 2709, 1250, 56, 540, 4047, 1400, 972, 1543, 2348, 1638, 3780, 4623, 3624, 3552, 2273, 3513, 3916, 3948, 3064, 279, 3119, 2603, 4458, 2548, 222, 1865, 4092, 801, 4133, 2017, 3438, 3269, 895, 2303, 908, 1023, 2498, 1299, 999, 4769, 1657, 1379, 929, 2797, 4816, 2455, 2458, 3274, 1317, 3996, 4231, 2786, 1163, 799, 3165, 1195, 1933, 75, 277, 1859, 2667, 2219, 553, 2969, 2740, 2197, 2457, 4835, 742, 1541, 1305, 3755, 313, 4498, 1187, 3050, 3707, 821, 1912, 1736, 1679, 112, 4583, 3811, 4299, 4194, 145, 4186, 240, 1134, 2300, 49, 3869, 4189, 3642, 1941, 2796, 2857, 750, 1123, 2679, 336, 4742, 2725, 1858, 3798, 996, 2253, 5003, 36, 3786, 815, 244, 1927, 2829, 1075, 2202, 2403, 1893, 4933, 1257, 1722, 1568, 1074, 1623, 962, 4476, 3840, 1992, 7, 4749, 233, 4946, 3791, 3760, 212, 1793, 791, 832, 2071, 3451, 100, 3157, 141, 2728, 2536, 3664, 3454, 3587, 530, 541, 3186, 3796, 4553, 3903, 3101, 2690, 3650, 4180, 1746, 3361, 4029, 1622, 1540, 463, 4156, 2250, 2389, 4726, 3313, 960, 2159, 2892, 4343, 3568, 1972, 3419, 1372, 1448, 4992, 232, 2561, 2800, 4883, 2255, 1813, 2540, 22, 2207, 4682, 2607, 2265, 720, 283, 2220, 444, 3089, 2002, 4108, 2763, 2616, 1976, 412, 3411, 4690, 1310, 386, 4674, 4040, 721, 4122, 2678, 2089, 3609, 2516, 1796, 4191, 3549, 3268, 1391, 2695, 1656, 4999, 536, 3535, 442, 3583, 1573, 3248, 4302, 736, 1484, 1326, 3546, 3716, 1000, 4780, 1122, 363, 1086, 3077, 349, 4022, 3294, 4884, 1410, 4839, 635, 4162, 4236, 4436, 2586, 2880, 3172, 26, 4395, 3328, 3912, 3037, 4250, 1148, 4444, 4192, 2062, 4605, 1078, 478, 3453, 2275, 4551, 1886, 1296, 1861, 4875, 18, 1544, 1817, 2905, 84, 4305, 577, 4637, 729, 1537, 4366, 2086, 471, 4611, 1216, 1869, 655, 539, 4561, 3941, 2302, 2047, 3852, 1493, 903, 4559, 3258, 2028, 2790, 4601, 2172, 3317, 3698, 2099, 4025, 2126, 2975, 4426, 2600, 216, 2861, 4729, 2950, 3913, 2756, 2743, 4606, 4387, 1253, 2893, 4783, 2612, 744, 3088, 3971, 3767, 3654, 879, 3802, 2805, 3065, 3919, 2947, 4479, 806, 937, 2395, 4535, 1739, 1021, 608, 4513, 3440, 4135, 1200, 979, 453, 613, 760, 838, 3420, 3161, 3582, 109, 672, 2562, 4537, 2682, 609, 2349, 4277, 2357, 2480, 2951, 2431, 2218, 4066, 1206, 2185, 2428, 4017, 2026, 2701, 1069, 316, 698, 4152, 623, 1482, 1786, 2007, 1818, 2529, 4113, 2958, 2665, 3153, 3325, 4791, 3383, 1530, 208, 3739, 2192, 3412, 5005, 4610, 4275, 2533, 2904, 3525, 96, 2257, 3706, 1091, 710, 4215, 4625, 3452, 1434, 3807, 4975, 346, 873, 322, 909, 959, 3345, 4130, 4592, 2384, 3872, 2651, 1149, 81, 2490, 4039, 62, 342, 1174, 4339, 5006, 1489, 3007, 21, 245, 4984, 627, 3278, 4788, 4059, 3023, 4850, 4080, 40, 2454, 1515, 495, 3640, 3871, 2640, 1970, 4679, 1293, 4684, 1546, 2624, 3397, 878, 3689, 4632, 2835, 1303, 4944, 3601, 154, 2598, 782, 3985, 2040, 2387, 4820, 696, 4308, 4638, 152, 4648, 3606, 2381, 1711, 894, 4672, 1152, 4653, 3370, 4403, 2650, 1157, 2249, 2323, 2691, 348, 4898, 2632, 803, 3732, 712, 3870, 2306, 502, 3794, 1738, 798, 4060, 4755, 1012, 4323, 1780, 4338, 4813, 1444, 1513, 3482, 3437, 110, 4720, 2541, 934, 1258, 726, 661, 4446, 3788, 3257, 2869, 602, 2369, 3955, 1418, 1047, 4272, 2325, 1841, 1097, 2037, 738, 686, 461, 4528, 1404, 1660, 4474, 4593, 422, 3073, 2715, 3264, 2741, 3646, 3772, 1698, 1776, 2694, 1249, 396, 1389, 567, 4111, 3588, 3567, 20, 3273, 394, 3382, 1605, 2393, 4771, 3180, 1579, 1627, 980, 1519, 1228, 416, 1438, 1180, 4857, 970, 251, 2382, 2528, 1044, 1986, 4819, 783, 2908, 1903, 3485, 3232, 2580, 2448, 3810, 3547, 4524, 2994, 4285, 2915, 2788, 2653, 2057, 628, 411, 1100, 722, 763, 3266, 2296, 4853, 4519, 4207, 413, 1266, 2849, 1677, 3320, 2155, 3662, 1553, 2193, 2698, 4119, 3354, 1463, 4881, 836, 2686, 4864, 4706, 933, 3029, 4324, 1304, 2719, 4110, 104, 2378, 4184, 144, 448, 2614, 358, 1144, 2248, 3333, 1343, 634, 1752, 1030, 2245, 1784, 1132, 3834, 3344, 2200, 4730, 2658, 900, 3631, 1895, 1119, 4166, 111, 1428, 3028, 2578, 209, 4138, 1387, 3929, 164, 317, 3154, 4477, 1385, 390, 3244, 578, 2485, 675, 3628, 3128, 687, 1798, 223, 941, 3722, 889, 4790, 2003, 1857, 3362, 4312, 618, 2819, 1339, 3081, 1042, 3056, 3553, 382, 3595, 3105, 3922, 3763, 320, 4137, 4799, 4708, 2781, 4667, 5011, 2636, 1121, 4178, 2247, 3731, 2346, 2282, 1265, 1289, 4645, 2523, 4569, 1812, 762, 2897, 2235, 3781, 4212, 1911, 2534, 2053, 1682, 205, 4782, 775, 1709, 2106, 4988, 4566, 1284, 1737, 44, 2084, 2907, 2013, 1557, 4074, 4626, 1179, 1048, 1483, 2504, 4118, 4851, 2776, 936, 2174, 2711, 619, 4669, 4124, 3839, 1356, 1088, 73, 1452, 3645, 3918, 3377, 3374, 1291, 4049, 4428, 1781, 169, 1981, 2110, 1337, 3652, 4064, 3021, 1764, 4310, 1209, 43, 2525, 757, 4965, 264, 674, 2579, 1033, 2163, 2631, 592, 4620, 4051, 1510, 2848, 2370, 989, 2987, 2628, 4368, 2911, 3537, 340, 4842, 1374, 32, 3651, 860, 3415, 1082, 1170, 3095, 4473, 359, 1398, 2415, 3298, 3148, 2697, 2872, 957, 4125, 1378, 1028, 4526, 1548, 514, 819, 3523, 2791, 2246, 1397, 2670, 1231, 381, 1596, 2187, 2271, 4276, 429, 2102, 4920, 404, 1997, 552, 3748, 1673, 3574, 2305, 3339, 3364, 3720, 2408, 1263, 981, 3648, 3076, 4306, 4356, 2493, 4647, 725, 1376, 1076, 3577, 1691, 167, 1013, 4379, 1947, 1143, 4827, 4876, 489, 1966, 2721, 4023, 3528, 3597, 2426, 2216, 3756, 2464, 4870, 4683, 4631, 1364, 3427, 4615, 2396, 523, 2590, 306, 3940, 2063, 606, 269, 1230, 1998, 558, 1959, 1527, 3600, 2394, 2238, 2167, 2188, 1588, 3873, 3039, 3054, 3683, 1488, 948, 865, 3137, 2720, 4413, 1866, 3896, 2070, 68, 1508, 4018, 2234, 3000, 714, 3385, 1569, 4578, 4644, 3102, 847, 708, 581, 1436, 1442, 2955, 864, 966, 4976, 3822, 1475, 3699, 755, 159, 4695, 1306, 2502, 3376, 814, 1686, 176, 1092, 1772, 3242, 287, 700, 4619, 3259, 3295, 4187, 3904, 1018, 3100, 4214, 2571, 116, 1730, 2153, 902, 1942, 161, 3150, 115, 470, 4506, 667, 733, 4011, 4630, 94, 2211, 605, 3590, 1003, 4050, 646, 3435, 2181, 4486, 4707, 4010, 1599, 3074, 809, 3483, 3665, 4717, 3455, 4143, 4772, 2321, 1567, 2527, 3783, 1614, 3543, 3114, 64, 4237, 462, 3496, 4216, 2270, 4239, 2777, 1313, 220, 2108, 479, 3905, 4326, 1336, 4917, 1522, 1116, 2406, 2111, 4420, 2022, 3575, 4533, 3855, 1420, 604, 1381, 4027, 848, 1096, 1383, 2898, 555, 2399, 4761, 4951, 3992, 1165, 921, 3751, 191, 4146, 1703, 3695, 4333, 4867, 3019, 786, 3974, 3728, 3152, 3500, 4828, 1652, 1108, 2812, 3025, 3407, 3921, 576, 3703, 2437, 4677, 33, 2927, 67, 2290, 4287, 3303, 1140, 4650, 1716, 3205, 4522, 4889, 2581, 1380, 3753, 3140, 4668, 3254, 1930, 2447, 2744, 575, 192, 4427, 3201, 2, 1396, 2157, 4849, 280, 1466, 3363, 2353, 1071, 660, 2117, 858, 4659, 4549, 874, 888, 2386, 2519, 4322, 90, 2551, 3170, 4375, 2935, 1176, 1267, 2993, 4581, 4927, 2896, 1004, 2822, 146, 3531, 2075, 2906, 3999, 3630, 2239, 2104, 1983, 1676, 2184, 2944, 835, 2804, 1334, 1985, 2427, 3166, 2745, 2748, 3214, 2558, 1512, 1318, 2161, 3469, 1778, 2840, 4609, 2079, 969, 2354, 2703, 4582, 3527, 2190, 45, 4434, 3893, 3222, 923, 4204, 2501, 2051, 4607, 1583, 3925, 1907, 1285, 4673, 3040, 984, 4142, 3657, 3818, 4879, 2654, 3049, 2967, 1457, 3113, 4814, 3581, 2331, 2509, 1767, 3679, 906, 3887, 236, 1177, 2093, 4407, 663, 1207, 4702, 4656, 2100, 282, 2123, 1789, 4431, 2210, 3675, 4907, 3862, 1114, 4548, 4311, 3945, 2966, 3121, 4911, 1470, 339, 2524, 1325, 551, 31, 4652, 1561, 2367, 2926, 2215, 4115, 3267, 2514, 1481, 2409, 1046, 2226, 1024, 2213, 3136, 549, 2462, 616, 916, 3185, 1957, 2752, 2377, 1252, 2680, 4534, 1565, 4026, 1641, 4545, 3824, 2269, 830, 4006, 2212, 4885, 2261, 3082, 2230, 3167, 17, 401, 121, 3846, 24, 1112, 2909, 1292, 699, 3080, 4838, 3669, 4990, 4280, 3598, 4746, 1692, 3031, 4264, 4262, 4763, 3189, 34, 3694, 813, 2882, 2175, 474, 347, 4811, 2986, 3002, 1477, 2538, 4554, 4144, 4168, 4358, 4509, 35, 611, 3968, 249, 421, 4177, 2933, 3433, 4954, 3159, 1421, 774, 3939, 1480, 534, 2004, 740, 4448, 2783, 72, 1760, 3541, 1050, 3069, 4540, 3421, 1058, 3256, 2392, 3511, 4400, 3988, 276, 4505, 1940, 4270, 826, 3127, 770, 4969, 3308, 1723, 4456, 3605, 4210, 2069, 1552, 657, 3507, 1107, 681, 1038, 1616, 2615, 4939, 1161, 1241, 3514, 2999, 1962, 2930, 3562, 3522, 516, 4290, 1360, 3490, 4087, 964, 1210, 1274, 1416, 4964, 4830, 3710, 827, 3473, 2681, 4158, 2366, 3085, 2125, 3117, 2510, 4107, 4044, 2291, 1578, 1700, 1172, 1188, 4760, 3714, 1301, 2718, 2535, 1057, 4003, 2371, 139, 2921, 301, 4966, 2802, 1193, 1269, 796, 378, 1550, 3817, 2019, 4948, 1340, 1773, 2838, 4176, 2018, 3220, 1341, 4282, 3712, 1065, 1852, 589, 48, 23, 3238, 3306, 1662, 2209, 1591, 4721, 1931, 4072, 4438, 4391, 1007, 2801, 392, 2787, 3938, 823, 123, 2196, 2042, 3472, 2956, 3384, 4170, 42, 1851, 2597, 653, 3093, 1455, 1139, 4544, 79, 2309, 3156, 991, 4597, 1936, 588, 1377, 1926, 1643, 3206, 2706, 816, 3789, 4355, 1211, 695, 2405, 2537, 2712, 1991, 1984, 4832, 3676, 707, 3417, 1440, 99, 2356, 3967, 3329, 4496, 4942, 1346, 571, 1885, 2730, 2865, 211, 1747, 947, 197, 2702, 1501, 4298, 3405, 2982, 59, 4671, 1615, 259, 2555, 1923, 3616, 3827, 4417, 1825, 4263, 1156, 590, 4380, 4910, 4658, 89, 2633, 2225, 1728, 1476, 2985, 114, 1707, 219, 2436, 2137, 1192, 4319, 3010, 2391, 844, 76, 4800, 3758, 2016, 2474, 183, 3321, 4856, 3001, 3708, 2762, 3188, 4259, 3672, 493, 2299, 4575, 3889, 2361, 3891, 2288, 4440, 2594, 357, 3692, 2494, 3599, 1392, 596, 872, 196, 1844, 1799, 4539, 508, 2938, 4076, 1036, 3927, 3215, 572, 3867, 2352, 2068, 1562, 2943, 173, 182, 1556, 2103, 3831, 3312, 1995, 1223, 772, 195, 1232, 1975, 2931, 3086, 488, 3, 982, 338, 310, 2714, 1756, 1098, 3678, 4801, 4860, 3357, 2050, 2201, 185, 3299, 3304, 1521, 133, 829, 2634, 12, 692, 3932, 3539, 1714, 3942, 4784, 4932, 4697, 3632, 1242, 2774, 2452, 469, 4995, 765, 3444, 594, 3843, 480, 4388, 4745, 2806, 3877, 204, 1494, 1159, 4396, 365, 4821, 1619, 2438, 1988, 341, 986, 4267, 4757, 60, 1726, 329, 2560, 3022, 919, 1528, 2854, 2280, 4381, 119, 3388, 1782, 1369, 4392, 3448, 1283, 1613, 2198, 1135, 3842, 4069, 3449, 2344, 583, 4903, 3034, 2635, 3894, 1227, 4516, 1271, 4466, 3062, 1597, 1055, 517, 157, 1413, 4453, 4980, 3335, 2112, 4019, 331, 3394, 2693, 1687, 1083, 3991, 992, 4008, 527, 561, 1464, 3626, 927, 2038, 4024, 3757, 1532, 4515, 1435, 3290, 2764, 607, 3389, 1855, 2827, 2468, 3762, 3882, 3784, 1154, 3291, 3774, 1497, 1160, 1800, 3741, 229, 753, 4347, 930, 497, 4398, 951, 3488, 395, 4409, 2964, 1020, 4661, 6, 1640, 526, 2251, 2101, 3943, 4457, 455, 257, 3946, 2059, 1980, 533, 3576, 1191, 1639, 87, 137, 807, 3341, 2611, 928, 3518, 1961, 1235, 2991, 2439, 869, 2484, 2067, 3350, 2487, 3667, 1450, 2035, 1244, 3270, 717, 1901, 4325, 4088, 3687, 4304, 3494, 2421, 1224, 926, 637, 4085, 3096, 3399, 3414, 384, 3045, 2362, 405, 3424, 2460, 4205, 1785, 3563, 4188, 1828, 5010, 3923, 3627, 124, 3956, 4230, 3020, 1197, 4576, 4082, 5008, 3519, 1664, 554, 3006, 3041, 3909, 754, 3224, 2817, 3251, 1910, 1724, 1902, 917, 4846, 1854, 2582, 1511, 955, 3886, 4293, 2734, 2473, 1820, 2012, 1987, 3430, 3391, 3425, 1035, 3928, 1279, 327, 2723, 2087, 300, 3311, 1145, 2268, 361, 1348, 4970, 688, 3009, 4953, 4394, 4141, 2662, 1766, 2424, 3727, 2340, 945, 4936, 3253, 1495, 565, 2962, 938, 1635, 3937, 1243, 2626, 1701, 4468, 2232, 697, 2277, 600, 2358, 3868, 1830, 3673, 3285, 741, 570, 2564, 2692, 4316, 2333, 2792, 766, 2846, 4704, 3393, 515, 3231, 4797, 263, 3225, 2486, 2031, 3443, 4020, 842, 4925, 1633, 3957, 4439, 2076, 1358, 3890, 2771, 1485, 4922, 4989, 1671, 1898, 1919, 737, 142, 3024, 3701, 3027, 2011, 1406, 3975, 573, 4314, 4705, 1525, 1590, 2749, 4454, 1079, 4536, 2412, 4422, 1680, 566, 1175, 2559, 1338, 649, 406, 3226, 4154, 4972, 3343, 1735, 3526, 2867, 2669, 3151, 3426, 4321, 2145, 3914, 3668, 1653, 647, 4622, 1277, 3233, 1120, 1323, 4981, 3691, 2318, 4414, 180, 2732, 4145, 4223, 3378, 2696, 2637, 2204, 2165, 3524, 1196, 2048, 4586, 166, 648, 2567, 2301, 3057, 138, 1875, 1871, 777, 732, 4378, 1015, 237, 1881, 767, 2274, 3139, 850, 2602, 53, 626, 3602, 2441, 1715, 3841, 4649, 3410, 1361, 3367, 4053, 2974, 3038, 3471, 749, 3994, 3369, 4320, 4564, 4636, 3529, 4271, 665, 1059, 1474, 3042, 2092, 943, 643, 2815, 2920, 1371, 4346, 4921, 3239, 1712, 1523, 1618, 4639, 797, 4090, 4200, 4655, 3230, 2772, 2780, 29, 2659, 3637, 434, 4472, 2746, 298, 2842, 3324, 3146, 2364, 345, 4713, 214, 3187, 2570, 2168, 2398, 1011, 4032, 3463, 4523, 4600, 1810, 1699, 3122, 1373, 3848, 4369, 3133, 1946, 30, 3375, 1539, 1424, 4266, 241, 3569, 2241, 1094, 3874, 1014, 4258, 436, 1979, 4445, 2418, 3557, 3501, 1695, 2826, 2319, 25, 3977, 3284, 1658, 2208, 302, 3428, 1606, 1740, 3879, 2266, 622, 4751, 3898, 4153, 852, 1840, 4768, 1831, 610, 3764, 4789, 4987, 689, 585, 4432, 2563, 1060, 4635, 3677, 1067, 1194, 2014, 106, 2294, 4775, 4016, 1690, 3163, 4802, 1308, 2233, 582, 3963, 1600, 3962, 326, 2049, 2316, 870, 1220, 356, 2526, 3995, 2339, 3540, 3183, 426, 861, 1362, 3158, 1909, 1388, 1894, 2446, 747, 2980, 4373, 4616, 1574, 1526, 2329, 4071, 4614, 1009, 194, 354, 3249, 318, 1620, 1037, 3639, 1720, 4491, 1566, 1150, 1646, 1498, 3277, 1386, 2717, 4429, 4349, 355, 2873, 1593, 5002, 4774, 4733, 2729, 3334, 563, 2023, 3236, 3470, 3773, 3301, 3372, 1151, 1577, 3351, 4000, 1053, 1913, 4926, 2973, 2608, 4499, 4715, 147, 4037, 1920, 4295, 1838, 4725, 1462, 3147, 1626, 2082, 2465, 2784, 1443, 2886, 1803, 3149, 2961, 3460, 3981, 4159, 824, 3263, 2713, 3289, 1563, 3644, 3520, 1663, 2052, 2942, 701, 4728, 4692, 1836, 1219, 148, 3051, 403, 531, 4252, 3984, 3072, 387, 4604, 509, 4383, 1697, 994, 3849, 1587, 2304, 186, 3486, 4331, 52, 424, 3690, 2983, 3761, 1316, 3892, 3436, 4688, 3340, 2705, 3559, 2469, 1350, 3770, 2642, 4247, 1125, 2440, 2240, 3641, 2435, 4847, 4663, 2337, 3953, 1864, 4148, 3593, 4501, 4079, 2820, 2754, 3016, 3915, 953, 3864, 2785, 924, 1468, 2317, 2434, 3828, 1352, 4915, 3199, 1877, 2151, 1598, 3571, 4105, 3145, 1215, 1222, 1090, 2828, 3302, 3459, 3184, 2120, 718, 3044, 3468, 1429, 1127, 4959, 4798, 727, 2008, 975, 4441, 3032, 3207, 3854, 1884, 2015, 3104, 2097, 2589, 1516, 4872, 3286, 4173, 1564, 2574, 3545, 3450, 337, 128, 2228, 2231, 4274, 3124, 4998, 3047, 4634, 1073, 3492, 3934, 1882, 2988, 1669, 187, 1684, 1095, 2336, 1607, 1847, 4004, 3196, 3466, 2902, 4565, 956, 1890, 4602, 4935, 4750, 2423, 3778, 3070, 1199, 1594, 1848, 4219, 2149, 4691, 3305, 3578, 2475, 4880, 2625, 2254, 745, 3204, 4858, 2077, 730, 3611, 4681, 3998, 1255, 102, 3297, 3744, 3515, 1538, 4450, 1611, 3829, 127, 664, 3381, 4384, 437, 3746, 1645, 2148, 1896, 1542, 2863, 3477, 2575, 4385, 1319, 939, 2610, 1788, 3358, 4701, 1384, 3596, 4754, 4862, 2742, 297, 1696, 299, 4773, 319, 3900, 500, 1298, 4603, 1999, 2500, 3217, 792, 3666, 2645, 676, 4334, 1432, 3589, 2874, 1359, 2959, 748, 351, 1702, 1630, 1354, 1052, 4175, 335, 4908, 2453, 2483, 2977, 1366, 3558, 971, 4940, 190, 911, 790, 2162, 3550, 2327, 14, 1850, 739, 261, 4452, 2549, 4825, 569, 366, 4758, 677, 369, 4354, 1239, 2144, 210, 1113, 1198, 3091, 255, 603, 658, 2878, 1824, 4960, 2445, 4956, 4120, 1816, 61, 2043, 3008, 2380, 4342, 4949, 134, 343, 2410, 1016, 2606, 2813, 3261, 2492, 1570, 2115, 2182, 1039, 2639, 4328, 1906, 891, 3015, 184, 293, 1, 10, 3779, 825, 2466, 2095, 2114, 4873, 3247, 2308, 2351, 1533, 1394, 1667, 2760, 1229, 4766, 1104, 3465, 3499, 490, 397, 3279, 3530, 267, 4919, 408, 3561, 4140, 2129, 4055, 4461, 3661, 486, 1261, 817, 3949, 1136, 2830, 632, 3012, 3823, 1141, 2736, 4418, 1399, 4104, 976, 818, 4283, 1681, 630, 1128, 2315, 899, 3129, 4899, 1324, 77, 4253, 460, 1822, 3845, 1446, 3413, 373, 1171, 4555, 391, 3243, 2515, 3169, 1126, 4075, 2891, 4497, 2021, 2809, 1158, 1581, 690, 1272, 3060, 1449, 3816, 19, 4101, 3814, 2224, 2825, 4852, 71, 227, 2894, 315, 2178, 1571, 4887, 3355, 3422, 2638, 4367, 913, 4012, 3030, 3386, 2532, 3223, 4584, 3604, 3729, 3809, 3737, 2949, 3094, 4897, 4163, 617, 3213, 2839, 2206, 3959, 3721, 4510, 2992, 4435, 1089, 1617, 885, 1460, 2862, 98, 2821, 805, 2140, 1945, 2156, 4986, 4291, 1770, 466, 2073, 4488, 4837, 2404, 291, 3878, 380, 4955, 2372, 780, 3241, 2160, 3004, 178, 1964, 4994, 4957, 410, 881, 456, 3168, 4099, 2767, 3876, 1880, 1110, 1245, 83, 2121, 2704, 332, 3610, 1045, 3175, 3745, 4106, 845, 2429, 800, 4292, 2814, 542, 1551, 2284, 1634, 839, 289, 2205, 4151, 2479, 2506, 4399, 1710, 4171, 3944, 3181, 2903, 4465, 3071, 1531, 4877, 4822, 4696, 4030, 2566, 170, 4587, 4928, 978, 1280, 3813, 1558, 165, 2591, 120, 113, 2508, 4689, 4901, 3579, 4206, 3920, 3356, 4815, 2472, 202, 1101, 3726, 545, 2243, 2360, 4217, 2976, 2884, 199, 1355, 4829, 4865, 2685, 3983, 4249, 3503, 1759, 2495, 1718, 1345, 2024, 1001, 4716, 1335, 2936, 2853, 1845, 477, 1186, 4220, 4061, 1837, 2154, 2334, 2735, 3193, 1928, 1717, 2689, 3899, 2060, 633, 1648, 4525, 3131, 3191, 1367, 0, 2150, 235, 270, 4808, 636, 135, 4905, 1870, 1178, 3653, 275, 2831, 4504, 1867, 2166, 334, 2945, 175, 4149, 3160, 242, 1805, 1878, 4226, 3534, 4767, 258, 2046, 2568, 2871, 4979, 2655, 4344, 3954, 3112, 1629, 400, 2649, 2443, 2518, 1155, 4172, 1819, 3952, 907, 4376, 108, 2648, 4098, 1034, 3432, 3618, 432, 768, 580, 2478, 560, 2039, 1401, 1790, 3719, 4786, 1490, 2471, 1775, 3398, 4234, 2365, 776, 1612, 2090, 2739, 2118, 3246, 3194, 3143, 1624, 1560, 1268, 107, 4895, 1636, 1899, 1809, 4585, 3875, 1846, 2971, 417, 820, 3400, 1414, 4855, 3173, 4640, 2009, 465, 1891, 4102, 822, 2341, 3125, 4371, 2674, 1922, 3120, 4007, 4185, 484, 4351, 2939, 4063, 4345, 2122, 4045, 1407, 4362, 4546, 4686, 2661, 3858, 4374, 3464, 3315, 1417, 225, 920, 1915, 3084, 85, 3950, 1426, 4401, 4805, 168, 2888, 1447, 3586, 1294, 2281, 3594, 1189, 3799, 2604, 629, 4083, 1006, 4817, 4710, 2630, 3353, 1118, 4854, 3423, 3853, 1433, 3759, 4492, 1322, 385, 2195, 2278, 1708, 1536, 1500, 4198, 644, 4242, 3018], "validation": [5324, 5182, 5456, 5348, 5090, 5315, 5356, 5385, 5502, 5120, 5083, 5495, 5230, 5398, 5400, 5234, 5399, 5369, 5063, 5053, 5154, 5288, 5278, 5039, 5394, 5282, 5105, 5422, 5205, 5489, 5434, 5426, 5015, 5036, 5050, 5404, 5471, 5366, 5150, 5458, 5292, 5195, 5281, 5381, 5383, 5034, 5499, 5393, 5164, 5222, 5106, 5032, 5273, 5467, 5277, 5491, 5225, 5072, 5481, 5314, 5462, 5440, 5380, 5181, 5086, 5373, 5048, 5219, 5303, 5444, 5445, 5028, 5255, 5115, 5094, 5146, 5180, 5077, 5472, 5073, 5246, 5504, 5210, 5014, 5460, 5092, 5410, 5302, 5100, 5238, 5065, 5482, 5478, 5244, 5507, 5465, 5102, 5117, 5131, 5012, 5031, 5095, 5242, 5365, 5030, 5271, 5346, 5291, 5390, 5459, 5505, 5493, 5037, 5156, 5486, 5449, 5232, 5326, 5062, 5294, 5317, 5187, 5233, 5112, 5436, 5475, 5419, 5179, 5020, 5334, 5299, 5453, 5354, 5485, 5265, 5239, 5254, 5025, 5193, 5293, 5208, 5330, 5397, 5013, 5171, 5231, 5240, 5178, 5161, 5395, 5256, 5257, 5215, 5396, 5235, 5165, 5411, 5089, 5332, 5492, 5296, 5170, 5096, 5312, 5125, 5311, 5260, 5168, 5206, 5046, 5443, 5437, 5101, 5252, 5342, 5047, 5097, 5040, 5044, 5428, 5151, 5081, 5107, 5351, 5261, 5349, 5301, 5335, 5392, 5438, 5343, 5018, 5503, 5307, 5124, 5185, 5484, 5370, 5337, 5391, 5209, 5290, 5403, 5216, 5033, 5200, 5068, 5432, 5283, 5228, 5266, 5433, 5183, 5285, 5427, 5435, 5196, 5259, 5202, 5286, 5506, 5243, 5341, 5074, 5192, 5454, 5128, 5023, 5017, 5121, 5263, 5136, 5412, 5172, 5130, 5508, 5189, 5103, 5203, 5071, 5204, 5214, 5041, 5127, 5474, 5415, 5035, 5158, 5284, 5297, 5353, 5224, 5267, 5498, 5153, 5211, 5368, 5364, 5417, 5060, 5223, 5408, 5431, 5043, 5406, 5483, 5087, 5469, 5401, 5169, 5184, 5076, 5123, 5201, 5316, 5331, 5080, 5363, 5319, 5500, 5355, 5328, 5051, 5374, 5186, 5108, 5054, 5155, 5447, 5295, 5253, 5159, 5270, 5345, 5464, 5347, 5372, 5442, 5042, 5176, 5152, 5402, 5509, 5361, 5188, 5300, 5144, 5405, 5190, 5162, 5249, 5466, 5236, 5289, 5384, 5279, 5199, 5149, 5064, 5026, 5024, 5111, 5055, 5344, 5141, 5333, 5069, 5212, 5247, 5313, 5269, 5079, 5375, 5143, 5029, 5336, 5409, 5358, 5118, 5175, 5138, 5323, 5220, 5217, 5059, 5382, 5487, 5114, 5129, 5350, 5423, 5237, 5511, 5325, 5352, 5413, 5142, 5049, 5088, 5298, 5248, 5441, 5197, 5218, 5258, 5338, 5268, 5376, 5194, 5306, 5461, 5145, 5280, 5098, 5251, 5308, 5490, 5477, 5229, 5470, 5241, 5414, 5457, 5418, 5250, 5496, 5085, 5058, 5221, 5245, 5132, 5310, 5371, 5451, 5021, 5066, 5056, 5430, 5494, 5116, 5135, 5227, 5497, 5327, 5137, 5045, 5126, 5501, 5407, 5276, 5304, 5173, 5139, 5091, 5388, 5052, 5262, 5110, 5174, 5207, 5339, 5287, 5274, 5160, 5468, 5322, 5019, 5177, 5340, 5134, 5075, 5321, 5379, 5479, 5093, 5386, 5439, 5016, 5446, 5067, 5476, 5022, 5198, 5084, 5157, 5166, 5213, 5057, 5027, 5309, 5389, 5082, 5421, 5387, 5061, 5378, 5377, 5318, 5275, 5420, 5488, 5272, 5329, 5510, 5226, 5113, 5480, 5167, 5264, 5140, 5416, 5305, 5367, 5362, 5191, 5360, 5463, 5455, 5038, 5099, 5122, 5450, 5429, 5448, 5320, 5133, 5147, 5070, 5424, 5359, 5104, 5148, 5357, 5473, 5452, 5163, 5425, 5078, 5119, 5109], "test": [5991, 5615, 5705, 5694, 5807, 5858, 5544, 5708, 5581, 5742, 5677, 5897, 5639, 5816, 5547, 6009, 5824, 5840, 5523, 5669, 5880, 5784, 5914, 5597, 5794, 5795, 5980, 5821, 5941, 5778, 5565, 5643, 5732, 5996, 5801, 5900, 5989, 5782, 5642, 5919, 5567, 5973, 5950, 5527, 5936, 5670, 5623, 5920, 5531, 5971, 5970, 5696, 5586, 5817, 5536, 5675, 5938, 5860, 5662, 5725, 5689, 5777, 5557, 5923, 5904, 5781, 5622, 5713, 5519, 5931, 5879, 5940, 5668, 5929, 5912, 5762, 5909, 5672, 5659, 5803, 5959, 5861, 5863, 5869, 5520, 5856, 5754, 5791, 5766, 5546, 5924, 5683, 5556, 5626, 5888, 5604, 5847, 5752, 5872, 5759, 5908, 5993, 5889, 5761, 5746, 5894, 5680, 5825, 5596, 5927, 5975, 5649, 5921, 5995, 5575, 5734, 5715, 5763, 5800, 5691, 5961, 5769, 5610, 5562, 6001, 5706, 5939, 5735, 5978, 5911, 5760, 5798, 5945, 5885, 5512, 5600, 5755, 5768, 6011, 5942, 5849, 5771, 5811, 5776, 5569, 5724, 5891, 5949, 5739, 5990, 5785, 5727, 5644, 5684, 6006, 5886, 5606, 5549, 5964, 5682, 5599, 5704, 5538, 5834, 5890, 5566, 5968, 5848, 5554, 5882, 5619, 5686, 5658, 5700, 5952, 5729, 5568, 5877, 5829, 5772, 5797, 5946, 5985, 5789, 5899, 5532, 5832, 5898, 5774, 5723, 5641, 5533, 5857, 5767, 5628, 5702, 5620, 5631, 5513, 5583, 5775, 5751, 5757, 5657, 5813, 5843, 5630, 5692, 5707, 5812, 5574, 5524, 5809, 5720, 5517, 5988, 5854, 5913, 5792, 5558, 5699, 5902, 5805, 5896, 6000, 5552, 5779, 5525, 5796, 5542, 5977, 5871, 5887, 5545, 5743, 5932, 5810, 5518, 5833, 5652, 5875, 5671, 5592, 5836, 5865, 5799, 5651, 5876, 5827, 5595, 5845, 5870, 5621, 5605, 5835, 5948, 5822, 5541, 5750, 5560, 5972, 5577, 5722, 5984, 5650, 5697, 5709, 5957, 5965, 5903, 5598, 5681, 5918, 5646, 5638, 5647, 5530, 5666, 5537, 5981, 5960, 5753, 5528, 5744, 5925, 5710, 5844, 6007, 5733, 5674, 5951, 5654, 5770, 5579, 5881, 5665, 5585, 5603, 5514, 5963, 5526, 5534, 5830, 6010, 5802, 5866, 5685, 5895, 5994, 5901, 5582, 5864, 5529, 5878, 5608, 5695, 5601, 5916, 5521, 5711, 5688, 5673, 5663, 5614, 5617, 5737, 5943, 5637, 5543, 6004, 5842, 5745, 5571, 5616, 5678, 5667, 6002, 5635, 5703, 5726, 5837, 5998, 5612, 5721, 5602, 5855, 5826, 5976, 5967, 5944, 5758, 5935, 5937, 5838, 5852, 5539, 5962, 5730, 5786, 5820, 5728, 5808, 5954, 5561, 5955, 5548, 5893, 5653, 5648, 5613, 5892, 5593, 5915, 5823, 5627, 5907, 5624, 5701, 5522, 5634, 5676, 5934, 5905, 5573, 5748, 5873, 5588, 5906, 5563, 5986, 5992, 5979, 5867, 5535, 5815, 5717, 5664, 5736, 5618, 5969, 6008, 5516, 5550, 5806, 5922, 5930, 5966, 5747, 5580, 5551, 5987, 5611, 5584, 5828, 5587, 5738, 5983, 5997, 5956, 5718, 5515, 5740, 5716, 5661, 5607, 5655, 5765, 5712, 5884, 5609, 5814, 5656, 5793, 5645, 5788, 5632, 5679, 5953, 5693, 5698, 5859, 5917, 5841, 5974, 5846, 5928, 5780, 5999, 5570, 5819, 5629, 5590, 5982, 5540, 5576, 5926, 5756, 5741, 5553, 5853, 5764, 5719, 5850, 5851, 5818, 6003, 5559, 5572, 5591, 5594, 5625, 5633, 5910, 6005, 5578, 5690, 5862, 5933, 5831, 5636, 5790, 5714, 5589, 5883, 5731, 5564, 5640, 5749, 5958, 5783, 5839, 5947, 5773, 5868, 5555, 5660, 5787, 5687, 5874, 5804]}, {"train": [2880, 602, 1898, 4055, 981, 958, 4853, 4772, 4794, 384, 3051, 2154, 3378, 247, 4167, 3937, 2340, 4514, 1711, 1769, 2703, 232, 1839, 2661, 734, 2232, 3949, 2982, 3545, 2790, 4478, 2681, 3871, 1730, 4268, 385, 3908, 3483, 658, 2214, 3096, 3275, 274, 1840, 1366, 455, 3473, 2416, 3978, 3602, 4936, 1585, 2379, 536, 4251, 1760, 3528, 4749, 1815, 1595, 1513, 2700, 4894, 3879, 1532, 2256, 1922, 4531, 2716, 1748, 1862, 2728, 2381, 3581, 2125, 1796, 1472, 4362, 1763, 1887, 1332, 2855, 3165, 3529, 4015, 2225, 884, 3639, 716, 1961, 2433, 3591, 2885, 1161, 1380, 3902, 2576, 4907, 2325, 258, 4270, 5004, 2050, 3319, 92, 3523, 2241, 3754, 4500, 4753, 209, 1806, 674, 1497, 2076, 108, 2335, 2757, 308, 2541, 2157, 233, 3932, 4329, 3074, 2733, 978, 4610, 3064, 2414, 3144, 4927, 1767, 4812, 559, 2233, 3632, 2609, 1104, 2354, 1080, 1710, 387, 4793, 3841, 1940, 3668, 1957, 687, 99, 661, 2126, 4741, 1706, 4682, 4756, 3397, 1187, 4890, 1959, 2693, 4643, 3829, 855, 2766, 3968, 2158, 3062, 689, 1809, 4701, 1194, 142, 2418, 5003, 4242, 426, 346, 2032, 4646, 2892, 2073, 1103, 4913, 1450, 1452, 3239, 3207, 4934, 1599, 2532, 4565, 4621, 726, 4792, 1782, 1791, 435, 3277, 356, 3715, 4919, 1919, 4131, 2760, 3408, 3832, 1414, 3159, 1384, 3272, 4965, 3093, 16, 2450, 4704, 653, 2664, 4955, 1214, 2498, 4292, 1051, 2645, 4019, 2551, 2336, 2742, 3243, 3741, 2111, 883, 3197, 1009, 4807, 2321, 885, 1197, 1036, 3441, 204, 1371, 1538, 223, 3898, 1779, 3266, 3410, 1527, 2136, 1178, 1666, 1628, 3805, 128, 2299, 1582, 4818, 4401, 3075, 4370, 4214, 4700, 2057, 4949, 1677, 1579, 1722, 2569, 546, 3762, 1795, 574, 761, 55, 2248, 476, 2186, 3069, 1564, 3995, 640, 112, 4722, 827, 220, 679, 2724, 4233, 3942, 1733, 1326, 758, 2585, 4122, 504, 3435, 3727, 4368, 3693, 3505, 1925, 4323, 2040, 2395, 3095, 3340, 1268, 3726, 451, 1984, 4040, 4098, 1550, 1202, 4604, 4460, 3708, 1588, 4626, 4272, 4751, 1360, 3757, 256, 90, 628, 2228, 125, 2489, 4243, 2914, 1911, 2640, 3975, 2776, 2726, 4379, 1647, 2114, 3560, 1811, 682, 3493, 2584, 2501, 4293, 503, 4518, 2529, 1296, 577, 4633, 3417, 1172, 3806, 795, 4042, 2331, 1546, 14, 3752, 4696, 192, 3765, 4578, 3952, 1428, 4030, 3824, 408, 80, 2628, 2981, 146, 2629, 4644, 1658, 4901, 3076, 1308, 3867, 3398, 3821, 1977, 2883, 2236, 460, 668, 3423, 1173, 906, 325, 1290, 4344, 3400, 2506, 436, 2407, 4353, 3925, 837, 3335, 2409, 3101, 984, 3740, 3377, 1682, 732, 3173, 4485, 3568, 4574, 4727, 1212, 2195, 4159, 3457, 324, 315, 4863, 4796, 303, 357, 4104, 318, 2350, 352, 3780, 3842, 1052, 2490, 4145, 47, 2113, 2020, 943, 4899, 4400, 4183, 4864, 2330, 603, 868, 2096, 1852, 1071, 3649, 4081, 2508, 3811, 3099, 4582, 4068, 365, 864, 2711, 878, 3479, 2184, 2517, 3895, 4874, 4076, 617, 776, 1847, 822, 1478, 3890, 2277, 4498, 4372, 2118, 2935, 4703, 171, 4462, 4324, 13, 3139, 3878, 1604, 2434, 3443, 4928, 2347, 3274, 4650, 2318, 1528, 4074, 1742, 621, 1093, 3644, 3794, 2357, 1382, 1719, 2953, 3624, 1281, 2272, 4017, 670, 1610, 2513, 3629, 1600, 4843, 1978, 4391, 1231, 4418, 936, 4978, 4267, 4865, 4230, 844, 3835, 2725, 1348, 4966, 4631, 2369, 1668, 1307, 518, 1715, 2509, 1857, 2188, 810, 2579, 83, 2643, 320, 1416, 1871, 938, 3366, 872, 2778, 4916, 1336, 2443, 403, 3016, 784, 4475, 2429, 1131, 3041, 4963, 2046, 2570, 3439, 1244, 2600, 2592, 468, 3077, 4172, 264, 4454, 4020, 4171, 519, 1717, 2589, 4496, 4187, 388, 4253, 278, 4512, 2127, 4388, 1070, 3257, 4555, 2491, 3569, 1207, 2536, 3625, 1545, 4580, 3684, 3521, 4053, 4887, 4115, 3476, 2428, 2743, 517, 166, 1522, 162, 4359, 1739, 161, 3300, 2911, 173, 2689, 3921, 4764, 4941, 383, 743, 3546, 1176, 1237, 3092, 870, 749, 724, 2886, 3759, 3753, 1373, 407, 596, 1891, 762, 3313, 3059, 2910, 2000, 4179, 4420, 1026, 480, 3206, 123, 478, 2897, 4039, 992, 4590, 2169, 770, 494, 60, 1489, 4457, 4351, 4798, 2960, 704, 350, 4206, 1973, 1729, 2002, 1264, 1735, 3459, 4991, 3634, 1955, 1794, 3514, 3172, 3561, 2117, 170, 3972, 1636, 4875, 1325, 2539, 3411, 2163, 720, 4168, 1019, 1347, 3901, 2312, 4663, 663, 3205, 4862, 4146, 1676, 2735, 4354, 4562, 502, 1759, 1151, 2378, 2291, 4339, 3073, 2063, 2485, 1411, 2129, 4481, 2923, 5006, 445, 1652, 1976, 527, 343, 1433, 3279, 4007, 3297, 3686, 4157, 1230, 1823, 2793, 4638, 2351, 1358, 1965, 3477, 48, 2840, 4538, 230, 2026, 2638, 4307, 2619, 1301, 2138, 514, 1171, 4872, 3055, 2151, 119, 2398, 1491, 3276, 524, 5, 2625, 1669, 1271, 2432, 1849, 1381, 2055, 1306, 2846, 3315, 2150, 4100, 1196, 1773, 1810, 3725, 4403, 667, 1322, 1776, 2889, 3421, 1438, 2290, 1485, 463, 3779, 1021, 394, 3888, 263, 738, 1065, 3766, 1022, 4759, 715, 911, 1725, 1559, 3278, 4852, 1899, 2763, 2053, 3168, 2932, 1633, 4632, 4598, 1573, 4390, 4480, 2942, 4830, 1484, 2301, 3770, 2288, 217, 3015, 2276, 1670, 889, 2307, 973, 2442, 1654, 3394, 1085, 2575, 2804, 744, 1878, 3839, 2363, 4787, 3573, 2083, 4073, 17, 114, 4006, 3729, 4489, 36, 3166, 2160, 3720, 3045, 379, 4057, 2456, 4935, 2945, 2677, 1067, 132, 4690, 4463, 4311, 515, 2956, 4589, 2837, 4240, 4859, 2759, 2271, 152, 1375, 4200, 735, 1738, 3336, 2651, 2211, 2317, 2731, 1512, 4630, 4925, 3309, 3252, 4458, 138, 4137, 1278, 4691, 2809, 997, 2801, 3128, 2400, 2774, 4358, 1804, 664, 21, 1695, 1792, 1126, 4723, 2803, 1038, 2931, 1156, 3808, 3379, 698, 3020, 3283, 2042, 1602, 3293, 4583, 3010, 645, 927, 2657, 4529, 644, 1309, 3520, 1298, 840, 2250, 1395, 4896, 887, 2448, 3628, 4860, 4769, 2162, 4321, 1357, 141, 798, 995, 2644, 3088, 1611, 3223, 869, 757, 3384, 1762, 3080, 775, 4952, 4092, 2905, 1656, 4754, 662, 904, 2873, 4680, 4527, 4409, 4336, 846, 4299, 2843, 2376, 4745, 1465, 924, 3785, 742, 5002, 1590, 1243, 2781, 1421, 2510, 2140, 2278, 3484, 2462, 4077, 4330, 2231, 2567, 3331, 3843, 3412, 4430, 1461, 231, 1066, 3189, 3481, 4677, 4655, 4639, 4371, 4728, 4758, 3769, 4322, 4147, 3913, 791, 2639, 3906, 2947, 3736, 3287, 2104, 1828, 1185, 4918, 4660, 4501, 3836, 243, 2665, 3768, 4199, 2991, 1548, 4679, 121, 3504, 2368, 2994, 1310, 1623, 32, 184, 4166, 4959, 3386, 3579, 957, 2867, 1141, 4557, 2018, 4446, 2120, 4262, 364, 2690, 2593, 2082, 126, 2303, 341, 1254, 2566, 2817, 450, 3548, 1778, 1580, 164, 3999, 597, 4781, 3185, 2598, 3600, 4239, 3138, 2227, 3810, 4517, 4326, 521, 1945, 1498, 4858, 4962, 409, 3831, 1099, 2802, 4833, 2906, 3604, 1262, 2399, 169, 1238, 2266, 2311, 1110, 1235, 1210, 234, 1679, 1303, 4343, 4716, 4725, 3089, 2094, 2596, 1587, 1225, 339, 1824, 1180, 4757, 3136, 1607, 4652, 420, 1788, 977, 1818, 2444, 2930, 3958, 1803, 647, 781, 4867, 67, 4440, 4735, 3316, 434, 1113, 3760, 2924, 4827, 824, 2505, 465, 1135, 1423, 1132, 1885, 876, 3134, 361, 402, 2511, 4614, 4204, 4520, 2206, 651, 4244, 3590, 2392, 2275, 3447, 3963, 2461, 1575, 252, 1817, 95, 3636, 30, 1856, 358, 2773, 4770, 1896, 1616, 1273, 703, 2992, 1456, 2103, 3519, 2367, 831, 4950, 3338, 3713, 713, 3369, 1391, 4920, 999, 3926, 2087, 2709, 2453, 3104, 2116, 4153, 4063, 1313, 612, 413, 4834, 330, 622, 399, 513, 2882, 591, 180, 1892, 2938, 3008, 3588, 1508, 928, 2985, 2207, 2618, 3589, 1505, 1106, 1372, 1378, 3229, 4802, 2218, 1807, 2814, 4552, 4738, 3977, 657, 1844, 1567, 3253, 3499, 1138, 3184, 2754, 2868, 3470, 4419, 4407, 1643, 1639, 1139, 236, 1226, 474, 4666, 2652, 1846, 3425, 2687, 3329, 3380, 3373, 120, 3677, 461, 4973, 1034, 3798, 4553, 2603, 2252, 2974, 3837, 1631, 3225, 3912, 1275, 773, 931, 2574, 2464, 2043, 3371, 4850, 3121, 1765, 2258, 1493, 3200, 934, 4013, 1096, 1327, 4822, 3930, 506, 777, 2090, 1157, 2650, 2014, 2516, 2189, 854, 1938, 2024, 1536, 4982, 3113, 3735, 1592, 1365, 2602, 4951, 3141, 4406, 4776, 1458, 140, 4411, 697, 507, 485, 4564, 2627, 601, 3420, 3491, 2470, 2993, 2933, 3048, 3541, 3490, 3663, 1044, 4513, 4228, 4542, 1216, 2741, 1477, 4283, 3167, 4385, 183, 3507, 1605, 2285, 3861, 3916, 226, 4476, 4138, 4676, 558, 2132, 2674, 4465, 3678, 4823, 4917, 2385, 2297, 604, 439, 1451, 1086, 1893, 700, 440, 545, 2435, 1436, 1166, 2697, 4149, 1917, 4747, 0, 4109, 2110, 555, 623, 1454, 297, 2653, 2084, 2879, 1552, 2360, 181, 3387, 4002, 4084, 1123, 26, 4164, 2662, 2240, 4434, 3814, 64, 3643, 3703, 3007, 976, 2095, 4732, 3475, 3945, 2573, 3355, 1203, 3517, 693, 618, 2934, 2595, 2253, 1924, 3097, 3456, 2476, 3414, 4484, 4118, 533, 1909, 3012, 4096, 4050, 1119, 825, 2599, 3914, 778, 2261, 2799, 2388, 2313, 4423, 1728, 3903, 2622, 609, 2270, 4992, 1245, 1321, 2249, 2678, 4915, 2663, 3690, 1944, 1232, 3361, 828, 516, 719, 932, 295, 4543, 3078, 257, 3181, 301, 3933, 2772, 4221, 2194, 966, 4415, 3749, 1285, 4337, 4979, 2346, 1073, 3682, 2494, 1734, 1192, 1566, 1764, 4414, 477, 3227, 2952, 569, 284, 4692, 3784, 3320, 918, 2247, 4045, 1819, 4814, 888, 4101, 2242, 1208, 1723, 1696, 1632, 996, 1078, 422, 2238, 3502, 87, 4219, 2683, 882, 709, 3648, 4545, 1902, 1334, 2826, 1766, 953, 964, 1583, 4355, 1688, 3130, 3248, 3924, 4060, 4325, 1396, 850, 4134, 4521, 3973, 2417, 4820, 3250, 1789, 2668, 4445, 1324, 211, 1125, 2751, 3152, 901, 4033, 2818, 382, 2601, 4797, 3374, 681, 1031, 1516, 3955, 1657, 1072, 2676, 3815, 2969, 2244, 2012, 4571, 4641, 2180, 1626, 4201, 3594, 4116, 3896, 3129, 3910, 988, 1209, 1316, 3063, 3883, 613, 4511, 2853, 2460, 4287, 551, 2295, 3180, 1650, 728, 1570, 2065, 1338, 1614, 4381, 2848, 2452, 3915, 3998, 3695, 2329, 4516, 1622, 4608, 1975, 1364, 319, 288, 160, 552, 3565, 820, 862, 1958, 1089, 2816, 1213, 2612, 4653, 4316, 706, 2535, 3233, 520, 3804, 1149, 4490, 3445, 4289, 3532, 3640, 583, 1746, 3303, 3848, 3900, 3747, 2479, 3268, 1678, 410, 672, 2075, 458, 1900, 2795, 1112, 3789, 1869, 1793, 1367, 3343, 488, 2611, 194, 1591, 1617, 4043, 4570, 4673, 4384, 1542, 541, 4836, 4908, 4114, 106, 696, 3929, 1074, 3469, 4377, 975, 262, 2302, 4010, 3919, 4189, 3679, 4026, 4683, 2015, 253, 2246, 501, 290, 2568, 2245, 3444, 2255, 2888, 4876, 2820, 4301, 2680, 429, 4662, 1991, 2292, 2122, 2091, 3402, 375, 994, 4616, 1987, 199, 1655, 3840, 2987, 381, 1453, 4165, 2081, 1047, 401, 4106, 2762, 338, 4649, 3986, 448, 307, 4035, 1028, 3216, 453, 1084, 4622, 4227, 4806, 4504, 2175, 4576, 2922, 1968, 3542, 367, 272, 2955, 4113, 1713, 3830, 4000, 4988, 1483, 3563, 4546, 2871, 1055, 4505, 2721, 2852, 858, 3655, 3454, 2694, 2898, 2578, 1683, 3822, 4897, 595, 3597, 2230, 3091, 4064, 2044, 511, 1829, 1929, 1988, 2865, 1045, 4659, 1877, 1675, 1241, 3102, 688, 727, 1012, 4849, 3269, 581, 1983, 175, 4998, 4635, 4402, 3718, 1481, 3153, 1920, 987, 2364, 4510, 1455, 635, 4308, 4577, 782, 1660, 1008, 1864, 4548, 2135, 3691, 599, 104, 2495, 2859, 690, 3892, 3111, 3326, 3112, 3116, 116, 2957, 3739, 1318, 4990, 2286, 1229, 3463, 4767, 3142, 4651, 2521, 4509, 3587, 4135, 3939, 2572, 4900, 3607, 444, 1578, 2212, 3311, 322, 2387, 1593, 3537, 1672, 3068, 2397, 2503, 1754, 23, 2902, 50, 3983, 3877, 2191, 195, 1946, 2010, 792, 4558, 3976, 2937, 1064, 2328, 4811, 2192, 3734, 2577, 568, 3084, 2287, 3951, 2217, 4845, 117, 1918, 1394, 2736, 4367, 3009, 2940, 2361, 910, 1153, 4052, 3577, 829, 2374, 336, 4573, 4958, 4953, 3132, 3984, 433, 954, 1443, 1029, 4067, 1033, 4152, 2750, 3037, 2221, 3719, 4665, 4342, 3194, 2966, 4755, 2971, 4488, 3175, 1721, 1159, 1608, 1555, 1470, 1543, 3605, 4263, 3669, 1223, 2747, 1915, 1422, 2305, 1800, 4877, 246, 908, 1234, 2768, 4386, 594, 1837, 3654, 2289, 3220, 4071, 467, 4788, 2458, 753, 344, 4985, 2062, 4054, 3501, 2229, 1781, 3671, 1821, 134, 412, 952, 1003, 3962, 4266, 4130, 165, 348, 1883, 598, 1353, 1551, 2895, 2597, 4284, 2047, 4855, 2377, 1095, 816, 4161, 3021, 678, 1905, 425, 2805, 1927, 991, 3518, 505, 110, 82, 2431, 3826, 2893, 15, 3312, 225, 3498, 2131, 1732, 4398, 2093, 1258, 1167, 2322, 741, 4058, 4972, 3480, 2352, 4572, 3001, 3025, 4, 4786, 4369, 4842, 4664, 1474, 528, 386, 3370, 3538, 1876, 3852, 4455, 1299, 2031, 3230, 2190, 2469, 229, 2165, 4022, 4312, 2488, 1393, 2366, 2695, 9, 642, 2016, 4378, 4615, 2355, 3598, 998, 2636, 1399, 2908, 3474, 1247, 2380, 2845, 4479, 3656, 4937, 3424, 2614, 1117, 4382, 2265, 4734, 2389, 2565, 3970, 3846, 2499, 2771, 1684, 377, 4903, 3618, 4345, 69, 745, 1042, 3372, 2467, 1750, 148, 619, 1627, 4537, 101, 1646, 2926, 156, 3536, 2857, 4838, 2472, 2961, 4095, 2649, 414, 4642, 2251, 2078, 3033, 2559, 615, 227, 3061, 2965, 1971, 4449, 3938, 294, 4779, 794, 1002, 4155, 4711, 4451, 3187, 1204, 2004, 1016, 65, 2011, 879, 1077, 2825, 897, 2925, 2549, 3081, 2243, 3675, 1549, 4394, 1744, 3562, 935, 2626, 1363, 3698, 3140, 1642, 4349, 4771, 168, 903, 2958, 176, 4870, 1936, 2413, 3448, 730, 5008, 1560, 4448, 3665, 4441, 3764, 3763, 4429, 347, 3295, 2080, 311, 1787, 788, 1091, 3610, 2139, 4237, 508, 3495, 3232, 755, 3497, 1288, 3083, 136, 3401, 3364, 2239, 4933, 4831, 3389, 3823, 3506, 856, 4674, 857, 4729, 3661, 2324, 4315, 423, 1701, 3559, 4234, 1387, 3273, 3178, 2512, 1412, 4624, 2672, 405, 2607, 1724, 2591, 832, 304, 1797, 1775, 4627, 4672, 4473, 2425, 1854, 1145, 3954, 849, 239, 4348, 3971, 2881, 2041, 2107, 3108, 1863, 4439, 369, 1533, 3356, 3157, 241, 4431, 1155, 3787, 2836, 1954, 783, 2333, 2878, 133, 3106, 895, 1468, 1860, 75, 3525, 472, 4169, 1385, 2785, 1603, 495, 2542, 3524, 147, 2900, 2675, 1494, 4082, 799, 3327, 1205, 2571, 1351, 3177, 2406, 1687, 2717, 2564, 3866, 3026, 1752, 292, 3985, 1090, 351, 4188, 3508, 3544, 3688, 2990, 560, 4305, 1329, 1312, 1755, 562, 1445, 3154, 2702, 2719, 3783, 185, 1408, 2504, 1671, 3646, 4184, 4222, 2854, 4784, 4005, 4661, 2403, 3795, 585, 3834, 2176, 893, 4249, 3637, 4229, 2710, 1771, 3453, 3755, 4375, 522, 2348, 1901, 3889, 4687, 4223, 4399, 1060, 1541, 1526, 4041, 3251, 3535, 441, 1661, 970, 2477, 1826, 2203, 158, 1217, 2688, 1737, 1059, 2048, 1272, 4182, 19, 4459, 2647, 2634, 3696, 912, 3750, 2555, 1709, 1183, 3767, 3415, 3880, 3066, 88, 1148, 4815, 1966, 4740, 3376, 1062, 4720, 3803, 306, 3578, 1931, 4038, 3217, 1525, 3965, 3953, 793, 345, 3086, 310, 159, 1827, 4946, 4426, 2148, 510, 2936, 3114, 4840, 4705, 1168, 2196, 4882, 4194, 1181, 63, 803, 3909, 1415, 723, 4922, 1147, 3442, 1510, 3182, 54, 3974, 4523, 1557, 1561, 2847, 3732, 1772, 841, 4541, 1990, 105, 3042, 3260, 3256, 3090, 4048, 3500, 3404, 1912, 340, 1256, 3701, 4957, 1389, 3630, 4989, 3202, 3028, 607, 941, 499, 113, 666, 1635, 3751, 471, 797, 4140, 3631, 881, 979, 182, 427, 177, 4943, 2918, 4563, 3351, 1882, 4832, 3352, 179, 2383, 4264, 139, 334, 3135, 3174, 963, 2056, 2831, 3198, 2552, 188, 1960, 3471, 1855, 4861, 1222, 1260, 1356, 1317, 3328, 3458, 866, 3825, 3342, 2471, 4205, 3800, 4256, 1836, 553, 1291, 3234, 1630, 3905, 4066, 2199, 74, 4947, 473, 3127, 2320, 1211, 282, 4314, 3249, 2973, 2375, 4654, 3931, 1731, 144, 4018, 2810, 2436, 4436, 2800, 3672, 3737, 4975, 4549, 4981, 4374, 2237, 4487, 3119, 627, 3827, 2874, 2842, 3004, 1726, 4245, 2013, 3263, 59, 4821, 432, 4302, 4357, 3359, 4210, 4707, 86, 1621, 1302, 3549, 1913, 4994, 4987, 817, 51, 4444, 2412, 3773, 637, 554, 3950, 4443, 4810, 2054, 826, 576, 944, 2005, 4285, 3996, 2660, 1046, 4088, 2493, 2823, 578, 1507, 1449, 3347, 2064, 993, 3460, 2455, 1444, 4156, 305, 736, 2009, 2156, 2173, 731, 2632, 1681, 4839, 699, 1747, 1989, 3662, 2761, 3196, 2273, 1577, 3375, 1410, 4575, 5001, 2777, 1612, 368, 1004, 3689, 1242, 3911, 2420, 3226, 1441, 1801, 3793, 4334, 3586, 4093, 2025, 3382, 2839, 3638, 4170, 4471, 2523, 3576, 4775, 1082, 1163, 3145, 1872, 4777, 2616, 363, 902, 392, 1589, 4647, 805, 1142, 1487, 2941, 2780, 1200, 3363, 4327, 1, 4014, 3179, 2944, 1011, 3582, 961, 4442, 1720, 4328, 4719, 1618, 98, 1018, 3288, 2787, 712, 4736, 3595, 2108, 2631, 3616, 1740, 418, 4969, 1662, 1942, 4585, 3564, 3758, 3224, 1930, 2705, 1848, 2996, 3928, 2007, 2410, 3023, 2100, 4857, 1879, 4871, 1597, 1190, 3606, 1799, 2358, 948, 332, 2556, 1514, 2006, 1486, 4102, 4144, 3730, 4466, 3599, 4748, 3488, 293, 3307, 1868, 276, 638, 1951, 1115, 768, 956, 571, 2894, 1017, 3388, 610, 4209, 3082, 847, 1101, 4247, 894, 3944, 91, 395, 1530, 3550, 3381, 2860, 1609, 1027, 1558, 1904, 3053, 3003, 4470, 2822, 3676, 4286, 2531, 3728, 974, 754, 809, 2306, 2887, 1403, 3241, 1376, 1640, 3711, 4869, 2482, 3614, 3748, 4059, 3943, 202, 1039, 2919, 4231, 5011, 1427, 4453, 1686, 767, 3429, 4024, 3651, 4623, 1517, 4468, 2783, 630, 804, 3571, 2098, 760, 942, 3738, 3406, 1488, 2146, 1349, 4121, 3362, 1447, 4910, 3047, 4547, 4027, 94, 3014, 2030, 3368, 4296, 4332, 1390, 2205, 1284, 2797, 4648, 2708, 3991, 3742, 2789, 2308, 2149, 40, 916, 4799, 1832, 3859, 1248, 4715, 3940, 416, 2528, 1160, 1471, 3818, 2451, 2481, 1469, 1407, 2550, 1188, 3344, 285, 3687, 3103, 3357, 1330, 786, 4970, 3170, 4331, 3692, 1300, 1335, 2483, 802, 2621, 4801, 535, 2525, 2309, 4544, 419, 2337, 4761, 298, 625, 4046, 2794, 4252, 254, 390, 886, 2332, 3146, 1531, 718, 3706, 1504, 1745, 796, 35, 2827, 3348, 1257, 3993, 4528, 2684, 3922, 1921, 3812, 3334, 1850, 2670, 3218, 4706, 4669, 3405, 4612, 4926, 683, 497, 652, 3570, 4259, 2829, 714, 3485, 946, 4750, 2170, 2679, 4645, 949, 1179, 1263, 4695, 500, 4011, 1191, 725, 4606, 2027, 1556, 193, 1908, 1286, 2257, 3446, 4497, 1529, 1875, 189, 1054, 1109, 871, 739, 1053, 2106, 3213, 313, 2983, 2642, 2445, 3304, 4694, 3228, 1007, 4120, 1409, 3704, 287, 57, 3433, 3899, 4396, 2279, 269, 4902, 4535, 1963, 4310, 801, 1120, 299, 53, 2722, 3489, 208, 3032, 811, 73, 2342, 4803, 2623, 608, 823, 2713, 2978, 3612, 5000, 686, 466, 717, 3540, 3539, 3143, 4416, 3036, 3160, 197, 2281, 565, 1523, 1083, 3756, 157, 96, 833, 2849, 2986, 4976, 4837, 3515, 3858, 248, 1520, 2109, 1442, 4246, 986, 1568, 2487, 212, 2617, 1812, 3650, 203, 3333, 3596, 3416, 238, 62, 4961, 634, 3744, 2235, 186, 3621, 1246, 4297, 759, 4923, 3282, 1499, 3267, 1343, 1287, 1953, 3887, 3992, 4670, 3345, 4051, 4365, 3242, 3085, 2480, 3566, 2606, 127, 1294, 4596, 245, 2457, 1289, 2696, 3714, 2028, 1490, 1355, 2393, 3060, 4203, 2179, 4309, 945, 985, 737, 312, 3707, 3583, 24, 2220, 493, 1377, 1361, 1480, 2561, 4873, 3530, 1150, 2553, 1189, 2718, 951, 4029, 2745, 1105, 4361, 2890, 2475, 4180, 4438, 4636, 4809, 454, 3286, 1193, 3413, 4766, 296, 4208, 1689, 3593, 3584, 779, 1932, 4658, 2844, 1130, 4713, 4502, 4765, 2869, 1665, 4760, 891, 2765, 3403, 2968, 4176, 1574, 4139, 39, 1144, 2594, 1716, 2715, 3555, 3683, 2074, 4492, 1562, 4689, 544, 5007, 4507, 2811, 3169, 641, 2356, 4001, 2474, 509, 3262, 4605, 2141, 1943, 328, 4178, 4235, 4294, 3137, 1107, 33, 3472, 2903, 1030, 1581, 4213, 286, 2268, 1554, 1463, 4087, 3203, 3235, 1266, 838, 4085, 3853, 3466, 2282, 1136, 2159, 1674, 2166, 4768, 566, 1934, 2119, 1383, 1500, 3554, 3264, 3782, 2035, 3775, 4110, 2145, 3393, 228, 1539, 1948, 4404, 4469, 3018, 4195, 3440, 1466, 4708, 2870, 1108, 930, 1785, 2060, 4065, 4491, 3743, 261, 4567, 1337, 4599, 3816, 538, 4190, 323, 815, 2526, 2177, 3133, 4828, 4601, 1888, 1283, 129, 4568, 1140, 2426, 4525, 1116, 3482, 3721, 2540, 3237, 3771, 2402, 4136, 2260, 1405, 834, 4107, 3609, 38, 1005, 268, 1087, 3298, 4829, 3118, 1718, 4945, 1970, 4569, 1601, 4350, 2134, 3437, 2283, 492, 4211, 437, 487, 4878, 694, 959, 684, 3642, 349, 2756, 2449, 1448, 4721, 1068, 2698, 2659, 580, 3709, 2915, 1690, 37, 512, 2371, 3246, 2749, 2497, 2951, 4996, 4534, 3031, 1398, 355, 1703, 1305, 2641, 335, 3716, 163, 4685, 3552, 3188, 2967, 1098, 4012, 2907, 4281, 673, 1158, 4762, 3221, 4269, 354, 1638, 2655, 1851, 4464, 4718, 3702, 4709, 650, 371, 2427, 1727, 3845, 2615, 2344, 2372, 523, 4540, 600, 4714, 960, 496, 207, 215, 3592, 3790, 111, 1274, 3622, 4889, 3918, 937, 289, 3098, 1741, 2105, 677, 567, 2484, 614, 4218, 2588, 206, 2580, 3882, 3948, 2851, 1435, 4472, 2353, 1534, 3385, 3430, 2833, 1170, 4202, 649, 890, 921, 2737, 2999, 4129, 4265, 3813, 4983, 2171, 2701, 4744, 4217, 2293, 1749, 4684, 2637, 3100, 4526, 4275, 4364, 4702, 4559, 49, 3772, 302, 611, 3308, 1025, 2917, 4634, 3040, 1923, 3358, 1594, 2518, 201, 470, 3904, 1374, 3427, 1175, 766, 821, 1916, 4198, 4995, 2770, 705, 1224, 1128, 1129, 4931, 3746, 2605, 3162, 1049, 3155, 3065, 1999, 331, 1830, 2739, 8, 2989, 3657, 764, 4826, 411, 2545, 327, 3436, 4280, 4956, 3894, 1339, 3522, 695, 1992, 3029, 4260, 3022, 2038, 1615, 1974, 2345, 1092, 1417, 131, 4533, 1886, 3503, 4940, 2045, 1227, 2544, 4595, 4693, 875, 1743, 266, 4884, 4893, 4432, 1798, 1834, 2707, 4898, 1174, 3854, 442, 1075, 3211, 61, 3451, 1783, 733, 1133, 3147, 154, 3301, 56, 4437, 4376, 3947, 3011, 421, 4056, 3163, 4456, 4506, 2658, 2896, 2534, 18, 751, 1346, 861, 2547, 4712, 219, 3553, 2891, 4637, 643, 1680, 1431, 2530, 326, 3941, 3788, 859, 2133, 3350, 3383, 1980, 2326, 819, 4726, 1634, 391, 2669, 2876, 1941, 4049, 1420, 3219, 1001, 45, 3865, 2514, 3645, 3627, 3487, 3109, 380, 1753, 3164, 4678, 2920, 2267, 3261, 1076, 1201, 1111, 1937, 122, 178, 905, 1282, 3697, 2168, 4532, 3935, 1388, 2988, 2077, 3957, 4273, 2548, 4681, 3255, 2052, 398, 2862, 626, 3987, 4554, 2152, 3204, 1233, 360, 4746, 1218, 2314, 3817, 4141, 4433, 1576, 3807, 3820, 2071, 333, 1137, 2533, 76, 28, 3396, 404, 2178, 3254, 4625, 2519, 2909, 4347, 4112, 2583, 1169, 2183, 1219, 2560, 2147, 900, 4062, 153, 3115, 3580, 2061, 526, 4967, 4539, 4508, 1515, 1625, 3317, 4317, 135, 4069, 1894, 909, 3294, 4356, 2405, 1341, 4791, 2815, 3367, 4909, 4193, 969, 4389, 3961, 3907, 1822, 4885, 222, 4883, 3070, 665, 3964, 3318, 1249, 3685, 3337, 2143, 4816, 4880, 1619, 2101, 3399, 4341, 3659, 2913, 839, 676, 1236, 1063, 4854, 149, 4003, 3131, 4421, 4174, 2727, 1495, 3994, 2, 539, 4083, 548, 3391, 4607, 3330, 3585, 1251, 198, 3850, 3323, 11, 214, 3791, 1843, 3881, 765, 2608, 1751, 3210, 4212, 4250, 2995, 4363, 70, 191, 1933, 789, 1947, 271, 1015, 3828, 1352, 1648, 2507, 2633, 2294, 877, 3464, 1519, 2954, 659, 2520, 1100, 462, 2850, 2769, 685, 2003, 2928, 702, 3035, 3245, 926, 3774, 1037, 865, 1050, 366, 1663, 4851, 1906, 549, 2899, 2033, 4319, 531, 3897, 2500, 3891, 4271, 1143, 3462, 2807, 446, 3885, 3240, 1198, 72, 1265, 2959, 1833, 4023, 1565, 1653, 1699, 1659, 1464, 4276, 1439, 853, 4593, 224, 4914, 3956, 2262, 675, 3512, 4037, 2806, 2546, 2538, 4824, 4300, 4964, 4424, 1354, 3280, 3125, 4123, 2604, 3884, 52, 925, 2796, 374, 701, 2562, 1967, 1345, 3776, 4070, 4844, 3608, 2274, 479, 680, 172, 2365, 2201, 588, 1430, 2137, 337, 143, 1982, 3516, 3699, 3626, 1250, 2948, 251, 3934, 4742, 431, 752, 359, 3418, 2964, 2691, 4960, 806, 4628, 564, 1280, 1402, 210, 2386, 2269, 255, 950, 3349, 1993, 3855, 3617, 1035, 990, 2446, 2646, 3094, 3543, 235, 2349, 3322, 982, 3409, 1094, 1873, 1853, 939, 570, 3551, 107, 4752, 3438, 79, 3801, 592, 3920, 4313, 3449, 1637, 71, 2916, 4482, 3265, 1702, 2963, 3653, 3615, 2067, 4412, 2753, 2001, 167, 400, 2421, 4034, 1586, 449, 722, 835, 3680, 2319, 2234, 1006, 1259, 4461, 3030, 1950, 1023, 3002, 2187, 1182, 1641, 3681, 4592, 2746, 4730, 1342, 4151, 3574, 1165, 1359, 4657, 4225, 1013, 3306, 46, 3468, 3745, 4515, 3212, 205, 965, 291, 4782, 955, 489, 1881, 2838, 3510, 632, 2939, 1134, 2390, 2582, 4717, 2808, 1761, 3875, 2943, 3558, 1790, 1344, 3027, 740, 2946, 4125, 2738, 2454, 3183, 2515, 707, 830, 438, 3674, 1707, 919, 2259, 2624, 3575, 1697, 4004, 3660, 3054, 873, 4086, 2066, 848, 1432, 2439, 1061, 3917, 3151, 4817, 2812, 4495, 4392, 1121, 4835, 2099, 3455, 2997, 1127, 2022, 3796, 2979, 4733, 457, 4009, 2323, 1985, 406, 4258, 2051, 3847, 4435, 3150, 5009, 443, 4380, 3122, 4091, 2949, 2036, 4032, 4255, 2264, 2123, 4906, 4148, 2527, 4333, 4295, 2699, 2298, 4318, 4173, 4261, 4028, 3156, 807, 2167, 4162, 3670, 1624, 378, 1698, 2927, 631, 1425, 4383, 1865, 771, 1620, 3426, 1831, 1253, 3980, 2422, 1972, 97, 174, 1164, 2198, 808, 2465, 4405, 1350, 2396, 1704, 4841, 1866, 2755, 1979, 200, 920, 3717, 1845, 3258, 1981, 1808, 1673, 4075, 1437, 389, 3419, 2155, 2704, 498, 772, 4602, 3724, 3325, 4124, 2792, 1206, 922, 2779, 1895, 2468, 2430, 2300, 283, 4597, 933, 4111, 4025, 280, 3513, 2706, 4335, 4550, 2209, 1014, 2558, 4584, 1467, 1462, 1861, 2284, 1057, 2466, 636, 2492, 3611, 249, 3799, 2085, 2029, 2685, 1693, 3291, 5010, 2813, 656, 68, 1020, 1537, 629, 1935, 259, 721, 2419, 3314, 3981, 3017, 1267, 4036, 4977, 3126, 940, 4924, 4866, 3149, 428, 3893, 2130, 1926, 3310, 561, 1154, 102, 3176, 2424, 4340, 4238, 729, 4671, 4825, 29, 1649, 710, 250, 486, 1041, 1692, 4127, 3886, 860, 4997, 2667, 3633, 3979, 1535, 3214, 4447, 3432, 842, 2174, 2654, 605, 4474, 1870, 6, 3864, 3013, 4192, 481, 3270, 1613, 2834, 3057, 4780, 3056, 2764, 3870, 3876, 145, 3467, 3005, 4594, 2976, 2673, 1664, 3434, 1459, 2791, 4808, 2752, 1524, 2144, 1297, 1569, 1261, 3353, 863, 3700, 2164, 4108, 2161, 2732, 748, 4428, 1379, 3792, 530, 151, 4819, 3809, 4021, 4774, 1292, 270, 4942, 4160, 213, 584, 316, 1186, 187, 2034, 3019, 1756, 4536, 2819, 2226, 2620, 557, 34, 1651, 646, 3641, 851, 4813, 4724, 2950, 671, 3039, 2263, 1424, 3960, 1392, 2858, 1774, 4804, 2901, 1032, 4905, 1540, 3071, 4452, 2522, 3201, 4911, 464, 491, 1521, 2088, 2581, 780, 4061, 4366, 3191, 2486, 4993, 1323, 3105, 2213, 1544, 1606, 31, 1598, 1858, 2223, 787, 4467, 2197, 4154, 4675, 1279, 669, 1503, 3723, 1446, 244, 2185, 275, 4422, 3603, 3208, 2671, 370, 2204, 3923, 1685, 265, 1956, 1986, 1705, 44, 314, 1712, 3778, 1122, 362, 3781, 190, 1553, 2729, 1043, 4196, 279, 1962, 2682, 267, 913, 4939, 3072, 4739, 4397, 2359, 93, 1770, 4117, 3124, 4248, 3786, 2394, 1114, 1406, 1629, 1270, 81, 216, 4789, 1518, 1939, 221, 1252, 277, 1277, 1768, 691, 4486, 1102, 2017, 1328, 372, 3872, 2782, 639, 4611, 2866, 1952, 2692, 4279, 3722, 3461, 4566, 899, 4417, 1269, 1118, 4031, 2182, 907, 3, 2554, 118, 1998, 2049, 3395, 3838, 4413, 2972, 4016, 2830, 547, 1333, 1889, 2904, 2072, 812, 2401, 750, 2070, 373, 3290, 3869, 3043, 4773, 4303, 4232, 4785, 880, 43, 3050, 3431, 469, 3321, 2537, 4938, 1397, 2478, 540, 1240, 537, 3407, 3236, 1419, 3494, 586, 1571, 4257, 137, 447, 240, 1813, 1596, 4886, 3761, 525, 3192, 4236, 4291, 273, 4306, 590, 1492, 1994, 3360, 1820, 2193, 4277, 2441, 4142, 2921, 3087, 4352, 84, 20, 218, 660, 3623, 747, 2142, 1048, 3186, 3299, 3346, 2496, 3712, 342, 1124, 3049, 843, 3572, 2613, 4044, 3531, 4408, 3171, 4710, 4847, 3567, 4373, 1777, 4097, 2841, 2216, 2058, 3666, 4427, 4008, 3731, 1884, 4493, 1482, 41, 4338, 4519, 2153, 2586, 3486, 4494, 490, 1476, 4805, 896, 4290, 1907, 2758, 4971, 2222, 4254, 3478, 654, 4186, 4089, 4800, 3465, 2587, 2975, 4600, 3802, 563, 4891, 396, 4954, 3658, 1331, 2970, 1386, 4241, 2524, 3982, 1910, 1056, 3856, 923, 4215, 3936, 1400, 3422, 616, 3966, 2856, 3862, 376, 1401, 2215, 1069, 1563, 3052, 1995, 1319, 4619, 353, 1969, 1506, 2798, 1890, 2304, 260, 3527, 1708, 4080, 1996, 5005, 4609, 1228, 3492, 3511, 3305, 3365, 4948, 424, 3613, 4143, 587, 4410, 1040, 1694, 196, 2208, 1440, 3635, 692, 2502, 874, 4603, 2438, 1418, 1162, 1152, 3244, 3997, 2962, 89, 2021, 852, 2373, 1802, 4132, 4094, 3694, 2102, 4163, 3556, 4974, 3664, 4904, 2557, 2767, 2423, 4207, 2411, 1199, 3533, 2086, 3496, 2341, 4395, 814, 3874, 3534, 1859, 3079, 2408, 4930, 4656, 85, 1362, 1460, 78, 3860, 529, 2172, 2124, 321, 915, 3123, 2824, 2440, 3710, 3107, 800, 397, 4846, 4561, 2872, 2115, 2121, 3390, 917, 3969, 2338, 1340, 3024, 2219, 3667, 2447, 2980, 3833, 1457, 4688, 1320, 4668, 1572, 4304, 4530, 2089, 3190, 2543, 393, 2863, 4274, 892, 813, 4224, 4105, 2473, 1088, 2788, 4181, 3652, 711, 1413, 1304, 655, 4613, 1475, 3601, 972, 1221, 2254, 4618, 4640, 4617, 4522, 2415, 1293, 929, 4778, 947, 2316, 4881, 3526, 2128, 309, 1964, 4856, 7, 3392, 1805, 2666, 3428, 4588, 2610, 1315, 1429, 4158, 103, 4216, 459, 2092, 1426, 2884, 2730, 3946, 3354, 4743, 3148, 1645, 4556, 2786, 3868, 2391, 2382, 2343, 2463, 4968, 2181, 3289, 4560, 769, 589, 2334, 1784, 1914, 3044, 1838, 4984, 4944, 1714, 3873, 483, 2720, 2112, 898, 4524, 2037, 3509, 115, 962, 3673, 3117, 2861, 4177, 281, 3863, 2362, 2370, 2200, 4226, 1473, 300, 22, 4360, 3620, 573, 3199, 4586, 572, 575, 2039, 2019, 4503, 3989, 1010, 1758, 3284, 3000, 2929, 4191, 3215, 4119, 4697, 1215, 1184, 4929, 4103, 2832, 1146, 1239, 3247, 1496, 2459, 4320, 4980, 534, 1369, 3927, 4620, 3647, 1700, 1220, 1511, 66, 1368, 579, 2877, 1897, 317, 2327, 3209, 1667, 1841, 3339, 2875, 2723, 593, 2998, 2339, 3857, 4581, 2748, 967, 1842, 4686, 3332, 2224, 4698, 4737, 1058, 130, 3158, 746, 4079, 4579, 1311, 1786, 3161, 3341, 417, 2912, 971, 2734, 2315, 3058, 3046, 1404, 4150, 4888, 836, 456, 242, 2775, 3222, 484, 1825, 4879, 3006, 4282, 329, 4450, 10, 4133, 1314, 1757, 1370, 3193, 2280, 77, 980, 415, 4197, 4387, 4099, 150, 1867, 2630, 4288, 2835, 2590, 1874, 27, 4587, 756, 4848, 3619, 708, 4425, 1736, 1255, 1814, 3452, 1691, 475, 532, 3110, 624, 3990, 3849, 3844, 3324, 2744, 2404, 2296, 3302, 4986, 2656, 4731, 785, 4551, 1816, 4868, 2821, 4078, 3959, 4763, 4185, 4783, 4090, 1097, 4892, 2714, 3120, 2202, 2686, 2563, 2784, 543, 2635, 2310, 3271, 542, 58, 4629, 1295, 3797, 124, 1903, 620, 2079, 1434, 4699, 452, 1780, 1276, 2023, 4790, 818, 3705, 155, 582, 4921, 1000, 3547, 2984, 3238, 42, 1081, 1928, 3851, 4128, 1479, 4667, 1547, 4393, 4278, 2097, 4298, 3557, 4499, 606, 1584, 3296, 4346, 2008, 2977, 2712, 4999, 989, 633, 968, 2648, 2740, 2210, 4912, 4047, 4895, 3195, 3034, 1177, 100, 763, 867, 3231, 648, 3988, 4220, 482, 109, 237, 4477, 774, 1997, 1644, 845, 3067, 2059, 1835, 1079, 4072, 3259, 2437, 3819, 25, 1195, 2384, 1509, 4932, 983, 4795, 430, 12, 1949, 1501, 3285, 1024, 3967, 1880, 3292, 3281, 1502, 4483, 4175, 790, 550, 3777, 3733, 556, 2864, 914, 2068, 4591, 3450, 3038, 2828, 4126, 2069], "validation": [5043, 5441, 5500, 5398, 5310, 5017, 5278, 5034, 5439, 5473, 5373, 5163, 5133, 5318, 5020, 5143, 5291, 5361, 5428, 5170, 5113, 5451, 5342, 5359, 5367, 5357, 5199, 5377, 5481, 5013, 5406, 5266, 5101, 5137, 5458, 5454, 5172, 5165, 5476, 5508, 5323, 5187, 5511, 5335, 5433, 5125, 5134, 5376, 5303, 5149, 5110, 5410, 5012, 5042, 5188, 5330, 5355, 5487, 5312, 5329, 5320, 5136, 5354, 5383, 5344, 5016, 5424, 5096, 5064, 5086, 5421, 5087, 5147, 5069, 5491, 5380, 5033, 5293, 5394, 5180, 5425, 5440, 5040, 5191, 5420, 5082, 5070, 5195, 5184, 5155, 5160, 5242, 5141, 5296, 5365, 5294, 5430, 5270, 5192, 5485, 5201, 5283, 5502, 5257, 5198, 5404, 5490, 5382, 5492, 5479, 5208, 5177, 5197, 5227, 5459, 5466, 5014, 5423, 5229, 5356, 5032, 5311, 5055, 5417, 5019, 5158, 5183, 5092, 5204, 5167, 5135, 5280, 5324, 5348, 5048, 5254, 5193, 5506, 5469, 5271, 5381, 5185, 5364, 5235, 5288, 5168, 5384, 5258, 5256, 5075, 5325, 5252, 5088, 5393, 5308, 5314, 5179, 5284, 5470, 5478, 5449, 5437, 5114, 5427, 5445, 5336, 5295, 5321, 5316, 5233, 5503, 5175, 5446, 5267, 5307, 5349, 5077, 5299, 5210, 5489, 5438, 5375, 5046, 5399, 5301, 5507, 5467, 5395, 5098, 5385, 5461, 5067, 5263, 5024, 5162, 5300, 5130, 5099, 5037, 5244, 5390, 5038, 5358, 5232, 5432, 5274, 5290, 5181, 5057, 5159, 5207, 5407, 5264, 5369, 5231, 5211, 5119, 5326, 5456, 5353, 5176, 5219, 5315, 5331, 5061, 5471, 5230, 5431, 5379, 5053, 5225, 5217, 5253, 5343, 5282, 5132, 5018, 5285, 5251, 5186, 5387, 5272, 5332, 5463, 5111, 5215, 5457, 5209, 5093, 5483, 5054, 5117, 5156, 5402, 5146, 5105, 5138, 5484, 5153, 5450, 5079, 5123, 5060, 5460, 5128, 5074, 5302, 5306, 5025, 5309, 5319, 5452, 5297, 5350, 5403, 5115, 5203, 5475, 5408, 5386, 5241, 5118, 5023, 5259, 5246, 5095, 5194, 5206, 5497, 5351, 5279, 5200, 5150, 5030, 5464, 5412, 5292, 5444, 5091, 5121, 5496, 5090, 5109, 5045, 5260, 5328, 5122, 5112, 5221, 5041, 5436, 5218, 5298, 5368, 5139, 5504, 5416, 5145, 5226, 5245, 5108, 5409, 5161, 5248, 5493, 5472, 5334, 5371, 5313, 5250, 5370, 5360, 5392, 5434, 5022, 5056, 5102, 5400, 5094, 5029, 5281, 5222, 5262, 5509, 5107, 5448, 5049, 5164, 5144, 5189, 5374, 5073, 5363, 5157, 5050, 5419, 5154, 5265, 5084, 5058, 5066, 5414, 5413, 5269, 5338, 5462, 5388, 5148, 5151, 5212, 5347, 5059, 5124, 5142, 5499, 5182, 5051, 5339, 5171, 5276, 5213, 5238, 5268, 5345, 5275, 5482, 5062, 5396, 5488, 5126, 5501, 5120, 5372, 5477, 5089, 5418, 5085, 5505, 5140, 5015, 5401, 5083, 5480, 5220, 5378, 5422, 5346, 5255, 5223, 5026, 5415, 5426, 5076, 5127, 5289, 5202, 5044, 5366, 5468, 5337, 5106, 5063, 5097, 5065, 5249, 5047, 5455, 5305, 5214, 5237, 5340, 5078, 5443, 5442, 5286, 5389, 5131, 5391, 5072, 5152, 5036, 5447, 5178, 5435, 5277, 5224, 5397, 5028, 5453, 5169, 5429, 5071, 5116, 5166, 5498, 5474, 5486, 5081, 5243, 5273, 5304, 5216, 5322, 5190, 5035, 5205, 5039, 5261, 5352, 5031, 5240, 5173, 5287, 5405, 5021, 5510, 5247, 5100, 5104, 5239, 5411, 5027, 5129, 5341, 5103, 5494, 5465, 5333, 5327, 5234, 5080, 5236, 5052, 5196, 5362, 5228, 5317, 5495, 5068, 5174], "test": [5990, 5650, 5624, 5802, 5931, 5541, 5675, 5716, 5602, 5763, 5629, 5578, 5693, 5795, 5974, 5523, 5761, 5740, 5529, 5784, 6003, 5904, 5609, 5546, 5512, 5810, 5528, 5527, 5807, 5697, 5662, 5787, 5856, 5626, 5686, 5533, 5712, 5993, 5644, 5642, 5890, 5700, 5769, 5746, 5667, 5873, 5727, 5793, 5661, 5731, 5927, 5610, 5753, 5522, 5989, 5940, 5928, 5864, 5970, 5575, 5925, 5957, 5984, 5748, 5571, 5851, 5625, 5951, 5930, 5581, 5805, 5962, 5730, 5790, 5576, 5579, 5660, 5824, 5976, 5782, 6004, 5670, 5903, 5823, 5889, 5801, 5935, 5964, 5549, 5703, 5803, 5980, 5537, 5881, 5556, 5636, 5628, 5808, 5728, 5781, 5898, 5939, 5897, 5982, 5998, 5583, 5653, 5913, 5845, 5619, 5582, 5981, 5825, 5967, 5723, 5611, 5519, 5840, 5766, 5648, 6006, 5588, 5941, 5820, 5936, 5880, 5838, 5849, 5739, 5563, 5551, 5770, 5704, 5752, 5613, 5884, 5634, 5637, 5598, 5811, 5620, 5553, 5915, 5997, 5603, 5685, 5804, 5580, 5869, 5942, 5979, 5954, 5992, 5639, 5654, 5531, 5778, 5888, 5882, 5596, 5862, 5841, 5707, 5949, 5589, 5706, 5741, 5535, 5655, 5617, 5779, 5972, 5559, 6007, 5887, 5552, 5835, 5664, 5751, 5668, 5534, 5682, 5604, 5961, 5584, 5952, 5907, 5843, 5994, 5948, 5875, 5587, 5969, 5886, 5755, 5946, 5991, 5680, 5517, 5836, 5959, 5772, 5860, 6009, 5878, 5879, 5955, 5720, 5544, 5594, 5569, 5757, 5960, 5865, 5985, 5777, 5977, 5798, 5555, 6011, 5863, 5759, 5649, 5615, 5834, 5816, 5783, 5710, 5565, 5796, 5709, 5780, 5674, 5696, 5585, 5822, 5988, 5826, 5745, 5905, 5735, 5947, 5699, 5968, 5758, 5799, 5831, 5910, 5812, 5800, 5543, 5813, 5540, 5645, 5606, 5958, 5842, 5513, 5677, 5859, 5608, 5771, 5848, 5618, 5762, 5846, 5853, 5847, 5926, 5687, 5857, 6001, 5656, 5892, 5726, 5631, 5872, 5514, 5666, 5883, 5916, 5852, 5524, 5934, 5632, 5767, 5917, 5943, 5933, 5538, 5536, 5797, 5658, 5641, 5765, 5695, 6008, 5562, 5657, 5738, 5754, 5630, 5900, 5895, 5870, 5789, 5818, 5747, 5614, 5844, 5806, 5971, 5633, 5975, 5918, 5564, 5622, 5773, 5729, 5520, 5983, 5623, 5966, 5530, 5855, 5830, 5515, 5673, 5833, 5891, 5768, 5963, 5539, 5568, 5627, 5791, 5717, 5814, 5965, 5821, 5776, 5518, 5854, 5908, 5995, 5679, 5827, 5521, 5832, 5548, 5871, 5893, 5607, 5736, 5672, 6010, 5737, 5698, 5894, 5945, 5689, 5526, 5684, 5691, 5986, 5591, 5861, 5901, 5920, 5987, 5713, 5595, 5956, 5839, 5597, 5651, 5876, 5953, 5688, 5749, 5586, 5896, 5545, 5532, 5774, 5724, 5560, 5683, 5828, 5744, 5592, 5542, 5570, 5694, 5638, 5676, 5561, 5938, 5906, 5829, 5743, 5937, 5973, 5708, 5902, 5605, 5547, 5669, 5566, 5950, 5516, 5932, 5557, 5978, 5599, 5794, 5944, 5817, 5701, 5756, 5775, 5652, 5550, 5616, 5868, 5866, 5734, 5721, 5659, 5671, 5665, 5899, 5792, 5921, 5600, 5919, 5646, 5558, 5719, 5858, 5678, 5690, 5567, 5640, 5635, 5702, 5850, 5705, 5999, 5909, 5764, 5725, 5750, 5837, 5733, 5593, 5572, 5732, 5760, 5929, 5785, 5714, 5525, 5601, 5819, 5911, 5577, 5574, 5924, 5590, 5815, 5809, 6002, 5722, 5612, 6000, 5715, 5923, 5718, 5554, 5996, 5914, 5788, 5877, 5786, 5643, 5647, 5681, 5742, 5663, 5922, 5621, 5692, 5573, 5912, 5867, 5885, 6005, 5874, 5711]}, {"train": [1873, 245, 647, 2497, 2132, 711, 76, 3473, 2720, 2454, 892, 974, 1028, 2476, 832, 958, 1307, 2378, 207, 1300, 3385, 4413, 1171, 2595, 532, 4290, 3065, 2041, 3207, 4271, 4249, 756, 2686, 2808, 2026, 474, 2747, 477, 364, 4734, 2133, 1888, 2681, 631, 4702, 2413, 3255, 858, 3686, 1744, 5009, 4806, 3747, 2059, 3835, 1530, 3859, 1216, 2837, 4319, 1368, 219, 1037, 4098, 3591, 3414, 2152, 752, 2708, 3554, 3563, 2253, 4013, 2536, 4467, 815, 1441, 2918, 421, 3366, 2028, 2360, 1932, 2432, 2693, 3049, 2931, 4061, 1318, 4497, 1253, 442, 1325, 2806, 3036, 4839, 1141, 3284, 3715, 119, 1948, 3846, 45, 757, 1355, 436, 2721, 4338, 1399, 2971, 1845, 2541, 4216, 603, 402, 718, 2335, 4591, 822, 739, 1893, 4616, 4068, 3402, 4247, 3682, 1674, 58, 3206, 1797, 4709, 3993, 424, 132, 1421, 3480, 3419, 4018, 3338, 2324, 2125, 2525, 2448, 2572, 3757, 4326, 3266, 779, 2082, 1433, 2128, 2424, 1023, 3026, 3556, 1226, 1255, 2446, 1854, 1121, 3815, 2123, 5006, 0, 1207, 4295, 2457, 4595, 4830, 640, 511, 1087, 2280, 4468, 3441, 1553, 452, 1785, 2409, 1117, 197, 2163, 1699, 4045, 2235, 2382, 3598, 2, 2972, 659, 950, 4418, 1081, 576, 1036, 3380, 1423, 4122, 3607, 1072, 1919, 24, 3973, 3620, 1174, 2558, 4889, 2750, 2899, 4486, 2566, 1764, 2405, 370, 903, 1115, 1453, 1280, 1690, 4947, 3306, 3412, 1824, 416, 2350, 941, 4649, 3147, 698, 1680, 2707, 1605, 2939, 3107, 1658, 2255, 2158, 679, 4195, 645, 88, 4570, 1562, 2364, 3905, 3334, 3444, 3557, 2172, 2305, 4255, 3509, 3865, 3731, 376, 519, 1516, 1049, 1811, 4568, 3566, 690, 3683, 1505, 3936, 908, 3201, 1084, 1567, 1885, 2542, 2212, 4385, 594, 2625, 3198, 1534, 3691, 3722, 395, 4103, 366, 192, 3220, 4256, 1669, 2339, 2938, 4820, 329, 2874, 3435, 4955, 3196, 4660, 1848, 4991, 4088, 4156, 1241, 4866, 2901, 4446, 365, 1506, 150, 2111, 543, 1301, 4257, 626, 1274, 3771, 4808, 279, 1743, 4400, 3746, 3739, 2869, 4735, 1953, 2889, 2799, 2507, 785, 4635, 2011, 4891, 1712, 2020, 4860, 2639, 2778, 1393, 4014, 4634, 1525, 3230, 562, 1012, 4886, 4871, 4723, 95, 570, 4531, 544, 4764, 3400, 4552, 2755, 4805, 2298, 4487, 1768, 4546, 3246, 839, 3748, 3125, 1527, 2189, 3507, 1450, 288, 2862, 846, 3240, 75, 2496, 2860, 807, 1678, 2194, 4133, 629, 1504, 2853, 4865, 3501, 4179, 1769, 3171, 4556, 1992, 2084, 1013, 3217, 661, 4017, 4483, 2056, 235, 4366, 4827, 38, 2791, 1443, 643, 1090, 4728, 1456, 3183, 3427, 1792, 1260, 250, 4813, 861, 2266, 3621, 3531, 1106, 4601, 922, 1286, 3242, 721, 843, 4176, 619, 3710, 3211, 2300, 836, 334, 2502, 3341, 517, 3361, 28, 2500, 542, 585, 4573, 1145, 4158, 1654, 1009, 473, 3532, 2061, 32, 2451, 4673, 2193, 4613, 555, 1294, 1628, 2154, 3927, 3538, 694, 346, 3387, 744, 654, 73, 1380, 1750, 3595, 2089, 2371, 493, 1608, 670, 538, 4931, 520, 3379, 951, 1362, 330, 1228, 1033, 1593, 3817, 2368, 900, 4302, 2695, 2713, 942, 2499, 1524, 4204, 2135, 4970, 3964, 4530, 782, 1637, 1080, 3761, 675, 962, 899, 3798, 2147, 2376, 2479, 1463, 4181, 4059, 1094, 4856, 801, 1994, 4751, 4933, 4324, 713, 579, 361, 1969, 3069, 741, 324, 1934, 3454, 2148, 3706, 4689, 1054, 2877, 662, 3966, 193, 4188, 2533, 2478, 732, 2313, 2336, 2369, 2254, 3269, 622, 4484, 3059, 4914, 4106, 2066, 2940, 2991, 4336, 4155, 4092, 3127, 1947, 1747, 21, 4353, 3571, 1550, 1591, 1522, 1951, 774, 4884, 1890, 1916, 2682, 2480, 4781, 4539, 3019, 864, 381, 1563, 3643, 4193, 1986, 3308, 3348, 1097, 1804, 460, 3567, 4725, 2352, 1210, 3221, 1035, 2838, 545, 66, 3527, 3799, 2090, 4799, 3485, 1831, 1000, 1966, 4675, 2729, 2870, 957, 3744, 4129, 1912, 2865, 22, 4542, 3304, 268, 2845, 1926, 3077, 1955, 3204, 3519, 2119, 1790, 1828, 4562, 2304, 4833, 1248, 2780, 3409, 3660, 1533, 2107, 468, 550, 4082, 2894, 971, 1602, 3818, 528, 4185, 2993, 2401, 3889, 3870, 3652, 4080, 1944, 4588, 4033, 3278, 251, 1738, 1147, 3176, 4873, 2265, 3145, 4509, 1352, 804, 3707, 2127, 3562, 3930, 4993, 189, 4182, 4521, 964, 1625, 2966, 3132, 4737, 3836, 2186, 2334, 1558, 1067, 2835, 1273, 102, 3058, 4629, 4391, 1353, 3087, 432, 4016, 3896, 4049, 2662, 1088, 2438, 1657, 3537, 2182, 1676, 4341, 2893, 3724, 3904, 2257, 1190, 2795, 3514, 3038, 4910, 1429, 1163, 2855, 537, 3989, 135, 2873, 4375, 2792, 4225, 3693, 2050, 4399, 1377, 1266, 170, 54, 4130, 2637, 3446, 3823, 623, 1643, 1688, 2986, 3199, 4493, 3081, 209, 2422, 955, 2981, 692, 4212, 4880, 3010, 3858, 4112, 1131, 1320, 3679, 491, 4205, 1714, 2537, 3797, 313, 3177, 1404, 1132, 3535, 2696, 2574, 4266, 2668, 2908, 4152, 1915, 2106, 344, 3910, 4672, 1204, 1894, 3544, 518, 3890, 202, 1536, 3188, 4051, 569, 405, 320, 2211, 4009, 2247, 2206, 1365, 823, 2068, 2929, 3166, 2898, 1800, 2840, 4698, 4544, 298, 306, 2358, 1931, 2482, 887, 4583, 1826, 2129, 820, 1420, 3120, 3140, 2177, 4795, 834, 1150, 1103, 3300, 3847, 857, 3358, 2114, 2924, 568, 1373, 3173, 1295, 3429, 4046, 3239, 2638, 2556, 1985, 4462, 461, 4401, 4543, 1007, 707, 4028, 2882, 1239, 1031, 1901, 1835, 4800, 3843, 1737, 1650, 4559, 1459, 4762, 3734, 3664, 2560, 4346, 92, 1927, 179, 1583, 3649, 1933, 3540, 4890, 4304, 3802, 2006, 412, 534, 1053, 1523, 4359, 2788, 4496, 863, 4618, 2577, 4906, 727, 333, 2598, 426, 3191, 2033, 1475, 873, 3828, 810, 451, 2618, 3499, 169, 2009, 363, 1642, 3399, 3181, 2554, 2289, 708, 758, 69, 979, 116, 968, 3894, 4528, 3624, 403, 3260, 4529, 1335, 3488, 2930, 3738, 1111, 3381, 1904, 1415, 1627, 1029, 4207, 3416, 373, 406, 833, 3313, 4494, 1786, 3466, 648, 2234, 2367, 3023, 222, 2299, 4438, 1384, 4174, 4077, 1629, 1160, 3148, 2545, 560, 1615, 4464, 3271, 3768, 2800, 1545, 2876, 255, 915, 2183, 4549, 755, 571, 3772, 3457, 4177, 343, 4816, 1850, 1961, 1005, 110, 4011, 1244, 2230, 2087, 3508, 1461, 3273, 1758, 1692, 479, 736, 3388, 1693, 1655, 3064, 2624, 3627, 4368, 3819, 672, 2976, 3104, 2748, 196, 3592, 2602, 2063, 2386, 4633, 666, 4708, 2943, 2314, 1908, 1044, 3216, 4157, 293, 4943, 161, 4905, 3237, 777, 1367, 4345, 484, 1202, 1794, 2505, 3243, 4594, 2742, 34, 2467, 1235, 4184, 1305, 1987, 3753, 841, 2384, 1725, 3585, 2002, 1970, 1187, 1455, 3640, 4638, 1696, 938, 2582, 74, 375, 1865, 657, 3282, 2659, 3675, 2173, 154, 667, 3099, 3642, 1639, 377, 644, 4325, 508, 1928, 580, 4967, 2428, 4694, 3800, 3205, 746, 136, 4763, 3701, 891, 1575, 3832, 2001, 4056, 2600, 2543, 3263, 4620, 4624, 2326, 2232, 2868, 2481, 642, 3985, 3913, 2271, 4040, 4516, 1842, 1577, 1740, 3360, 997, 4825, 2887, 174, 3558, 602, 653, 4517, 2164, 4440, 2830, 1473, 1665, 2224, 4031, 4776, 486, 2903, 1858, 4329, 4315, 3072, 2612, 819, 4687, 4309, 3560, 4427, 3755, 3900, 2227, 4948, 4669, 1668, 2987, 3472, 1923, 3883, 2926, 4979, 1074, 2567, 1519, 2776, 2284, 2049, 2650, 4331, 2221, 4233, 3803, 16, 1477, 4982, 220, 4089, 3042, 2591, 3651, 4868, 3031, 1689, 340, 105, 875, 2735, 4007, 3891, 2301, 4604, 2604, 2309, 1922, 1333, 3670, 795, 96, 4869, 11, 4973, 1871, 590, 717, 3285, 2362, 384, 1346, 4245, 4198, 1911, 1424, 2363, 1634, 1146, 4987, 3618, 3907, 701, 1952, 1448, 2589, 1142, 1618, 3653, 3367, 2517, 1840, 3241, 966, 3448, 2150, 3956, 4070, 1128, 1045, 3948, 1777, 932, 2768, 3697, 4792, 2450, 4897, 4659, 176, 33, 4262, 3550, 266, 910, 3143, 1681, 1371, 3316, 4284, 1720, 94, 2726, 2596, 2345, 4857, 3599, 3443, 4060, 4658, 2998, 3770, 2325, 738, 1389, 885, 4452, 2606, 1436, 913, 448, 2828, 151, 3820, 1339, 917, 2812, 1521, 4288, 2040, 2911, 1982, 865, 317, 3716, 2745, 3773, 463, 2699, 387, 1739, 630, 211, 665, 27, 3730, 2539, 4095, 4578, 4582, 4190, 4504, 2763, 3549, 4536, 2661, 1561, 2316, 1610, 434, 790, 476, 4308, 3390, 4149, 2657, 4237, 2447, 2508, 2453, 2226, 1595, 2233, 4023, 2592, 4453, 1855, 2719, 1809, 4333, 4847, 1660, 1838, 490, 1971, 1957, 2632, 4642, 4832, 2442, 837, 308, 408, 1104, 4579, 2361, 3804, 115, 1233, 4147, 3689, 1414, 3420, 2093, 97, 1118, 2851, 2075, 1387, 4664, 1256, 4015, 3493, 4477, 3423, 638, 1517, 2356, 4111, 1302, 3394, 613, 274, 2393, 3950, 3102, 813, 2576, 4421, 3135, 936, 1161, 4495, 3703, 3906, 2912, 1675, 1206, 4614, 2565, 4985, 4355, 2641, 84, 230, 4972, 2578, 4627, 4755, 2411, 427, 3774, 1100, 3860, 143, 2927, 3920, 2436, 1123, 2473, 3536, 2727, 3877, 1140, 3908, 4567, 4073, 2743, 772, 794, 4283, 1458, 3482, 2704, 4770, 2663, 1837, 4235, 4750, 4328, 4592, 3524, 3373, 2561, 3548, 2452, 2925, 1151, 2103, 3931, 4692, 4716, 1921, 3089, 2881, 4926, 1220, 677, 871, 2690, 3342, 4197, 3152, 113, 3809, 1394, 1581, 615, 2633, 180, 991, 240, 1263, 1478, 3117, 821, 4665, 893, 2923, 393, 4231, 184, 4553, 3124, 2520, 4323, 712, 1652, 1381, 524, 4840, 3024, 327, 3489, 3795, 4878, 1195, 4054, 3728, 697, 4564, 3317, 186, 1507, 4142, 1408, 2236, 4118, 3073, 1716, 4718, 130, 2753, 1042, 4907, 3673, 1490, 2498, 4882, 3440, 4810, 4123, 620, 4349, 2017, 4189, 139, 766, 465, 1746, 3422, 1978, 481, 1576, 3788, 3337, 467, 3169, 770, 223, 816, 1898, 4139, 2829, 2789, 2274, 1547, 1771, 1611, 1844, 4317, 1208, 2287, 1791, 2390, 259, 760, 3849, 1832, 2909, 2698, 3839, 3694, 4126, 745, 1230, 1566, 2759, 784, 2169, 4434, 3887, 2143, 4653, 3515, 3265, 4566, 1015, 923, 3483, 4585, 3924, 2120, 2892, 77, 4888, 64, 4409, 716, 4631, 2417, 920, 99, 1347, 3233, 1734, 1070, 867, 581, 1209, 2418, 4524, 609, 3098, 981, 2866, 3848, 2621, 1820, 339, 4258, 4777, 1983, 349, 2485, 1154, 2579, 2590, 2294, 4990, 1170, 1062, 618, 4332, 4534, 886, 4066, 2772, 1172, 934, 2547, 4641, 1363, 1529, 2241, 835, 870, 2941, 2245, 4811, 4732, 686, 3370, 4134, 1129, 1314, 3633, 4791, 3811, 495, 2997, 4214, 1641, 3791, 2801, 1542, 600, 719, 535, 3155, 2627, 398, 47, 1913, 2168, 2160, 1224, 3824, 3339, 4143, 3498, 2674, 2470, 3146, 1685, 4757, 1959, 2749, 3195, 4812, 134, 2039, 4160, 4960, 3276, 345, 2952, 1313, 2331, 4084, 3320, 3307, 1194, 3779, 4347, 1649, 4206, 194, 4849, 680, 1069, 851, 2412, 4460, 3880, 1765, 3576, 1288, 302, 2917, 1659, 3970, 3101, 614, 4920, 709, 1836, 3115, 4609, 898, 4606, 3330, 3302, 1434, 521, 982, 2827, 2455, 4908, 3543, 4916, 2261, 3600, 2171, 3605, 3106, 890, 1666, 2310, 566, 4719, 2344, 574, 4451, 1511, 85, 4971, 148, 3992, 2515, 2343, 1883, 1549, 300, 1425, 768, 3826, 1753, 4893, 2522, 507, 3684, 536, 1729, 967, 2272, 314, 4958, 3840, 2640, 2779, 53, 1821, 2031, 1711, 1225, 1422, 1254, 844, 3013, 2258, 2631, 3174, 516, 3397, 1683, 3267, 5005, 806, 4350, 4518, 1755, 2843, 4917, 2645, 814, 4965, 1515, 4146, 558, 1616, 4966, 4662, 2513, 3392, 4124, 529, 1646, 2586, 478, 4121, 3256, 798, 1684, 831, 264, 2199, 3661, 1271, 2353, 4278, 1105, 1003, 1760, 2859, 3336, 3810, 3767, 4279, 449, 1437, 1697, 4575, 4384, 901, 1859, 632, 232, 881, 3898, 1884, 458, 4104, 1889, 2773, 4900, 4874, 488, 4974, 3321, 3721, 2945, 234, 3091, 2022, 3294, 2042, 939, 3601, 1109, 80, 733, 4858, 3438, 3404, 4426, 2756, 1882, 4934, 2953, 2691, 217, 4192, 3945, 4456, 2394, 4608, 2023, 1041, 131, 4022, 141, 3857, 866, 3572, 4094, 4194, 203, 156, 4864, 1827, 3814, 2204, 612, 906, 4695, 2667, 4307, 3464, 206, 2397, 930, 2905, 59, 390, 3136, 4647, 2373, 275, 4369, 658, 1059, 1144, 4186, 400, 3837, 4037, 40, 89, 549, 1847, 2086, 2521, 3086, 2584, 15, 3628, 3208, 2281, 1617, 3766, 246, 3805, 1731, 3218, 2043, 104, 2005, 4457, 3384, 2202, 1918, 3310, 1589, 4281, 3244, 4676, 582, 4526, 4978, 748, 3425, 3202, 3144, 2677, 3533, 3641, 238, 4178, 1411, 3516, 1565, 431, 112, 995, 4753, 2375, 2858, 2322, 1465, 1718, 1903, 1340, 4253, 4050, 4500, 4087, 4803, 4065, 2328, 200, 382, 1651, 2207, 4141, 2179, 35, 3247, 3034, 1724, 4643, 4303, 3782, 2935, 397, 1498, 3189, 1540, 4892, 2340, 599, 2687, 2904, 2702, 120, 126, 1483, 138, 3192, 824, 1338, 4475, 4986, 2053, 1633, 4885, 4298, 4356, 2805, 874, 3926, 624, 2660, 4802, 3323, 1485, 1417, 1303, 3967, 3745, 3636, 4930, 1400, 3084, 4020, 2761, 4135, 1030, 3933, 2444, 3975, 3375, 3978, 4164, 1502, 1181, 407, 3677, 272, 3776, 4717, 993, 4226, 1385, 2623, 1357, 634, 1531, 3525, 2036, 1143, 1479, 3829, 2153, 2222, 3481, 715, 969, 3471, 3574, 2477, 1095, 526, 1805, 2752, 2174, 309, 2622, 1447, 591, 4707, 98, 1034, 1727, 4946, 3888, 4428, 4322, 869, 4842, 4043, 3553, 984, 1469, 4422, 3616, 737, 2504, 1193, 4313, 50, 4915, 4929, 4200, 551, 2321, 3386, 4996, 3137, 552, 4114, 586, 4491, 2047, 4560, 2733, 2038, 4747, 4150, 876, 244, 503, 3030, 19, 2774, 2218, 3588, 2433, 802, 494, 1708, 1492, 1500, 2035, 3350, 380, 2263, 1052, 3885, 3831, 1672, 70, 4678, 2955, 1899, 1332, 399, 2330, 4834, 2794, 296, 1234, 4394, 4997, 3680, 1597, 3100, 2441, 1189, 1822, 2510, 4285, 4102, 2166, 2922, 4872, 4395, 1930, 1834, 475, 3303, 4170, 1972, 2653, 1251, 700, 3672, 2097, 4154, 4458, 783, 3965, 2004, 1937, 4909, 2978, 3778, 39, 1572, 1636, 3596, 1343, 3129, 3979, 256, 3168, 3012, 761, 1979, 4466, 1316, 3961, 2757, 4654, 2279, 4396, 4393, 1520, 1006, 735, 505, 3657, 4623, 4259, 3268, 3097, 1257, 3022, 1366, 1326, 723, 588, 4116, 311, 3329, 780, 3490, 3193, 2223, 3288, 1354, 1861, 4841, 91, 918, 2398, 1996, 3130, 3671, 1019, 4297, 68, 2725, 2781, 2228, 4690, 509, 1462, 2647, 1756, 1872, 2195, 2740, 3497, 1383, 4074, 3048, 2933, 4365, 3845, 2267, 1323, 4039, 4030, 164, 3393, 2875, 2252, 978, 3938, 1843, 3051, 3259, 3459, 4944, 199, 3116, 1014, 3593, 868, 2611, 2585, 1351, 1388, 445, 208, 82, 1742, 1026, 128, 4382, 706, 3078, 2198, 4276, 283, 1180, 977, 2188, 2213, 4412, 1626, 3737, 673, 1376, 3487, 928, 1229, 911, 4912, 1350, 1925, 3226, 4670, 414, 3517, 72, 4172, 1078, 2259, 2614, 2037, 3645, 4398, 1909, 1337, 4415, 3760, 4105, 4705, 1043, 1125, 4855, 3452, 2104, 287, 671, 557, 2003, 3996, 1047, 4898, 2985, 1551, 1819, 4587, 4919, 2176, 2404, 4655, 1795, 907, 2025, 453, 3873, 4876, 3037, 2989, 2509, 3090, 1471, 2724, 4523, 1200, 684, 4558, 4691, 1773, 975, 4780, 4668, 3121, 2902, 2102, 1058, 1348, 3830, 2810, 1526, 4741, 3383, 1001, 3326, 4519, 4482, 4055, 1060, 1176, 4274, 1704, 4293, 1706, 9, 3290, 1817, 1853, 1749, 4809, 2771, 1342, 3093, 4680, 1236, 1481, 1924, 2648, 3740, 3709, 1077, 2652, 4619, 2957, 2705, 4364, 2947, 173, 168, 4505, 4501, 485, 3579, 610, 3578, 4064, 1135, 3451, 4904, 2010, 1849, 2205, 2403, 4224, 2250, 3696, 2435, 1897, 3318, 2387, 3736, 4387, 2029, 4472, 2518, 1863, 225, 2175, 14, 3935, 3074, 2573, 3937, 1733, 1002, 1412, 1488, 3322, 704, 952, 1815, 4822, 998, 945, 4724, 4251, 3919, 3577, 2080, 3291, 2105, 4036, 2857, 919, 3406, 3272, 2680, 3733, 947, 3343, 2804, 3752, 3297, 2051, 4166, 849, 4838, 1334, 1092, 4922, 2956, 2388, 882, 3775, 4229, 4796, 3813, 1569, 3714, 1102, 2597, 2019, 36, 325, 1574, 2888, 4223, 4417, 1833, 3315, 3182, 3615, 4165, 940, 1700, 4265, 2914, 3235, 3750, 155, 4191, 3083, 247, 4952, 4790, 466, 2210, 3150, 3763, 4628, 691, 724, 1336, 3951, 263, 2529, 2921, 3821, 3219, 2203, 848, 4272, 1370, 276, 2815, 954, 1494, 2354, 4201, 2201, 1039, 3654, 7, 3151, 989, 856, 103, 2846, 3203, 178, 1379, 1715, 4682, 294, 616, 3928, 726, 4234, 4448, 4248, 635, 3946, 4683, 318, 2965, 598, 953, 729, 4419, 1736, 3103, 3325, 292, 1395, 3844, 282, 853, 799, 3528, 214, 1345, 4527, 4128, 137, 1830, 2487, 2821, 4473, 2825, 3298, 1289, 1120, 3723, 1369, 4435, 4576, 5011, 3916, 3333, 1093, 44, 825, 775, 3586, 1866, 1457, 1580, 4057, 3046, 3876, 3014, 4232, 4488, 2116, 2096, 1722, 3629, 2293, 228, 1541, 3289, 3274, 1732, 3362, 2290, 2526, 4153, 4852, 2311, 4846, 122, 299, 2248, 1452, 2425, 4959, 1555, 1496, 3161, 3587, 4639, 3570, 3331, 1810, 1480, 350, 1584, 1177, 2658, 1056, 4789, 2109, 1285, 3632, 4383, 3647, 3122, 1687, 2155, 2437, 1974, 374, 2744, 4752, 1596, 1793, 792, 1134, 4759, 46, 924, 1293, 2094, 1535, 2741, 4320, 1879, 2385, 3968, 3990, 1360, 817, 1082, 4280, 4242, 2469, 2967, 4343, 497, 1156, 4238, 3972, 2644, 1482, 514, 4361, 2419, 4163, 731, 1721, 669, 4951, 4942, 3850, 1539, 401, 2818, 437, 1439, 956, 2665, 1061, 4736, 1445, 4311, 1781, 1182, 2359, 980, 121, 3555, 1998, 4236, 2495, 3494, 3590, 5010, 2464, 4508, 4, 1136, 1221, 4793, 4067, 803, 1212, 2165, 652, 3575, 2728, 3552, 3224, 1557, 1579, 1277, 4563, 3929, 1223, 2895, 4246, 3407, 2415, 2484, 513, 4674, 4041, 3088, 2664, 4769, 1468, 3614, 3020, 2190, 2841, 2700, 4598, 3355, 4136, 4314, 2456, 3178, 2783, 759, 3841, 4774, 441, 1719, 3108, 3878, 158, 976, 498, 1431, 1196, 4199, 2581, 2977, 4983, 4008, 2962, 2884, 4430, 4557, 4899, 26, 5001, 3914, 3295, 4580, 4778, 10, 3153, 201, 2291, 3909, 4938, 3879, 4788, 1192, 4420, 1024, 4499, 3777, 2994, 1419, 3700, 1588, 606, 4221, 1601, 2890, 1782, 4109, 502, 728, 2494, 4222, 4263, 4962, 3354, 1702, 3580, 4503, 4380, 3801, 1570, 2136, 3439, 1297, 3396, 3749, 4713, 1514, 417, 3500, 2117, 2785, 2099, 303, 2420, 826, 2568, 4211, 2317, 4275, 4210, 3421, 2503, 617, 185, 1083, 3646, 420, 351, 1328, 1122, 86, 3213, 3504, 1958, 4344, 2414, 1757, 1587, 1219, 2016, 4722, 2130, 2964, 191, 108, 1205, 2984, 2124, 1949, 2192, 4731, 450, 1438, 1397, 1101, 2527, 17, 2285, 190, 1283, 4742, 1976, 1186, 1413, 4883, 1825, 4378, 4879, 2824, 842, 734, 3542, 889, 1237, 4581, 3939, 4617, 1403, 4964, 3110, 4569, 2101, 418, 3510, 3861, 2737, 212, 4603, 2048, 5004, 3827, 965, 1308, 368, 2671, 4489, 4733, 3352, 4171, 1571, 2315, 2429, 2817, 595, 4406, 2942, 4851, 3225, 828, 1772, 2706, 428, 554, 1410, 4887, 4131, 3159, 261, 4894, 1073, 1396, 1510, 4219, 3253, 4335, 2121, 3007, 3977, 3705, 2045, 2115, 2277, 2949, 3882, 3299, 3468, 2593, 4696, 4848, 1620, 1010, 3713, 3539, 4921, 986, 291, 3223, 4625, 2489, 4844, 1606, 4239, 611, 4097, 3699, 4875, 4903, 4327, 3637, 902, 249, 3597, 4423, 4402, 1349, 1211, 1484, 4950, 1440, 3698, 660, 2146, 4115, 3003, 4685, 2963, 4490, 2883, 106, 2891, 2225, 4337, 3436, 2181, 3175, 4316, 1735, 63, 1528, 1816, 12, 109, 281, 2008, 456, 3982, 1509, 2861, 3545, 2320, 587, 3610, 2471, 1880, 2249, 2676, 668, 4461, 3941, 4137, 1705, 3639, 1222, 1942, 2490, 1279, 3925, 107, 1264, 2787, 4063, 1089, 1644, 5000, 1568, 4547, 2342, 65, 2242, 1789, 572, 3999, 2196, 4924, 2396, 4108, 2238, 1149, 3603, 3644, 2410, 3853, 676, 175, 3349, 4902, 2349, 4646, 4300, 523, 2180, 3940, 4388, 4289, 4159, 4407, 3154, 1497, 2067, 1179, 4079, 3918, 4513, 2534, 277, 561, 2970, 1878, 2275, 3312, 996, 4870, 1027, 921, 2848, 1309, 489, 970, 3418, 1841, 3685, 307, 1713, 916, 4334, 2634, 1851, 3346, 1487, 3398, 1327, 425, 3057, 1372, 383, 4957, 2551, 3729, 1578, 1900, 3862, 4744, 1231, 880, 927, 4085, 2219, 3475, 4961, 457, 1640, 87, 1940, 1185, 4786, 4650, 3002, 649, 2936, 1938, 960, 1564, 4465, 1386, 2871, 1467, 3666, 3469, 878, 3215, 531, 4463, 4593, 3080, 295, 3687, 419, 1906, 127, 3875, 2790, 959, 4720, 4025, 3922, 3988, 3648, 49, 3530, 3382, 687, 2897, 4515, 3478, 4241, 3281, 3513, 262, 3484, 710, 4379, 355, 51, 862, 4699, 2216, 3340, 2703, 2516, 1960, 2628, 3450, 4144, 2896, 3465, 650, 4927, 3976, 2822, 812, 4376, 3581, 3534, 4099, 1573, 123, 1759, 805, 4932, 4321, 4215, 226, 3486, 3055, 4424, 1265, 3719, 827, 1472, 5003, 2506, 4667, 1270, 159, 2214, 2746, 4459, 3783, 3952, 1112, 4663, 3009, 1857, 2599, 2460, 4703, 1262, 3052, 3808, 3986, 258, 685, 3004, 601, 1775, 90, 4644, 1329, 2900, 4831, 3141, 3474, 1613, 3547, 3792, 265, 3980, 1748, 2197, 3437, 3461, 4076, 4729, 229, 504, 4069, 4739, 1813, 1993, 83, 3522, 1126, 4478, 3565, 1096, 1619, 254, 944, 4414, 1364, 2960, 1281, 3780, 1762, 2332, 1406, 224, 1356, 4168, 702, 252, 2052, 4561, 1598, 546, 559, 3681, 4661, 3794, 2854, 3035, 592, 3604, 1048, 3032, 3790, 2969, 2088, 2208, 237, 2264, 2319, 182, 4404, 3434, 4371, 3405, 1138, 2323, 4745, 747, 4640, 1967, 1050, 1407, 1984, 4989, 2666, 1164, 3816, 4896, 2402, 4437, 3309, 4227, 3424, 1247, 4120, 4310, 4721, 1962, 2092, 2307, 3786, 4525, 3016, 4455, 2427, 118, 1113, 4086, 1405, 4804, 2709, 2251, 4470, 2646, 181, 2185, 4449, 2157, 879, 338, 1116, 4918, 3523, 541, 725, 3720, 3279, 4029, 2831, 2992, 1694, 3194, 4254, 800, 3995, 4352, 2548, 4213, 4381, 3872, 1554, 2669, 3874, 1215, 1390, 4853, 1374, 2872, 2161, 3732, 1621, 4004, 3619, 2406, 2069, 4783, 2809, 4187, 3505, 4666, 3359, 1169, 3974, 3611, 1710, 3662, 4411, 2793, 4035, 2074, 1799, 93, 1552, 1432, 4571, 888, 1544, 2523, 1065, 948, 4746, 243, 4807, 2844, 4148, 4410, 1703, 4937, 946, 153, 3061, 3025, 4100, 1745, 2796, 1513, 3594, 1246, 4771, 2847, 3476, 3796, 3942, 278, 3674, 2886, 548, 5008, 1085, 3559, 847, 852, 2126, 2395, 2934, 1110, 3328, 2973, 114, 4044, 929, 3056, 4671, 4363, 62, 539, 2244, 1284, 3453, 1268, 195, 3197, 1936, 1375, 1382, 4939, 3917, 4360, 1272, 607, 3123, 1638, 1818, 3368, 3185, 4956, 4301, 3018, 358, 4945, 2856, 1017, 1860, 3602, 1537, 4354, 2000, 1614, 1086, 1989, 1489, 776, 297, 413, 3622, 4968, 1213, 1943, 4162, 386, 2656, 2141, 4390, 3222, 2683, 3413, 2483, 2736, 2524, 3496, 3702, 4773, 2013, 372, 2654, 323, 3609, 1868, 743, 2027, 1823, 2714, 270, 4657, 2990, 1990, 1954, 2055, 359, 1310, 1895, 4550, 4761, 567, 4273, 4610, 3401, 3871, 3962, 4439, 3028, 4140, 2712, 2462, 1428, 3852, 556, 2731, 1055, 4027, 3812, 4711, 2675, 4706, 1980, 3851, 2811, 2095, 4161, 3311, 999, 764, 1173, 4768, 2021, 3503, 3128, 2327, 4270, 3286, 378, 2616, 3041, 1401, 1040, 3250, 2501, 4370, 3165, 2530, 2601, 4342, 1991, 3626, 2306, 1875, 4006, 3864, 3625, 1968, 4963, 3209, 3296, 872, 4127, 2408, 3248, 3410, 3667, 1887, 4511, 3869, 2070, 1543, 227, 1032, 1723, 4173, 1242, 218, 3403, 1914, 1877, 4196, 4881, 2217, 3584, 1287, 2071, 3623, 3114, 2138, 4208, 4651, 1787, 4992, 3085, 2826, 1322, 988, 1331, 4405, 3126, 3991, 1201, 2377, 4701, 3589, 2341, 4339, 730, 2260, 1631, 2073, 3495, 4416, 2167, 4469, 2995, 625, 3164, 1319, 2642, 25, 43, 4362, 681, 1667, 2738, 3551, 2366, 2079, 1312, 720, 3892, 3678, 987, 4854, 510, 2915, 1261, 273, 4012, 1232, 2557, 3045, 3060, 4730, 3521, 4612, 2018, 3280, 2461, 162, 3293, 608, 4715, 4941, 2928, 2701, 3901, 877, 3663, 1470, 1464, 4954, 4479, 3119, 3292, 4001, 2961, 4749, 3833, 2229, 379, 992, 1623, 111, 850, 422, 3210, 286, 3327, 2365, 3960, 315, 2519, 2538, 2098, 1682, 129, 4520, 1630, 29, 678, 1203, 389, 2007, 3113, 459, 3886, 4514, 2269, 205, 4758, 146, 1491, 487, 3157, 1020, 1950, 796, 1276, 1402, 187, 2220, 4101, 2553, 894, 2142, 2864, 4481, 4119, 4782, 3411, 3742, 859, 931, 1168, 3635, 2607, 4019, 348, 328, 2587, 1474, 4760, 444, 4053, 2443, 4533, 3460, 3658, 4925, 4299, 4010, 2588, 2077, 2988, 2570, 3275, 1814, 1774, 4038, 3458, 3726, 3502, 2151, 1512, 3033, 4554, 3903, 56, 914, 781, 627, 4485, 4710, 1679, 4113, 4071, 4471, 4817, 1964, 2786, 786, 2187, 3067, 4538, 4548, 4981, 3511, 71, 1910, 2191, 4901, 2762, 360, 3764, 2134, 3039, 4743, 4450, 682, 3015, 204, 904, 2609, 3236, 3369, 2351, 4572, 2679, 183, 949, 3751, 2550, 57, 3001, 430, 2996, 4078, 438, 4220, 4648, 30, 762, 750, 778, 316, 1560, 1292, 533, 3932, 3834, 4306, 2458, 2946, 4083, 2122, 2034, 3172, 3447, 2237, 3258, 3759, 285, 1867, 100, 3021, 4863, 1290, 1184, 157, 763, 1929, 4032, 1648, 688, 4243, 4532, 1493, 3526, 771, 2608, 1344, 2139, 1250, 3162, 3655, 3582, 2760, 2797, 2044, 2370, 3372, 305, 884, 1051, 188, 808, 1133, 4021, 1691, 4686, 3357, 2064, 655, 699, 1870, 1218, 3867, 3131, 1217, 522, 2380, 2723, 2959, 1778, 4444, 443, 4294, 3249, 3335, 440, 933, 2571, 215, 2528, 2270, 2145, 1779, 2399, 1317, 1896, 3912, 3345, 1920, 4132, 435, 4264, 3463, 1663, 3868, 3754, 3902, 2372, 3442, 937, 4287, 4117, 2983, 3095, 1812, 2549, 2113, 4545, 1582, 1726, 4826, 4748, 469, 1604, 6, 3432, 577, 4431, 4072, 2140, 20, 429, 4551, 2465, 3374, 4145, 1609, 656, 4107, 2580, 754, 663, 2423, 4024, 530, 2555, 3741, 4785, 37, 4000, 628, 1064, 4677, 961, 2347, 4202, 1435, 335, 3029, 2775, 1018, 4374, 167, 3200, 501, 695, 2765, 3229, 3212, 5, 3954, 3781, 4798, 3971, 2381, 651, 3762, 4091, 788, 1499, 838, 23, 3180, 1451, 1886, 2569, 1546, 3082, 2617, 749, 1076, 3981, 3998, 664, 1259, 4697, 1107, 4726, 4632, 3430, 4372, 2256, 2346, 3631, 2879, 4656, 1299, 396, 4138, 3324, 1677, 4767, 269, 1148, 3838, 353, 2426, 4062, 4936, 5002, 3923, 4034, 1776, 101, 2137, 3158, 2430, 3983, 1358, 4998, 3997, 3356, 3445, 3583, 2672, 3431, 2302, 1449, 2678, 829, 1695, 2440, 597, 596, 2649, 446, 4779, 3787, 1661, 326, 117, 2819, 3433, 1157, 1874, 895, 4312, 4502, 767, 2243, 1796, 3112, 1139, 2754, 4454, 1075, 472, 2948, 2407, 2563, 4949, 4003, 2770, 4815, 4589, 2246, 1426, 149, 2421, 1430, 1066, 4330, 241, 2732, 2531, 4498, 172, 2112, 3005, 2620, 1398, 3287, 163, 3987, 4574, 840, 1501, 2184, 500, 1298, 4492, 4843, 31, 3470, 4940, 3006, 1808, 2880, 2697, 3351, 2820, 2108, 3319, 2286, 564, 3142, 1311, 1130, 4480, 1227, 4565, 4318, 578, 1590, 2751, 391, 2383, 1852, 1876, 1167, 2603, 142, 1997, 4433, 4408, 3047, 13, 1304, 3234, 2439, 4999, 3963, 2329, 3305, 1442, 2449, 4432, 312, 3934, 4975, 4507, 1198, 1188, 2718, 4829, 4445, 4794, 3634, 3378, 147, 753, 133, 2863, 3711, 1306, 1763, 2968, 3608, 496, 4217, 4522, 905, 2739, 4636, 4784, 2564, 2651, 3228, 4125, 4756, 2730, 3462, 404, 2619, 2065, 1178, 2231, 3477, 683, 1124, 3283, 4442, 3139, 4607, 455, 3676, 124, 1476, 3245, 2445, 2689, 4250, 2468, 703, 1585, 4005, 2015, 2239, 257, 3264, 1801, 2878, 4765, 4754, 239, 4537, 3825, 2823, 2932, 1622, 765, 3606, 3391, 4704, 4652, 4602, 2081, 4429, 3712, 2684, 1891, 48, 2816, 4775, 1761, 4911, 1068, 284, 1418, 525, 793, 1935, 2240, 2916, 3456, 4622, 2159, 636, 3718, 3053, 1839, 1022, 319, 3947, 943, 646, 4052, 332, 573, 3769, 4645, 3096, 2338, 633, 362, 4535, 4994, 4637, 3050, 4096, 3806, 1939, 3613, 2162, 1391, 4727, 565, 3262, 3094, 394, 18, 4230, 2282, 3944, 3365, 4861, 1829, 4555, 2784, 3881, 4512, 2091, 1975, 1946, 4286, 3756, 2276, 280, 1751, 1709, 2979, 1454, 2273, 3793, 152, 4980, 1798, 4787, 3668, 78, 4269, 2110, 2782, 4850, 3897, 248, 2673, 2937, 2400, 4348, 1671, 4837, 553, 1446, 4175, 4828, 2535, 2178, 2295, 4823, 4373, 321, 2734, 3105, 1249, 4058, 2950, 1127, 4047, 4075, 447, 3842, 2630, 2492, 2982, 773, 2807, 1686, 4586, 409, 4026, 1016, 3784, 4218, 1495, 1324, 1008, 4110, 1099, 4714, 855, 1361, 3408, 3994, 883, 985, 171, 3426, 3428, 2262, 990, 1864, 3270, 2466, 3364, 897, 3363, 4590, 213, 1021, 2974, 637, 492, 1902, 2491, 3573, 166, 604, 267, 854, 4679, 367, 1359, 689, 1486, 1155, 342, 2769, 1662, 1905, 4048, 4209, 1079, 3257, 4282, 8, 2766, 1392, 1214, 4895, 2711, 3688, 2512, 2292, 3953, 1754, 2814, 742, 714, 641, 2170, 1701, 433, 1091, 1269, 2849, 1965, 1586, 3650, 3344, 4167, 3569, 464, 1183, 4814, 818, 722, 2670, 2764, 1409, 2635, 2303, 860, 983, 791, 1238, 3665, 2474, 1416, 3656, 1917, 2144, 1245, 3856, 4541, 2913, 2057, 2032, 3921, 926, 1025, 3529, 4935, 177, 2852, 4819, 1846, 4835, 3866, 2283, 3232, 1945, 3863, 1165, 797, 4183, 2688, 1267, 2920, 216, 4824, 2337, 2054, 3008, 336, 1153, 3332, 471, 2710, 925, 4584, 1607, 4615, 3062, 1159, 2717, 1158, 483, 3377, 3984, 1166, 3, 3044, 3758, 4801, 1717, 347, 4928, 3955, 2839, 4597, 3612, 3170, 3314, 769, 2215, 3915, 1556, 4766, 3353, 1592, 233, 470, 3561, 4351, 3506, 3717, 2200, 3415, 415, 3179, 2357, 3899, 4681, 1856, 4296, 1999, 4357, 1046, 3231, 935, 61, 2562, 4738, 301, 2836, 589, 4913, 55, 2722, 1378, 4772, 1645, 1988, 3725, 2209, 2278, 2643, 1741, 3043, 2434, 789, 2629, 4630, 354, 454, 2951, 1175, 1038, 140, 3789, 4977, 3895, 410, 3884, 3467, 1995, 2149, 2906, 1632, 1004, 4268, 2024, 2552, 1673, 2802, 2655, 2610, 1503, 1881, 1559, 1071, 3708, 2389, 3347, 2308, 2832, 3564, 3704, 3118, 1282, 3252, 3959, 2379, 2312, 972, 3066, 3092, 1137, 3727, 2391, 4740, 705, 3109, 3184, 2910, 352, 4818, 1098, 845, 4923, 3669, 3455, 1508, 3040, 3167, 2842, 60, 1981, 696, 2156, 1460, 357, 1162, 369, 2813, 4180, 2615, 3160, 4367, 621, 527, 3957, 4995, 2980, 4797, 3301, 4340, 751, 2078, 811, 506, 2511, 2540, 1114, 4291, 290, 1803, 165, 3111, 1518, 4277, 2999, 2767, 4386, 909, 439, 2486, 2374, 4240, 4441, 2777, 1670, 4621, 3807, 4042, 4169, 4151, 3854, 4867, 4476, 210, 4081, 2463, 3227, 3690, 1653, 4540, 3011, 3969, 2919, 3389, 236, 1291, 3692, 4093, 2867, 1330, 2514, 3638, 584, 3156, 3785, 2392, 1466, 2833, 499, 2954, 3659, 809, 1656, 1806, 2559, 371, 1752, 1956, 3070, 2076, 4443, 2692, 4700, 3958, 3254, 4684, 830, 1191, 4626, 4845, 462, 3491, 2758, 2685, 3520, 583, 2355, 2472, 4305, 4392, 2333, 3134, 4599, 2030, 1707, 231, 4002, 912, 480, 4836, 4389, 2062, 2288, 2613, 1698, 2944, 3943, 1973, 310, 1599, 2085, 4611, 1252, 2544, 341, 1278, 1341, 1258, 1427, 1728, 1770, 198, 2694, 1862, 1802, 3017, 4425, 356, 3068, 4436, 3027, 575, 1011, 3479, 963, 411, 4506, 2296, 3238, 4474, 388, 4510, 2575, 1766, 2318, 2060, 67, 52, 2594, 3186, 392, 4260, 3187, 3695, 2636, 540, 1603, 3765, 1784, 260, 3512, 547, 2626, 482, 4252, 4600, 385, 3079, 423, 3822, 2072, 1600, 1199, 1647, 1548, 4577, 2716, 1063, 2546, 2083, 3376, 3735, 1788, 2850, 3546, 2975, 3163, 2532, 4447, 4969, 2715, 304, 896, 1119, 4203, 2297, 1444, 1977, 242, 1624, 3541, 4267, 1780, 605, 2907, 5007, 674, 2885, 3743, 4984, 1907, 4712, 2046, 2834, 1321, 639, 42, 289, 2012, 331, 1664, 4688, 3949, 4859, 4244, 512, 4397, 973, 4976, 79, 3568, 3893, 3617, 1635, 693, 2058, 2459, 4403, 1538, 337, 3190, 3417, 3063, 4377, 1807, 221, 1152, 2118, 3395, 160, 4261, 994, 1963, 1197, 3214, 2605, 253, 1941, 4693, 145, 740, 1612, 3855, 4228, 1296, 3133, 1275, 3149, 125, 3449, 144, 3071, 3261, 4605, 2583, 563, 2488, 2493, 3277, 2416, 2958, 1767, 1057, 3371, 4292, 2100, 1892, 1730, 1108, 1315, 515, 3518, 81, 787, 322, 2131, 4988, 2014, 3251, 1532, 1869, 3138, 3076, 2431, 4953, 1594, 3054, 3000, 2798, 271, 1240, 4358, 4877, 2268, 3630, 2803, 1, 593, 41, 3075, 4596, 1243, 3492, 1783, 2475, 2348, 4090, 3911, 4821, 4862], "validation": [5186, 5288, 5248, 5440, 5372, 5235, 5435, 5097, 5150, 5472, 5148, 5436, 5218, 5289, 5337, 5340, 5475, 5135, 5060, 5066, 5219, 5168, 5375, 5434, 5050, 5363, 5442, 5083, 5036, 5172, 5351, 5137, 5029, 5383, 5143, 5298, 5461, 5278, 5252, 5213, 5156, 5349, 5127, 5230, 5282, 5121, 5486, 5038, 5214, 5476, 5377, 5188, 5302, 5242, 5092, 5227, 5406, 5290, 5264, 5133, 5045, 5190, 5224, 5444, 5115, 5321, 5396, 5228, 5129, 5285, 5364, 5459, 5408, 5200, 5279, 5464, 5458, 5154, 5205, 5462, 5294, 5237, 5072, 5381, 5031, 5446, 5415, 5247, 5016, 5284, 5249, 5152, 5276, 5465, 5119, 5470, 5502, 5024, 5358, 5220, 5312, 5422, 5112, 5267, 5499, 5090, 5184, 5113, 5145, 5454, 5310, 5433, 5049, 5155, 5177, 5015, 5160, 5157, 5102, 5474, 5046, 5457, 5306, 5494, 5398, 5058, 5355, 5485, 5365, 5151, 5268, 5449, 5075, 5386, 5118, 5335, 5416, 5438, 5295, 5367, 5159, 5480, 5093, 5144, 5099, 5192, 5297, 5250, 5504, 5035, 5234, 5360, 5317, 5179, 5334, 5023, 5221, 5487, 5413, 5431, 5063, 5189, 5122, 5057, 5511, 5269, 5239, 5291, 5286, 5084, 5068, 5203, 5328, 5341, 5018, 5331, 5223, 5162, 5384, 5271, 5484, 5393, 5402, 5226, 5165, 5111, 5215, 5479, 5110, 5081, 5326, 5501, 5187, 5507, 5460, 5492, 5082, 5339, 5123, 5369, 5178, 5166, 5217, 5456, 5320, 5354, 5468, 5076, 5471, 5373, 5153, 5445, 5211, 5411, 5185, 5182, 5251, 5361, 5265, 5091, 5141, 5313, 5138, 5103, 5315, 5095, 5176, 5263, 5366, 5130, 5225, 5020, 5345, 5430, 5409, 5209, 5272, 5198, 5421, 5324, 5210, 5098, 5379, 5087, 5325, 5376, 5404, 5019, 5510, 5114, 5013, 5089, 5483, 5105, 5509, 5074, 5423, 5378, 5356, 5319, 5051, 5171, 5374, 5032, 5254, 5040, 5496, 5053, 5104, 5270, 5149, 5419, 5073, 5481, 5139, 5229, 5033, 5163, 5108, 5026, 5495, 5180, 5131, 5491, 5065, 5062, 5316, 5414, 5096, 5059, 5469, 5136, 5207, 5259, 5425, 5208, 5069, 5100, 5330, 5455, 5241, 5477, 5357, 5380, 5308, 5134, 5323, 5022, 5307, 5417, 5021, 5183, 5410, 5080, 5488, 5353, 5255, 5042, 5030, 5418, 5222, 5432, 5266, 5473, 5401, 5167, 5261, 5296, 5256, 5027, 5352, 5191, 5064, 5146, 5505, 5094, 5350, 5448, 5232, 5370, 5174, 5498, 5126, 5260, 5128, 5336, 5299, 5390, 5333, 5407, 5071, 5034, 5437, 5253, 5012, 5318, 5140, 5343, 5204, 5314, 5017, 5429, 5107, 5412, 5043, 5304, 5161, 5389, 5344, 5245, 5132, 5441, 5025, 5257, 5490, 5338, 5054, 5447, 5388, 5077, 5240, 5451, 5194, 5292, 5244, 5047, 5088, 5061, 5497, 5078, 5382, 5348, 5085, 5028, 5309, 5101, 5246, 5109, 5385, 5079, 5055, 5274, 5399, 5305, 5175, 5197, 5443, 5281, 5039, 5206, 5231, 5067, 5482, 5277, 5117, 5405, 5273, 5327, 5201, 5420, 5243, 5322, 5332, 5086, 5041, 5452, 5169, 5173, 5193, 5500, 5439, 5368, 5466, 5329, 5508, 5181, 5120, 5359, 5311, 5044, 5426, 5196, 5347, 5427, 5428, 5280, 5216, 5394, 5387, 5056, 5258, 5503, 5158, 5236, 5106, 5147, 5275, 5116, 5493, 5199, 5037, 5070, 5467, 5403, 5478, 5400, 5346, 5238, 5142, 5048, 5342, 5395, 5450, 5287, 5300, 5262, 5283, 5392, 5424, 5463, 5362, 5397, 5391, 5489, 5014, 5124, 5371, 5164, 5506, 5125, 5453, 5052, 5212, 5301, 5303, 5293, 5233, 5195, 5202, 5170], "test": [5739, 5551, 5880, 5604, 5886, 5917, 5845, 5556, 5755, 5809, 5985, 5687, 5637, 5588, 6000, 5677, 5913, 5730, 5554, 5912, 5600, 5727, 5628, 5905, 6004, 5742, 5881, 6006, 5520, 6008, 5967, 5791, 5868, 5690, 5521, 5928, 5927, 5618, 5803, 5788, 5762, 5753, 5771, 5804, 5824, 5883, 5537, 5567, 5893, 5805, 5647, 5530, 5557, 5576, 5664, 5902, 5736, 5591, 5655, 5978, 5682, 5698, 5885, 5901, 5794, 5943, 5781, 5989, 5842, 5800, 5877, 5658, 5934, 5773, 5700, 5579, 5818, 5683, 5630, 5540, 5724, 5645, 5915, 5577, 5853, 5735, 5744, 5546, 5865, 5900, 5659, 5790, 5514, 5680, 5714, 5932, 5512, 5536, 5871, 6010, 5933, 5711, 5513, 5656, 5819, 5929, 5641, 5832, 5594, 5695, 5896, 6007, 5990, 5936, 5614, 6011, 5571, 5766, 5836, 5531, 5639, 5666, 5891, 5916, 5652, 5816, 5748, 5969, 5918, 5661, 5713, 5672, 5774, 5617, 5515, 5740, 5813, 5921, 5904, 5761, 5837, 5560, 5898, 5582, 5767, 6001, 5941, 5667, 5830, 5605, 5982, 5882, 5578, 5889, 5958, 5846, 5632, 5980, 5729, 5543, 5555, 5801, 5721, 5759, 5723, 5752, 5895, 5780, 5589, 5764, 5821, 5833, 5733, 5596, 5769, 5697, 5563, 5887, 5720, 5526, 5675, 5679, 5930, 5534, 5542, 6009, 5674, 5631, 5590, 5538, 5535, 5994, 5942, 5802, 5878, 5786, 5906, 5998, 5857, 5726, 5991, 5968, 5856, 5909, 5559, 5619, 5778, 5844, 5908, 5860, 5584, 5975, 5704, 5694, 5541, 5732, 5633, 5587, 5952, 5750, 5783, 5976, 5706, 5811, 5663, 5843, 5903, 5709, 5999, 5650, 5725, 5799, 5789, 5796, 5988, 5610, 5606, 5763, 5812, 5612, 5995, 5572, 5765, 5670, 5524, 5793, 5715, 5615, 5701, 5719, 5689, 5841, 5624, 5911, 5850, 5867, 5550, 5772, 5599, 5894, 5613, 5518, 5564, 5539, 5548, 5731, 5947, 5703, 5561, 5876, 5925, 5984, 5866, 5945, 5970, 5870, 5872, 5840, 5552, 5678, 5525, 5528, 5699, 5966, 5951, 5562, 5671, 5668, 5607, 5955, 5779, 5640, 5646, 5519, 5691, 5684, 5657, 5792, 5575, 5961, 5922, 5785, 5716, 5573, 5517, 5784, 5707, 5829, 5696, 5848, 5847, 5950, 5873, 5981, 5601, 5948, 5751, 5974, 5644, 5583, 5620, 5593, 5585, 5702, 5993, 5625, 5746, 5643, 5807, 5920, 5960, 5602, 5858, 5747, 5965, 5685, 5749, 5549, 5914, 5987, 5558, 5728, 5935, 5676, 5938, 5717, 5527, 5839, 5738, 5770, 5997, 5621, 5629, 5627, 5708, 5888, 5831, 5608, 5864, 5522, 5851, 5705, 5673, 5806, 5959, 5875, 5931, 5586, 5592, 5979, 5581, 5940, 5532, 5533, 5597, 6003, 5890, 5926, 5822, 5756, 5669, 5874, 5745, 5923, 5648, 5623, 5757, 5692, 5566, 5937, 5869, 5946, 5854, 5983, 5722, 5743, 5649, 5787, 5826, 5852, 5834, 5603, 5710, 5754, 5553, 5996, 5638, 5817, 5516, 5944, 5823, 5636, 5545, 5642, 5956, 5820, 5907, 5953, 5622, 5523, 5568, 5838, 5892, 5569, 5827, 5972, 5626, 5971, 5609, 5957, 5547, 5808, 6002, 5986, 5924, 5660, 5835, 5712, 5758, 5653, 5734, 5574, 5815, 5776, 5737, 5662, 5635, 5992, 5825, 5954, 5686, 5855, 6005, 5962, 5611, 5651, 5595, 5919, 5598, 5949, 5863, 5693, 5782, 5861, 5939, 5828, 5741, 5570, 5899, 5580, 5665, 5964, 5634, 5775, 5798, 5897, 5849, 5529, 5910, 5654, 5879, 5681, 5862, 5688, 5810, 5884, 5616, 5797, 5718, 5963, 5973, 5795, 5859, 5768, 5760, 5777, 5565, 5544, 5814, 5977]}]
\ No newline at end of file
diff --git a/data/unified_datasets/multiwoz21/database.py b/data/unified_datasets/multiwoz21/database.py
index a23908ee13b460614213ecaad3f747fc791deff3..f599f95acfb9e89efc11572f3fb779f25900604e 100644
--- a/data/unified_datasets/multiwoz21/database.py
+++ b/data/unified_datasets/multiwoz21/database.py
@@ -57,18 +57,23 @@ class Database(BaseDatabase):
             for key, val in state:
                 if key == 'department':
                     department = val
+            if not department:
+                for key, val in soft_contraints:
+                    if key == 'department':
+                        department = val
             if not department:
                 return deepcopy(self.dbs['hospital'])
             else:
                 return [deepcopy(x) for x in self.dbs['hospital'] if x['department'].lower() == department.strip().lower()]
         state = list(map(lambda ele: (self.slot2dbattr.get(ele[0], ele[0]), ele[1]) if not(ele[0] == 'area' and ele[1] == 'center') else ('area', 'centre'), state))
+        soft_contraints = list(map(lambda ele: (self.slot2dbattr.get(ele[0], ele[0]), ele[1]) if not(ele[0] == 'area' and ele[1] == 'center') else ('area', 'centre'), soft_contraints))
 
         found = []
         for i, record in enumerate(self.dbs[domain]):
             constraints_iterator = zip(state, [False] * len(state))
             soft_contraints_iterator = zip(soft_contraints, [True] * len(soft_contraints))
             for (key, val), fuzzy_match in chain(constraints_iterator, soft_contraints_iterator):
-                if val in ["", "dont care", 'not mentioned', "don't care", "dontcare", "do n't care"]:
+                if val in ["", "dont care", 'not mentioned', "don't care", "dontcare", "do n't care", "do not care"]:
                     pass
                 else:
                     try:
diff --git a/docs/Makefile b/docs/Makefile
deleted file mode 100644
index d0c3cbf1020d5c292abdedf27627c6abe25e2293..0000000000000000000000000000000000000000
--- a/docs/Makefile
+++ /dev/null
@@ -1,20 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = source
-BUILDDIR      = build
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/make.bat b/docs/make.bat
deleted file mode 100644
index 6247f7e231716482115f34084ac61030743e0715..0000000000000000000000000000000000000000
--- a/docs/make.bat
+++ /dev/null
@@ -1,35 +0,0 @@
-@ECHO OFF
-
-pushd %~dp0
-
-REM Command file for Sphinx documentation
-
-if "%SPHINXBUILD%" == "" (
-	set SPHINXBUILD=sphinx-build
-)
-set SOURCEDIR=source
-set BUILDDIR=build
-
-if "%1" == "" goto help
-
-%SPHINXBUILD% >NUL 2>NUL
-if errorlevel 9009 (
-	echo.
-	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
-	echo.installed, then set the SPHINXBUILD environment variable to point
-	echo.to the full path of the 'sphinx-build' executable. Alternatively you
-	echo.may add the Sphinx directory to PATH.
-	echo.
-	echo.If you don't have Sphinx installed, grab it from
-	echo.http://sphinx-doc.org/
-	exit /b 1
-)
-
-%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-goto end
-
-:help
-%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
-
-:end
-popd
diff --git a/docs/source/conf.py b/docs/source/conf.py
deleted file mode 100644
index 0a4ab653e6fcef85f60b933a7352a7b4dfc9da9b..0000000000000000000000000000000000000000
--- a/docs/source/conf.py
+++ /dev/null
@@ -1,199 +0,0 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# This file only contains a selection of the most common options. For a full
-# list see the documentation:
-# http://www.sphinx-doc.org/en/master/config
-
-# -- Path setup --------------------------------------------------------------
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-import os
-import sys
-sys.path.insert(0, os.path.abspath('../..'))
-
-
-# -- Project information -----------------------------------------------------
-
-project = 'ConvLab-2'
-copyright = '2020, thu-coai'
-author = 'thu-coai'
-
-# The full version, including alpha/beta/rc tags
-release = '1.0.0'
-
-
-# -- General configuration ---------------------------------------------------
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.doctest',
-    'sphinx.ext.intersphinx',
-    'sphinx.ext.todo',
-    'sphinx.ext.coverage',
-    'sphinx.ext.mathjax',
-    'sphinx.ext.ifconfig',
-    'sphinx.ext.githubpages',
-]
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
-
-# The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
-#
-# source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
-
-# The master toctree document.
-master_doc = 'index'
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
-language = None
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = []
-
-
-# -- Options for HTML output -------------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
-html_theme = 'sphinx_rtd_theme'
-html_theme_path = ["_themes", ]
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#
-# html_theme_options = {}
-html_theme_options = {
-    'canonical_url': '',
-    #'analytics_id': 'UA-XXXXXXX-1',  #  Provided by Google in your dashboard
-    'logo_only': False,
-    'display_version': True,
-    'prev_next_buttons_location': 'bottom',
-    'style_external_links': False,
-    #'vcs_pageview_mode': '',
-    # Toc options
-    'collapse_navigation': True,
-    'sticky_navigation': True,
-    'navigation_depth': 4,
-    'includehidden': True,
-    'titles_only': False
-}
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-
-# Custom sidebar templates, must be a dictionary that maps document names
-# to template names.
-#
-# The default sidebars (for documents that don't match any pattern) are
-# defined by theme itself.  Builtin themes are using these templates by
-# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
-# 'searchbox.html']``.
-#
-# html_sidebars = {}
-
-
-# -- Options for HTMLHelp output ---------------------------------------------
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'ConvLab-2 doc'
-
-
-# -- Options for LaTeX output ------------------------------------------------
-
-latex_elements = {
-    # The paper size ('letterpaper' or 'a4paper').
-    #
-    # 'papersize': 'letterpaper',
-
-    # The font size ('10pt', '11pt' or '12pt').
-    #
-    # 'pointsize': '10pt',
-
-    # Additional stuff for the LaTeX preamble.
-    #
-    # 'preamble': '',
-
-    # Latex figure (float) alignment
-    #
-    # 'figure_align': 'htbp',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-#  author, documentclass [howto, manual, or own class]).
-# latex_documents = [
-#     (master_doc, 'tatk.tex', 'tatk Documentation',
-#      'thu-coai', 'manual'),
-# ]
-
-
-# -- Options for manual page output ------------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-# man_pages = [
-#     (master_doc, project, project + ' Documentation',
-#      [author], 1)
-# ]
-
-
-# -- Options for Texinfo output ----------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-#  dir menu entry, description, category)
-# texinfo_documents = [
-#     (master_doc, project, project + ' Documentation',
-#      author, 'tatk', 'One line description of project.',
-#      'Miscellaneous'),
-# ]
-
-
-# -- Options for Epub output -------------------------------------------------
-
-# Bibliographic Dublin Core info.
-epub_title = project
-
-# The unique identifier of the text. This can be a ISBN number
-# or the project homepage.
-#
-# epub_identifier = ''
-
-# A unique identification for the text.
-#
-# epub_uid = ''
-
-# A list of files that should not be packed into the epub file.
-epub_exclude_files = ['search.html']
-
-
-# -- Extension configuration -------------------------------------------------
-
-# -- Options for intersphinx extension ---------------------------------------
-
-# Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'https://docs.python.org/': None}
-
-# -- Options for todo extension ----------------------------------------------
-
-# If true, `todo` and `todoList` produce output, else they produce nothing.
-todo_include_todos = True
diff --git a/docs/source/convlab2.dialog_agent.rst b/docs/source/convlab2.dialog_agent.rst
deleted file mode 100644
index ca42d2adf1b628f61eb38cb01fe542daac412488..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.dialog_agent.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.dialog\_agent package
-==============================
-
-Submodules
-----------
-
-convlab.dialog\_agent.agent module
------------------------------------
-
-.. automodule:: convlab.dialog_agent.agent
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.dialog\_agent.env module
----------------------------------
-
-.. automodule:: convlab.dialog_agent.env
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.dialog\_agent.session module
--------------------------------------
-
-.. automodule:: convlab.dialog_agent.session
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.dialog_agent
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.dst.comer.rst b/docs/source/convlab2.dst.comer.rst
deleted file mode 100644
index aa3b933babe0cf6779ba6301009d83efaa243c13..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.dst.comer.rst
+++ /dev/null
@@ -1,17 +0,0 @@
-convlab.dst.comer package
-==========================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.dst.comer.multiwoz
-
-Module contents
----------------
-
-.. automodule:: convlab.dst.comer
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.dst.mdbt.multiwoz.rst b/docs/source/convlab2.dst.mdbt.multiwoz.rst
deleted file mode 100644
index df23278b816482148573b80cb9d1ee7a2aa95842..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.dst.mdbt.multiwoz.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-convlab.dst.mdbt.multiwoz package
-==================================
-
-Submodules
-----------
-
-convlab.dst.mdbt.multiwoz.dst module
--------------------------------------
-
-.. automodule:: convlab.dst.mdbt.multiwoz.dst
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.dst.mdbt.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.dst.mdbt.rst b/docs/source/convlab2.dst.mdbt.rst
deleted file mode 100644
index c693a7f2bd578970984388e7f3285d1735f3f460..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.dst.mdbt.rst
+++ /dev/null
@@ -1,37 +0,0 @@
-convlab.dst.mdbt package
-=========================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.dst.mdbt.multiwoz
-
-Submodules
-----------
-
-convlab.dst.mdbt.mdbt module
------------------------------
-
-.. automodule:: convlab.dst.mdbt.mdbt
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.dst.mdbt.mdbt\_util module
------------------------------------
-
-.. automodule:: convlab.dst.mdbt.mdbt_util
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.dst.mdbt
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.dst.rst b/docs/source/convlab2.dst.rst
deleted file mode 100644
index 5d68b6be77ec92a214f16660e079dcb02ac5c54b..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.dst.rst
+++ /dev/null
@@ -1,41 +0,0 @@
-convlab.dst package
-====================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.dst.comer
-   convlab.dst.mdbt
-   convlab.dst.rule
-   convlab.dst.sumbt
-   convlab.dst.trade
-
-Submodules
-----------
-
-convlab.dst.dst module
------------------------
-
-.. automodule:: convlab.dst.dst
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.dst.evaluate module
-----------------------------
-
-.. automodule:: convlab.dst.evaluate
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.dst
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.dst.rule.camrest.rst b/docs/source/convlab2.dst.rule.camrest.rst
deleted file mode 100644
index 434973e140361a6421ca19ef335a722f78e31f16..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.dst.rule.camrest.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-convlab.dst.rule.camrest package
-=================================
-
-Submodules
-----------
-
-convlab.dst.rule.camrest.dst module
-------------------------------------
-
-.. automodule:: convlab.dst.rule.camrest.dst
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.dst.rule.camrest
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.dst.rule.crosswoz.rst b/docs/source/convlab2.dst.rule.crosswoz.rst
deleted file mode 100644
index 2cb5894050dfd49df59bdf8013737419c8f778c2..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.dst.rule.crosswoz.rst
+++ /dev/null
@@ -1,30 +0,0 @@
-convlab.dst.rule.crosswoz package
-==================================
-
-Submodules
-----------
-
-convlab.dst.rule.crosswoz.dst module
--------------------------------------
-
-.. automodule:: convlab.dst.rule.crosswoz.dst
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.dst.rule.crosswoz.evaluate module
-------------------------------------------
-
-.. automodule:: convlab.dst.rule.crosswoz.evaluate
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.dst.rule.crosswoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.dst.rule.multiwoz.rst b/docs/source/convlab2.dst.rule.multiwoz.rst
deleted file mode 100644
index 702c1cabd943323b8ba4864ad6cee00ef8c18080..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.dst.rule.multiwoz.rst
+++ /dev/null
@@ -1,30 +0,0 @@
-convlab.dst.rule.multiwoz package
-==================================
-
-Submodules
-----------
-
-convlab.dst.rule.multiwoz.dst module
--------------------------------------
-
-.. automodule:: convlab.dst.rule.multiwoz.dst
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.dst.rule.multiwoz.dst\_util module
--------------------------------------------
-
-.. automodule:: convlab.dst.rule.multiwoz.dst_util
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.dst.rule.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.dst.rule.rst b/docs/source/convlab2.dst.rule.rst
deleted file mode 100644
index ea858e16f46945241db5fa3e7bc3958bdff3936a..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.dst.rule.rst
+++ /dev/null
@@ -1,19 +0,0 @@
-convlab.dst.rule package
-=========================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.dst.rule.camrest
-   convlab.dst.rule.crosswoz
-   convlab.dst.rule.multiwoz
-
-Module contents
----------------
-
-.. automodule:: convlab.dst.rule
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.dst.sumbt.multiwoz.rst b/docs/source/convlab2.dst.sumbt.multiwoz.rst
deleted file mode 100644
index 76125b5482deb9ff66c4f99c35cb1aae46e007d2..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.dst.sumbt.multiwoz.rst
+++ /dev/null
@@ -1,46 +0,0 @@
-convlab.dst.sumbt.multiwoz package
-===================================
-
-Submodules
-----------
-
-convlab.dst.sumbt.multiwoz.convert\_to\_glue\_format module
-------------------------------------------------------------
-
-.. automodule:: convlab.dst.sumbt.multiwoz.convert_to_glue_format
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.dst.sumbt.multiwoz.sumbt module
-----------------------------------------
-
-.. automodule:: convlab.dst.sumbt.multiwoz.sumbt
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.dst.sumbt.multiwoz.sumbt\_config module
-------------------------------------------------
-
-.. automodule:: convlab.dst.sumbt.multiwoz.sumbt_config
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.dst.sumbt.multiwoz.sumbt\_utils module
------------------------------------------------
-
-.. automodule:: convlab.dst.sumbt.multiwoz.sumbt_utils
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.dst.sumbt.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.dst.sumbt.rst b/docs/source/convlab2.dst.sumbt.rst
deleted file mode 100644
index 17b1080484960fef4da0bb332fc560fcaa8e5110..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.dst.sumbt.rst
+++ /dev/null
@@ -1,29 +0,0 @@
-convlab.dst.sumbt package
-==========================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.dst.sumbt.multiwoz
-
-Submodules
-----------
-
-convlab.dst.sumbt.BeliefTrackerSlotQueryMultiSlot module
----------------------------------------------------------
-
-.. automodule:: convlab.dst.sumbt.BeliefTrackerSlotQueryMultiSlot
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.dst.sumbt
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.dst.trade.rst b/docs/source/convlab2.dst.trade.rst
deleted file mode 100644
index ae5d305a322573818739e578824a9e16ed56c827..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.dst.trade.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-convlab.dst.trade package
-==========================
-
-Submodules
-----------
-
-convlab.dst.trade.trade module
--------------------------------
-
-.. automodule:: convlab.dst.trade.trade
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.dst.trade
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.e2e.damd.multiwoz.rst b/docs/source/convlab2.e2e.damd.multiwoz.rst
deleted file mode 100644
index 754453c3ee8f156a115ac13237fa3c04af12905b..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.e2e.damd.multiwoz.rst
+++ /dev/null
@@ -1,78 +0,0 @@
-convlab.e2e.damd.multiwoz package
-==================================
-
-Submodules
-----------
-
-convlab.e2e.damd.multiwoz.clean\_dataset module
-------------------------------------------------
-
-.. automodule:: convlab.e2e.damd.multiwoz.clean_dataset
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.damd.multiwoz.config module
-----------------------------------------
-
-.. automodule:: convlab.e2e.damd.multiwoz.config
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.damd.multiwoz.damd module
---------------------------------------
-
-.. automodule:: convlab.e2e.damd.multiwoz.damd
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.damd.multiwoz.damd\_net module
--------------------------------------------
-
-.. automodule:: convlab.e2e.damd.multiwoz.damd_net
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.damd.multiwoz.db\_ops module
------------------------------------------
-
-.. automodule:: convlab.e2e.damd.multiwoz.db_ops
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.damd.multiwoz.ontology module
-------------------------------------------
-
-.. automodule:: convlab.e2e.damd.multiwoz.ontology
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.damd.multiwoz.reader module
-----------------------------------------
-
-.. automodule:: convlab.e2e.damd.multiwoz.reader
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.damd.multiwoz.utils module
----------------------------------------
-
-.. automodule:: convlab.e2e.damd.multiwoz.utils
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.e2e.damd.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.e2e.damd.rst b/docs/source/convlab2.e2e.damd.rst
deleted file mode 100644
index a9ce3e528be39a350c3e0d702bb2cbb564dac69c..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.e2e.damd.rst
+++ /dev/null
@@ -1,17 +0,0 @@
-convlab.e2e.damd package
-=========================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.e2e.damd.multiwoz
-
-Module contents
----------------
-
-.. automodule:: convlab.e2e.damd
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.e2e.rnn_rollout.deal_or_not.rst b/docs/source/convlab2.e2e.rnn_rollout.deal_or_not.rst
deleted file mode 100644
index fd6e64fbab417e7b8d13fe98189853b00f2fc75e..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.e2e.rnn_rollout.deal_or_not.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-convlab.e2e.rnn\_rollout.deal\_or\_not package
-===============================================
-
-Submodules
-----------
-
-convlab.e2e.rnn\_rollout.deal\_or\_not.model module
-----------------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.deal_or_not.model
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.e2e.rnn_rollout.deal_or_not
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.e2e.rnn_rollout.engines.rst b/docs/source/convlab2.e2e.rnn_rollout.engines.rst
deleted file mode 100644
index 922c6139ff4114fe1f4f621ac988f42f52dbba4c..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.e2e.rnn_rollout.engines.rst
+++ /dev/null
@@ -1,46 +0,0 @@
-convlab.e2e.rnn\_rollout.engines package
-=========================================
-
-Submodules
-----------
-
-convlab.e2e.rnn\_rollout.engines.engine module
------------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.engines.engine
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.engines.latent\_clustering\_engine module
--------------------------------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.engines.latent_clustering_engine
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.engines.rnn\_engine module
-----------------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.engines.rnn_engine
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.engines.selection\_engine module
-----------------------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.engines.selection_engine
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.e2e.rnn_rollout.engines
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.e2e.rnn_rollout.models.rst b/docs/source/convlab2.e2e.rnn_rollout.models.rst
deleted file mode 100644
index fa1fdc43f973c296f865214536474b8324c1e6cd..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.e2e.rnn_rollout.models.rst
+++ /dev/null
@@ -1,70 +0,0 @@
-convlab.e2e.rnn\_rollout.models package
-========================================
-
-Submodules
-----------
-
-convlab.e2e.rnn\_rollout.models.attn module
---------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.models.attn
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.models.ctx\_encoder module
-----------------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.models.ctx_encoder
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.models.latent\_clustering\_model module
------------------------------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.models.latent_clustering_model
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.models.modules module
------------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.models.modules
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.models.rnn\_model module
---------------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.models.rnn_model
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.models.selection\_model module
---------------------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.models.selection_model
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.models.utils module
----------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.models.utils
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.e2e.rnn_rollout.models
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.e2e.rnn_rollout.rst b/docs/source/convlab2.e2e.rnn_rollout.rst
deleted file mode 100644
index 57f1a6a8b7a0b7c4999c0dfd334468e003832ddb..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.e2e.rnn_rollout.rst
+++ /dev/null
@@ -1,143 +0,0 @@
-convlab.e2e.rnn\_rollout package
-=================================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.e2e.rnn_rollout.deal_or_not
-   convlab.e2e.rnn_rollout.engines
-   convlab.e2e.rnn_rollout.models
-
-Submodules
-----------
-
-convlab.e2e.rnn\_rollout.agent module
---------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.agent
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.avg\_rank module
-------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.avg_rank
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.chat module
--------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.chat
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.config module
----------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.config
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.data module
--------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.data
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.dialog module
----------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.dialog
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.domain module
----------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.domain
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.eval\_selfplay module
------------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.eval_selfplay
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.metric module
----------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.metric
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.reinforce module
-------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.reinforce
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.rnn\_model module
--------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.rnn_model
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.rnnrollout module
--------------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.rnnrollout
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.split module
---------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.split
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.utils module
---------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.utils
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.rnn\_rollout.vis module
-------------------------------------
-
-.. automodule:: convlab.e2e.rnn_rollout.vis
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.e2e.rnn_rollout
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.e2e.rst b/docs/source/convlab2.e2e.rst
deleted file mode 100644
index e5993e95ea81363243bdcd807e6a528df85f0b84..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.e2e.rst
+++ /dev/null
@@ -1,19 +0,0 @@
-convlab.e2e package
-====================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.e2e.damd
-   convlab.e2e.rnn_rollout
-   convlab.e2e.sequicity
-
-Module contents
----------------
-
-.. automodule:: convlab.e2e
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.e2e.sequicity.camrest.rst b/docs/source/convlab2.e2e.sequicity.camrest.rst
deleted file mode 100644
index cfbaac61023e9181622f151e853f420307cff07a..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.e2e.sequicity.camrest.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-convlab.e2e.sequicity.camrest package
-======================================
-
-Submodules
-----------
-
-convlab.e2e.sequicity.camrest.sequicity module
------------------------------------------------
-
-.. automodule:: convlab.e2e.sequicity.camrest.sequicity
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.e2e.sequicity.camrest
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.e2e.sequicity.multiwoz.rst b/docs/source/convlab2.e2e.sequicity.multiwoz.rst
deleted file mode 100644
index c38b55e5f1fd3dd454224cd6a68ee82683356087..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.e2e.sequicity.multiwoz.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-convlab.e2e.sequicity.multiwoz package
-=======================================
-
-Submodules
-----------
-
-convlab.e2e.sequicity.multiwoz.sequicity module
-------------------------------------------------
-
-.. automodule:: convlab.e2e.sequicity.multiwoz.sequicity
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.e2e.sequicity.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.e2e.sequicity.rst b/docs/source/convlab2.e2e.sequicity.rst
deleted file mode 100644
index e2f3e9c0e6750a703167d292a5b6853d165f6e12..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.e2e.sequicity.rst
+++ /dev/null
@@ -1,62 +0,0 @@
-convlab.e2e.sequicity package
-==============================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.e2e.sequicity.camrest
-   convlab.e2e.sequicity.multiwoz
-
-Submodules
-----------
-
-convlab.e2e.sequicity.config module
-------------------------------------
-
-.. automodule:: convlab.e2e.sequicity.config
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.sequicity.metric module
-------------------------------------
-
-.. automodule:: convlab.e2e.sequicity.metric
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.sequicity.model module
------------------------------------
-
-.. automodule:: convlab.e2e.sequicity.model
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.sequicity.reader module
-------------------------------------
-
-.. automodule:: convlab.e2e.sequicity.reader
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.e2e.sequicity.tsd\_net module
---------------------------------------
-
-.. automodule:: convlab.e2e.sequicity.tsd_net
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.e2e.sequicity
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.evaluator.rst b/docs/source/convlab2.evaluator.rst
deleted file mode 100644
index 6cb325a95085a5cedda659a2b724f8d4c60e4807..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.evaluator.rst
+++ /dev/null
@@ -1,30 +0,0 @@
-convlab.evaluator package
-==========================
-
-Submodules
-----------
-
-convlab.evaluator.evaluator module
------------------------------------
-
-.. automodule:: convlab.evaluator.evaluator
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.evaluator.multiwoz\_eval module
-----------------------------------------
-
-.. automodule:: convlab.evaluator.multiwoz_eval
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.evaluator
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlg.rst b/docs/source/convlab2.nlg.rst
deleted file mode 100644
index e936d91a8b96bdd18cc9a1e8ea8bffdaeeb67163..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlg.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.nlg package
-====================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.nlg.sclstm
-   convlab.nlg.template
-
-Submodules
-----------
-
-convlab.nlg.evaluate module
-----------------------------
-
-.. automodule:: convlab.nlg.evaluate
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlg.nlg module
------------------------
-
-.. automodule:: convlab.nlg.nlg
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlg
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlg.sclstm.camrest.rst b/docs/source/convlab2.nlg.sclstm.camrest.rst
deleted file mode 100644
index 0e8f0a0dd6fe41f7c8500ee92cd1de97eb15dcc0..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlg.sclstm.camrest.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.nlg.sclstm.camrest package
-===================================
-
-Submodules
-----------
-
-convlab.nlg.sclstm.camrest.evaluate module
--------------------------------------------
-
-.. automodule:: convlab.nlg.sclstm.camrest.evaluate
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlg.sclstm.camrest.sc\_lstm module
--------------------------------------------
-
-.. automodule:: convlab.nlg.sclstm.camrest.sc_lstm
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlg.sclstm.camrest.train module
-----------------------------------------
-
-.. automodule:: convlab.nlg.sclstm.camrest.train
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlg.sclstm.camrest
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlg.sclstm.crosswoz.rst b/docs/source/convlab2.nlg.sclstm.crosswoz.rst
deleted file mode 100644
index 7f8f5134a3e321b2635e7a9937f5878c2d97d2a7..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlg.sclstm.crosswoz.rst
+++ /dev/null
@@ -1,46 +0,0 @@
-convlab.nlg.sclstm.crosswoz package
-====================================
-
-Submodules
-----------
-
-convlab.nlg.sclstm.crosswoz.evaluate module
---------------------------------------------
-
-.. automodule:: convlab.nlg.sclstm.crosswoz.evaluate
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlg.sclstm.crosswoz.generate\_resources module
--------------------------------------------------------
-
-.. automodule:: convlab.nlg.sclstm.crosswoz.generate_resources
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlg.sclstm.crosswoz.sc\_lstm module
---------------------------------------------
-
-.. automodule:: convlab.nlg.sclstm.crosswoz.sc_lstm
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlg.sclstm.crosswoz.train module
------------------------------------------
-
-.. automodule:: convlab.nlg.sclstm.crosswoz.train
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlg.sclstm.crosswoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlg.sclstm.multiwoz.rst b/docs/source/convlab2.nlg.sclstm.multiwoz.rst
deleted file mode 100644
index 044c1c1cc932049c2a6c4ae65bb11b771b6d9fa2..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlg.sclstm.multiwoz.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.nlg.sclstm.multiwoz package
-====================================
-
-Submodules
-----------
-
-convlab.nlg.sclstm.multiwoz.evaluate module
---------------------------------------------
-
-.. automodule:: convlab.nlg.sclstm.multiwoz.evaluate
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlg.sclstm.multiwoz.sc\_lstm module
---------------------------------------------
-
-.. automodule:: convlab.nlg.sclstm.multiwoz.sc_lstm
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlg.sclstm.multiwoz.train module
------------------------------------------
-
-.. automodule:: convlab.nlg.sclstm.multiwoz.train
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlg.sclstm.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlg.sclstm.rst b/docs/source/convlab2.nlg.sclstm.rst
deleted file mode 100644
index 411e4538c65267fdada38eb21936659649059281..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlg.sclstm.rst
+++ /dev/null
@@ -1,31 +0,0 @@
-convlab.nlg.sclstm package
-===========================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.nlg.sclstm.camrest
-   convlab.nlg.sclstm.crosswoz
-   convlab.nlg.sclstm.multiwoz
-
-Submodules
-----------
-
-convlab.nlg.sclstm.bleu module
--------------------------------
-
-.. automodule:: convlab.nlg.sclstm.bleu
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlg.sclstm
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlg.template.camrest.rst b/docs/source/convlab2.nlg.template.camrest.rst
deleted file mode 100644
index 015c1aa3f9046ddb8657209564161cb936db4e98..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlg.template.camrest.rst
+++ /dev/null
@@ -1,30 +0,0 @@
-convlab.nlg.template.camrest package
-=====================================
-
-Submodules
-----------
-
-convlab.nlg.template.camrest.evaluate module
----------------------------------------------
-
-.. automodule:: convlab.nlg.template.camrest.evaluate
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlg.template.camrest.nlg module
-----------------------------------------
-
-.. automodule:: convlab.nlg.template.camrest.nlg
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlg.template.camrest
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlg.template.crosswoz.rst b/docs/source/convlab2.nlg.template.crosswoz.rst
deleted file mode 100644
index 569fe598ece0f2266546bd4b8a82823da42008ec..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlg.template.crosswoz.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.nlg.template.crosswoz package
-======================================
-
-Submodules
-----------
-
-convlab.nlg.template.crosswoz.evaluate module
-----------------------------------------------
-
-.. automodule:: convlab.nlg.template.crosswoz.evaluate
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlg.template.crosswoz.generate\_auto\_template module
---------------------------------------------------------------
-
-.. automodule:: convlab.nlg.template.crosswoz.generate_auto_template
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlg.template.crosswoz.nlg module
------------------------------------------
-
-.. automodule:: convlab.nlg.template.crosswoz.nlg
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlg.template.crosswoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlg.template.multiwoz.rst b/docs/source/convlab2.nlg.template.multiwoz.rst
deleted file mode 100644
index 8e86f2905baeacfcc9357f2ff359bfe4775a7d49..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlg.template.multiwoz.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-convlab.nlg.template.multiwoz package
-======================================
-
-Submodules
-----------
-
-convlab.nlg.template.multiwoz.nlg module
------------------------------------------
-
-.. automodule:: convlab.nlg.template.multiwoz.nlg
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlg.template.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlg.template.rst b/docs/source/convlab2.nlg.template.rst
deleted file mode 100644
index 20d5d1f2b6f1268faf527a12a81ac92168d65492..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlg.template.rst
+++ /dev/null
@@ -1,19 +0,0 @@
-convlab.nlg.template package
-=============================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.nlg.template.camrest
-   convlab.nlg.template.crosswoz
-   convlab.nlg.template.multiwoz
-
-Module contents
----------------
-
-.. automodule:: convlab.nlg.template
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlu.jointBERT.camrest.rst b/docs/source/convlab2.nlu.jointBERT.camrest.rst
deleted file mode 100644
index 5053706b49686124729c92a4ef30122eca653577..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlu.jointBERT.camrest.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.nlu.jointBERT.camrest package
-======================================
-
-Submodules
-----------
-
-convlab.nlu.jointBERT.camrest.nlu module
------------------------------------------
-
-.. automodule:: convlab.nlu.jointBERT.camrest.nlu
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.jointBERT.camrest.postprocess module
--------------------------------------------------
-
-.. automodule:: convlab.nlu.jointBERT.camrest.postprocess
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.jointBERT.camrest.preprocess module
-------------------------------------------------
-
-.. automodule:: convlab.nlu.jointBERT.camrest.preprocess
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlu.jointBERT.camrest
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlu.jointBERT.crosswoz.rst b/docs/source/convlab2.nlu.jointBERT.crosswoz.rst
deleted file mode 100644
index a875026bb376703fd073a30c0bebc8fbed1a8715..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlu.jointBERT.crosswoz.rst
+++ /dev/null
@@ -1,46 +0,0 @@
-convlab.nlu.jointBERT.crosswoz package
-=======================================
-
-Submodules
-----------
-
-convlab.nlu.jointBERT.crosswoz.analyse module
-----------------------------------------------
-
-.. automodule:: convlab.nlu.jointBERT.crosswoz.analyse
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.jointBERT.crosswoz.nlu module
-------------------------------------------
-
-.. automodule:: convlab.nlu.jointBERT.crosswoz.nlu
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.jointBERT.crosswoz.postprocess module
---------------------------------------------------
-
-.. automodule:: convlab.nlu.jointBERT.crosswoz.postprocess
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.jointBERT.crosswoz.preprocess module
--------------------------------------------------
-
-.. automodule:: convlab.nlu.jointBERT.crosswoz.preprocess
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlu.jointBERT.crosswoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlu.jointBERT.multiwoz.rst b/docs/source/convlab2.nlu.jointBERT.multiwoz.rst
deleted file mode 100644
index de513a34dfefa1cfebc9715bfc27b736300b4cc2..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlu.jointBERT.multiwoz.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.nlu.jointBERT.multiwoz package
-=======================================
-
-Submodules
-----------
-
-convlab.nlu.jointBERT.multiwoz.nlu module
-------------------------------------------
-
-.. automodule:: convlab.nlu.jointBERT.multiwoz.nlu
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.jointBERT.multiwoz.postprocess module
---------------------------------------------------
-
-.. automodule:: convlab.nlu.jointBERT.multiwoz.postprocess
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.jointBERT.multiwoz.preprocess module
--------------------------------------------------
-
-.. automodule:: convlab.nlu.jointBERT.multiwoz.preprocess
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlu.jointBERT.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlu.jointBERT.rst b/docs/source/convlab2.nlu.jointBERT.rst
deleted file mode 100644
index 187458ab3ade0589d965133ac5d6dc9521a66181..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlu.jointBERT.rst
+++ /dev/null
@@ -1,55 +0,0 @@
-convlab.nlu.jointBERT package
-==============================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.nlu.jointBERT.camrest
-   convlab.nlu.jointBERT.crosswoz
-   convlab.nlu.jointBERT.multiwoz
-
-Submodules
-----------
-
-convlab.nlu.jointBERT.dataloader module
-----------------------------------------
-
-.. automodule:: convlab.nlu.jointBERT.dataloader
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.jointBERT.jointBERT module
----------------------------------------
-
-.. automodule:: convlab.nlu.jointBERT.jointBERT
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.jointBERT.test module
-----------------------------------
-
-.. automodule:: convlab.nlu.jointBERT.test
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.jointBERT.train module
------------------------------------
-
-.. automodule:: convlab.nlu.jointBERT.train
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlu.jointBERT
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlu.milu.multiwoz.rst b/docs/source/convlab2.nlu.milu.multiwoz.rst
deleted file mode 100644
index 51dda55566651e8ce8b7be09a3f60206dcb0e7aa..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlu.milu.multiwoz.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-convlab.nlu.milu.multiwoz package
-==================================
-
-Submodules
-----------
-
-convlab.nlu.milu.multiwoz.nlu module
--------------------------------------
-
-.. automodule:: convlab.nlu.milu.multiwoz.nlu
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlu.milu.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlu.milu.rst b/docs/source/convlab2.nlu.milu.rst
deleted file mode 100644
index 438ea6631bc88055e3849e5eabadf3fbd598d858..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlu.milu.rst
+++ /dev/null
@@ -1,69 +0,0 @@
-convlab.nlu.milu package
-=========================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.nlu.milu.multiwoz
-
-Submodules
-----------
-
-convlab.nlu.milu.dai\_f1\_measure module
------------------------------------------
-
-.. automodule:: convlab.nlu.milu.dai_f1_measure
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.milu.dataset\_reader module
-----------------------------------------
-
-.. automodule:: convlab.nlu.milu.dataset_reader
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.milu.evaluate module
----------------------------------
-
-.. automodule:: convlab.nlu.milu.evaluate
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.milu.model module
-------------------------------
-
-.. automodule:: convlab.nlu.milu.model
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.milu.multilabel\_f1\_measure module
-------------------------------------------------
-
-.. automodule:: convlab.nlu.milu.multilabel_f1_measure
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.milu.train module
-------------------------------
-
-.. automodule:: convlab.nlu.milu.train
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlu.milu
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlu.rst b/docs/source/convlab2.nlu.rst
deleted file mode 100644
index 890d0071efc7b8249b1f74817aa1460b30400ab6..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlu.rst
+++ /dev/null
@@ -1,39 +0,0 @@
-convlab.nlu package
-====================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.nlu.jointBERT
-   convlab.nlu.milu
-   convlab.nlu.svm
-
-Submodules
-----------
-
-convlab.nlu.evaluate module
-----------------------------
-
-.. automodule:: convlab.nlu.evaluate
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.nlu module
------------------------
-
-.. automodule:: convlab.nlu.nlu
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlu
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlu.svm.camrest.rst b/docs/source/convlab2.nlu.svm.camrest.rst
deleted file mode 100644
index 87da8e5865067fa00421e687cc44d004f536a46c..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlu.svm.camrest.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.nlu.svm.camrest package
-================================
-
-Submodules
-----------
-
-convlab.nlu.svm.camrest.evaluate module
-----------------------------------------
-
-.. automodule:: convlab.nlu.svm.camrest.evaluate
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.svm.camrest.nlu module
------------------------------------
-
-.. automodule:: convlab.nlu.svm.camrest.nlu
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.svm.camrest.preprocess module
-------------------------------------------
-
-.. automodule:: convlab.nlu.svm.camrest.preprocess
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlu.svm.camrest
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlu.svm.multiwoz.rst b/docs/source/convlab2.nlu.svm.multiwoz.rst
deleted file mode 100644
index 1cf584298aa87611c134457b73893250a6081f6a..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlu.svm.multiwoz.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.nlu.svm.multiwoz package
-=================================
-
-Submodules
-----------
-
-convlab.nlu.svm.multiwoz.evaluate module
------------------------------------------
-
-.. automodule:: convlab.nlu.svm.multiwoz.evaluate
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.svm.multiwoz.nlu module
-------------------------------------
-
-.. automodule:: convlab.nlu.svm.multiwoz.nlu
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.svm.multiwoz.preprocess module
--------------------------------------------
-
-.. automodule:: convlab.nlu.svm.multiwoz.preprocess
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlu.svm.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.nlu.svm.rst b/docs/source/convlab2.nlu.svm.rst
deleted file mode 100644
index 1e76053eb86c1593e9d8049084aabe7ac3310100..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.nlu.svm.rst
+++ /dev/null
@@ -1,70 +0,0 @@
-convlab.nlu.svm package
-========================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.nlu.svm.camrest
-   convlab.nlu.svm.multiwoz
-
-Submodules
-----------
-
-convlab.nlu.svm.Classifier module
-----------------------------------
-
-.. automodule:: convlab.nlu.svm.Classifier
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.svm.Features module
---------------------------------
-
-.. automodule:: convlab.nlu.svm.Features
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.svm.Tuples module
-------------------------------
-
-.. automodule:: convlab.nlu.svm.Tuples
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.svm.dataset\_walker module
----------------------------------------
-
-.. automodule:: convlab.nlu.svm.dataset_walker
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.svm.sutils module
-------------------------------
-
-.. automodule:: convlab.nlu.svm.sutils
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.nlu.svm.train module
------------------------------
-
-.. automodule:: convlab.nlu.svm.train
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.nlu.svm
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.gdpl.multiwoz.rst b/docs/source/convlab2.policy.gdpl.multiwoz.rst
deleted file mode 100644
index e7d2baf16c54691e8a2c46be9dc3dbb5963535fe..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.gdpl.multiwoz.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-convlab.policy.gdpl.multiwoz package
-=====================================
-
-Submodules
-----------
-
-convlab.policy.gdpl.multiwoz.gdpl\_policy module
--------------------------------------------------
-
-.. automodule:: convlab.policy.gdpl.multiwoz.gdpl_policy
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.gdpl.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.gdpl.rst b/docs/source/convlab2.policy.gdpl.rst
deleted file mode 100644
index 9fbf92e9b72909f522309f8939e9f4eabe721224..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.gdpl.rst
+++ /dev/null
@@ -1,45 +0,0 @@
-convlab.policy.gdpl package
-============================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.policy.gdpl.multiwoz
-
-Submodules
-----------
-
-convlab.policy.gdpl.estimator module
--------------------------------------
-
-.. automodule:: convlab.policy.gdpl.estimator
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.gdpl.gdpl module
---------------------------------
-
-.. automodule:: convlab.policy.gdpl.gdpl
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.gdpl.train module
----------------------------------
-
-.. automodule:: convlab.policy.gdpl.train
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.gdpl
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.mdrg.rst b/docs/source/convlab2.policy.mdrg.rst
deleted file mode 100644
index 42475946e600aa5b8e14225e2fe9751782e2e238..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.mdrg.rst
+++ /dev/null
@@ -1,17 +0,0 @@
-convlab.policy.mdrg package
-============================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.policy.mdrg.multiwoz
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.mdrg
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.mle.camrest.rst b/docs/source/convlab2.policy.mle.camrest.rst
deleted file mode 100644
index 534a176d1b616ff304eb1df8c8489c2a1b9195f2..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.mle.camrest.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.policy.mle.camrest package
-===================================
-
-Submodules
-----------
-
-convlab.policy.mle.camrest.loader module
------------------------------------------
-
-.. automodule:: convlab.policy.mle.camrest.loader
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.mle.camrest.mle module
---------------------------------------
-
-.. automodule:: convlab.policy.mle.camrest.mle
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.mle.camrest.train module
-----------------------------------------
-
-.. automodule:: convlab.policy.mle.camrest.train
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.mle.camrest
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.mle.crosswoz.rst b/docs/source/convlab2.policy.mle.crosswoz.rst
deleted file mode 100644
index 7bdb5dd4ed66bff145e7909c3af8c4b597570514..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.mle.crosswoz.rst
+++ /dev/null
@@ -1,46 +0,0 @@
-convlab.policy.mle.crosswoz package
-====================================
-
-Submodules
-----------
-
-convlab.policy.mle.crosswoz.evaluate module
---------------------------------------------
-
-.. automodule:: convlab.policy.mle.crosswoz.evaluate
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.mle.crosswoz.loader module
-------------------------------------------
-
-.. automodule:: convlab.policy.mle.crosswoz.loader
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.mle.crosswoz.mle module
----------------------------------------
-
-.. automodule:: convlab.policy.mle.crosswoz.mle
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.mle.crosswoz.train module
------------------------------------------
-
-.. automodule:: convlab.policy.mle.crosswoz.train
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.mle.crosswoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.mle.multiwoz.rst b/docs/source/convlab2.policy.mle.multiwoz.rst
deleted file mode 100644
index 9c97544797e935864a6621738ff1f7823c7551d6..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.mle.multiwoz.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.policy.mle.multiwoz package
-====================================
-
-Submodules
-----------
-
-convlab.policy.mle.multiwoz.loader module
-------------------------------------------
-
-.. automodule:: convlab.policy.mle.multiwoz.loader
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.mle.multiwoz.mle module
----------------------------------------
-
-.. automodule:: convlab.policy.mle.multiwoz.mle
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.mle.multiwoz.train module
------------------------------------------
-
-.. automodule:: convlab.policy.mle.multiwoz.train
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.mle.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.mle.rst b/docs/source/convlab2.policy.mle.rst
deleted file mode 100644
index c2a6333372b98036a733cbdc1a160f6c23511689..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.mle.rst
+++ /dev/null
@@ -1,47 +0,0 @@
-convlab.policy.mle package
-===========================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.policy.mle.camrest
-   convlab.policy.mle.crosswoz
-   convlab.policy.mle.multiwoz
-
-Submodules
-----------
-
-convlab.policy.mle.loader module
----------------------------------
-
-.. automodule:: convlab.policy.mle.loader
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.mle.mle module
-------------------------------
-
-.. automodule:: convlab.policy.mle.mle
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.mle.train module
---------------------------------
-
-.. automodule:: convlab.policy.mle.train
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.mle
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.pg.multiwoz.rst b/docs/source/convlab2.policy.pg.multiwoz.rst
deleted file mode 100644
index 6743536bf4a1af6675b76c97a6bf29054f435283..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.pg.multiwoz.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-convlab.policy.pg.multiwoz package
-===================================
-
-Submodules
-----------
-
-convlab.policy.pg.multiwoz.pg\_policy module
----------------------------------------------
-
-.. automodule:: convlab.policy.pg.multiwoz.pg_policy
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.pg.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.pg.rst b/docs/source/convlab2.policy.pg.rst
deleted file mode 100644
index 4a0438565fe2879daa5842058cedefb27822ebed..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.pg.rst
+++ /dev/null
@@ -1,37 +0,0 @@
-convlab.policy.pg package
-==========================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.policy.pg.multiwoz
-
-Submodules
-----------
-
-convlab.policy.pg.pg module
-----------------------------
-
-.. automodule:: convlab.policy.pg.pg
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.pg.train module
--------------------------------
-
-.. automodule:: convlab.policy.pg.train
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.pg
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.ppo.multiwoz.rst b/docs/source/convlab2.policy.ppo.multiwoz.rst
deleted file mode 100644
index c8f6a40f8f1a11cefd53fb639e9ff80810d8871c..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.ppo.multiwoz.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-convlab.policy.ppo.multiwoz package
-====================================
-
-Submodules
-----------
-
-convlab.policy.ppo.multiwoz.ppo\_policy module
------------------------------------------------
-
-.. automodule:: convlab.policy.ppo.multiwoz.ppo_policy
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.ppo.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.ppo.rst b/docs/source/convlab2.policy.ppo.rst
deleted file mode 100644
index 021684a87a78c6fe58578c061e071d2cd281fb7e..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.ppo.rst
+++ /dev/null
@@ -1,37 +0,0 @@
-convlab.policy.ppo package
-===========================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.policy.ppo.multiwoz
-
-Submodules
-----------
-
-convlab.policy.ppo.ppo module
-------------------------------
-
-.. automodule:: convlab.policy.ppo.ppo
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.ppo.train module
---------------------------------
-
-.. automodule:: convlab.policy.ppo.train
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.ppo
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.rst b/docs/source/convlab2.policy.rst
deleted file mode 100644
index fea1046151fd58646738d9bbeb98d3270d0484a1..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.rst
+++ /dev/null
@@ -1,60 +0,0 @@
-convlab.policy package
-=======================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.policy.gdpl
-   convlab.policy.mdrg
-   convlab.policy.mle
-   convlab.policy.pg
-   convlab.policy.ppo
-   convlab.policy.rule
-   convlab.policy.vector
-   convlab.policy.vhus
-
-Submodules
-----------
-
-convlab.policy.evaluate module
--------------------------------
-
-.. automodule:: convlab.policy.evaluate
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.policy module
------------------------------
-
-.. automodule:: convlab.policy.policy
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.rlmodule module
--------------------------------
-
-.. automodule:: convlab.policy.rlmodule
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.vec module
---------------------------
-
-.. automodule:: convlab.policy.vec
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.rule.camrest.rst b/docs/source/convlab2.policy.rule.camrest.rst
deleted file mode 100644
index 00bccc9d8b2c15b3b5d942dad951f7edd287f968..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.rule.camrest.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.policy.rule.camrest package
-====================================
-
-Submodules
-----------
-
-convlab.policy.rule.camrest.policy\_agenda\_camrest module
------------------------------------------------------------
-
-.. automodule:: convlab.policy.rule.camrest.policy_agenda_camrest
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.rule.camrest.rule module
-----------------------------------------
-
-.. automodule:: convlab.policy.rule.camrest.rule
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.rule.camrest.rule\_based\_camrest\_bot module
--------------------------------------------------------------
-
-.. automodule:: convlab.policy.rule.camrest.rule_based_camrest_bot
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.rule.camrest
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.rule.crosswoz.rst b/docs/source/convlab2.policy.rule.crosswoz.rst
deleted file mode 100644
index 416c5d0cebe621b9f8935c69ea216b051ed352a2..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.rule.crosswoz.rst
+++ /dev/null
@@ -1,30 +0,0 @@
-convlab.policy.rule.crosswoz package
-=====================================
-
-Submodules
-----------
-
-convlab.policy.rule.crosswoz.evaluate module
----------------------------------------------
-
-.. automodule:: convlab.policy.rule.crosswoz.evaluate
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.rule.crosswoz.rule\_simulator module
-----------------------------------------------------
-
-.. automodule:: convlab.policy.rule.crosswoz.rule_simulator
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.rule.crosswoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.rule.multiwoz.rst b/docs/source/convlab2.policy.rule.multiwoz.rst
deleted file mode 100644
index 4af738e06767cfaf31c5050f159dbb55c201b45f..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.rule.multiwoz.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.policy.rule.multiwoz package
-=====================================
-
-Submodules
-----------
-
-convlab.policy.rule.multiwoz.policy\_agenda\_multiwoz module
--------------------------------------------------------------
-
-.. automodule:: convlab.policy.rule.multiwoz.policy_agenda_multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.rule.multiwoz.rule module
------------------------------------------
-
-.. automodule:: convlab.policy.rule.multiwoz.rule
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.rule.multiwoz.rule\_based\_multiwoz\_bot module
----------------------------------------------------------------
-
-.. automodule:: convlab.policy.rule.multiwoz.rule_based_multiwoz_bot
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.rule.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.rule.rst b/docs/source/convlab2.policy.rule.rst
deleted file mode 100644
index 5dc2ca91425d70affa9e1ebc2f986d7694005e53..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.rule.rst
+++ /dev/null
@@ -1,19 +0,0 @@
-convlab.policy.rule package
-============================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.policy.rule.camrest
-   convlab.policy.rule.crosswoz
-   convlab.policy.rule.multiwoz
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.rule
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.vector.rst b/docs/source/convlab2.policy.vector.rst
deleted file mode 100644
index 9abd78174e9af1109d3ed51b5bd5e59466ca96e8..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.vector.rst
+++ /dev/null
@@ -1,46 +0,0 @@
-convlab.policy.vector package
-==============================
-
-Submodules
-----------
-
-convlab.policy.vector.dataset module
--------------------------------------
-
-.. automodule:: convlab.policy.vector.dataset
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.vector.vector\_camrest module
----------------------------------------------
-
-.. automodule:: convlab.policy.vector.vector_camrest
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.vector.vector\_crosswoz module
-----------------------------------------------
-
-.. automodule:: convlab.policy.vector.vector_crosswoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.vector.vector\_multiwoz module
-----------------------------------------------
-
-.. automodule:: convlab.policy.vector.vector_multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.vector
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.vhus.camrest.rst b/docs/source/convlab2.policy.vhus.camrest.rst
deleted file mode 100644
index 63e420277df2bd736fa34c94ece40ab282af64dc..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.vhus.camrest.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.policy.vhus.camrest package
-====================================
-
-Submodules
-----------
-
-convlab.policy.vhus.camrest.train module
------------------------------------------
-
-.. automodule:: convlab.policy.vhus.camrest.train
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.vhus.camrest.usermanager module
------------------------------------------------
-
-.. automodule:: convlab.policy.vhus.camrest.usermanager
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.vhus.camrest.vhus module
-----------------------------------------
-
-.. automodule:: convlab.policy.vhus.camrest.vhus
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.vhus.camrest
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.vhus.multiwoz.rst b/docs/source/convlab2.policy.vhus.multiwoz.rst
deleted file mode 100644
index 9faa27dec3789497cf108de077808988da838735..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.vhus.multiwoz.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.policy.vhus.multiwoz package
-=====================================
-
-Submodules
-----------
-
-convlab.policy.vhus.multiwoz.train module
-------------------------------------------
-
-.. automodule:: convlab.policy.vhus.multiwoz.train
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.vhus.multiwoz.usermanager module
-------------------------------------------------
-
-.. automodule:: convlab.policy.vhus.multiwoz.usermanager
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.vhus.multiwoz.vhus module
------------------------------------------
-
-.. automodule:: convlab.policy.vhus.multiwoz.vhus
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.vhus.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.policy.vhus.rst b/docs/source/convlab2.policy.vhus.rst
deleted file mode 100644
index 54eb0fa7335d7802a0c8f7e8944ae6f25c227dde..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.policy.vhus.rst
+++ /dev/null
@@ -1,54 +0,0 @@
-convlab.policy.vhus package
-============================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.policy.vhus.camrest
-   convlab.policy.vhus.multiwoz
-
-Submodules
-----------
-
-convlab.policy.vhus.train module
----------------------------------
-
-.. automodule:: convlab.policy.vhus.train
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.vhus.usermodule module
---------------------------------------
-
-.. automodule:: convlab.policy.vhus.usermodule
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.vhus.util module
---------------------------------
-
-.. automodule:: convlab.policy.vhus.util
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.policy.vhus.vhus module
---------------------------------
-
-.. automodule:: convlab.policy.vhus.vhus
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.policy.vhus
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.rst b/docs/source/convlab2.rst
deleted file mode 100644
index c05fce52bb0e643f8013df6dcfb9784c63236f72..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.rst
+++ /dev/null
@@ -1,25 +0,0 @@
-convlab package
-================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.dialog_agent
-   convlab.dst
-   convlab.e2e
-   convlab.evaluator
-   convlab.nlg
-   convlab.nlu
-   convlab.policy
-   convlab.task
-   convlab.util
-
-Module contents
----------------
-
-.. automodule:: convlab
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.task.camrest.rst b/docs/source/convlab2.task.camrest.rst
deleted file mode 100644
index f26a58371c97ea711ddccc31969a075a05a32b09..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.task.camrest.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-convlab.task.camrest package
-=============================
-
-Submodules
-----------
-
-convlab.task.camrest.goal\_generator module
---------------------------------------------
-
-.. automodule:: convlab.task.camrest.goal_generator
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.task.camrest
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.task.crosswoz.rst b/docs/source/convlab2.task.crosswoz.rst
deleted file mode 100644
index 8c2ab1474e7377993128fa12d128e165d48609e0..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.task.crosswoz.rst
+++ /dev/null
@@ -1,78 +0,0 @@
-convlab.task.crosswoz package
-==============================
-
-Submodules
-----------
-
-convlab.task.crosswoz.attraction\_generator module
----------------------------------------------------
-
-.. automodule:: convlab.task.crosswoz.attraction_generator
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.task.crosswoz.goal\_generator module
----------------------------------------------
-
-.. automodule:: convlab.task.crosswoz.goal_generator
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.task.crosswoz.hotel\_generator module
-----------------------------------------------
-
-.. automodule:: convlab.task.crosswoz.hotel_generator
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.task.crosswoz.metro\_generator module
-----------------------------------------------
-
-.. automodule:: convlab.task.crosswoz.metro_generator
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.task.crosswoz.reorder module
--------------------------------------
-
-.. automodule:: convlab.task.crosswoz.reorder
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.task.crosswoz.restaurant\_generator module
----------------------------------------------------
-
-.. automodule:: convlab.task.crosswoz.restaurant_generator
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.task.crosswoz.sentence\_generator module
--------------------------------------------------
-
-.. automodule:: convlab.task.crosswoz.sentence_generator
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.task.crosswoz.taxi\_generator module
----------------------------------------------
-
-.. automodule:: convlab.task.crosswoz.taxi_generator
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.task.crosswoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.task.multiwoz.rst b/docs/source/convlab2.task.multiwoz.rst
deleted file mode 100644
index eb6f80e018c26b674cc4ac54cd5bb12dc2467089..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.task.multiwoz.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-convlab.task.multiwoz package
-==============================
-
-Submodules
-----------
-
-convlab.task.multiwoz.goal\_generator module
----------------------------------------------
-
-.. automodule:: convlab.task.multiwoz.goal_generator
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.task.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.task.rst b/docs/source/convlab2.task.rst
deleted file mode 100644
index 30609493b8a4808409a61ab4023e6d04ee745baa..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.task.rst
+++ /dev/null
@@ -1,19 +0,0 @@
-convlab.task package
-=====================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.task.camrest
-   convlab.task.crosswoz
-   convlab.task.multiwoz
-
-Module contents
----------------
-
-.. automodule:: convlab.task
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.util.camrest.rst b/docs/source/convlab2.util.camrest.rst
deleted file mode 100644
index 7553f24cfd61bbf399887b25ca6e83f417473854..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.util.camrest.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.util.camrest package
-=============================
-
-Submodules
-----------
-
-convlab.util.camrest.dbquery module
-------------------------------------
-
-.. automodule:: convlab.util.camrest.dbquery
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.util.camrest.lexicalize module
----------------------------------------
-
-.. automodule:: convlab.util.camrest.lexicalize
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.util.camrest.state module
-----------------------------------
-
-.. automodule:: convlab.util.camrest.state
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.util.camrest
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.util.crosswoz.rst b/docs/source/convlab2.util.crosswoz.rst
deleted file mode 100644
index 4f7f3ad7b461f86e7dcf5c6e8cce72bf1f78218d..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.util.crosswoz.rst
+++ /dev/null
@@ -1,38 +0,0 @@
-convlab.util.crosswoz package
-==============================
-
-Submodules
-----------
-
-convlab.util.crosswoz.dbquery module
--------------------------------------
-
-.. automodule:: convlab.util.crosswoz.dbquery
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.util.crosswoz.lexicalize module
-----------------------------------------
-
-.. automodule:: convlab.util.crosswoz.lexicalize
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.util.crosswoz.state module
------------------------------------
-
-.. automodule:: convlab.util.crosswoz.state
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.util.crosswoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.util.dataloader.rst b/docs/source/convlab2.util.dataloader.rst
deleted file mode 100644
index ce6307e968fd32714472de06bffffa2f0a87a48e..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.util.dataloader.rst
+++ /dev/null
@@ -1,30 +0,0 @@
-convlab.util.dataloader package
-================================
-
-Submodules
-----------
-
-convlab.util.dataloader.dataset\_dataloader module
----------------------------------------------------
-
-.. automodule:: convlab.util.dataloader.dataset_dataloader
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.util.dataloader.module\_dataloader module
---------------------------------------------------
-
-.. automodule:: convlab.util.dataloader.module_dataloader
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.util.dataloader
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.util.multiwoz.rst b/docs/source/convlab2.util.multiwoz.rst
deleted file mode 100644
index b0e44d79751d645d690c231fe07083aa49e4caee..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.util.multiwoz.rst
+++ /dev/null
@@ -1,46 +0,0 @@
-convlab.util.multiwoz package
-==============================
-
-Submodules
-----------
-
-convlab.util.multiwoz.dbquery module
--------------------------------------
-
-.. automodule:: convlab.util.multiwoz.dbquery
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.util.multiwoz.lexicalize module
-----------------------------------------
-
-.. automodule:: convlab.util.multiwoz.lexicalize
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.util.multiwoz.multiwoz\_slot\_trans module
----------------------------------------------------
-
-.. automodule:: convlab.util.multiwoz.multiwoz_slot_trans
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.util.multiwoz.state module
------------------------------------
-
-.. automodule:: convlab.util.multiwoz.state
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.util.multiwoz
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/convlab2.util.rst b/docs/source/convlab2.util.rst
deleted file mode 100644
index e73afeae04576af4bc7c4f8209eaf424e7f268bd..0000000000000000000000000000000000000000
--- a/docs/source/convlab2.util.rst
+++ /dev/null
@@ -1,56 +0,0 @@
-convlab.util package
-=====================
-
-Subpackages
------------
-
-.. toctree::
-
-   convlab.util.camrest
-   convlab.util.crosswoz
-   convlab.util.dataloader
-   convlab.util.multiwoz
-
-Submodules
-----------
-
-convlab.util.allennlp\_file\_utils module
-------------------------------------------
-
-.. automodule:: convlab.util.allennlp_file_utils
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.util.file\_util module
--------------------------------
-
-.. automodule:: convlab.util.file_util
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.util.module module
----------------------------
-
-.. automodule:: convlab.util.module
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-convlab.util.train\_util module
---------------------------------
-
-.. automodule:: convlab.util.train_util
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-
-Module contents
----------------
-
-.. automodule:: convlab.util
-   :members:
-   :undoc-members:
-   :show-inheritance:
diff --git a/docs/source/index.rst b/docs/source/index.rst
deleted file mode 100644
index ca951cd7f5cbb2b1b2fe9b82aa81def9bb1cb153..0000000000000000000000000000000000000000
--- a/docs/source/index.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-.. convlab documentation master file, created by
-   sphinx-quickstart on Thu May  7 12:16:27 2020.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
-
-Welcome to convlab's documentation!
-====================================
-
-.. toctree::
-   :maxdepth: 2
-   :caption: Contents:
-
-
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
diff --git a/docs/source/modules.rst b/docs/source/modules.rst
deleted file mode 100644
index 39cf0072e6c2d2ee5c663aae18da59f8bbe89162..0000000000000000000000000000000000000000
--- a/docs/source/modules.rst
+++ /dev/null
@@ -1,7 +0,0 @@
-convlab
-========
-
-.. toctree::
-   :maxdepth: 4
-
-   convlab
diff --git a/examples/agent_examples/test_LAVA.py b/examples/agent_examples/test_LAVA.py
index 38379072d5c3611e349925a64cefa2c8b9560be6..d5b592c2e09f8c7f73ebf925abdf52b1a4be24b6 100755
--- a/examples/agent_examples/test_LAVA.py
+++ b/examples/agent_examples/test_LAVA.py
@@ -8,13 +8,14 @@ import json
 import random
 from pprint import pprint
 from argparse import ArgumentParser
-from convlab.nlu.jointBERT.multiwoz import BERTNLU
+from convlab.nlu.jointBERT.unified_datasets import BERTNLU
+# from convlab.nlu.jointBERT.multiwoz import BERTNLU as BERTNLU_woz
 # from convlab.nlu.milu.multiwoz import MILU
 # available DST models
 from convlab.dst.rule.multiwoz import RuleDST
 # from convlab.dst.mdbt.multiwoz import MDBT
 # from convlab.dst.sumbt.multiwoz import SUMBT
-from convlab.dst.setsumbt.multiwoz.Tracker import SetSUMBTTracker
+# from convlab.dst.setsumbt.multiwoz.Tracker import SetSUMBTTracker
 # from convlab.dst.trippy.multiwoz import TRIPPY
 # from convlab.dst.trade.multiwoz import TRADE
 # from convlab.dst.comer.multiwoz import COMER
@@ -31,7 +32,7 @@ from convlab.policy.rule.multiwoz import RulePolicy
 from convlab.policy.lava.multiwoz import LAVA
 # available NLG models
 from convlab.nlg.template.multiwoz import TemplateNLG
-from convlab.nlg.sclstm.multiwoz import SCLSTM
+# from convlab.nlg.sclstm.multiwoz import SCLSTM
 # available E2E models
 # from convlab.e2e.sequicity.multiwoz import Sequicity
 # from convlab.e2e.damd.multiwoz import Damd
@@ -60,7 +61,7 @@ def set_seed(r_seed):
 def test_end2end(args, model_dir):
     # BERT nlu
     if args.dst_type=="bertnlu_rule":
-        sys_nlu = BERTNLU()
+        sys_nlu = BERTNLU("user", config_file="multiwoz21_user_context3.json", model_file="bertnlu_unified_multiwoz21_user_context3")
     elif args.dst_type in ["setsumbt", "trippy"]:
         sys_nlu = None
     
@@ -79,10 +80,10 @@ def test_end2end(args, model_dir):
 
 
     # where the models are saved from training
-    lava_dir = "/gpfs/project/lubis/ConvLab-3/convlab/policy/lava/multiwoz/experiments_woz/sys_config_log_model/"
+    # lava_dir = "/gpfs/project/lubis/ConvLab-3/convlab/policy/lava/multiwoz/experiments_woz/sys_config_log_model/"
+    lava_dir = "/gpfs/project/lubis/LAVA_code/LAVA_published/experiments_woz/sys_config_log_model/"
 
     if "rl" in model_dir:
-        # lava_path = "{}/{}/reward_best.model".format(lava_dir, model_path[args.lava_model_type])
         lava_path = "{}/{}/reward_best.model".format(lava_dir, model_dir)
     else:
         # default saved model format
@@ -96,14 +97,9 @@ def test_end2end(args, model_dir):
 
     # template NLG
     sys_nlg = None
-    # assemble
-    sys_agent = PipelineAgent(
-        sys_nlu, sys_dst, sys_policy, sys_nlg, name='sys')
-    sys_agent.add_booking_info = False
 
     # BERT nlu trained on sys utterance
-    user_nlu = BERTNLU(mode='sys', config_file='multiwoz_sys_context.json',
-                       model_file='https://convlab.blob.core.windows.net/convlab-2/bert_multiwoz_sys_context.zip')
+    user_nlu = BERTNLU("sys", config_file="multiwoz21_system_context3_new.json", model_file="bertnlu_unified_multiwoz21_system_context3")
     if args.US_type == "ABUS":
         # not use dst
         user_dst = None
@@ -122,9 +118,14 @@ def test_end2end(args, model_dir):
         user_policy = UserPolicy(user_config)
     # template NLG
     user_nlg = TemplateNLG(is_user=True)
-    # assemble
+    # assemble agents
     user_agent = PipelineAgent(
         user_nlu, user_dst, user_policy, user_nlg, name='user')
+    sys_agent = PipelineAgent(
+        sys_nlu, sys_dst, sys_policy, sys_nlg, name='sys')
+
+    sys_agent.add_booking_info = False
+
 
     analyzer = Analyzer(user_agent=user_agent, dataset='multiwoz')
 
@@ -132,15 +133,15 @@ def test_end2end(args, model_dir):
     set_seed(args.seed)
 
     model_name = '{}_{}_lava_{}'.format(args.US_type, args.dst_type, model_dir)
-    analyzer.comprehensive_analyze(sys_agent=sys_agent, model_name=model_name, total_dialog=1000)
+    analyzer.comprehensive_analyze(sys_agent=sys_agent, model_name=model_name, total_dialog=500)
 
 if __name__ == '__main__':
     parser = ArgumentParser()
-    parser.add_argument("--lava_dir", type=str, default="2020-05-12-14-51-49-actz_cat")
+    parser.add_argument("--lava_dir", type=str, default="2020-05-12-14-51-49-actz_cat/rl-2020-05-18-10-50-48")
     parser.add_argument("--US_trained", type=bool, default=False, help="whether to use model trained on US or not")
     parser.add_argument("--seed", type=int, default=20200202, help="seed for random processes")
     parser.add_argument("--US_type", type=str, default="ABUS", help="which user simulator to us, ABUS or TUS")
-    parser.add_argument("--dst_type", type=str, default="setsumbt", help="which DST to use, bertnlu_rule, setsumbt, or trippy")
+    parser.add_argument("--dst_type", type=str, default="bertnlu_rule", help="which DST to use, bertnlu_rule, setsumbt, or trippy")
     args = parser.parse_args()
 
     test_end2end(args, args.lava_dir)
diff --git a/requirements.txt b/requirements.txt
index 1f2ab21fe01047f604424fcb2674fa3c1e1a0bcc..26b14729071e602dcb2d7891eeb5f6b32cd3397e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,114 +1,125 @@
-absl-py==1.1.0
-accelerate==0.10.0
-aiohttp==3.8.1
-aiosignal==1.2.0
+absl-py==1.3.0
+accelerate==0.14.0
+aiohttp==3.8.3
+aiosignal==1.3.1
 async-timeout==4.0.2
-attrs==21.4.0
-blis==0.7.8
-boto3==1.24.22
-botocore==1.27.22
+attrs==22.1.0
+blis==0.7.9
+boto3==1.26.18
+botocore==1.29.18
 cachetools==5.2.0
-catalogue==2.0.7
-certifi==2022.5.18.1
-charset-normalizer==2.1.0
+catalogue==2.0.8
+certifi==2022.9.24
+charset-normalizer==2.1.1
 click==8.1.3
-colorama==0.4.5
+colorama==0.4.6
+confection==0.0.3
+contourpy==1.0.6
 cycler==0.11.0
-cymem==2.0.6
-datasets==2.3.2
+cymem==2.0.7
+datasets==2.7.1
 deepspeech==0.9.3
-dill==0.3.5.1
+dill==0.3.6
 embeddings==0.0.8
-filelock==3.7.1
-fonttools==4.33.3
-frozenlist==1.3.0
-fsspec==2022.5.0
+filelock==3.8.0
+fonttools==4.38.0
+frozenlist==1.3.3
+fsspec==2022.11.0
 fuzzywuzzy==0.18.0
-google-auth==2.9.0
+google-auth==2.14.1
 google-auth-oauthlib==0.4.6
-grpcio==1.47.0
-gTTS==2.2.4
-huggingface-hub==0.8.1
-idna==3.3
-importlib-metadata==4.12.0
+grpcio==1.50.0
+gTTS==2.3.0
+huggingface-hub==0.11.1
+idna==3.4
+importlib-metadata==5.1.0
 jieba==0.42.1
 Jinja2==3.1.2
 jmespath==1.0.1
-joblib==1.1.0
+joblib==1.2.0
 json-lines==0.5.0
 jsonpatch==1.32
 jsonpointer==2.3
-kiwisolver==1.4.3
+kiwisolver==1.4.4
 langcodes==3.3.0
-Markdown==3.3.7
+Levenshtein==0.20.8
+lxml==4.9.1
+Markdown==3.4.1
 MarkupSafe==2.1.1
-matplotlib==3.5.2
+matplotlib==3.6.2
 multidict==6.0.2
-multiprocess==0.70.13
-murmurhash==1.0.7
+multiprocess==0.70.14
+murmurhash==1.0.9
+networkx==2.8.8
 nltk==3.7
-numpy==1.23.0
-oauthlib==3.2.0
+numpy==1.23.5
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+oauthlib==3.2.2
 packaging==21.3
-pandas==1.4.3
-pathy==0.6.2
-Pillow==9.2.0
-portalocker==2.4.0
-preshed==3.0.6
-protobuf==3.19.4
-psutil==5.9.1
-pyarrow==8.0.0
+pandas==1.5.2
+pathy==0.10.0
+Pillow==9.3.0
+portalocker==2.6.0
+preshed==3.0.8
+protobuf==3.20.2
+psutil==5.9.4
+pyarrow==10.0.1
 pyasn1==0.4.8
 pyasn1-modules==0.2.8
-pydantic==1.8.2
+pydantic==1.10.2
 pydub==0.25.1
 pyparsing==3.0.9
 python-dateutil==2.8.2
-python-Levenshtein==0.12.2
-pytz==2022.1
+python-Levenshtein==0.20.8
+pytz==2022.6
 PyYAML==6.0
-pyzmq==23.2.0
 quadprog==0.1.11
-regex==2022.6.2
+rapidfuzz==2.13.2
+regex==2022.10.31
 requests==2.28.1
 requests-oauthlib==1.3.1
 responses==0.18.0
-rouge-score==0.0.4
-rsa==4.8
+rouge-score==0.1.2
+rsa==4.9
 s3transfer==0.6.0
-sacrebleu==2.1.0
-scikit-learn==1.1.1
-scipy==1.8.1
+sacrebleu==2.3.1
+scikit-learn==1.1.3
+scipy==1.9.3
+seaborn==0.12.1
 sentence-transformers==2.2.2
+sentencepiece==0.1.97
 seqeval==1.2.2
-simplejson==3.17.6
+simplejson==3.18.0
 six==1.16.0
 smart-open==5.2.1
-spacy==3.3.1
-spacy-legacy==3.0.9
-spacy-loggers==1.0.2
-srsly==2.4.3
-tabulate==0.8.10
-tensorboard==2.9.1
+spacy==3.4.3
+spacy-legacy==3.0.10
+spacy-loggers==1.0.3
+srsly==2.4.5
+tabulate==0.9.0
+tensorboard==2.11.0
 tensorboard-data-server==0.6.1
 tensorboard-plugin-wit==1.8.1
-tensorboardX==2.5.1
-thinc==8.0.17
+tensorboardX==2.5
+thinc==8.1.5
 threadpoolctl==3.1.0
-tokenizers==0.12.1
-torch==1.12.0
-torchfile==0.1.0
+tokenizers==0.13.2
+torch==1.11.0
+torchvision==0.12.0
 tornado==6.2
-tqdm==4.64.0
-transformers==4.20.1
-typer==0.4.2
-typing_extensions==4.3.0
-Unidecode==1.3.4
-urllib3==1.26.9
-visdom==0.1.8.9
-wasabi==0.9.1
-websocket-client==1.3.3
-Werkzeug==2.1.2
-xxhash==3.0.0
-yarl==1.7.2
-zipp==3.8.0
+tqdm==4.64.1
+transformers==4.24.0
+typer==0.7.0
+typing_extensions==4.4.0
+Unidecode==1.3.6
+urllib3==1.26.13
+visdom==0.2.3
+wasabi==0.10.1
+websocket-client==1.4.2
+Werkzeug==2.2.2
+xxhash==3.1.0
+yarl==1.8.1
+zipp==3.11.0
diff --git a/setup.py b/setup.py
index d195fdcc10bc83cd7a1ab1fcf85e68ae1156cdec..c2cae13c857561899c9b71f68e236f30d705f5ce 100755
--- a/setup.py
+++ b/setup.py
@@ -8,14 +8,12 @@ setup(
     version='3.0.0',
     packages=find_packages(),
     license='Apache',
-    description='An Open-source Dialog System Platform',
+    description='An Open-source Dialog System Toolkit',
     long_description=open('README.md', encoding='UTF-8').read(),
     long_description_content_type="text/markdown",
     classifiers=[
         'Development Status :: 3 - Alpha',
         'License :: OSI Approved :: Apache Software License',
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Intended Audience :: Science/Research',
@@ -24,12 +22,17 @@ setup(
     ],
     setup_requires=['setuptools-git'],
     install_requires=[
+        'joblib>=1.2.0',
+        'pillow>=9.3.0',
+        'protobuf>=3.20.2',
+        'oauthlib>=3.2.1',
         'accelerate',
         'rouge-score',
         'sacrebleu',
         'tensorboardX',
         'boto3',
         'matplotlib',
+        'seaborn',
         'tabulate',
         'python-Levenshtein',
         'requests',
@@ -37,10 +40,10 @@ setup(
         'nltk',
         'scipy',
         'tensorboard',
-        'torch>=1.6',
-        'transformers>=4.0',
-        'sentence-transformers',
-        'datasets>=1.8',
+        'torch>=1.10.1,<=1.13',
+        'transformers>=4.17.0,<=4.24.0',
+        'sentence-transformers>=2.2.2',
+        'datasets>=2.0',
         'seqeval',
         'spacy',
         'simplejson',
@@ -74,6 +77,6 @@ setup(
     url='https://github.com/ConvLab/ConvLab-3',
     author='convlab',
     author_email='convlab@googlegroups.com',
-    python_requires='>=3.6',
+    python_requires='>=3.8',
     zip_safe=False
 )