diff --git a/convlab/policy/evaluate.py b/convlab/policy/evaluate.py index ecdfaddbcb204ec465688e17b49595eebc231af0..7a692261869f35e587c34a26e425d1489abdcf56 100755 --- a/convlab/policy/evaluate.py +++ b/convlab/policy/evaluate.py @@ -58,7 +58,7 @@ def evaluate(config_path, model_name, verbose=False, model_path="", goals_from_d from convlab.policy.gdpl import GDPL policy_sys = GDPL(vectorizer=conf['vectorizer_sys_activated']) elif model_name == "DDPT": - from convlab2.policy.vtrace_DPT import VTRACE + from convlab.policy.vtrace_DPT import VTRACE policy_sys = VTRACE(is_train=False, vectorizer=conf['vectorizer_sys_activated']) try: diff --git a/convlab/policy/evaluate_distributed.py b/convlab/policy/evaluate_distributed.py index fb363d3d739a42a0de5259965909692c345e1525..969c13b24d39592a174d7d5f3fab7af1a54895e8 100644 --- a/convlab/policy/evaluate_distributed.py +++ b/convlab/policy/evaluate_distributed.py @@ -2,13 +2,9 @@ import random import torch -import sys -import torch -from pprint import pprint - -import matplotlib.pyplot as plt import numpy as np -from convlab.policy.rlmodule import Memory_evaluator, Transition + +from convlab.policy.rlmodule import Memory_evaluator from torch import multiprocessing as mp @@ -47,6 +43,7 @@ def sampler(pid, queue, evt, sess, seed_range, goals): request = 0 select = 0 offer = 0 + recommend = 0 task_success = {} for i in range(40): @@ -71,6 +68,8 @@ def sampler(pid, queue, evt, sess, seed_range, goals): select += 1 if intent.lower() == 'offerbook': offer += 1 + if intent.lower() == 'recommend': + recommend += 1 if session_over is True: success = sess.evaluator.task_success() @@ -85,7 +84,7 @@ def sampler(pid, queue, evt, sess, seed_range, goals): task_success[key].append(success_strict) buff.push(complete, success, success_strict, total_return_complete, total_return_success, turns, avg_actions / turns, - task_success, book, inform, request, select, offer) + task_success, book, inform, request, select, offer, recommend) # this is end of sampling all batchsz of items. # when sampling is over, push all buff data into queue @@ -140,7 +139,7 @@ def evaluate_distributed(sess, seed_range, process_num, goals): return np.average(batch.complete), np.average(batch.success), np.average(batch.success_strict), \ np.average(batch.total_return_success), np.average(batch.turns), np.average(batch.avg_actions), \ batch.task_success, np.average(batch.book_actions), np.average(batch.inform_actions), np.average(batch.request_actions), \ - np.average(batch.select_actions), np.average(batch.offer_actions) + np.average(batch.select_actions), np.average(batch.offer_actions), np.average(batch.recommend_actions) if __name__ == "__main__": diff --git a/convlab/policy/gdpl/train.py b/convlab/policy/gdpl/train.py index 3d560b5650d5eb87e02334a42d62b4bdfd097279..c2c607fc855ddfb50799480ae2295b812a13d12c 100755 --- a/convlab/policy/gdpl/train.py +++ b/convlab/policy/gdpl/train.py @@ -186,7 +186,7 @@ if __name__ == '__main__': parser = ArgumentParser() parser.add_argument("--path", type=str, default='convlab/policy/gdpl/semantic_level_config.json', help="Load path for config file") - parser.add_argument("--seed", type=int, default=0, + parser.add_argument("--seed", type=int, default=None, help="Seed for the policy parameter initialization") parser.add_argument("--pretrain", action='store_true', help="whether to pretrain the reward estimator") parser.add_argument("--mode", type=str, default='info', @@ -202,7 +202,7 @@ if __name__ == '__main__': logger, tb_writer, current_time, save_path, config_save_path, dir_path, log_save_path = \ init_logging(os.path.dirname(os.path.abspath(__file__)), mode) - args = [('model', 'seed', seed)] + args = [('model', 'seed', seed)] if seed is not None else list() environment_config = load_config_file(path) save_config(vars(parser.parse_args()), environment_config, config_save_path) diff --git a/convlab/policy/pg/train.py b/convlab/policy/pg/train.py index 05d51015b38072ffc00b72ae11f9d1ba0ee0ade0..a8400c23a61a758c6866c5660ebcf1e1eefb004d 100755 --- a/convlab/policy/pg/train.py +++ b/convlab/policy/pg/train.py @@ -184,7 +184,7 @@ if __name__ == '__main__': parser = ArgumentParser() parser.add_argument("--path", type=str, default='convlab/policy/pg/semantic_level_config.json', help="Load path for config file") - parser.add_argument("--seed", type=int, default=0, + parser.add_argument("--seed", type=int, default=None, help="Seed for the policy parameter initialization") parser.add_argument("--mode", type=str, default='info', help="Set level for logger") @@ -199,7 +199,7 @@ if __name__ == '__main__': logger, tb_writer, current_time, save_path, config_save_path, dir_path, log_save_path = \ init_logging(os.path.dirname(os.path.abspath(__file__)), mode) - args = [('model', 'seed', seed)] + args = [('model', 'seed', seed)] if seed is not None else list() environment_config = load_config_file(path) save_config(vars(parser.parse_args()), environment_config, config_save_path) diff --git a/convlab/policy/ppo/train.py b/convlab/policy/ppo/train.py index 415b0b27477599248fc31c4f92e9232dc422b47b..1fb3e39759488f6b95c57ad9403abf0e3bd33079 100755 --- a/convlab/policy/ppo/train.py +++ b/convlab/policy/ppo/train.py @@ -184,7 +184,7 @@ if __name__ == '__main__': parser = ArgumentParser() parser.add_argument("--path", type=str, default='convlab/policy/ppo/semantic_level_config.json', help="Load path for config file") - parser.add_argument("--seed", type=int, default=0, + parser.add_argument("--seed", type=int, default=None, help="Seed for the policy parameter initialization") parser.add_argument("--mode", type=str, default='info', help="Set level for logger") @@ -199,7 +199,7 @@ if __name__ == '__main__': logger, tb_writer, current_time, save_path, config_save_path, dir_path, log_save_path = \ init_logging(os.path.dirname(os.path.abspath(__file__)), mode) - args = [('model', 'seed', seed)] + args = [('model', 'seed', seed)] if seed is not None else list() environment_config = load_config_file(path) save_config(vars(parser.parse_args()), environment_config, config_save_path) diff --git a/convlab/policy/rlmodule.py b/convlab/policy/rlmodule.py index db46026656d908b2453a9143b3f482ce7378e382..18844ca375967cb9524dacd3a81a4f85a2cf9041 100755 --- a/convlab/policy/rlmodule.py +++ b/convlab/policy/rlmodule.py @@ -287,7 +287,7 @@ class Value(nn.Module): Transition_evaluator = namedtuple('Transition_evaluator', ('complete', 'success', 'success_strict', 'total_return_complete', 'total_return_success', 'turns', 'avg_actions', 'task_success', 'book_actions', 'inform_actions', 'request_actions', 'select_actions', - 'offer_actions')) + 'offer_actions', 'recommend_actions')) class Memory_evaluator(object): diff --git a/convlab/policy/vector/dataset.py b/convlab/policy/vector/dataset.py index 0aa1b7ad879f2d814cece3a98bb457b71ad99033..b1481854fb784ce5f29b24549b44eda93150fb27 100755 --- a/convlab/policy/vector/dataset.py +++ b/convlab/policy/vector/dataset.py @@ -18,6 +18,26 @@ class ActDataset(data.Dataset): return self.num_total +class ActDatasetKG(data.Dataset): + def __init__(self, action_batch, a_masks, current_domain_mask_batch, non_current_domain_mask_batch): + self.action_batch = action_batch + self.a_masks = a_masks + self.current_domain_mask_batch = current_domain_mask_batch + self.non_current_domain_mask_batch = non_current_domain_mask_batch + self.num_total = len(action_batch) + + def __getitem__(self, index): + action = self.action_batch[index] + action_mask = self.a_masks[index] + current_domain_mask = self.current_domain_mask_batch[index] + non_current_domain_mask = self.non_current_domain_mask_batch[index] + + return action, action_mask, current_domain_mask, non_current_domain_mask, index + + def __len__(self): + return self.num_total + + class ActStateDataset(data.Dataset): def __init__(self, s_s, a_s, next_s): self.s_s = s_s diff --git a/convlab/policy/vector/vector_base.py b/convlab/policy/vector/vector_base.py index 8b7d8ff0ddafed41efc91b249003ae55c525bc93..89f222036945c40658496d52291cbb137f1ca9fe 100644 --- a/convlab/policy/vector/vector_base.py +++ b/convlab/policy/vector/vector_base.py @@ -2,10 +2,11 @@ import os import sys import numpy as np +import logging from copy import deepcopy from convlab.policy.vec import Vector -from convlab.util.custom_util import flatten_acts +from convlab.util.custom_util import flatten_acts, timeout from convlab.util.multiwoz.lexicalize import delexicalize_da, flat_da, deflat_da, lexicalize_da from convlab.util import load_ontology, load_database, load_dataset @@ -22,18 +23,20 @@ class VectorBase(Vector): super().__init__() + logging.info(f"Vectorizer: Data set used is {dataset_name}") self.set_seed(seed) self.ontology = load_ontology(dataset_name) try: # execute to make sure that the database exists or is downloaded otherwise - load_database(dataset_name) + if dataset_name == "multiwoz21": + load_database(dataset_name) # the following two lines are needed for pickling correctly during multi-processing exec(f'from data.unified_datasets.{dataset_name}.database import Database') self.db = eval('Database()') self.db_domains = self.db.domains except Exception as e: self.db = None - self.db_domains = None + self.db_domains = [] print(f"VectorBase: {e}") self.dataset_name = dataset_name @@ -272,6 +275,10 @@ class VectorBase(Vector): 2. If there is an entity available, can not say NoOffer or NoBook ''' mask_list = np.zeros(self.da_dim) + + if number_entities_dict is None: + return mask_list + for i in range(self.da_dim): action = self.vec2act[i] domain, intent, slot, value = action.split('-') diff --git a/convlab/policy/vector/vector_nodes.py b/convlab/policy/vector/vector_nodes.py index c2f6258f48dfc7b27f2bce6c17b7c6e1f92e7705..2e073669effc518cee4efd1f03d25bbd501b65af 100644 --- a/convlab/policy/vector/vector_nodes.py +++ b/convlab/policy/vector/vector_nodes.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import sys import numpy as np +import logging + from convlab.util.multiwoz.lexicalize import delexicalize_da, flat_da from .vector_base import VectorBase @@ -8,9 +10,11 @@ from .vector_base import VectorBase class VectorNodes(VectorBase): def __init__(self, dataset_name='multiwoz21', character='sys', use_masking=False, manually_add_entity_names=True, - seed=0): + seed=0, filter_state=True): super().__init__(dataset_name, character, use_masking, manually_add_entity_names, seed) + self.filter_state = filter_state + logging.info(f"We filter state by active domains: {self.filter_state}") def get_state_dim(self): self.belief_state_dim = 0 @@ -56,9 +60,16 @@ class VectorNodes(VectorBase): self.get_user_act_feature(state) self.get_sys_act_feature(state) domain_active_dict = self.get_user_goal_feature(state, domain_active_dict) - number_entities_dict = self.get_db_features() self.get_general_features(state, domain_active_dict) + if self.db is not None: + number_entities_dict = self.get_db_features() + else: + number_entities_dict = None + + if self.filter_state: + self.kg_info = self.filter_inactive_domains(domain_active_dict) + if self.use_mask: mask = self.get_mask(domain_active_dict, number_entities_dict) for i in range(self.da_dim): @@ -89,13 +100,15 @@ class VectorNodes(VectorBase): feature_type = 'user goal' for domain in self.belief_domains: - for slot, value in state['belief_state'][domain].items(): - description = f"user goal-{domain}-{slot}".lower() - value = 1.0 if (value and value != "not mentioned") else 0.0 - self.add_graph_node(domain, feature_type, description, value) - - if [slot for slot, value in state['belief_state'][domain].items() if value]: - domain_active_dict[domain] = True + # the if case is needed because SGD only saves the dialogue state info for active domains + if domain in state['belief_state']: + for slot, value in state['belief_state'][domain].items(): + description = f"user goal-{domain}-{slot}".lower() + value = 1.0 if (value and value != "not mentioned") else 0.0 + self.add_graph_node(domain, feature_type, description, value) + + if [slot for slot, value in state['belief_state'][domain].items() if value]: + domain_active_dict[domain] = True return domain_active_dict def get_sys_act_feature(self, state): @@ -128,11 +141,12 @@ class VectorNodes(VectorBase): def get_general_features(self, state, domain_active_dict): feature_type = 'general' - for i, domain in enumerate(self.db_domains): - if domain in state['booked']: - description = f"general-{domain}-booked".lower() - value = 1.0 if state['booked'][domain] else 0.0 - self.add_graph_node(domain, feature_type, description, value) + if 'booked' in state: + for i, domain in enumerate(self.db_domains): + if domain in state['booked']: + description = f"general-{domain}-booked".lower() + value = 1.0 if state['booked'][domain] else 0.0 + self.add_graph_node(domain, feature_type, description, value) for domain in self.domains: if domain == 'general': @@ -140,3 +154,17 @@ class VectorNodes(VectorBase): value = 1.0 if domain_active_dict[domain] else 0 description = f"general-{domain}".lower() self.add_graph_node(domain, feature_type, description, value) + + def filter_inactive_domains(self, domain_active_dict): + + kg_filtered = [] + for node in self.kg_info: + domain = node['domain'] + if domain in domain_active_dict: + if domain_active_dict[domain]: + kg_filtered.append(node) + else: + kg_filtered.append(node) + + return kg_filtered + diff --git a/convlab/policy/vtrace_DPT/config.json b/convlab/policy/vtrace_DPT/config.json index 5b28472a2b74896af1da11c3fa5addfab6b51a7b..13362e74f11fd3705820028366b145c7d66ea4d9 100644 --- a/convlab/policy/vtrace_DPT/config.json +++ b/convlab/policy/vtrace_DPT/config.json @@ -1,8 +1,9 @@ { "batchsz": 64, - "epoch": 16, + "epoch": 40, "gamma": 0.99, - "policy_lr": 0.00005, + "policy_lr": 5e-06, + "supervised_lr": 1e-05, "entropy_weight": 0.01, "value_lr": 0.0001, "save_dir": "save", @@ -17,16 +18,21 @@ "supervised_weight": 0.0, "online_offline_ratio": 0.20, "smoothed_value_function": false, - "use_reservoir_sampling": true, + "use_reservoir_sampling": false, "seed": 0, "lambda": 1, "tau": 0.001, - "policy_freq": 2, + "policy_freq": 1, "print_per_batch": 400, "c": 1.0, "rho_bar": 1, "max_length": 10, "noisy_linear": false, + "dataset_name": "multiwoz21", + "data_percentage": 1.0, + "dialogue_order": 0, + "multiwoz_like": false, + "regularization_weight": 0.0, "enc_input_dim": 128, "enc_nhead": 2, @@ -66,6 +72,5 @@ "independent": true, "old_critic": false, "pos_weight": 5, - "lr_supervised": 0.0001, "weight_decay": 0.00001 } \ No newline at end of file diff --git a/convlab/policy/vtrace_DPT/create_descriptions.py b/convlab/policy/vtrace_DPT/create_descriptions.py index 6a6778048668d8c3a4eb1c43a8ed9e389ac6557a..5357fafafcf4972aa8eebaecf8c56e9fba79afe7 100644 --- a/convlab/policy/vtrace_DPT/create_descriptions.py +++ b/convlab/policy/vtrace_DPT/create_descriptions.py @@ -3,6 +3,7 @@ import json from convlab.policy.vector.vector_binary import VectorBinary from convlab.util import load_ontology, load_database +from convlab.util.custom_util import timeout def create_description_dicts(name='multiwoz21'): @@ -11,8 +12,13 @@ def create_description_dicts(name='multiwoz21'): ontology = load_ontology(name) default_state = ontology['state'] domains = list(ontology['domains'].keys()) - db = load_database(name) - db_domains = db.domains + + if name == "multiwoz21": + db = load_database(name) + db_domains = db.domains + else: + db = None + db_domains = [] root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) voc_file = os.path.join(root_dir, f'vector/action_dicts/{name}_VectorBinary/sys_da_voc.txt') @@ -27,25 +33,32 @@ def create_description_dicts(name='multiwoz21'): for domain in default_state: for slot in default_state[domain]: + domain = domain.lower() description_dict_semantic[f"user goal-{domain}-{slot.lower()}"] = f"user goal {domain} {slot}" - for domain in db_domains: - description_dict_semantic[f"db-{domain}-entities"] = f"data base {domain} number of entities" - description_dict_semantic[f"general-{domain}-booked"] = f"general {domain} booked" + if db_domains: + for domain in db_domains: + domain = domain.lower() + description_dict_semantic[f"db-{domain}-entities"] = f"data base {domain} number of entities" + description_dict_semantic[f"general-{domain}-booked"] = f"general {domain} booked" for domain in domains: + domain = domain.lower() description_dict_semantic[f"general-{domain}"] = f"domain {domain}" for act in da_voc: domain, intent, slot, value = act.split("-") + domain = domain.lower() description_dict_semantic["system-"+act.lower()] = f"last system act {domain} {intent} {slot} {value}" for act in da_voc_opp: domain, intent, slot, value = [item.lower() for item in act.split("-")] + domain = domain.lower() description_dict_semantic["user-"+act.lower()] = f"user act {domain} {intent} {slot} {value}" root_dir = os.path.dirname(os.path.abspath(__file__)) - with open(os.path.join(root_dir, 'semantic_information_descriptions.json'), "w") as f: + os.makedirs(os.path.join(root_dir, "descriptions"), exist_ok=True) + with open(os.path.join(root_dir, 'descriptions', f'semantic_information_descriptions_{name}.json'), "w") as f: json.dump(description_dict_semantic, f) diff --git a/convlab/policy/vtrace_DPT/semantic_information_descriptions.json b/convlab/policy/vtrace_DPT/descriptions/semantic_information_descriptions_multiwoz21.json similarity index 100% rename from convlab/policy/vtrace_DPT/semantic_information_descriptions.json rename to convlab/policy/vtrace_DPT/descriptions/semantic_information_descriptions_multiwoz21.json diff --git a/convlab/policy/vtrace_DPT/descriptions/semantic_information_descriptions_sgd.json b/convlab/policy/vtrace_DPT/descriptions/semantic_information_descriptions_sgd.json new file mode 100644 index 0000000000000000000000000000000000000000..2539067dfd07e25e35359218ed4487a6da057910 --- /dev/null +++ b/convlab/policy/vtrace_DPT/descriptions/semantic_information_descriptions_sgd.json @@ -0,0 +1 @@ +{"user goal-banks_1-account_type": "user goal banks_1 account_type", "user goal-banks_1-recipient_account_type": "user goal banks_1 recipient_account_type", "user goal-banks_1-balance": "user goal banks_1 balance", "user goal-banks_1-amount": "user goal banks_1 amount", "user goal-banks_1-recipient_account_name": "user goal banks_1 recipient_account_name", "user goal-buses_1-from_location": "user goal buses_1 from_location", "user goal-buses_1-to_location": "user goal buses_1 to_location", "user goal-buses_1-from_station": "user goal buses_1 from_station", "user goal-buses_1-to_station": "user goal buses_1 to_station", "user goal-buses_1-leaving_date": "user goal buses_1 leaving_date", "user goal-buses_1-leaving_time": "user goal buses_1 leaving_time", "user goal-buses_1-fare": "user goal buses_1 fare", "user goal-buses_1-travelers": "user goal buses_1 travelers", "user goal-buses_1-transfers": "user goal buses_1 transfers", "user goal-buses_2-origin": "user goal buses_2 origin", "user goal-buses_2-destination": "user goal buses_2 destination", "user goal-buses_2-origin_station_name": "user goal buses_2 origin_station_name", "user goal-buses_2-destination_station_name": "user goal buses_2 destination_station_name", "user goal-buses_2-departure_date": "user goal buses_2 departure_date", "user goal-buses_2-price": "user goal buses_2 price", "user goal-buses_2-departure_time": "user goal buses_2 departure_time", "user goal-buses_2-group_size": "user goal buses_2 group_size", "user goal-buses_2-fare_type": "user goal buses_2 fare_type", "user goal-calendar_1-event_date": "user goal calendar_1 event_date", "user goal-calendar_1-event_time": "user goal calendar_1 event_time", "user goal-calendar_1-event_location": "user goal calendar_1 event_location", "user goal-calendar_1-event_name": "user goal calendar_1 event_name", "user goal-calendar_1-available_start_time": "user goal calendar_1 available_start_time", "user goal-calendar_1-available_end_time": "user goal calendar_1 available_end_time", "user goal-events_1-category": "user goal events_1 category", "user goal-events_1-subcategory": "user goal events_1 subcategory", "user goal-events_1-event_name": "user goal events_1 event_name", "user goal-events_1-date": "user goal events_1 date", "user goal-events_1-time": "user goal events_1 time", "user goal-events_1-number_of_seats": "user goal events_1 number_of_seats", "user goal-events_1-city_of_event": "user goal events_1 city_of_event", "user goal-events_1-event_location": "user goal events_1 event_location", "user goal-events_1-address_of_location": "user goal events_1 address_of_location", "user goal-events_2-event_type": "user goal events_2 event_type", "user goal-events_2-category": "user goal events_2 category", "user goal-events_2-event_name": "user goal events_2 event_name", "user goal-events_2-date": "user goal events_2 date", "user goal-events_2-time": "user goal events_2 time", "user goal-events_2-number_of_tickets": "user goal events_2 number_of_tickets", "user goal-events_2-city": "user goal events_2 city", "user goal-events_2-venue": "user goal events_2 venue", "user goal-events_2-venue_address": "user goal events_2 venue_address", "user goal-flights_1-passengers": "user goal flights_1 passengers", "user goal-flights_1-seating_class": "user goal flights_1 seating_class", "user goal-flights_1-origin_city": "user goal flights_1 origin_city", "user goal-flights_1-destination_city": "user goal flights_1 destination_city", "user goal-flights_1-origin_airport": "user goal flights_1 origin_airport", "user goal-flights_1-destination_airport": "user goal flights_1 destination_airport", "user goal-flights_1-departure_date": "user goal flights_1 departure_date", "user goal-flights_1-return_date": "user goal flights_1 return_date", "user goal-flights_1-number_stops": "user goal flights_1 number_stops", "user goal-flights_1-outbound_departure_time": "user goal flights_1 outbound_departure_time", "user goal-flights_1-outbound_arrival_time": "user goal flights_1 outbound_arrival_time", "user goal-flights_1-inbound_arrival_time": "user goal flights_1 inbound_arrival_time", "user goal-flights_1-inbound_departure_time": "user goal flights_1 inbound_departure_time", "user goal-flights_1-price": "user goal flights_1 price", "user goal-flights_1-refundable": "user goal flights_1 refundable", "user goal-flights_1-airlines": "user goal flights_1 airlines", "user goal-flights_2-passengers": "user goal flights_2 passengers", "user goal-flights_2-seating_class": "user goal flights_2 seating_class", "user goal-flights_2-origin": "user goal flights_2 origin", "user goal-flights_2-destination": "user goal flights_2 destination", "user goal-flights_2-origin_airport": "user goal flights_2 origin_airport", "user goal-flights_2-destination_airport": "user goal flights_2 destination_airport", "user goal-flights_2-departure_date": "user goal flights_2 departure_date", "user goal-flights_2-return_date": "user goal flights_2 return_date", "user goal-flights_2-number_stops": "user goal flights_2 number_stops", "user goal-flights_2-outbound_departure_time": "user goal flights_2 outbound_departure_time", "user goal-flights_2-outbound_arrival_time": "user goal flights_2 outbound_arrival_time", "user goal-flights_2-inbound_arrival_time": "user goal flights_2 inbound_arrival_time", "user goal-flights_2-inbound_departure_time": "user goal flights_2 inbound_departure_time", "user goal-flights_2-fare": "user goal flights_2 fare", "user goal-flights_2-is_redeye": "user goal flights_2 is_redeye", "user goal-flights_2-airlines": "user goal flights_2 airlines", "user goal-homes_1-area": "user goal homes_1 area", "user goal-homes_1-address": "user goal homes_1 address", "user goal-homes_1-property_name": "user goal homes_1 property_name", "user goal-homes_1-phone_number": "user goal homes_1 phone_number", "user goal-homes_1-furnished": "user goal homes_1 furnished", "user goal-homes_1-pets_allowed": "user goal homes_1 pets_allowed", "user goal-homes_1-rent": "user goal homes_1 rent", "user goal-homes_1-visit_date": "user goal homes_1 visit_date", "user goal-homes_1-number_of_beds": "user goal homes_1 number_of_beds", "user goal-homes_1-number_of_baths": "user goal homes_1 number_of_baths", "user goal-hotels_1-destination": "user goal hotels_1 destination", "user goal-hotels_1-number_of_rooms": "user goal hotels_1 number_of_rooms", "user goal-hotels_1-check_in_date": "user goal hotels_1 check_in_date", "user goal-hotels_1-number_of_days": "user goal hotels_1 number_of_days", "user goal-hotels_1-star_rating": "user goal hotels_1 star_rating", "user goal-hotels_1-hotel_name": "user goal hotels_1 hotel_name", "user goal-hotels_1-street_address": "user goal hotels_1 street_address", "user goal-hotels_1-phone_number": "user goal hotels_1 phone_number", "user goal-hotels_1-price_per_night": "user goal hotels_1 price_per_night", "user goal-hotels_1-has_wifi": "user goal hotels_1 has_wifi", "user goal-hotels_2-where_to": "user goal hotels_2 where_to", "user goal-hotels_2-number_of_adults": "user goal hotels_2 number_of_adults", "user goal-hotels_2-check_in_date": "user goal hotels_2 check_in_date", "user goal-hotels_2-check_out_date": "user goal hotels_2 check_out_date", "user goal-hotels_2-rating": "user goal hotels_2 rating", "user goal-hotels_2-address": "user goal hotels_2 address", "user goal-hotels_2-phone_number": "user goal hotels_2 phone_number", "user goal-hotels_2-total_price": "user goal hotels_2 total_price", "user goal-hotels_2-has_laundry_service": "user goal hotels_2 has_laundry_service", "user goal-hotels_3-location": "user goal hotels_3 location", "user goal-hotels_3-number_of_rooms": "user goal hotels_3 number_of_rooms", "user goal-hotels_3-check_in_date": "user goal hotels_3 check_in_date", "user goal-hotels_3-check_out_date": "user goal hotels_3 check_out_date", "user goal-hotels_3-average_rating": "user goal hotels_3 average_rating", "user goal-hotels_3-hotel_name": "user goal hotels_3 hotel_name", "user goal-hotels_3-street_address": "user goal hotels_3 street_address", "user goal-hotels_3-phone_number": "user goal hotels_3 phone_number", "user goal-hotels_3-price": "user goal hotels_3 price", "user goal-hotels_3-pets_welcome": "user goal hotels_3 pets_welcome", "user goal-media_1-title": "user goal media_1 title", "user goal-media_1-genre": "user goal media_1 genre", "user goal-media_1-subtitles": "user goal media_1 subtitles", "user goal-media_1-directed_by": "user goal media_1 directed_by", "user goal-movies_1-price": "user goal movies_1 price", "user goal-movies_1-number_of_tickets": "user goal movies_1 number_of_tickets", "user goal-movies_1-show_type": "user goal movies_1 show_type", "user goal-movies_1-theater_name": "user goal movies_1 theater_name", "user goal-movies_1-show_time": "user goal movies_1 show_time", "user goal-movies_1-show_date": "user goal movies_1 show_date", "user goal-movies_1-genre": "user goal movies_1 genre", "user goal-movies_1-street_address": "user goal movies_1 street_address", "user goal-movies_1-location": "user goal movies_1 location", "user goal-movies_1-movie_name": "user goal movies_1 movie_name", "user goal-music_1-song_name": "user goal music_1 song_name", "user goal-music_1-artist": "user goal music_1 artist", "user goal-music_1-album": "user goal music_1 album", "user goal-music_1-genre": "user goal music_1 genre", "user goal-music_1-year": "user goal music_1 year", "user goal-music_1-playback_device": "user goal music_1 playback_device", "user goal-music_2-song_name": "user goal music_2 song_name", "user goal-music_2-artist": "user goal music_2 artist", "user goal-music_2-album": "user goal music_2 album", "user goal-music_2-genre": "user goal music_2 genre", "user goal-music_2-playback_device": "user goal music_2 playback_device", "user goal-rentalcars_1-type": "user goal rentalcars_1 type", "user goal-rentalcars_1-car_name": "user goal rentalcars_1 car_name", "user goal-rentalcars_1-pickup_location": "user goal rentalcars_1 pickup_location", "user goal-rentalcars_1-pickup_date": "user goal rentalcars_1 pickup_date", "user goal-rentalcars_1-pickup_time": "user goal rentalcars_1 pickup_time", "user goal-rentalcars_1-pickup_city": "user goal rentalcars_1 pickup_city", "user goal-rentalcars_1-dropoff_date": "user goal rentalcars_1 dropoff_date", "user goal-rentalcars_1-total_price": "user goal rentalcars_1 total_price", "user goal-rentalcars_2-car_type": "user goal rentalcars_2 car_type", "user goal-rentalcars_2-car_name": "user goal rentalcars_2 car_name", "user goal-rentalcars_2-pickup_location": "user goal rentalcars_2 pickup_location", "user goal-rentalcars_2-pickup_date": "user goal rentalcars_2 pickup_date", "user goal-rentalcars_2-pickup_time": "user goal rentalcars_2 pickup_time", "user goal-rentalcars_2-pickup_city": "user goal rentalcars_2 pickup_city", "user goal-rentalcars_2-dropoff_date": "user goal rentalcars_2 dropoff_date", "user goal-rentalcars_2-total_price": "user goal rentalcars_2 total_price", "user goal-restaurants_1-restaurant_name": "user goal restaurants_1 restaurant_name", "user goal-restaurants_1-date": "user goal restaurants_1 date", "user goal-restaurants_1-time": "user goal restaurants_1 time", "user goal-restaurants_1-serves_alcohol": "user goal restaurants_1 serves_alcohol", "user goal-restaurants_1-has_live_music": "user goal restaurants_1 has_live_music", "user goal-restaurants_1-phone_number": "user goal restaurants_1 phone_number", "user goal-restaurants_1-street_address": "user goal restaurants_1 street_address", "user goal-restaurants_1-party_size": "user goal restaurants_1 party_size", "user goal-restaurants_1-price_range": "user goal restaurants_1 price_range", "user goal-restaurants_1-city": "user goal restaurants_1 city", "user goal-restaurants_1-cuisine": "user goal restaurants_1 cuisine", "user goal-ridesharing_1-destination": "user goal ridesharing_1 destination", "user goal-ridesharing_1-shared_ride": "user goal ridesharing_1 shared_ride", "user goal-ridesharing_1-ride_fare": "user goal ridesharing_1 ride_fare", "user goal-ridesharing_1-approximate_ride_duration": "user goal ridesharing_1 approximate_ride_duration", "user goal-ridesharing_1-number_of_riders": "user goal ridesharing_1 number_of_riders", "user goal-ridesharing_2-destination": "user goal ridesharing_2 destination", "user goal-ridesharing_2-ride_type": "user goal ridesharing_2 ride_type", "user goal-ridesharing_2-ride_fare": "user goal ridesharing_2 ride_fare", "user goal-ridesharing_2-wait_time": "user goal ridesharing_2 wait_time", "user goal-ridesharing_2-number_of_seats": "user goal ridesharing_2 number_of_seats", "user goal-services_1-stylist_name": "user goal services_1 stylist_name", "user goal-services_1-phone_number": "user goal services_1 phone_number", "user goal-services_1-average_rating": "user goal services_1 average_rating", "user goal-services_1-is_unisex": "user goal services_1 is_unisex", "user goal-services_1-street_address": "user goal services_1 street_address", "user goal-services_1-city": "user goal services_1 city", "user goal-services_1-appointment_date": "user goal services_1 appointment_date", "user goal-services_1-appointment_time": "user goal services_1 appointment_time", "user goal-services_2-dentist_name": "user goal services_2 dentist_name", "user goal-services_2-phone_number": "user goal services_2 phone_number", "user goal-services_2-address": "user goal services_2 address", "user goal-services_2-city": "user goal services_2 city", "user goal-services_2-appointment_date": "user goal services_2 appointment_date", "user goal-services_2-appointment_time": "user goal services_2 appointment_time", "user goal-services_2-offers_cosmetic_services": "user goal services_2 offers_cosmetic_services", "user goal-services_3-doctor_name": "user goal services_3 doctor_name", "user goal-services_3-phone_number": "user goal services_3 phone_number", "user goal-services_3-average_rating": "user goal services_3 average_rating", "user goal-services_3-street_address": "user goal services_3 street_address", "user goal-services_3-city": "user goal services_3 city", "user goal-services_3-appointment_date": "user goal services_3 appointment_date", "user goal-services_3-appointment_time": "user goal services_3 appointment_time", "user goal-services_3-type": "user goal services_3 type", "user goal-travel_1-location": "user goal travel_1 location", "user goal-travel_1-attraction_name": "user goal travel_1 attraction_name", "user goal-travel_1-category": "user goal travel_1 category", "user goal-travel_1-phone_number": "user goal travel_1 phone_number", "user goal-travel_1-free_entry": "user goal travel_1 free_entry", "user goal-travel_1-good_for_kids": "user goal travel_1 good_for_kids", "user goal-weather_1-precipitation": "user goal weather_1 precipitation", "user goal-weather_1-humidity": "user goal weather_1 humidity", "user goal-weather_1-wind": "user goal weather_1 wind", "user goal-weather_1-temperature": "user goal weather_1 temperature", "user goal-weather_1-city": "user goal weather_1 city", "user goal-weather_1-date": "user goal weather_1 date", "user goal-alarm_1-alarm_time": "user goal alarm_1 alarm_time", "user goal-alarm_1-alarm_name": "user goal alarm_1 alarm_name", "user goal-alarm_1-new_alarm_time": "user goal alarm_1 new_alarm_time", "user goal-alarm_1-new_alarm_name": "user goal alarm_1 new_alarm_name", "user goal-banks_2-account_type": "user goal banks_2 account_type", "user goal-banks_2-recipient_account_type": "user goal banks_2 recipient_account_type", "user goal-banks_2-account_balance": "user goal banks_2 account_balance", "user goal-banks_2-transfer_amount": "user goal banks_2 transfer_amount", "user goal-banks_2-recipient_name": "user goal banks_2 recipient_name", "user goal-banks_2-transfer_time": "user goal banks_2 transfer_time", "user goal-flights_3-passengers": "user goal flights_3 passengers", "user goal-flights_3-flight_class": "user goal flights_3 flight_class", "user goal-flights_3-origin_city": "user goal flights_3 origin_city", "user goal-flights_3-destination_city": "user goal flights_3 destination_city", "user goal-flights_3-origin_airport_name": "user goal flights_3 origin_airport_name", "user goal-flights_3-destination_airport_name": "user goal flights_3 destination_airport_name", "user goal-flights_3-departure_date": "user goal flights_3 departure_date", "user goal-flights_3-return_date": "user goal flights_3 return_date", "user goal-flights_3-number_stops": "user goal flights_3 number_stops", "user goal-flights_3-outbound_departure_time": "user goal flights_3 outbound_departure_time", "user goal-flights_3-outbound_arrival_time": "user goal flights_3 outbound_arrival_time", "user goal-flights_3-inbound_arrival_time": "user goal flights_3 inbound_arrival_time", "user goal-flights_3-inbound_departure_time": "user goal flights_3 inbound_departure_time", "user goal-flights_3-price": "user goal flights_3 price", "user goal-flights_3-number_checked_bags": "user goal flights_3 number_checked_bags", "user goal-flights_3-airlines": "user goal flights_3 airlines", "user goal-flights_3-arrives_next_day": "user goal flights_3 arrives_next_day", "user goal-hotels_4-location": "user goal hotels_4 location", "user goal-hotels_4-number_of_rooms": "user goal hotels_4 number_of_rooms", "user goal-hotels_4-check_in_date": "user goal hotels_4 check_in_date", "user goal-hotels_4-stay_length": "user goal hotels_4 stay_length", "user goal-hotels_4-star_rating": "user goal hotels_4 star_rating", "user goal-hotels_4-place_name": "user goal hotels_4 place_name", "user goal-hotels_4-street_address": "user goal hotels_4 street_address", "user goal-hotels_4-phone_number": "user goal hotels_4 phone_number", "user goal-hotels_4-price_per_night": "user goal hotels_4 price_per_night", "user goal-hotels_4-smoking_allowed": "user goal hotels_4 smoking_allowed", "user goal-media_2-movie_name": "user goal media_2 movie_name", "user goal-media_2-genre": "user goal media_2 genre", "user goal-media_2-subtitle_language": "user goal media_2 subtitle_language", "user goal-media_2-director": "user goal media_2 director", "user goal-media_2-actors": "user goal media_2 actors", "user goal-media_2-price": "user goal media_2 price", "user goal-movies_2-title": "user goal movies_2 title", "user goal-movies_2-genre": "user goal movies_2 genre", "user goal-movies_2-aggregate_rating": "user goal movies_2 aggregate_rating", "user goal-movies_2-starring": "user goal movies_2 starring", "user goal-movies_2-director": "user goal movies_2 director", "user goal-restaurants_2-restaurant_name": "user goal restaurants_2 restaurant_name", "user goal-restaurants_2-date": "user goal restaurants_2 date", "user goal-restaurants_2-time": "user goal restaurants_2 time", "user goal-restaurants_2-has_seating_outdoors": "user goal restaurants_2 has_seating_outdoors", "user goal-restaurants_2-has_vegetarian_options": "user goal restaurants_2 has_vegetarian_options", "user goal-restaurants_2-phone_number": "user goal restaurants_2 phone_number", "user goal-restaurants_2-rating": "user goal restaurants_2 rating", "user goal-restaurants_2-address": "user goal restaurants_2 address", "user goal-restaurants_2-number_of_seats": "user goal restaurants_2 number_of_seats", "user goal-restaurants_2-price_range": "user goal restaurants_2 price_range", "user goal-restaurants_2-location": "user goal restaurants_2 location", "user goal-restaurants_2-category": "user goal restaurants_2 category", "user goal-services_4-therapist_name": "user goal services_4 therapist_name", "user goal-services_4-phone_number": "user goal services_4 phone_number", "user goal-services_4-address": "user goal services_4 address", "user goal-services_4-city": "user goal services_4 city", "user goal-services_4-appointment_date": "user goal services_4 appointment_date", "user goal-services_4-appointment_time": "user goal services_4 appointment_time", "user goal-services_4-type": "user goal services_4 type", "user goal-buses_3-from_city": "user goal buses_3 from_city", "user goal-buses_3-to_city": "user goal buses_3 to_city", "user goal-buses_3-from_station": "user goal buses_3 from_station", "user goal-buses_3-to_station": "user goal buses_3 to_station", "user goal-buses_3-departure_date": "user goal buses_3 departure_date", "user goal-buses_3-departure_time": "user goal buses_3 departure_time", "user goal-buses_3-price": "user goal buses_3 price", "user goal-buses_3-additional_luggage": "user goal buses_3 additional_luggage", "user goal-buses_3-num_passengers": "user goal buses_3 num_passengers", "user goal-buses_3-category": "user goal buses_3 category", "user goal-events_3-event_type": "user goal events_3 event_type", "user goal-events_3-event_name": "user goal events_3 event_name", "user goal-events_3-date": "user goal events_3 date", "user goal-events_3-time": "user goal events_3 time", "user goal-events_3-number_of_tickets": "user goal events_3 number_of_tickets", "user goal-events_3-price_per_ticket": "user goal events_3 price_per_ticket", "user goal-events_3-city": "user goal events_3 city", "user goal-events_3-venue": "user goal events_3 venue", "user goal-events_3-venue_address": "user goal events_3 venue_address", "user goal-flights_4-number_of_tickets": "user goal flights_4 number_of_tickets", "user goal-flights_4-seating_class": "user goal flights_4 seating_class", "user goal-flights_4-origin_airport": "user goal flights_4 origin_airport", "user goal-flights_4-destination_airport": "user goal flights_4 destination_airport", "user goal-flights_4-departure_date": "user goal flights_4 departure_date", "user goal-flights_4-return_date": "user goal flights_4 return_date", "user goal-flights_4-is_nonstop": "user goal flights_4 is_nonstop", "user goal-flights_4-outbound_departure_time": "user goal flights_4 outbound_departure_time", "user goal-flights_4-outbound_arrival_time": "user goal flights_4 outbound_arrival_time", "user goal-flights_4-inbound_arrival_time": "user goal flights_4 inbound_arrival_time", "user goal-flights_4-inbound_departure_time": "user goal flights_4 inbound_departure_time", "user goal-flights_4-price": "user goal flights_4 price", "user goal-flights_4-airlines": "user goal flights_4 airlines", "user goal-homes_2-intent": "user goal homes_2 intent", "user goal-homes_2-area": "user goal homes_2 area", "user goal-homes_2-address": "user goal homes_2 address", "user goal-homes_2-property_name": "user goal homes_2 property_name", "user goal-homes_2-phone_number": "user goal homes_2 phone_number", "user goal-homes_2-has_garage": "user goal homes_2 has_garage", "user goal-homes_2-in_unit_laundry": "user goal homes_2 in_unit_laundry", "user goal-homes_2-price": "user goal homes_2 price", "user goal-homes_2-visit_date": "user goal homes_2 visit_date", "user goal-homes_2-number_of_beds": "user goal homes_2 number_of_beds", "user goal-homes_2-number_of_baths": "user goal homes_2 number_of_baths", "user goal-media_3-title": "user goal media_3 title", "user goal-media_3-genre": "user goal media_3 genre", "user goal-media_3-subtitle_language": "user goal media_3 subtitle_language", "user goal-media_3-starring": "user goal media_3 starring", "user goal-messaging_1-location": "user goal messaging_1 location", "user goal-messaging_1-contact_name": "user goal messaging_1 contact_name", "user goal-movies_3-movie_title": "user goal movies_3 movie_title", "user goal-movies_3-genre": "user goal movies_3 genre", "user goal-movies_3-percent_rating": "user goal movies_3 percent_rating", "user goal-movies_3-cast": "user goal movies_3 cast", "user goal-movies_3-directed_by": "user goal movies_3 directed_by", "user goal-music_3-track": "user goal music_3 track", "user goal-music_3-artist": "user goal music_3 artist", "user goal-music_3-album": "user goal music_3 album", "user goal-music_3-genre": "user goal music_3 genre", "user goal-music_3-year": "user goal music_3 year", "user goal-music_3-device": "user goal music_3 device", "user goal-payment_1-payment_method": "user goal payment_1 payment_method", "user goal-payment_1-amount": "user goal payment_1 amount", "user goal-payment_1-receiver": "user goal payment_1 receiver", "user goal-payment_1-private_visibility": "user goal payment_1 private_visibility", "user goal-rentalcars_3-car_type": "user goal rentalcars_3 car_type", "user goal-rentalcars_3-car_name": "user goal rentalcars_3 car_name", "user goal-rentalcars_3-pickup_location": "user goal rentalcars_3 pickup_location", "user goal-rentalcars_3-start_date": "user goal rentalcars_3 start_date", "user goal-rentalcars_3-pickup_time": "user goal rentalcars_3 pickup_time", "user goal-rentalcars_3-city": "user goal rentalcars_3 city", "user goal-rentalcars_3-end_date": "user goal rentalcars_3 end_date", "user goal-rentalcars_3-price_per_day": "user goal rentalcars_3 price_per_day", "user goal-rentalcars_3-add_insurance": "user goal rentalcars_3 add_insurance", "user goal-trains_1-from": "user goal trains_1 from", "user goal-trains_1-to": "user goal trains_1 to", "user goal-trains_1-from_station": "user goal trains_1 from_station", "user goal-trains_1-to_station": "user goal trains_1 to_station", "user goal-trains_1-date_of_journey": "user goal trains_1 date_of_journey", "user goal-trains_1-journey_start_time": "user goal trains_1 journey_start_time", "user goal-trains_1-total": "user goal trains_1 total", "user goal-trains_1-number_of_adults": "user goal trains_1 number_of_adults", "user goal-trains_1-class": "user goal trains_1 class", "user goal-trains_1-trip_protection": "user goal trains_1 trip_protection", "general-banks_1": "domain banks_1", "general-buses_1": "domain buses_1", "general-buses_2": "domain buses_2", "general-calendar_1": "domain calendar_1", "general-events_1": "domain events_1", "general-events_2": "domain events_2", "general-flights_1": "domain flights_1", "general-flights_2": "domain flights_2", "general-homes_1": "domain homes_1", "general-hotels_1": "domain hotels_1", "general-hotels_2": "domain hotels_2", "general-hotels_3": "domain hotels_3", "general-media_1": "domain media_1", "general-movies_1": "domain movies_1", "general-music_1": "domain music_1", "general-music_2": "domain music_2", "general-rentalcars_1": "domain rentalcars_1", "general-rentalcars_2": "domain rentalcars_2", "general-restaurants_1": "domain restaurants_1", "general-ridesharing_1": "domain ridesharing_1", "general-ridesharing_2": "domain ridesharing_2", "general-services_1": "domain services_1", "general-services_2": "domain services_2", "general-services_3": "domain services_3", "general-travel_1": "domain travel_1", "general-weather_1": "domain weather_1", "general-alarm_1": "domain alarm_1", "general-banks_2": "domain banks_2", "general-flights_3": "domain flights_3", "general-hotels_4": "domain hotels_4", "general-media_2": "domain media_2", "general-movies_2": "domain movies_2", "general-restaurants_2": "domain restaurants_2", "general-services_4": "domain services_4", "general-buses_3": "domain buses_3", "general-events_3": "domain events_3", "general-flights_4": "domain flights_4", "general-homes_2": "domain homes_2", "general-media_3": "domain media_3", "general-messaging_1": "domain messaging_1", "general-movies_3": "domain movies_3", "general-music_3": "domain music_3", "general-payment_1": "domain payment_1", "general-rentalcars_3": "domain rentalcars_3", "general-trains_1": "domain trains_1", "system--goodbye-none-none": "last system act goodbye none none", "system--req_more-none-none": "last system act req_more none none", "system-alarm_1-confirm-new_alarm_name-1": "last system act alarm_1 confirm new_alarm_name 1", "system-alarm_1-confirm-new_alarm_time-1": "last system act alarm_1 confirm new_alarm_time 1", "system-alarm_1-inform_count-count-1": "last system act alarm_1 inform_count count 1", "system-alarm_1-notify_success-none-none": "last system act alarm_1 notify_success none none", "system-alarm_1-offer-alarm_name-1": "last system act alarm_1 offer alarm_name 1", "system-alarm_1-offer-alarm_time-1": "last system act alarm_1 offer alarm_time 1", "system-alarm_1-offer_intent-addalarm-1": "last system act alarm_1 offer_intent AddAlarm 1", "system-alarm_1-request-new_alarm_time-?": "last system act alarm_1 request new_alarm_time ?", "system-banks_1-confirm-account_type-1": "last system act banks_1 confirm account_type 1", "system-banks_1-confirm-amount-1": "last system act banks_1 confirm amount 1", "system-banks_1-confirm-recipient_account_name-1": "last system act banks_1 confirm recipient_account_name 1", "system-banks_1-confirm-recipient_account_type-1": "last system act banks_1 confirm recipient_account_type 1", "system-banks_1-notify_success-none-none": "last system act banks_1 notify_success none none", "system-banks_1-offer-account_type-1": "last system act banks_1 offer account_type 1", "system-banks_1-offer-balance-1": "last system act banks_1 offer balance 1", "system-banks_1-offer_intent-transfermoney-1": "last system act banks_1 offer_intent TransferMoney 1", "system-banks_1-request-account_type-?": "last system act banks_1 request account_type ?", "system-banks_1-request-amount-?": "last system act banks_1 request amount ?", "system-banks_1-request-recipient_account_name-?": "last system act banks_1 request recipient_account_name ?", "system-banks_2-confirm-account_type-1": "last system act banks_2 confirm account_type 1", "system-banks_2-confirm-recipient_account_type-1": "last system act banks_2 confirm recipient_account_type 1", "system-banks_2-confirm-recipient_name-1": "last system act banks_2 confirm recipient_name 1", "system-banks_2-confirm-transfer_amount-1": "last system act banks_2 confirm transfer_amount 1", "system-banks_2-inform-transfer_time-1": "last system act banks_2 inform transfer_time 1", "system-banks_2-notify_success-none-none": "last system act banks_2 notify_success none none", "system-banks_2-offer-account_balance-1": "last system act banks_2 offer account_balance 1", "system-banks_2-offer-account_type-1": "last system act banks_2 offer account_type 1", "system-banks_2-offer_intent-transfermoney-1": "last system act banks_2 offer_intent TransferMoney 1", "system-banks_2-request-account_type-?": "last system act banks_2 request account_type ?", "system-banks_2-request-recipient_name-?": "last system act banks_2 request recipient_name ?", "system-banks_2-request-transfer_amount-?": "last system act banks_2 request transfer_amount ?", "system-buses_1-confirm-from_location-1": "last system act buses_1 confirm from_location 1", "system-buses_1-confirm-leaving_date-1": "last system act buses_1 confirm leaving_date 1", "system-buses_1-confirm-leaving_time-1": "last system act buses_1 confirm leaving_time 1", "system-buses_1-confirm-to_location-1": "last system act buses_1 confirm to_location 1", "system-buses_1-confirm-travelers-1": "last system act buses_1 confirm travelers 1", "system-buses_1-inform-from_station-1": "last system act buses_1 inform from_station 1", "system-buses_1-inform-to_station-1": "last system act buses_1 inform to_station 1", "system-buses_1-inform-transfers-1": "last system act buses_1 inform transfers 1", "system-buses_1-inform_count-count-1": "last system act buses_1 inform_count count 1", "system-buses_1-notify_failure-none-none": "last system act buses_1 notify_failure none none", "system-buses_1-notify_success-none-none": "last system act buses_1 notify_success none none", "system-buses_1-offer-fare-1": "last system act buses_1 offer fare 1", "system-buses_1-offer-leaving_time-1": "last system act buses_1 offer leaving_time 1", "system-buses_1-offer-transfers-1": "last system act buses_1 offer transfers 1", "system-buses_1-offer_intent-buybusticket-1": "last system act buses_1 offer_intent BuyBusTicket 1", "system-buses_1-request-from_location-?": "last system act buses_1 request from_location ?", "system-buses_1-request-leaving_date-?": "last system act buses_1 request leaving_date ?", "system-buses_1-request-leaving_time-?": "last system act buses_1 request leaving_time ?", "system-buses_1-request-to_location-?": "last system act buses_1 request to_location ?", "system-buses_1-request-travelers-?": "last system act buses_1 request travelers ?", "system-buses_2-confirm-departure_date-1": "last system act buses_2 confirm departure_date 1", "system-buses_2-confirm-departure_time-1": "last system act buses_2 confirm departure_time 1", "system-buses_2-confirm-destination-1": "last system act buses_2 confirm destination 1", "system-buses_2-confirm-fare_type-1": "last system act buses_2 confirm fare_type 1", "system-buses_2-confirm-group_size-1": "last system act buses_2 confirm group_size 1", "system-buses_2-confirm-origin-1": "last system act buses_2 confirm origin 1", "system-buses_2-inform-destination_station_name-1": "last system act buses_2 inform destination_station_name 1", "system-buses_2-inform-origin_station_name-1": "last system act buses_2 inform origin_station_name 1", "system-buses_2-inform_count-count-1": "last system act buses_2 inform_count count 1", "system-buses_2-notify_failure-none-none": "last system act buses_2 notify_failure none none", "system-buses_2-notify_success-none-none": "last system act buses_2 notify_success none none", "system-buses_2-offer-departure_time-1": "last system act buses_2 offer departure_time 1", "system-buses_2-offer-fare_type-1": "last system act buses_2 offer fare_type 1", "system-buses_2-offer-price-1": "last system act buses_2 offer price 1", "system-buses_2-offer_intent-buybusticket-1": "last system act buses_2 offer_intent BuyBusTicket 1", "system-buses_2-request-departure_date-?": "last system act buses_2 request departure_date ?", "system-buses_2-request-departure_time-?": "last system act buses_2 request departure_time ?", "system-buses_2-request-destination-?": "last system act buses_2 request destination ?", "system-buses_2-request-group_size-?": "last system act buses_2 request group_size ?", "system-buses_2-request-origin-?": "last system act buses_2 request origin ?", "system-buses_3-confirm-additional_luggage-1": "last system act buses_3 confirm additional_luggage 1", "system-buses_3-confirm-departure_date-1": "last system act buses_3 confirm departure_date 1", "system-buses_3-confirm-departure_time-1": "last system act buses_3 confirm departure_time 1", "system-buses_3-confirm-from_city-1": "last system act buses_3 confirm from_city 1", "system-buses_3-confirm-num_passengers-1": "last system act buses_3 confirm num_passengers 1", "system-buses_3-confirm-to_city-1": "last system act buses_3 confirm to_city 1", "system-buses_3-inform-category-1": "last system act buses_3 inform category 1", "system-buses_3-inform-from_station-1": "last system act buses_3 inform from_station 1", "system-buses_3-inform-to_station-1": "last system act buses_3 inform to_station 1", "system-buses_3-inform_count-count-1": "last system act buses_3 inform_count count 1", "system-buses_3-notify_failure-none-none": "last system act buses_3 notify_failure none none", "system-buses_3-notify_success-none-none": "last system act buses_3 notify_success none none", "system-buses_3-offer-departure_time-1": "last system act buses_3 offer departure_time 1", "system-buses_3-offer-price-1": "last system act buses_3 offer price 1", "system-buses_3-offer_intent-buybusticket-1": "last system act buses_3 offer_intent BuyBusTicket 1", "system-buses_3-request-departure_date-?": "last system act buses_3 request departure_date ?", "system-buses_3-request-departure_time-?": "last system act buses_3 request departure_time ?", "system-buses_3-request-from_city-?": "last system act buses_3 request from_city ?", "system-buses_3-request-num_passengers-?": "last system act buses_3 request num_passengers ?", "system-buses_3-request-to_city-?": "last system act buses_3 request to_city ?", "system-calendar_1-confirm-event_date-1": "last system act calendar_1 confirm event_date 1", "system-calendar_1-confirm-event_location-1": "last system act calendar_1 confirm event_location 1", "system-calendar_1-confirm-event_name-1": "last system act calendar_1 confirm event_name 1", "system-calendar_1-confirm-event_time-1": "last system act calendar_1 confirm event_time 1", "system-calendar_1-inform_count-count-1": "last system act calendar_1 inform_count count 1", "system-calendar_1-notify_success-none-none": "last system act calendar_1 notify_success none none", "system-calendar_1-offer-available_end_time-1": "last system act calendar_1 offer available_end_time 1", "system-calendar_1-offer-available_start_time-1": "last system act calendar_1 offer available_start_time 1", "system-calendar_1-offer-event_date-1": "last system act calendar_1 offer event_date 1", "system-calendar_1-offer-event_name-1": "last system act calendar_1 offer event_name 1", "system-calendar_1-offer-event_time-1": "last system act calendar_1 offer event_time 1", "system-calendar_1-offer_intent-addevent-1": "last system act calendar_1 offer_intent AddEvent 1", "system-calendar_1-request-event_date-?": "last system act calendar_1 request event_date ?", "system-calendar_1-request-event_location-?": "last system act calendar_1 request event_location ?", "system-calendar_1-request-event_name-?": "last system act calendar_1 request event_name ?", "system-calendar_1-request-event_time-?": "last system act calendar_1 request event_time ?", "system-events_1-confirm-city_of_event-1": "last system act events_1 confirm city_of_event 1", "system-events_1-confirm-date-1": "last system act events_1 confirm date 1", "system-events_1-confirm-event_name-1": "last system act events_1 confirm event_name 1", "system-events_1-confirm-number_of_seats-1": "last system act events_1 confirm number_of_seats 1", "system-events_1-inform-address_of_location-1": "last system act events_1 inform address_of_location 1", "system-events_1-inform-event_location-1": "last system act events_1 inform event_location 1", "system-events_1-inform-subcategory-1": "last system act events_1 inform subcategory 1", "system-events_1-inform-time-1": "last system act events_1 inform time 1", "system-events_1-inform_count-count-1": "last system act events_1 inform_count count 1", "system-events_1-notify_failure-none-none": "last system act events_1 notify_failure none none", "system-events_1-notify_success-none-none": "last system act events_1 notify_success none none", "system-events_1-offer-date-1": "last system act events_1 offer date 1", "system-events_1-offer-event_location-1": "last system act events_1 offer event_location 1", "system-events_1-offer-event_name-1": "last system act events_1 offer event_name 1", "system-events_1-offer-time-1": "last system act events_1 offer time 1", "system-events_1-offer_intent-buyeventtickets-1": "last system act events_1 offer_intent BuyEventTickets 1", "system-events_1-request-category-?": "last system act events_1 request category ?", "system-events_1-request-city_of_event-?": "last system act events_1 request city_of_event ?", "system-events_1-request-date-?": "last system act events_1 request date ?", "system-events_1-request-event_name-?": "last system act events_1 request event_name ?", "system-events_1-request-number_of_seats-?": "last system act events_1 request number_of_seats ?", "system-events_2-confirm-city-1": "last system act events_2 confirm city 1", "system-events_2-confirm-date-1": "last system act events_2 confirm date 1", "system-events_2-confirm-event_name-1": "last system act events_2 confirm event_name 1", "system-events_2-confirm-number_of_tickets-1": "last system act events_2 confirm number_of_tickets 1", "system-events_2-inform-category-1": "last system act events_2 inform category 1", "system-events_2-inform-time-1": "last system act events_2 inform time 1", "system-events_2-inform-venue-1": "last system act events_2 inform venue 1", "system-events_2-inform-venue_address-1": "last system act events_2 inform venue_address 1", "system-events_2-inform_count-count-1": "last system act events_2 inform_count count 1", "system-events_2-notify_success-none-none": "last system act events_2 notify_success none none", "system-events_2-offer-date-1": "last system act events_2 offer date 1", "system-events_2-offer-event_name-1": "last system act events_2 offer event_name 1", "system-events_2-offer-venue-1": "last system act events_2 offer venue 1", "system-events_2-offer_intent-buyeventtickets-1": "last system act events_2 offer_intent BuyEventTickets 1", "system-events_2-request-city-?": "last system act events_2 request city ?", "system-events_2-request-date-?": "last system act events_2 request date ?", "system-events_2-request-event_name-?": "last system act events_2 request event_name ?", "system-events_2-request-event_type-?": "last system act events_2 request event_type ?", "system-events_2-request-number_of_tickets-?": "last system act events_2 request number_of_tickets ?", "system-events_3-confirm-city-1": "last system act events_3 confirm city 1", "system-events_3-confirm-date-1": "last system act events_3 confirm date 1", "system-events_3-confirm-event_name-1": "last system act events_3 confirm event_name 1", "system-events_3-confirm-number_of_tickets-1": "last system act events_3 confirm number_of_tickets 1", "system-events_3-inform-price_per_ticket-1": "last system act events_3 inform price_per_ticket 1", "system-events_3-inform-venue_address-1": "last system act events_3 inform venue_address 1", "system-events_3-inform_count-count-1": "last system act events_3 inform_count count 1", "system-events_3-notify_success-none-none": "last system act events_3 notify_success none none", "system-events_3-offer-date-1": "last system act events_3 offer date 1", "system-events_3-offer-event_name-1": "last system act events_3 offer event_name 1", "system-events_3-offer-time-1": "last system act events_3 offer time 1", "system-events_3-offer-venue-1": "last system act events_3 offer venue 1", "system-events_3-offer_intent-buyeventtickets-1": "last system act events_3 offer_intent BuyEventTickets 1", "system-events_3-request-city-?": "last system act events_3 request city ?", "system-events_3-request-date-?": "last system act events_3 request date ?", "system-events_3-request-event_name-?": "last system act events_3 request event_name ?", "system-events_3-request-event_type-?": "last system act events_3 request event_type ?", "system-events_3-request-number_of_tickets-?": "last system act events_3 request number_of_tickets ?", "system-flights_1-confirm-airlines-1": "last system act flights_1 confirm airlines 1", "system-flights_1-confirm-departure_date-1": "last system act flights_1 confirm departure_date 1", "system-flights_1-confirm-destination_city-1": "last system act flights_1 confirm destination_city 1", "system-flights_1-confirm-inbound_departure_time-1": "last system act flights_1 confirm inbound_departure_time 1", "system-flights_1-confirm-origin_city-1": "last system act flights_1 confirm origin_city 1", "system-flights_1-confirm-outbound_departure_time-1": "last system act flights_1 confirm outbound_departure_time 1", "system-flights_1-confirm-passengers-1": "last system act flights_1 confirm passengers 1", "system-flights_1-confirm-return_date-1": "last system act flights_1 confirm return_date 1", "system-flights_1-confirm-seating_class-1": "last system act flights_1 confirm seating_class 1", "system-flights_1-inform-destination_airport-1": "last system act flights_1 inform destination_airport 1", "system-flights_1-inform-inbound_arrival_time-1": "last system act flights_1 inform inbound_arrival_time 1", "system-flights_1-inform-number_stops-1": "last system act flights_1 inform number_stops 1", "system-flights_1-inform-origin_airport-1": "last system act flights_1 inform origin_airport 1", "system-flights_1-inform-outbound_arrival_time-1": "last system act flights_1 inform outbound_arrival_time 1", "system-flights_1-inform-refundable-1": "last system act flights_1 inform refundable 1", "system-flights_1-inform_count-count-1": "last system act flights_1 inform_count count 1", "system-flights_1-notify_failure-none-none": "last system act flights_1 notify_failure none none", "system-flights_1-notify_success-none-none": "last system act flights_1 notify_success none none", "system-flights_1-offer-airlines-1": "last system act flights_1 offer airlines 1", "system-flights_1-offer-inbound_departure_time-1": "last system act flights_1 offer inbound_departure_time 1", "system-flights_1-offer-number_stops-1": "last system act flights_1 offer number_stops 1", "system-flights_1-offer-outbound_departure_time-1": "last system act flights_1 offer outbound_departure_time 1", "system-flights_1-offer-price-1": "last system act flights_1 offer price 1", "system-flights_1-offer_intent-reserveonewayflight-1": "last system act flights_1 offer_intent ReserveOnewayFlight 1", "system-flights_1-offer_intent-reserveroundtripflights-1": "last system act flights_1 offer_intent ReserveRoundtripFlights 1", "system-flights_1-request-airlines-?": "last system act flights_1 request airlines ?", "system-flights_1-request-departure_date-?": "last system act flights_1 request departure_date ?", "system-flights_1-request-destination_city-?": "last system act flights_1 request destination_city ?", "system-flights_1-request-inbound_departure_time-?": "last system act flights_1 request inbound_departure_time ?", "system-flights_1-request-origin_city-?": "last system act flights_1 request origin_city ?", "system-flights_1-request-outbound_departure_time-?": "last system act flights_1 request outbound_departure_time ?", "system-flights_1-request-return_date-?": "last system act flights_1 request return_date ?", "system-flights_2-inform-destination_airport-1": "last system act flights_2 inform destination_airport 1", "system-flights_2-inform-is_redeye-1": "last system act flights_2 inform is_redeye 1", "system-flights_2-inform-origin_airport-1": "last system act flights_2 inform origin_airport 1", "system-flights_2-inform-outbound_arrival_time-1": "last system act flights_2 inform outbound_arrival_time 1", "system-flights_2-inform_count-count-1": "last system act flights_2 inform_count count 1", "system-flights_2-offer-airlines-1": "last system act flights_2 offer airlines 1", "system-flights_2-offer-fare-1": "last system act flights_2 offer fare 1", "system-flights_2-offer-inbound_departure_time-1": "last system act flights_2 offer inbound_departure_time 1", "system-flights_2-offer-number_stops-1": "last system act flights_2 offer number_stops 1", "system-flights_2-offer-outbound_departure_time-1": "last system act flights_2 offer outbound_departure_time 1", "system-flights_2-request-departure_date-?": "last system act flights_2 request departure_date ?", "system-flights_2-request-destination-?": "last system act flights_2 request destination ?", "system-flights_2-request-origin-?": "last system act flights_2 request origin ?", "system-flights_2-request-return_date-?": "last system act flights_2 request return_date ?", "system-flights_3-inform-arrives_next_day-1": "last system act flights_3 inform arrives_next_day 1", "system-flights_3-inform-destination_airport_name-1": "last system act flights_3 inform destination_airport_name 1", "system-flights_3-inform-origin_airport_name-1": "last system act flights_3 inform origin_airport_name 1", "system-flights_3-inform-outbound_arrival_time-1": "last system act flights_3 inform outbound_arrival_time 1", "system-flights_3-inform_count-count-1": "last system act flights_3 inform_count count 1", "system-flights_3-offer-airlines-1": "last system act flights_3 offer airlines 1", "system-flights_3-offer-inbound_departure_time-1": "last system act flights_3 offer inbound_departure_time 1", "system-flights_3-offer-number_stops-1": "last system act flights_3 offer number_stops 1", "system-flights_3-offer-outbound_departure_time-1": "last system act flights_3 offer outbound_departure_time 1", "system-flights_3-offer-price-1": "last system act flights_3 offer price 1", "system-flights_3-request-departure_date-?": "last system act flights_3 request departure_date ?", "system-flights_3-request-destination_city-?": "last system act flights_3 request destination_city ?", "system-flights_3-request-origin_city-?": "last system act flights_3 request origin_city ?", "system-flights_3-request-return_date-?": "last system act flights_3 request return_date ?", "system-flights_4-inform-inbound_arrival_time-1": "last system act flights_4 inform inbound_arrival_time 1", "system-flights_4-inform-number_of_tickets-1": "last system act flights_4 inform number_of_tickets 1", "system-flights_4-inform-outbound_arrival_time-1": "last system act flights_4 inform outbound_arrival_time 1", "system-flights_4-inform-seating_class-1": "last system act flights_4 inform seating_class 1", "system-flights_4-inform_count-count-1": "last system act flights_4 inform_count count 1", "system-flights_4-offer-airlines-1": "last system act flights_4 offer airlines 1", "system-flights_4-offer-inbound_departure_time-1": "last system act flights_4 offer inbound_departure_time 1", "system-flights_4-offer-is_nonstop-1": "last system act flights_4 offer is_nonstop 1", "system-flights_4-offer-outbound_departure_time-1": "last system act flights_4 offer outbound_departure_time 1", "system-flights_4-offer-price-1": "last system act flights_4 offer price 1", "system-flights_4-request-departure_date-?": "last system act flights_4 request departure_date ?", "system-flights_4-request-destination_airport-?": "last system act flights_4 request destination_airport ?", "system-flights_4-request-origin_airport-?": "last system act flights_4 request origin_airport ?", "system-flights_4-request-return_date-?": "last system act flights_4 request return_date ?", "system-homes_1-confirm-property_name-1": "last system act homes_1 confirm property_name 1", "system-homes_1-confirm-visit_date-1": "last system act homes_1 confirm visit_date 1", "system-homes_1-inform-furnished-1": "last system act homes_1 inform furnished 1", "system-homes_1-inform-pets_allowed-1": "last system act homes_1 inform pets_allowed 1", "system-homes_1-inform-phone_number-1": "last system act homes_1 inform phone_number 1", "system-homes_1-inform_count-count-1": "last system act homes_1 inform_count count 1", "system-homes_1-notify_failure-none-none": "last system act homes_1 notify_failure none none", "system-homes_1-notify_success-none-none": "last system act homes_1 notify_success none none", "system-homes_1-offer-address-1": "last system act homes_1 offer address 1", "system-homes_1-offer-number_of_baths-1": "last system act homes_1 offer number_of_baths 1", "system-homes_1-offer-number_of_beds-1": "last system act homes_1 offer number_of_beds 1", "system-homes_1-offer-property_name-1": "last system act homes_1 offer property_name 1", "system-homes_1-offer-rent-1": "last system act homes_1 offer rent 1", "system-homes_1-offer_intent-schedulevisit-1": "last system act homes_1 offer_intent ScheduleVisit 1", "system-homes_1-request-area-?": "last system act homes_1 request area ?", "system-homes_1-request-number_of_beds-?": "last system act homes_1 request number_of_beds ?", "system-homes_1-request-visit_date-?": "last system act homes_1 request visit_date ?", "system-homes_2-confirm-property_name-1": "last system act homes_2 confirm property_name 1", "system-homes_2-confirm-visit_date-1": "last system act homes_2 confirm visit_date 1", "system-homes_2-inform-has_garage-1": "last system act homes_2 inform has_garage 1", "system-homes_2-inform-in_unit_laundry-1": "last system act homes_2 inform in_unit_laundry 1", "system-homes_2-inform-phone_number-1": "last system act homes_2 inform phone_number 1", "system-homes_2-inform_count-count-1": "last system act homes_2 inform_count count 1", "system-homes_2-notify_success-none-none": "last system act homes_2 notify_success none none", "system-homes_2-offer-address-1": "last system act homes_2 offer address 1", "system-homes_2-offer-price-1": "last system act homes_2 offer price 1", "system-homes_2-offer-property_name-1": "last system act homes_2 offer property_name 1", "system-homes_2-offer_intent-schedulevisit-1": "last system act homes_2 offer_intent ScheduleVisit 1", "system-homes_2-request-area-?": "last system act homes_2 request area ?", "system-homes_2-request-intent-?": "last system act homes_2 request intent ?", "system-homes_2-request-number_of_baths-?": "last system act homes_2 request number_of_baths ?", "system-homes_2-request-number_of_beds-?": "last system act homes_2 request number_of_beds ?", "system-homes_2-request-visit_date-?": "last system act homes_2 request visit_date ?", "system-hotels_1-confirm-check_in_date-1": "last system act hotels_1 confirm check_in_date 1", "system-hotels_1-confirm-destination-1": "last system act hotels_1 confirm destination 1", "system-hotels_1-confirm-hotel_name-1": "last system act hotels_1 confirm hotel_name 1", "system-hotels_1-confirm-number_of_days-1": "last system act hotels_1 confirm number_of_days 1", "system-hotels_1-confirm-number_of_rooms-1": "last system act hotels_1 confirm number_of_rooms 1", "system-hotels_1-inform-has_wifi-1": "last system act hotels_1 inform has_wifi 1", "system-hotels_1-inform-phone_number-1": "last system act hotels_1 inform phone_number 1", "system-hotels_1-inform-price_per_night-1": "last system act hotels_1 inform price_per_night 1", "system-hotels_1-inform-street_address-1": "last system act hotels_1 inform street_address 1", "system-hotels_1-inform_count-count-1": "last system act hotels_1 inform_count count 1", "system-hotels_1-notify_success-none-none": "last system act hotels_1 notify_success none none", "system-hotels_1-offer-hotel_name-1": "last system act hotels_1 offer hotel_name 1", "system-hotels_1-offer-star_rating-1": "last system act hotels_1 offer star_rating 1", "system-hotels_1-offer_intent-reservehotel-1": "last system act hotels_1 offer_intent ReserveHotel 1", "system-hotels_1-request-check_in_date-?": "last system act hotels_1 request check_in_date ?", "system-hotels_1-request-destination-?": "last system act hotels_1 request destination ?", "system-hotels_1-request-hotel_name-?": "last system act hotels_1 request hotel_name ?", "system-hotels_1-request-number_of_days-?": "last system act hotels_1 request number_of_days ?", "system-hotels_2-confirm-check_in_date-1": "last system act hotels_2 confirm check_in_date 1", "system-hotels_2-confirm-check_out_date-1": "last system act hotels_2 confirm check_out_date 1", "system-hotels_2-confirm-number_of_adults-1": "last system act hotels_2 confirm number_of_adults 1", "system-hotels_2-confirm-where_to-1": "last system act hotels_2 confirm where_to 1", "system-hotels_2-inform-has_laundry_service-1": "last system act hotels_2 inform has_laundry_service 1", "system-hotels_2-inform-phone_number-1": "last system act hotels_2 inform phone_number 1", "system-hotels_2-inform-total_price-1": "last system act hotels_2 inform total_price 1", "system-hotels_2-inform_count-count-1": "last system act hotels_2 inform_count count 1", "system-hotels_2-notify_success-none-none": "last system act hotels_2 notify_success none none", "system-hotels_2-offer-address-1": "last system act hotels_2 offer address 1", "system-hotels_2-offer-rating-1": "last system act hotels_2 offer rating 1", "system-hotels_2-offer_intent-bookhouse-1": "last system act hotels_2 offer_intent BookHouse 1", "system-hotels_2-request-check_in_date-?": "last system act hotels_2 request check_in_date ?", "system-hotels_2-request-check_out_date-?": "last system act hotels_2 request check_out_date ?", "system-hotels_2-request-number_of_adults-?": "last system act hotels_2 request number_of_adults ?", "system-hotels_2-request-where_to-?": "last system act hotels_2 request where_to ?", "system-hotels_3-confirm-check_in_date-1": "last system act hotels_3 confirm check_in_date 1", "system-hotels_3-confirm-check_out_date-1": "last system act hotels_3 confirm check_out_date 1", "system-hotels_3-confirm-hotel_name-1": "last system act hotels_3 confirm hotel_name 1", "system-hotels_3-confirm-location-1": "last system act hotels_3 confirm location 1", "system-hotels_3-confirm-number_of_rooms-1": "last system act hotels_3 confirm number_of_rooms 1", "system-hotels_3-inform-pets_welcome-1": "last system act hotels_3 inform pets_welcome 1", "system-hotels_3-inform-phone_number-1": "last system act hotels_3 inform phone_number 1", "system-hotels_3-inform-price-1": "last system act hotels_3 inform price 1", "system-hotels_3-inform-street_address-1": "last system act hotels_3 inform street_address 1", "system-hotels_3-inform_count-count-1": "last system act hotels_3 inform_count count 1", "system-hotels_3-notify_success-none-none": "last system act hotels_3 notify_success none none", "system-hotels_3-offer-average_rating-1": "last system act hotels_3 offer average_rating 1", "system-hotels_3-offer-hotel_name-1": "last system act hotels_3 offer hotel_name 1", "system-hotels_3-offer_intent-reservehotel-1": "last system act hotels_3 offer_intent ReserveHotel 1", "system-hotels_3-request-check_in_date-?": "last system act hotels_3 request check_in_date ?", "system-hotels_3-request-check_out_date-?": "last system act hotels_3 request check_out_date ?", "system-hotels_3-request-hotel_name-?": "last system act hotels_3 request hotel_name ?", "system-hotels_3-request-location-?": "last system act hotels_3 request location ?", "system-hotels_4-confirm-check_in_date-1": "last system act hotels_4 confirm check_in_date 1", "system-hotels_4-confirm-location-1": "last system act hotels_4 confirm location 1", "system-hotels_4-confirm-number_of_rooms-1": "last system act hotels_4 confirm number_of_rooms 1", "system-hotels_4-confirm-place_name-1": "last system act hotels_4 confirm place_name 1", "system-hotels_4-confirm-stay_length-1": "last system act hotels_4 confirm stay_length 1", "system-hotels_4-inform-phone_number-1": "last system act hotels_4 inform phone_number 1", "system-hotels_4-inform-price_per_night-1": "last system act hotels_4 inform price_per_night 1", "system-hotels_4-inform-smoking_allowed-1": "last system act hotels_4 inform smoking_allowed 1", "system-hotels_4-inform-street_address-1": "last system act hotels_4 inform street_address 1", "system-hotels_4-inform_count-count-1": "last system act hotels_4 inform_count count 1", "system-hotels_4-notify_success-none-none": "last system act hotels_4 notify_success none none", "system-hotels_4-offer-place_name-1": "last system act hotels_4 offer place_name 1", "system-hotels_4-offer-star_rating-1": "last system act hotels_4 offer star_rating 1", "system-hotels_4-offer_intent-reservehotel-1": "last system act hotels_4 offer_intent ReserveHotel 1", "system-hotels_4-request-check_in_date-?": "last system act hotels_4 request check_in_date ?", "system-hotels_4-request-location-?": "last system act hotels_4 request location ?", "system-hotels_4-request-stay_length-?": "last system act hotels_4 request stay_length ?", "system-media_1-confirm-subtitles-1": "last system act media_1 confirm subtitles 1", "system-media_1-confirm-title-1": "last system act media_1 confirm title 1", "system-media_1-inform-directed_by-1": "last system act media_1 inform directed_by 1", "system-media_1-inform-genre-1": "last system act media_1 inform genre 1", "system-media_1-inform_count-count-1": "last system act media_1 inform_count count 1", "system-media_1-notify_failure-none-none": "last system act media_1 notify_failure none none", "system-media_1-notify_success-none-none": "last system act media_1 notify_success none none", "system-media_1-offer-title-1": "last system act media_1 offer title 1", "system-media_1-offer-title-2": "last system act media_1 offer title 2", "system-media_1-offer-title-3": "last system act media_1 offer title 3", "system-media_1-offer_intent-playmovie-1": "last system act media_1 offer_intent PlayMovie 1", "system-media_1-request-genre-?": "last system act media_1 request genre ?", "system-media_1-request-title-?": "last system act media_1 request title ?", "system-media_2-confirm-movie_name-1": "last system act media_2 confirm movie_name 1", "system-media_2-confirm-subtitle_language-1": "last system act media_2 confirm subtitle_language 1", "system-media_2-inform-price-1": "last system act media_2 inform price 1", "system-media_2-inform_count-count-1": "last system act media_2 inform_count count 1", "system-media_2-notify_success-none-none": "last system act media_2 notify_success none none", "system-media_2-offer-movie_name-1": "last system act media_2 offer movie_name 1", "system-media_2-offer-movie_name-2": "last system act media_2 offer movie_name 2", "system-media_2-offer-movie_name-3": "last system act media_2 offer movie_name 3", "system-media_2-offer_intent-rentmovie-1": "last system act media_2 offer_intent RentMovie 1", "system-media_2-request-genre-?": "last system act media_2 request genre ?", "system-media_3-confirm-subtitle_language-1": "last system act media_3 confirm subtitle_language 1", "system-media_3-confirm-title-1": "last system act media_3 confirm title 1", "system-media_3-inform-starring-1": "last system act media_3 inform starring 1", "system-media_3-inform_count-count-1": "last system act media_3 inform_count count 1", "system-media_3-notify_success-none-none": "last system act media_3 notify_success none none", "system-media_3-offer-title-1": "last system act media_3 offer title 1", "system-media_3-offer-title-2": "last system act media_3 offer title 2", "system-media_3-offer-title-3": "last system act media_3 offer title 3", "system-media_3-offer_intent-playmovie-1": "last system act media_3 offer_intent PlayMovie 1", "system-media_3-request-genre-?": "last system act media_3 request genre ?", "system-messaging_1-confirm-contact_name-1": "last system act messaging_1 confirm contact_name 1", "system-messaging_1-confirm-location-1": "last system act messaging_1 confirm location 1", "system-messaging_1-notify_success-none-none": "last system act messaging_1 notify_success none none", "system-messaging_1-request-contact_name-?": "last system act messaging_1 request contact_name ?", "system-messaging_1-request-location-?": "last system act messaging_1 request location ?", "system-movies_1-confirm-location-1": "last system act movies_1 confirm location 1", "system-movies_1-confirm-movie_name-1": "last system act movies_1 confirm movie_name 1", "system-movies_1-confirm-number_of_tickets-1": "last system act movies_1 confirm number_of_tickets 1", "system-movies_1-confirm-show_date-1": "last system act movies_1 confirm show_date 1", "system-movies_1-confirm-show_time-1": "last system act movies_1 confirm show_time 1", "system-movies_1-confirm-show_type-1": "last system act movies_1 confirm show_type 1", "system-movies_1-inform-genre-1": "last system act movies_1 inform genre 1", "system-movies_1-inform-price-1": "last system act movies_1 inform price 1", "system-movies_1-inform-street_address-1": "last system act movies_1 inform street_address 1", "system-movies_1-inform_count-count-1": "last system act movies_1 inform_count count 1", "system-movies_1-notify_failure-none-none": "last system act movies_1 notify_failure none none", "system-movies_1-notify_success-none-none": "last system act movies_1 notify_success none none", "system-movies_1-offer-movie_name-1": "last system act movies_1 offer movie_name 1", "system-movies_1-offer-movie_name-2": "last system act movies_1 offer movie_name 2", "system-movies_1-offer-movie_name-3": "last system act movies_1 offer movie_name 3", "system-movies_1-offer-show_time-1": "last system act movies_1 offer show_time 1", "system-movies_1-offer-theater_name-1": "last system act movies_1 offer theater_name 1", "system-movies_1-offer_intent-buymovietickets-1": "last system act movies_1 offer_intent BuyMovieTickets 1", "system-movies_1-request-location-?": "last system act movies_1 request location ?", "system-movies_1-request-movie_name-?": "last system act movies_1 request movie_name ?", "system-movies_1-request-number_of_tickets-?": "last system act movies_1 request number_of_tickets ?", "system-movies_1-request-show_date-?": "last system act movies_1 request show_date ?", "system-movies_1-request-show_time-?": "last system act movies_1 request show_time ?", "system-movies_1-request-show_type-?": "last system act movies_1 request show_type ?", "system-movies_2-inform_count-count-1": "last system act movies_2 inform_count count 1", "system-movies_2-offer-aggregate_rating-1": "last system act movies_2 offer aggregate_rating 1", "system-movies_2-offer-title-1": "last system act movies_2 offer title 1", "system-movies_3-inform-cast-1": "last system act movies_3 inform cast 1", "system-movies_3-inform-directed_by-1": "last system act movies_3 inform directed_by 1", "system-movies_3-inform-genre-1": "last system act movies_3 inform genre 1", "system-movies_3-inform_count-count-1": "last system act movies_3 inform_count count 1", "system-movies_3-offer-movie_title-1": "last system act movies_3 offer movie_title 1", "system-movies_3-offer-percent_rating-1": "last system act movies_3 offer percent_rating 1", "system-music_1-confirm-playback_device-1": "last system act music_1 confirm playback_device 1", "system-music_1-confirm-song_name-1": "last system act music_1 confirm song_name 1", "system-music_1-inform-album-1": "last system act music_1 inform album 1", "system-music_1-inform-genre-1": "last system act music_1 inform genre 1", "system-music_1-inform-year-1": "last system act music_1 inform year 1", "system-music_1-inform_count-count-1": "last system act music_1 inform_count count 1", "system-music_1-notify_success-none-none": "last system act music_1 notify_success none none", "system-music_1-offer-album-1": "last system act music_1 offer album 1", "system-music_1-offer-artist-1": "last system act music_1 offer artist 1", "system-music_1-offer-song_name-1": "last system act music_1 offer song_name 1", "system-music_1-offer_intent-playsong-1": "last system act music_1 offer_intent PlaySong 1", "system-music_1-request-song_name-?": "last system act music_1 request song_name ?", "system-music_2-confirm-playback_device-1": "last system act music_2 confirm playback_device 1", "system-music_2-confirm-song_name-1": "last system act music_2 confirm song_name 1", "system-music_2-inform-genre-1": "last system act music_2 inform genre 1", "system-music_2-inform_count-count-1": "last system act music_2 inform_count count 1", "system-music_2-notify_success-none-none": "last system act music_2 notify_success none none", "system-music_2-offer-album-1": "last system act music_2 offer album 1", "system-music_2-offer-artist-1": "last system act music_2 offer artist 1", "system-music_2-offer-song_name-1": "last system act music_2 offer song_name 1", "system-music_2-offer_intent-playmedia-1": "last system act music_2 offer_intent PlayMedia 1", "system-music_2-request-song_name-?": "last system act music_2 request song_name ?", "system-music_3-confirm-device-1": "last system act music_3 confirm device 1", "system-music_3-confirm-track-1": "last system act music_3 confirm track 1", "system-music_3-inform-genre-1": "last system act music_3 inform genre 1", "system-music_3-inform-year-1": "last system act music_3 inform year 1", "system-music_3-inform_count-count-1": "last system act music_3 inform_count count 1", "system-music_3-notify_success-none-none": "last system act music_3 notify_success none none", "system-music_3-offer-album-1": "last system act music_3 offer album 1", "system-music_3-offer-artist-1": "last system act music_3 offer artist 1", "system-music_3-offer-track-1": "last system act music_3 offer track 1", "system-music_3-offer_intent-playmedia-1": "last system act music_3 offer_intent PlayMedia 1", "system-payment_1-confirm-amount-1": "last system act payment_1 confirm amount 1", "system-payment_1-confirm-payment_method-1": "last system act payment_1 confirm payment_method 1", "system-payment_1-confirm-private_visibility-1": "last system act payment_1 confirm private_visibility 1", "system-payment_1-confirm-receiver-1": "last system act payment_1 confirm receiver 1", "system-payment_1-notify_success-none-none": "last system act payment_1 notify_success none none", "system-payment_1-request-amount-?": "last system act payment_1 request amount ?", "system-payment_1-request-payment_method-?": "last system act payment_1 request payment_method ?", "system-payment_1-request-receiver-?": "last system act payment_1 request receiver ?", "system-rentalcars_1-confirm-dropoff_date-1": "last system act rentalcars_1 confirm dropoff_date 1", "system-rentalcars_1-confirm-pickup_date-1": "last system act rentalcars_1 confirm pickup_date 1", "system-rentalcars_1-confirm-pickup_location-1": "last system act rentalcars_1 confirm pickup_location 1", "system-rentalcars_1-confirm-pickup_time-1": "last system act rentalcars_1 confirm pickup_time 1", "system-rentalcars_1-confirm-type-1": "last system act rentalcars_1 confirm type 1", "system-rentalcars_1-inform-car_name-1": "last system act rentalcars_1 inform car_name 1", "system-rentalcars_1-inform-total_price-1": "last system act rentalcars_1 inform total_price 1", "system-rentalcars_1-inform_count-count-1": "last system act rentalcars_1 inform_count count 1", "system-rentalcars_1-notify_success-none-none": "last system act rentalcars_1 notify_success none none", "system-rentalcars_1-offer-car_name-1": "last system act rentalcars_1 offer car_name 1", "system-rentalcars_1-offer-pickup_date-1": "last system act rentalcars_1 offer pickup_date 1", "system-rentalcars_1-offer-pickup_location-1": "last system act rentalcars_1 offer pickup_location 1", "system-rentalcars_1-offer-type-1": "last system act rentalcars_1 offer type 1", "system-rentalcars_1-offer_intent-reservecar-1": "last system act rentalcars_1 offer_intent ReserveCar 1", "system-rentalcars_1-request-dropoff_date-?": "last system act rentalcars_1 request dropoff_date ?", "system-rentalcars_1-request-pickup_city-?": "last system act rentalcars_1 request pickup_city ?", "system-rentalcars_1-request-pickup_date-?": "last system act rentalcars_1 request pickup_date ?", "system-rentalcars_1-request-pickup_location-?": "last system act rentalcars_1 request pickup_location ?", "system-rentalcars_1-request-pickup_time-?": "last system act rentalcars_1 request pickup_time ?", "system-rentalcars_1-request-type-?": "last system act rentalcars_1 request type ?", "system-rentalcars_2-confirm-car_type-1": "last system act rentalcars_2 confirm car_type 1", "system-rentalcars_2-confirm-dropoff_date-1": "last system act rentalcars_2 confirm dropoff_date 1", "system-rentalcars_2-confirm-pickup_date-1": "last system act rentalcars_2 confirm pickup_date 1", "system-rentalcars_2-confirm-pickup_location-1": "last system act rentalcars_2 confirm pickup_location 1", "system-rentalcars_2-confirm-pickup_time-1": "last system act rentalcars_2 confirm pickup_time 1", "system-rentalcars_2-inform-car_name-1": "last system act rentalcars_2 inform car_name 1", "system-rentalcars_2-inform-total_price-1": "last system act rentalcars_2 inform total_price 1", "system-rentalcars_2-inform_count-count-1": "last system act rentalcars_2 inform_count count 1", "system-rentalcars_2-notify_success-none-none": "last system act rentalcars_2 notify_success none none", "system-rentalcars_2-offer-car_name-1": "last system act rentalcars_2 offer car_name 1", "system-rentalcars_2-offer-car_type-1": "last system act rentalcars_2 offer car_type 1", "system-rentalcars_2-offer-pickup_date-1": "last system act rentalcars_2 offer pickup_date 1", "system-rentalcars_2-offer-pickup_location-1": "last system act rentalcars_2 offer pickup_location 1", "system-rentalcars_2-offer_intent-reservecar-1": "last system act rentalcars_2 offer_intent ReserveCar 1", "system-rentalcars_2-request-car_type-?": "last system act rentalcars_2 request car_type ?", "system-rentalcars_2-request-dropoff_date-?": "last system act rentalcars_2 request dropoff_date ?", "system-rentalcars_2-request-pickup_city-?": "last system act rentalcars_2 request pickup_city ?", "system-rentalcars_2-request-pickup_date-?": "last system act rentalcars_2 request pickup_date ?", "system-rentalcars_2-request-pickup_location-?": "last system act rentalcars_2 request pickup_location ?", "system-rentalcars_2-request-pickup_time-?": "last system act rentalcars_2 request pickup_time ?", "system-rentalcars_3-confirm-add_insurance-1": "last system act rentalcars_3 confirm add_insurance 1", "system-rentalcars_3-confirm-car_type-1": "last system act rentalcars_3 confirm car_type 1", "system-rentalcars_3-confirm-end_date-1": "last system act rentalcars_3 confirm end_date 1", "system-rentalcars_3-confirm-pickup_location-1": "last system act rentalcars_3 confirm pickup_location 1", "system-rentalcars_3-confirm-pickup_time-1": "last system act rentalcars_3 confirm pickup_time 1", "system-rentalcars_3-confirm-start_date-1": "last system act rentalcars_3 confirm start_date 1", "system-rentalcars_3-inform-car_name-1": "last system act rentalcars_3 inform car_name 1", "system-rentalcars_3-inform-price_per_day-1": "last system act rentalcars_3 inform price_per_day 1", "system-rentalcars_3-inform_count-count-1": "last system act rentalcars_3 inform_count count 1", "system-rentalcars_3-notify_success-none-none": "last system act rentalcars_3 notify_success none none", "system-rentalcars_3-offer-car_name-1": "last system act rentalcars_3 offer car_name 1", "system-rentalcars_3-offer-car_type-1": "last system act rentalcars_3 offer car_type 1", "system-rentalcars_3-offer-pickup_location-1": "last system act rentalcars_3 offer pickup_location 1", "system-rentalcars_3-offer_intent-reservecar-1": "last system act rentalcars_3 offer_intent ReserveCar 1", "system-rentalcars_3-request-add_insurance-?": "last system act rentalcars_3 request add_insurance ?", "system-rentalcars_3-request-car_type-?": "last system act rentalcars_3 request car_type ?", "system-rentalcars_3-request-city-?": "last system act rentalcars_3 request city ?", "system-rentalcars_3-request-end_date-?": "last system act rentalcars_3 request end_date ?", "system-rentalcars_3-request-pickup_location-?": "last system act rentalcars_3 request pickup_location ?", "system-rentalcars_3-request-pickup_time-?": "last system act rentalcars_3 request pickup_time ?", "system-rentalcars_3-request-start_date-?": "last system act rentalcars_3 request start_date ?", "system-restaurants_1-confirm-city-1": "last system act restaurants_1 confirm city 1", "system-restaurants_1-confirm-date-1": "last system act restaurants_1 confirm date 1", "system-restaurants_1-confirm-party_size-1": "last system act restaurants_1 confirm party_size 1", "system-restaurants_1-confirm-restaurant_name-1": "last system act restaurants_1 confirm restaurant_name 1", "system-restaurants_1-confirm-time-1": "last system act restaurants_1 confirm time 1", "system-restaurants_1-inform-cuisine-1": "last system act restaurants_1 inform cuisine 1", "system-restaurants_1-inform-has_live_music-1": "last system act restaurants_1 inform has_live_music 1", "system-restaurants_1-inform-phone_number-1": "last system act restaurants_1 inform phone_number 1", "system-restaurants_1-inform-price_range-1": "last system act restaurants_1 inform price_range 1", "system-restaurants_1-inform-serves_alcohol-1": "last system act restaurants_1 inform serves_alcohol 1", "system-restaurants_1-inform-street_address-1": "last system act restaurants_1 inform street_address 1", "system-restaurants_1-inform_count-count-1": "last system act restaurants_1 inform_count count 1", "system-restaurants_1-notify_failure-none-none": "last system act restaurants_1 notify_failure none none", "system-restaurants_1-notify_success-none-none": "last system act restaurants_1 notify_success none none", "system-restaurants_1-offer-city-1": "last system act restaurants_1 offer city 1", "system-restaurants_1-offer-date-1": "last system act restaurants_1 offer date 1", "system-restaurants_1-offer-party_size-1": "last system act restaurants_1 offer party_size 1", "system-restaurants_1-offer-restaurant_name-1": "last system act restaurants_1 offer restaurant_name 1", "system-restaurants_1-offer-time-1": "last system act restaurants_1 offer time 1", "system-restaurants_1-offer_intent-reserverestaurant-1": "last system act restaurants_1 offer_intent ReserveRestaurant 1", "system-restaurants_1-request-city-?": "last system act restaurants_1 request city ?", "system-restaurants_1-request-cuisine-?": "last system act restaurants_1 request cuisine ?", "system-restaurants_1-request-restaurant_name-?": "last system act restaurants_1 request restaurant_name ?", "system-restaurants_1-request-time-?": "last system act restaurants_1 request time ?", "system-restaurants_2-confirm-date-1": "last system act restaurants_2 confirm date 1", "system-restaurants_2-confirm-location-1": "last system act restaurants_2 confirm location 1", "system-restaurants_2-confirm-number_of_seats-1": "last system act restaurants_2 confirm number_of_seats 1", "system-restaurants_2-confirm-restaurant_name-1": "last system act restaurants_2 confirm restaurant_name 1", "system-restaurants_2-confirm-time-1": "last system act restaurants_2 confirm time 1", "system-restaurants_2-inform-address-1": "last system act restaurants_2 inform address 1", "system-restaurants_2-inform-has_seating_outdoors-1": "last system act restaurants_2 inform has_seating_outdoors 1", "system-restaurants_2-inform-has_vegetarian_options-1": "last system act restaurants_2 inform has_vegetarian_options 1", "system-restaurants_2-inform-phone_number-1": "last system act restaurants_2 inform phone_number 1", "system-restaurants_2-inform-price_range-1": "last system act restaurants_2 inform price_range 1", "system-restaurants_2-inform-rating-1": "last system act restaurants_2 inform rating 1", "system-restaurants_2-inform_count-count-1": "last system act restaurants_2 inform_count count 1", "system-restaurants_2-notify_failure-none-none": "last system act restaurants_2 notify_failure none none", "system-restaurants_2-notify_success-none-none": "last system act restaurants_2 notify_success none none", "system-restaurants_2-offer-date-1": "last system act restaurants_2 offer date 1", "system-restaurants_2-offer-location-1": "last system act restaurants_2 offer location 1", "system-restaurants_2-offer-number_of_seats-1": "last system act restaurants_2 offer number_of_seats 1", "system-restaurants_2-offer-restaurant_name-1": "last system act restaurants_2 offer restaurant_name 1", "system-restaurants_2-offer-time-1": "last system act restaurants_2 offer time 1", "system-restaurants_2-offer_intent-reserverestaurant-1": "last system act restaurants_2 offer_intent ReserveRestaurant 1", "system-restaurants_2-request-category-?": "last system act restaurants_2 request category ?", "system-restaurants_2-request-location-?": "last system act restaurants_2 request location ?", "system-restaurants_2-request-restaurant_name-?": "last system act restaurants_2 request restaurant_name ?", "system-restaurants_2-request-time-?": "last system act restaurants_2 request time ?", "system-ridesharing_1-confirm-destination-1": "last system act ridesharing_1 confirm destination 1", "system-ridesharing_1-confirm-number_of_riders-1": "last system act ridesharing_1 confirm number_of_riders 1", "system-ridesharing_1-confirm-shared_ride-1": "last system act ridesharing_1 confirm shared_ride 1", "system-ridesharing_1-inform-approximate_ride_duration-1": "last system act ridesharing_1 inform approximate_ride_duration 1", "system-ridesharing_1-inform-ride_fare-1": "last system act ridesharing_1 inform ride_fare 1", "system-ridesharing_1-notify_success-none-none": "last system act ridesharing_1 notify_success none none", "system-ridesharing_1-request-destination-?": "last system act ridesharing_1 request destination ?", "system-ridesharing_1-request-number_of_riders-?": "last system act ridesharing_1 request number_of_riders ?", "system-ridesharing_1-request-shared_ride-?": "last system act ridesharing_1 request shared_ride ?", "system-ridesharing_2-confirm-destination-1": "last system act ridesharing_2 confirm destination 1", "system-ridesharing_2-confirm-number_of_seats-1": "last system act ridesharing_2 confirm number_of_seats 1", "system-ridesharing_2-confirm-ride_type-1": "last system act ridesharing_2 confirm ride_type 1", "system-ridesharing_2-inform-ride_fare-1": "last system act ridesharing_2 inform ride_fare 1", "system-ridesharing_2-inform-wait_time-1": "last system act ridesharing_2 inform wait_time 1", "system-ridesharing_2-notify_success-none-none": "last system act ridesharing_2 notify_success none none", "system-ridesharing_2-request-destination-?": "last system act ridesharing_2 request destination ?", "system-ridesharing_2-request-number_of_seats-?": "last system act ridesharing_2 request number_of_seats ?", "system-ridesharing_2-request-ride_type-?": "last system act ridesharing_2 request ride_type ?", "system-services_1-confirm-appointment_date-1": "last system act services_1 confirm appointment_date 1", "system-services_1-confirm-appointment_time-1": "last system act services_1 confirm appointment_time 1", "system-services_1-confirm-stylist_name-1": "last system act services_1 confirm stylist_name 1", "system-services_1-inform-average_rating-1": "last system act services_1 inform average_rating 1", "system-services_1-inform-is_unisex-1": "last system act services_1 inform is_unisex 1", "system-services_1-inform-phone_number-1": "last system act services_1 inform phone_number 1", "system-services_1-inform-street_address-1": "last system act services_1 inform street_address 1", "system-services_1-inform_count-count-1": "last system act services_1 inform_count count 1", "system-services_1-notify_failure-none-none": "last system act services_1 notify_failure none none", "system-services_1-notify_success-none-none": "last system act services_1 notify_success none none", "system-services_1-offer-appointment_date-1": "last system act services_1 offer appointment_date 1", "system-services_1-offer-appointment_time-1": "last system act services_1 offer appointment_time 1", "system-services_1-offer-city-1": "last system act services_1 offer city 1", "system-services_1-offer-stylist_name-1": "last system act services_1 offer stylist_name 1", "system-services_1-offer_intent-bookappointment-1": "last system act services_1 offer_intent BookAppointment 1", "system-services_1-request-appointment_date-?": "last system act services_1 request appointment_date ?", "system-services_1-request-appointment_time-?": "last system act services_1 request appointment_time ?", "system-services_1-request-city-?": "last system act services_1 request city ?", "system-services_2-confirm-appointment_date-1": "last system act services_2 confirm appointment_date 1", "system-services_2-confirm-appointment_time-1": "last system act services_2 confirm appointment_time 1", "system-services_2-confirm-dentist_name-1": "last system act services_2 confirm dentist_name 1", "system-services_2-inform-address-1": "last system act services_2 inform address 1", "system-services_2-inform-offers_cosmetic_services-1": "last system act services_2 inform offers_cosmetic_services 1", "system-services_2-inform-phone_number-1": "last system act services_2 inform phone_number 1", "system-services_2-inform_count-count-1": "last system act services_2 inform_count count 1", "system-services_2-notify_failure-none-none": "last system act services_2 notify_failure none none", "system-services_2-notify_success-none-none": "last system act services_2 notify_success none none", "system-services_2-offer-appointment_date-1": "last system act services_2 offer appointment_date 1", "system-services_2-offer-appointment_time-1": "last system act services_2 offer appointment_time 1", "system-services_2-offer-city-1": "last system act services_2 offer city 1", "system-services_2-offer-dentist_name-1": "last system act services_2 offer dentist_name 1", "system-services_2-offer_intent-bookappointment-1": "last system act services_2 offer_intent BookAppointment 1", "system-services_2-request-appointment_date-?": "last system act services_2 request appointment_date ?", "system-services_2-request-appointment_time-?": "last system act services_2 request appointment_time ?", "system-services_2-request-city-?": "last system act services_2 request city ?", "system-services_3-confirm-appointment_date-1": "last system act services_3 confirm appointment_date 1", "system-services_3-confirm-appointment_time-1": "last system act services_3 confirm appointment_time 1", "system-services_3-confirm-doctor_name-1": "last system act services_3 confirm doctor_name 1", "system-services_3-inform-average_rating-1": "last system act services_3 inform average_rating 1", "system-services_3-inform-phone_number-1": "last system act services_3 inform phone_number 1", "system-services_3-inform-street_address-1": "last system act services_3 inform street_address 1", "system-services_3-inform_count-count-1": "last system act services_3 inform_count count 1", "system-services_3-notify_failure-none-none": "last system act services_3 notify_failure none none", "system-services_3-notify_success-none-none": "last system act services_3 notify_success none none", "system-services_3-offer-appointment_date-1": "last system act services_3 offer appointment_date 1", "system-services_3-offer-appointment_time-1": "last system act services_3 offer appointment_time 1", "system-services_3-offer-city-1": "last system act services_3 offer city 1", "system-services_3-offer-doctor_name-1": "last system act services_3 offer doctor_name 1", "system-services_3-offer-type-1": "last system act services_3 offer type 1", "system-services_3-offer_intent-bookappointment-1": "last system act services_3 offer_intent BookAppointment 1", "system-services_3-request-appointment_date-?": "last system act services_3 request appointment_date ?", "system-services_3-request-appointment_time-?": "last system act services_3 request appointment_time ?", "system-services_3-request-city-?": "last system act services_3 request city ?", "system-services_3-request-type-?": "last system act services_3 request type ?", "system-services_4-confirm-appointment_date-1": "last system act services_4 confirm appointment_date 1", "system-services_4-confirm-appointment_time-1": "last system act services_4 confirm appointment_time 1", "system-services_4-confirm-therapist_name-1": "last system act services_4 confirm therapist_name 1", "system-services_4-inform-address-1": "last system act services_4 inform address 1", "system-services_4-inform-phone_number-1": "last system act services_4 inform phone_number 1", "system-services_4-inform_count-count-1": "last system act services_4 inform_count count 1", "system-services_4-notify_failure-none-none": "last system act services_4 notify_failure none none", "system-services_4-notify_success-none-none": "last system act services_4 notify_success none none", "system-services_4-offer-appointment_date-1": "last system act services_4 offer appointment_date 1", "system-services_4-offer-appointment_time-1": "last system act services_4 offer appointment_time 1", "system-services_4-offer-city-1": "last system act services_4 offer city 1", "system-services_4-offer-therapist_name-1": "last system act services_4 offer therapist_name 1", "system-services_4-offer-type-1": "last system act services_4 offer type 1", "system-services_4-offer_intent-bookappointment-1": "last system act services_4 offer_intent BookAppointment 1", "system-services_4-request-appointment_date-?": "last system act services_4 request appointment_date ?", "system-services_4-request-appointment_time-?": "last system act services_4 request appointment_time ?", "system-services_4-request-city-?": "last system act services_4 request city ?", "system-services_4-request-type-?": "last system act services_4 request type ?", "system-trains_1-confirm-class-1": "last system act trains_1 confirm class 1", "system-trains_1-confirm-date_of_journey-1": "last system act trains_1 confirm date_of_journey 1", "system-trains_1-confirm-from-1": "last system act trains_1 confirm from 1", "system-trains_1-confirm-journey_start_time-1": "last system act trains_1 confirm journey_start_time 1", "system-trains_1-confirm-number_of_adults-1": "last system act trains_1 confirm number_of_adults 1", "system-trains_1-confirm-to-1": "last system act trains_1 confirm to 1", "system-trains_1-confirm-trip_protection-1": "last system act trains_1 confirm trip_protection 1", "system-trains_1-inform-from_station-1": "last system act trains_1 inform from_station 1", "system-trains_1-inform-to_station-1": "last system act trains_1 inform to_station 1", "system-trains_1-inform_count-count-1": "last system act trains_1 inform_count count 1", "system-trains_1-notify_success-none-none": "last system act trains_1 notify_success none none", "system-trains_1-offer-journey_start_time-1": "last system act trains_1 offer journey_start_time 1", "system-trains_1-offer-total-1": "last system act trains_1 offer total 1", "system-trains_1-offer_intent-gettraintickets-1": "last system act trains_1 offer_intent GetTrainTickets 1", "system-trains_1-request-date_of_journey-?": "last system act trains_1 request date_of_journey ?", "system-trains_1-request-from-?": "last system act trains_1 request from ?", "system-trains_1-request-number_of_adults-?": "last system act trains_1 request number_of_adults ?", "system-trains_1-request-to-?": "last system act trains_1 request to ?", "system-trains_1-request-trip_protection-?": "last system act trains_1 request trip_protection ?", "system-travel_1-inform-free_entry-1": "last system act travel_1 inform free_entry 1", "system-travel_1-inform-good_for_kids-1": "last system act travel_1 inform good_for_kids 1", "system-travel_1-inform-phone_number-1": "last system act travel_1 inform phone_number 1", "system-travel_1-inform_count-count-1": "last system act travel_1 inform_count count 1", "system-travel_1-offer-attraction_name-1": "last system act travel_1 offer attraction_name 1", "system-travel_1-offer-category-1": "last system act travel_1 offer category 1", "system-travel_1-request-location-?": "last system act travel_1 request location ?", "system-weather_1-inform-humidity-1": "last system act weather_1 inform humidity 1", "system-weather_1-inform-wind-1": "last system act weather_1 inform wind 1", "system-weather_1-offer-precipitation-1": "last system act weather_1 offer precipitation 1", "system-weather_1-offer-temperature-1": "last system act weather_1 offer temperature 1", "system-weather_1-request-city-?": "last system act weather_1 request city ?", "user--affirm-none-none": "user act affirm none none", "user--goodbye-none-none": "user act goodbye none none", "user--negate-none-none": "user act negate none none", "user--thank_you-none-none": "user act thank_you none none", "user-alarm_1-affirm_intent-none-none": "user act alarm_1 affirm_intent none none", "user-alarm_1-inform-new_alarm_name-1": "user act alarm_1 inform new_alarm_name 1", "user-alarm_1-inform-new_alarm_time-1": "user act alarm_1 inform new_alarm_time 1", "user-alarm_1-inform_intent-addalarm-1": "user act alarm_1 inform_intent addalarm 1", "user-alarm_1-inform_intent-getalarms-1": "user act alarm_1 inform_intent getalarms 1", "user-alarm_1-negate_intent-none-none": "user act alarm_1 negate_intent none none", "user-alarm_1-select-none-none": "user act alarm_1 select none none", "user-banks_1-inform-account_type-1": "user act banks_1 inform account_type 1", "user-banks_1-inform-amount-1": "user act banks_1 inform amount 1", "user-banks_1-inform-recipient_account_name-1": "user act banks_1 inform recipient_account_name 1", "user-banks_1-inform-recipient_account_type-1": "user act banks_1 inform recipient_account_type 1", "user-banks_1-inform_intent-checkbalance-1": "user act banks_1 inform_intent checkbalance 1", "user-banks_1-inform_intent-transfermoney-1": "user act banks_1 inform_intent transfermoney 1", "user-banks_1-negate_intent-none-none": "user act banks_1 negate_intent none none", "user-banks_1-request_alts-none-none": "user act banks_1 request_alts none none", "user-banks_1-select-none-none": "user act banks_1 select none none", "user-banks_2-affirm_intent-none-none": "user act banks_2 affirm_intent none none", "user-banks_2-inform-account_type-1": "user act banks_2 inform account_type 1", "user-banks_2-inform-recipient_account_type-1": "user act banks_2 inform recipient_account_type 1", "user-banks_2-inform-recipient_name-1": "user act banks_2 inform recipient_name 1", "user-banks_2-inform-transfer_amount-1": "user act banks_2 inform transfer_amount 1", "user-banks_2-inform_intent-checkbalance-1": "user act banks_2 inform_intent checkbalance 1", "user-banks_2-inform_intent-transfermoney-1": "user act banks_2 inform_intent transfermoney 1", "user-banks_2-negate_intent-none-none": "user act banks_2 negate_intent none none", "user-banks_2-request-transfer_time-?": "user act banks_2 request transfer_time ?", "user-banks_2-request_alts-none-none": "user act banks_2 request_alts none none", "user-banks_2-select-none-none": "user act banks_2 select none none", "user-buses_1-affirm_intent-none-none": "user act buses_1 affirm_intent none none", "user-buses_1-inform-from_location-1": "user act buses_1 inform from_location 1", "user-buses_1-inform-leaving_date-1": "user act buses_1 inform leaving_date 1", "user-buses_1-inform-leaving_time-1": "user act buses_1 inform leaving_time 1", "user-buses_1-inform-to_location-1": "user act buses_1 inform to_location 1", "user-buses_1-inform-travelers-1": "user act buses_1 inform travelers 1", "user-buses_1-inform_intent-buybusticket-1": "user act buses_1 inform_intent buybusticket 1", "user-buses_1-inform_intent-findbus-1": "user act buses_1 inform_intent findbus 1", "user-buses_1-negate_intent-none-none": "user act buses_1 negate_intent none none", "user-buses_1-request-from_station-?": "user act buses_1 request from_station ?", "user-buses_1-request-to_station-?": "user act buses_1 request to_station ?", "user-buses_1-request-transfers-?": "user act buses_1 request transfers ?", "user-buses_1-request_alts-none-none": "user act buses_1 request_alts none none", "user-buses_1-select-none-none": "user act buses_1 select none none", "user-buses_2-affirm_intent-none-none": "user act buses_2 affirm_intent none none", "user-buses_2-inform-departure_date-1": "user act buses_2 inform departure_date 1", "user-buses_2-inform-departure_time-1": "user act buses_2 inform departure_time 1", "user-buses_2-inform-destination-1": "user act buses_2 inform destination 1", "user-buses_2-inform-fare_type-1": "user act buses_2 inform fare_type 1", "user-buses_2-inform-group_size-1": "user act buses_2 inform group_size 1", "user-buses_2-inform-origin-1": "user act buses_2 inform origin 1", "user-buses_2-inform_intent-buybusticket-1": "user act buses_2 inform_intent buybusticket 1", "user-buses_2-inform_intent-findbus-1": "user act buses_2 inform_intent findbus 1", "user-buses_2-negate_intent-none-none": "user act buses_2 negate_intent none none", "user-buses_2-request-destination_station_name-?": "user act buses_2 request destination_station_name ?", "user-buses_2-request-origin_station_name-?": "user act buses_2 request origin_station_name ?", "user-buses_2-request-price-?": "user act buses_2 request price ?", "user-buses_2-request_alts-none-none": "user act buses_2 request_alts none none", "user-buses_2-select-none-none": "user act buses_2 select none none", "user-buses_3-inform-additional_luggage-1": "user act buses_3 inform additional_luggage 1", "user-buses_3-inform-category-1": "user act buses_3 inform category 1", "user-buses_3-inform-departure_date-1": "user act buses_3 inform departure_date 1", "user-buses_3-inform-departure_time-1": "user act buses_3 inform departure_time 1", "user-buses_3-inform-from_city-1": "user act buses_3 inform from_city 1", "user-buses_3-inform-num_passengers-1": "user act buses_3 inform num_passengers 1", "user-buses_3-inform-to_city-1": "user act buses_3 inform to_city 1", "user-buses_3-inform_intent-buybusticket-1": "user act buses_3 inform_intent buybusticket 1", "user-buses_3-inform_intent-findbus-1": "user act buses_3 inform_intent findbus 1", "user-buses_3-negate_intent-none-none": "user act buses_3 negate_intent none none", "user-buses_3-request-category-?": "user act buses_3 request category ?", "user-buses_3-request-from_station-?": "user act buses_3 request from_station ?", "user-buses_3-request-to_station-?": "user act buses_3 request to_station ?", "user-buses_3-request_alts-none-none": "user act buses_3 request_alts none none", "user-buses_3-select-none-none": "user act buses_3 select none none", "user-calendar_1-inform-event_date-1": "user act calendar_1 inform event_date 1", "user-calendar_1-inform-event_location-1": "user act calendar_1 inform event_location 1", "user-calendar_1-inform-event_name-1": "user act calendar_1 inform event_name 1", "user-calendar_1-inform-event_time-1": "user act calendar_1 inform event_time 1", "user-calendar_1-inform_intent-addevent-1": "user act calendar_1 inform_intent addevent 1", "user-calendar_1-inform_intent-getavailabletime-1": "user act calendar_1 inform_intent getavailabletime 1", "user-calendar_1-inform_intent-getevents-1": "user act calendar_1 inform_intent getevents 1", "user-calendar_1-negate_intent-none-none": "user act calendar_1 negate_intent none none", "user-calendar_1-request_alts-none-none": "user act calendar_1 request_alts none none", "user-calendar_1-select-none-none": "user act calendar_1 select none none", "user-events_1-affirm_intent-none-none": "user act events_1 affirm_intent none none", "user-events_1-inform-category-1": "user act events_1 inform category 1", "user-events_1-inform-city_of_event-1": "user act events_1 inform city_of_event 1", "user-events_1-inform-date-1": "user act events_1 inform date 1", "user-events_1-inform-event_name-1": "user act events_1 inform event_name 1", "user-events_1-inform-number_of_seats-1": "user act events_1 inform number_of_seats 1", "user-events_1-inform-subcategory-1": "user act events_1 inform subcategory 1", "user-events_1-inform_intent-buyeventtickets-1": "user act events_1 inform_intent buyeventtickets 1", "user-events_1-inform_intent-findevents-1": "user act events_1 inform_intent findevents 1", "user-events_1-negate_intent-none-none": "user act events_1 negate_intent none none", "user-events_1-request-address_of_location-?": "user act events_1 request address_of_location ?", "user-events_1-request-event_location-?": "user act events_1 request event_location ?", "user-events_1-request-subcategory-?": "user act events_1 request subcategory ?", "user-events_1-request-time-?": "user act events_1 request time ?", "user-events_1-request_alts-none-none": "user act events_1 request_alts none none", "user-events_1-select-none-none": "user act events_1 select none none", "user-events_2-affirm_intent-none-none": "user act events_2 affirm_intent none none", "user-events_2-inform-category-1": "user act events_2 inform category 1", "user-events_2-inform-city-1": "user act events_2 inform city 1", "user-events_2-inform-date-1": "user act events_2 inform date 1", "user-events_2-inform-event_name-1": "user act events_2 inform event_name 1", "user-events_2-inform-event_type-1": "user act events_2 inform event_type 1", "user-events_2-inform-number_of_tickets-1": "user act events_2 inform number_of_tickets 1", "user-events_2-inform_intent-buyeventtickets-1": "user act events_2 inform_intent buyeventtickets 1", "user-events_2-inform_intent-findevents-1": "user act events_2 inform_intent findevents 1", "user-events_2-inform_intent-geteventdates-1": "user act events_2 inform_intent geteventdates 1", "user-events_2-negate_intent-none-none": "user act events_2 negate_intent none none", "user-events_2-request-category-?": "user act events_2 request category ?", "user-events_2-request-time-?": "user act events_2 request time ?", "user-events_2-request-venue-?": "user act events_2 request venue ?", "user-events_2-request-venue_address-?": "user act events_2 request venue_address ?", "user-events_2-request_alts-none-none": "user act events_2 request_alts none none", "user-events_2-select-none-none": "user act events_2 select none none", "user-events_3-inform-city-1": "user act events_3 inform city 1", "user-events_3-inform-date-1": "user act events_3 inform date 1", "user-events_3-inform-event_name-1": "user act events_3 inform event_name 1", "user-events_3-inform-event_type-1": "user act events_3 inform event_type 1", "user-events_3-inform-number_of_tickets-1": "user act events_3 inform number_of_tickets 1", "user-events_3-inform_intent-buyeventtickets-1": "user act events_3 inform_intent buyeventtickets 1", "user-events_3-inform_intent-findevents-1": "user act events_3 inform_intent findevents 1", "user-events_3-negate_intent-none-none": "user act events_3 negate_intent none none", "user-events_3-request-price_per_ticket-?": "user act events_3 request price_per_ticket ?", "user-events_3-request-venue_address-?": "user act events_3 request venue_address ?", "user-events_3-request_alts-none-none": "user act events_3 request_alts none none", "user-events_3-select-none-none": "user act events_3 select none none", "user-flights_1-affirm_intent-none-none": "user act flights_1 affirm_intent none none", "user-flights_1-inform-airlines-1": "user act flights_1 inform airlines 1", "user-flights_1-inform-departure_date-1": "user act flights_1 inform departure_date 1", "user-flights_1-inform-destination_city-1": "user act flights_1 inform destination_city 1", "user-flights_1-inform-inbound_departure_time-1": "user act flights_1 inform inbound_departure_time 1", "user-flights_1-inform-origin_city-1": "user act flights_1 inform origin_city 1", "user-flights_1-inform-outbound_departure_time-1": "user act flights_1 inform outbound_departure_time 1", "user-flights_1-inform-passengers-1": "user act flights_1 inform passengers 1", "user-flights_1-inform-refundable-1": "user act flights_1 inform refundable 1", "user-flights_1-inform-return_date-1": "user act flights_1 inform return_date 1", "user-flights_1-inform-seating_class-1": "user act flights_1 inform seating_class 1", "user-flights_1-inform_intent-reserveonewayflight-1": "user act flights_1 inform_intent reserveonewayflight 1", "user-flights_1-inform_intent-reserveroundtripflights-1": "user act flights_1 inform_intent reserveroundtripflights 1", "user-flights_1-inform_intent-searchonewayflight-1": "user act flights_1 inform_intent searchonewayflight 1", "user-flights_1-inform_intent-searchroundtripflights-1": "user act flights_1 inform_intent searchroundtripflights 1", "user-flights_1-negate_intent-none-none": "user act flights_1 negate_intent none none", "user-flights_1-request-destination_airport-?": "user act flights_1 request destination_airport ?", "user-flights_1-request-inbound_arrival_time-?": "user act flights_1 request inbound_arrival_time ?", "user-flights_1-request-number_stops-?": "user act flights_1 request number_stops ?", "user-flights_1-request-origin_airport-?": "user act flights_1 request origin_airport ?", "user-flights_1-request-outbound_arrival_time-?": "user act flights_1 request outbound_arrival_time ?", "user-flights_1-request-refundable-?": "user act flights_1 request refundable ?", "user-flights_1-request_alts-none-none": "user act flights_1 request_alts none none", "user-flights_1-select-none-none": "user act flights_1 select none none", "user-flights_2-inform-airlines-1": "user act flights_2 inform airlines 1", "user-flights_2-inform-departure_date-1": "user act flights_2 inform departure_date 1", "user-flights_2-inform-destination-1": "user act flights_2 inform destination 1", "user-flights_2-inform-origin-1": "user act flights_2 inform origin 1", "user-flights_2-inform-passengers-1": "user act flights_2 inform passengers 1", "user-flights_2-inform-return_date-1": "user act flights_2 inform return_date 1", "user-flights_2-inform-seating_class-1": "user act flights_2 inform seating_class 1", "user-flights_2-inform_intent-searchonewayflight-1": "user act flights_2 inform_intent searchonewayflight 1", "user-flights_2-inform_intent-searchroundtripflights-1": "user act flights_2 inform_intent searchroundtripflights 1", "user-flights_2-request-destination_airport-?": "user act flights_2 request destination_airport ?", "user-flights_2-request-is_redeye-?": "user act flights_2 request is_redeye ?", "user-flights_2-request-origin_airport-?": "user act flights_2 request origin_airport ?", "user-flights_2-request-outbound_arrival_time-?": "user act flights_2 request outbound_arrival_time ?", "user-flights_2-request_alts-none-none": "user act flights_2 request_alts none none", "user-flights_2-select-none-none": "user act flights_2 select none none", "user-flights_3-inform-airlines-1": "user act flights_3 inform airlines 1", "user-flights_3-inform-departure_date-1": "user act flights_3 inform departure_date 1", "user-flights_3-inform-destination_city-1": "user act flights_3 inform destination_city 1", "user-flights_3-inform-flight_class-1": "user act flights_3 inform flight_class 1", "user-flights_3-inform-number_checked_bags-1": "user act flights_3 inform number_checked_bags 1", "user-flights_3-inform-origin_city-1": "user act flights_3 inform origin_city 1", "user-flights_3-inform-passengers-1": "user act flights_3 inform passengers 1", "user-flights_3-inform-return_date-1": "user act flights_3 inform return_date 1", "user-flights_3-inform_intent-searchonewayflight-1": "user act flights_3 inform_intent searchonewayflight 1", "user-flights_3-inform_intent-searchroundtripflights-1": "user act flights_3 inform_intent searchroundtripflights 1", "user-flights_3-request-arrives_next_day-?": "user act flights_3 request arrives_next_day ?", "user-flights_3-request-destination_airport_name-?": "user act flights_3 request destination_airport_name ?", "user-flights_3-request-origin_airport_name-?": "user act flights_3 request origin_airport_name ?", "user-flights_3-request-outbound_arrival_time-?": "user act flights_3 request outbound_arrival_time ?", "user-flights_3-request_alts-none-none": "user act flights_3 request_alts none none", "user-flights_3-select-none-none": "user act flights_3 select none none", "user-flights_4-inform-airlines-1": "user act flights_4 inform airlines 1", "user-flights_4-inform-departure_date-1": "user act flights_4 inform departure_date 1", "user-flights_4-inform-destination_airport-1": "user act flights_4 inform destination_airport 1", "user-flights_4-inform-number_of_tickets-1": "user act flights_4 inform number_of_tickets 1", "user-flights_4-inform-origin_airport-1": "user act flights_4 inform origin_airport 1", "user-flights_4-inform-return_date-1": "user act flights_4 inform return_date 1", "user-flights_4-inform-seating_class-1": "user act flights_4 inform seating_class 1", "user-flights_4-inform_intent-searchonewayflight-1": "user act flights_4 inform_intent searchonewayflight 1", "user-flights_4-inform_intent-searchroundtripflights-1": "user act flights_4 inform_intent searchroundtripflights 1", "user-flights_4-request-inbound_arrival_time-?": "user act flights_4 request inbound_arrival_time ?", "user-flights_4-request-number_of_tickets-?": "user act flights_4 request number_of_tickets ?", "user-flights_4-request-outbound_arrival_time-?": "user act flights_4 request outbound_arrival_time ?", "user-flights_4-request-seating_class-?": "user act flights_4 request seating_class ?", "user-flights_4-request_alts-none-none": "user act flights_4 request_alts none none", "user-flights_4-select-none-none": "user act flights_4 select none none", "user-homes_1-affirm_intent-none-none": "user act homes_1 affirm_intent none none", "user-homes_1-inform-area-1": "user act homes_1 inform area 1", "user-homes_1-inform-furnished-1": "user act homes_1 inform furnished 1", "user-homes_1-inform-number_of_baths-1": "user act homes_1 inform number_of_baths 1", "user-homes_1-inform-number_of_beds-1": "user act homes_1 inform number_of_beds 1", "user-homes_1-inform-pets_allowed-1": "user act homes_1 inform pets_allowed 1", "user-homes_1-inform-visit_date-1": "user act homes_1 inform visit_date 1", "user-homes_1-inform_intent-findapartment-1": "user act homes_1 inform_intent findapartment 1", "user-homes_1-inform_intent-schedulevisit-1": "user act homes_1 inform_intent schedulevisit 1", "user-homes_1-negate_intent-none-none": "user act homes_1 negate_intent none none", "user-homes_1-request-furnished-?": "user act homes_1 request furnished ?", "user-homes_1-request-pets_allowed-?": "user act homes_1 request pets_allowed ?", "user-homes_1-request-phone_number-?": "user act homes_1 request phone_number ?", "user-homes_1-request_alts-none-none": "user act homes_1 request_alts none none", "user-homes_1-select-none-none": "user act homes_1 select none none", "user-homes_2-affirm_intent-none-none": "user act homes_2 affirm_intent none none", "user-homes_2-inform-area-1": "user act homes_2 inform area 1", "user-homes_2-inform-intent-1": "user act homes_2 inform intent 1", "user-homes_2-inform-number_of_baths-1": "user act homes_2 inform number_of_baths 1", "user-homes_2-inform-number_of_beds-1": "user act homes_2 inform number_of_beds 1", "user-homes_2-inform-property_name-1": "user act homes_2 inform property_name 1", "user-homes_2-inform-visit_date-1": "user act homes_2 inform visit_date 1", "user-homes_2-inform_intent-findhomebyarea-1": "user act homes_2 inform_intent findhomebyarea 1", "user-homes_2-inform_intent-schedulevisit-1": "user act homes_2 inform_intent schedulevisit 1", "user-homes_2-request-has_garage-?": "user act homes_2 request has_garage ?", "user-homes_2-request-in_unit_laundry-?": "user act homes_2 request in_unit_laundry ?", "user-homes_2-request-phone_number-?": "user act homes_2 request phone_number ?", "user-homes_2-request_alts-none-none": "user act homes_2 request_alts none none", "user-homes_2-select-none-none": "user act homes_2 select none none", "user-hotels_1-affirm_intent-none-none": "user act hotels_1 affirm_intent none none", "user-hotels_1-inform-check_in_date-1": "user act hotels_1 inform check_in_date 1", "user-hotels_1-inform-destination-1": "user act hotels_1 inform destination 1", "user-hotels_1-inform-has_wifi-1": "user act hotels_1 inform has_wifi 1", "user-hotels_1-inform-hotel_name-1": "user act hotels_1 inform hotel_name 1", "user-hotels_1-inform-number_of_days-1": "user act hotels_1 inform number_of_days 1", "user-hotels_1-inform-number_of_rooms-1": "user act hotels_1 inform number_of_rooms 1", "user-hotels_1-inform-star_rating-1": "user act hotels_1 inform star_rating 1", "user-hotels_1-inform_intent-reservehotel-1": "user act hotels_1 inform_intent reservehotel 1", "user-hotels_1-inform_intent-searchhotel-1": "user act hotels_1 inform_intent searchhotel 1", "user-hotels_1-negate_intent-none-none": "user act hotels_1 negate_intent none none", "user-hotels_1-request-has_wifi-?": "user act hotels_1 request has_wifi ?", "user-hotels_1-request-phone_number-?": "user act hotels_1 request phone_number ?", "user-hotels_1-request-price_per_night-?": "user act hotels_1 request price_per_night ?", "user-hotels_1-request-street_address-?": "user act hotels_1 request street_address ?", "user-hotels_1-request_alts-none-none": "user act hotels_1 request_alts none none", "user-hotels_1-select-none-none": "user act hotels_1 select none none", "user-hotels_2-affirm_intent-none-none": "user act hotels_2 affirm_intent none none", "user-hotels_2-inform-check_in_date-1": "user act hotels_2 inform check_in_date 1", "user-hotels_2-inform-check_out_date-1": "user act hotels_2 inform check_out_date 1", "user-hotels_2-inform-has_laundry_service-1": "user act hotels_2 inform has_laundry_service 1", "user-hotels_2-inform-number_of_adults-1": "user act hotels_2 inform number_of_adults 1", "user-hotels_2-inform-rating-1": "user act hotels_2 inform rating 1", "user-hotels_2-inform-where_to-1": "user act hotels_2 inform where_to 1", "user-hotels_2-inform_intent-bookhouse-1": "user act hotels_2 inform_intent bookhouse 1", "user-hotels_2-inform_intent-searchhouse-1": "user act hotels_2 inform_intent searchhouse 1", "user-hotels_2-negate_intent-none-none": "user act hotels_2 negate_intent none none", "user-hotels_2-request-has_laundry_service-?": "user act hotels_2 request has_laundry_service ?", "user-hotels_2-request-phone_number-?": "user act hotels_2 request phone_number ?", "user-hotels_2-request-total_price-?": "user act hotels_2 request total_price ?", "user-hotels_2-request_alts-none-none": "user act hotels_2 request_alts none none", "user-hotels_2-select-none-none": "user act hotels_2 select none none", "user-hotels_3-affirm_intent-none-none": "user act hotels_3 affirm_intent none none", "user-hotels_3-inform-check_in_date-1": "user act hotels_3 inform check_in_date 1", "user-hotels_3-inform-check_out_date-1": "user act hotels_3 inform check_out_date 1", "user-hotels_3-inform-hotel_name-1": "user act hotels_3 inform hotel_name 1", "user-hotels_3-inform-location-1": "user act hotels_3 inform location 1", "user-hotels_3-inform-number_of_rooms-1": "user act hotels_3 inform number_of_rooms 1", "user-hotels_3-inform-pets_welcome-1": "user act hotels_3 inform pets_welcome 1", "user-hotels_3-inform_intent-reservehotel-1": "user act hotels_3 inform_intent reservehotel 1", "user-hotels_3-inform_intent-searchhotel-1": "user act hotels_3 inform_intent searchhotel 1", "user-hotels_3-negate_intent-none-none": "user act hotels_3 negate_intent none none", "user-hotels_3-request-pets_welcome-?": "user act hotels_3 request pets_welcome ?", "user-hotels_3-request-phone_number-?": "user act hotels_3 request phone_number ?", "user-hotels_3-request-price-?": "user act hotels_3 request price ?", "user-hotels_3-request-street_address-?": "user act hotels_3 request street_address ?", "user-hotels_3-request_alts-none-none": "user act hotels_3 request_alts none none", "user-hotels_3-select-none-none": "user act hotels_3 select none none", "user-hotels_4-affirm_intent-none-none": "user act hotels_4 affirm_intent none none", "user-hotels_4-inform-check_in_date-1": "user act hotels_4 inform check_in_date 1", "user-hotels_4-inform-location-1": "user act hotels_4 inform location 1", "user-hotels_4-inform-number_of_rooms-1": "user act hotels_4 inform number_of_rooms 1", "user-hotels_4-inform-smoking_allowed-1": "user act hotels_4 inform smoking_allowed 1", "user-hotels_4-inform-star_rating-1": "user act hotels_4 inform star_rating 1", "user-hotels_4-inform-stay_length-1": "user act hotels_4 inform stay_length 1", "user-hotels_4-inform_intent-reservehotel-1": "user act hotels_4 inform_intent reservehotel 1", "user-hotels_4-inform_intent-searchhotel-1": "user act hotels_4 inform_intent searchhotel 1", "user-hotels_4-negate_intent-none-none": "user act hotels_4 negate_intent none none", "user-hotels_4-request-phone_number-?": "user act hotels_4 request phone_number ?", "user-hotels_4-request-price_per_night-?": "user act hotels_4 request price_per_night ?", "user-hotels_4-request-smoking_allowed-?": "user act hotels_4 request smoking_allowed ?", "user-hotels_4-request-street_address-?": "user act hotels_4 request street_address ?", "user-hotels_4-request_alts-none-none": "user act hotels_4 request_alts none none", "user-hotels_4-select-none-none": "user act hotels_4 select none none", "user-media_1-affirm_intent-none-none": "user act media_1 affirm_intent none none", "user-media_1-inform-directed_by-1": "user act media_1 inform directed_by 1", "user-media_1-inform-genre-1": "user act media_1 inform genre 1", "user-media_1-inform-subtitles-1": "user act media_1 inform subtitles 1", "user-media_1-inform-title-1": "user act media_1 inform title 1", "user-media_1-inform_intent-findmovies-1": "user act media_1 inform_intent findmovies 1", "user-media_1-inform_intent-playmovie-1": "user act media_1 inform_intent playmovie 1", "user-media_1-negate_intent-none-none": "user act media_1 negate_intent none none", "user-media_1-request-directed_by-?": "user act media_1 request directed_by ?", "user-media_1-request-genre-?": "user act media_1 request genre ?", "user-media_1-request_alts-none-none": "user act media_1 request_alts none none", "user-media_1-select-title-1": "user act media_1 select title 1", "user-media_2-affirm_intent-none-none": "user act media_2 affirm_intent none none", "user-media_2-inform-actors-1": "user act media_2 inform actors 1", "user-media_2-inform-director-1": "user act media_2 inform director 1", "user-media_2-inform-genre-1": "user act media_2 inform genre 1", "user-media_2-inform-subtitle_language-1": "user act media_2 inform subtitle_language 1", "user-media_2-inform_intent-findmovies-1": "user act media_2 inform_intent findmovies 1", "user-media_2-inform_intent-rentmovie-1": "user act media_2 inform_intent rentmovie 1", "user-media_2-request-price-?": "user act media_2 request price ?", "user-media_2-select-movie_name-1": "user act media_2 select movie_name 1", "user-media_3-affirm_intent-none-none": "user act media_3 affirm_intent none none", "user-media_3-inform-genre-1": "user act media_3 inform genre 1", "user-media_3-inform-starring-1": "user act media_3 inform starring 1", "user-media_3-inform-subtitle_language-1": "user act media_3 inform subtitle_language 1", "user-media_3-inform-title-1": "user act media_3 inform title 1", "user-media_3-inform_intent-findmovies-1": "user act media_3 inform_intent findmovies 1", "user-media_3-inform_intent-playmovie-1": "user act media_3 inform_intent playmovie 1", "user-media_3-negate_intent-none-none": "user act media_3 negate_intent none none", "user-media_3-request-starring-?": "user act media_3 request starring ?", "user-media_3-request_alts-none-none": "user act media_3 request_alts none none", "user-media_3-select-title-1": "user act media_3 select title 1", "user-messaging_1-inform-contact_name-1": "user act messaging_1 inform contact_name 1", "user-messaging_1-inform-location-1": "user act messaging_1 inform location 1", "user-messaging_1-inform_intent-sharelocation-1": "user act messaging_1 inform_intent sharelocation 1", "user-movies_1-inform-genre-1": "user act movies_1 inform genre 1", "user-movies_1-inform-location-1": "user act movies_1 inform location 1", "user-movies_1-inform-movie_name-1": "user act movies_1 inform movie_name 1", "user-movies_1-inform-number_of_tickets-1": "user act movies_1 inform number_of_tickets 1", "user-movies_1-inform-show_date-1": "user act movies_1 inform show_date 1", "user-movies_1-inform-show_time-1": "user act movies_1 inform show_time 1", "user-movies_1-inform-show_type-1": "user act movies_1 inform show_type 1", "user-movies_1-inform-theater_name-1": "user act movies_1 inform theater_name 1", "user-movies_1-inform_intent-buymovietickets-1": "user act movies_1 inform_intent buymovietickets 1", "user-movies_1-inform_intent-findmovies-1": "user act movies_1 inform_intent findmovies 1", "user-movies_1-inform_intent-gettimesformovie-1": "user act movies_1 inform_intent gettimesformovie 1", "user-movies_1-negate_intent-none-none": "user act movies_1 negate_intent none none", "user-movies_1-request-genre-?": "user act movies_1 request genre ?", "user-movies_1-request-price-?": "user act movies_1 request price ?", "user-movies_1-request-street_address-?": "user act movies_1 request street_address ?", "user-movies_1-request_alts-none-none": "user act movies_1 request_alts none none", "user-movies_1-select-movie_name-1": "user act movies_1 select movie_name 1", "user-movies_1-select-none-none": "user act movies_1 select none none", "user-movies_2-inform-director-1": "user act movies_2 inform director 1", "user-movies_2-inform-genre-1": "user act movies_2 inform genre 1", "user-movies_2-inform-starring-1": "user act movies_2 inform starring 1", "user-movies_2-inform_intent-findmovies-1": "user act movies_2 inform_intent findmovies 1", "user-movies_2-request_alts-none-none": "user act movies_2 request_alts none none", "user-movies_2-select-none-none": "user act movies_2 select none none", "user-movies_3-inform-cast-1": "user act movies_3 inform cast 1", "user-movies_3-inform-directed_by-1": "user act movies_3 inform directed_by 1", "user-movies_3-inform-genre-1": "user act movies_3 inform genre 1", "user-movies_3-inform_intent-findmovies-1": "user act movies_3 inform_intent findmovies 1", "user-movies_3-request-cast-?": "user act movies_3 request cast ?", "user-movies_3-request-directed_by-?": "user act movies_3 request directed_by ?", "user-movies_3-request-genre-?": "user act movies_3 request genre ?", "user-movies_3-select-none-none": "user act movies_3 select none none", "user-music_1-affirm_intent-none-none": "user act music_1 affirm_intent none none", "user-music_1-inform-album-1": "user act music_1 inform album 1", "user-music_1-inform-artist-1": "user act music_1 inform artist 1", "user-music_1-inform-genre-1": "user act music_1 inform genre 1", "user-music_1-inform-playback_device-1": "user act music_1 inform playback_device 1", "user-music_1-inform-song_name-1": "user act music_1 inform song_name 1", "user-music_1-inform-year-1": "user act music_1 inform year 1", "user-music_1-inform_intent-lookupsong-1": "user act music_1 inform_intent lookupsong 1", "user-music_1-inform_intent-playsong-1": "user act music_1 inform_intent playsong 1", "user-music_1-request-album-?": "user act music_1 request album ?", "user-music_1-request-genre-?": "user act music_1 request genre ?", "user-music_1-request-year-?": "user act music_1 request year ?", "user-music_1-request_alts-none-none": "user act music_1 request_alts none none", "user-music_1-select-none-none": "user act music_1 select none none", "user-music_2-affirm_intent-none-none": "user act music_2 affirm_intent none none", "user-music_2-inform-album-1": "user act music_2 inform album 1", "user-music_2-inform-artist-1": "user act music_2 inform artist 1", "user-music_2-inform-genre-1": "user act music_2 inform genre 1", "user-music_2-inform-playback_device-1": "user act music_2 inform playback_device 1", "user-music_2-inform-song_name-1": "user act music_2 inform song_name 1", "user-music_2-inform_intent-lookupmusic-1": "user act music_2 inform_intent lookupmusic 1", "user-music_2-inform_intent-playmedia-1": "user act music_2 inform_intent playmedia 1", "user-music_2-request-genre-?": "user act music_2 request genre ?", "user-music_2-request_alts-none-none": "user act music_2 request_alts none none", "user-music_2-select-none-none": "user act music_2 select none none", "user-music_3-affirm_intent-none-none": "user act music_3 affirm_intent none none", "user-music_3-inform-album-1": "user act music_3 inform album 1", "user-music_3-inform-artist-1": "user act music_3 inform artist 1", "user-music_3-inform-device-1": "user act music_3 inform device 1", "user-music_3-inform-genre-1": "user act music_3 inform genre 1", "user-music_3-inform-year-1": "user act music_3 inform year 1", "user-music_3-inform_intent-lookupmusic-1": "user act music_3 inform_intent lookupmusic 1", "user-music_3-inform_intent-playmedia-1": "user act music_3 inform_intent playmedia 1", "user-music_3-negate_intent-none-none": "user act music_3 negate_intent none none", "user-music_3-request-genre-?": "user act music_3 request genre ?", "user-music_3-request-year-?": "user act music_3 request year ?", "user-music_3-request_alts-none-none": "user act music_3 request_alts none none", "user-music_3-select-none-none": "user act music_3 select none none", "user-payment_1-inform-amount-1": "user act payment_1 inform amount 1", "user-payment_1-inform-payment_method-1": "user act payment_1 inform payment_method 1", "user-payment_1-inform-private_visibility-1": "user act payment_1 inform private_visibility 1", "user-payment_1-inform-receiver-1": "user act payment_1 inform receiver 1", "user-payment_1-inform_intent-makepayment-1": "user act payment_1 inform_intent makepayment 1", "user-payment_1-inform_intent-requestpayment-1": "user act payment_1 inform_intent requestpayment 1", "user-rentalcars_1-affirm_intent-none-none": "user act rentalcars_1 affirm_intent none none", "user-rentalcars_1-inform-dropoff_date-1": "user act rentalcars_1 inform dropoff_date 1", "user-rentalcars_1-inform-pickup_city-1": "user act rentalcars_1 inform pickup_city 1", "user-rentalcars_1-inform-pickup_date-1": "user act rentalcars_1 inform pickup_date 1", "user-rentalcars_1-inform-pickup_location-1": "user act rentalcars_1 inform pickup_location 1", "user-rentalcars_1-inform-pickup_time-1": "user act rentalcars_1 inform pickup_time 1", "user-rentalcars_1-inform-type-1": "user act rentalcars_1 inform type 1", "user-rentalcars_1-inform_intent-getcarsavailable-1": "user act rentalcars_1 inform_intent getcarsavailable 1", "user-rentalcars_1-inform_intent-reservecar-1": "user act rentalcars_1 inform_intent reservecar 1", "user-rentalcars_1-negate_intent-none-none": "user act rentalcars_1 negate_intent none none", "user-rentalcars_1-request-car_name-?": "user act rentalcars_1 request car_name ?", "user-rentalcars_1-request-total_price-?": "user act rentalcars_1 request total_price ?", "user-rentalcars_1-request_alts-none-none": "user act rentalcars_1 request_alts none none", "user-rentalcars_1-select-none-none": "user act rentalcars_1 select none none", "user-rentalcars_2-affirm_intent-none-none": "user act rentalcars_2 affirm_intent none none", "user-rentalcars_2-inform-car_type-1": "user act rentalcars_2 inform car_type 1", "user-rentalcars_2-inform-dropoff_date-1": "user act rentalcars_2 inform dropoff_date 1", "user-rentalcars_2-inform-pickup_city-1": "user act rentalcars_2 inform pickup_city 1", "user-rentalcars_2-inform-pickup_date-1": "user act rentalcars_2 inform pickup_date 1", "user-rentalcars_2-inform-pickup_location-1": "user act rentalcars_2 inform pickup_location 1", "user-rentalcars_2-inform-pickup_time-1": "user act rentalcars_2 inform pickup_time 1", "user-rentalcars_2-inform_intent-getcarsavailable-1": "user act rentalcars_2 inform_intent getcarsavailable 1", "user-rentalcars_2-inform_intent-reservecar-1": "user act rentalcars_2 inform_intent reservecar 1", "user-rentalcars_2-negate_intent-none-none": "user act rentalcars_2 negate_intent none none", "user-rentalcars_2-request-car_name-?": "user act rentalcars_2 request car_name ?", "user-rentalcars_2-request-total_price-?": "user act rentalcars_2 request total_price ?", "user-rentalcars_2-request_alts-none-none": "user act rentalcars_2 request_alts none none", "user-rentalcars_2-select-none-none": "user act rentalcars_2 select none none", "user-rentalcars_3-affirm_intent-none-none": "user act rentalcars_3 affirm_intent none none", "user-rentalcars_3-inform-add_insurance-1": "user act rentalcars_3 inform add_insurance 1", "user-rentalcars_3-inform-car_type-1": "user act rentalcars_3 inform car_type 1", "user-rentalcars_3-inform-city-1": "user act rentalcars_3 inform city 1", "user-rentalcars_3-inform-end_date-1": "user act rentalcars_3 inform end_date 1", "user-rentalcars_3-inform-pickup_location-1": "user act rentalcars_3 inform pickup_location 1", "user-rentalcars_3-inform-pickup_time-1": "user act rentalcars_3 inform pickup_time 1", "user-rentalcars_3-inform-start_date-1": "user act rentalcars_3 inform start_date 1", "user-rentalcars_3-inform_intent-getcarsavailable-1": "user act rentalcars_3 inform_intent getcarsavailable 1", "user-rentalcars_3-inform_intent-reservecar-1": "user act rentalcars_3 inform_intent reservecar 1", "user-rentalcars_3-negate_intent-none-none": "user act rentalcars_3 negate_intent none none", "user-rentalcars_3-request-car_name-?": "user act rentalcars_3 request car_name ?", "user-rentalcars_3-request-price_per_day-?": "user act rentalcars_3 request price_per_day ?", "user-rentalcars_3-request_alts-none-none": "user act rentalcars_3 request_alts none none", "user-rentalcars_3-select-none-none": "user act rentalcars_3 select none none", "user-restaurants_1-affirm_intent-none-none": "user act restaurants_1 affirm_intent none none", "user-restaurants_1-inform-city-1": "user act restaurants_1 inform city 1", "user-restaurants_1-inform-cuisine-1": "user act restaurants_1 inform cuisine 1", "user-restaurants_1-inform-date-1": "user act restaurants_1 inform date 1", "user-restaurants_1-inform-has_live_music-1": "user act restaurants_1 inform has_live_music 1", "user-restaurants_1-inform-party_size-1": "user act restaurants_1 inform party_size 1", "user-restaurants_1-inform-price_range-1": "user act restaurants_1 inform price_range 1", "user-restaurants_1-inform-restaurant_name-1": "user act restaurants_1 inform restaurant_name 1", "user-restaurants_1-inform-serves_alcohol-1": "user act restaurants_1 inform serves_alcohol 1", "user-restaurants_1-inform-time-1": "user act restaurants_1 inform time 1", "user-restaurants_1-inform_intent-findrestaurants-1": "user act restaurants_1 inform_intent findrestaurants 1", "user-restaurants_1-inform_intent-reserverestaurant-1": "user act restaurants_1 inform_intent reserverestaurant 1", "user-restaurants_1-negate_intent-none-none": "user act restaurants_1 negate_intent none none", "user-restaurants_1-request-cuisine-?": "user act restaurants_1 request cuisine ?", "user-restaurants_1-request-has_live_music-?": "user act restaurants_1 request has_live_music ?", "user-restaurants_1-request-phone_number-?": "user act restaurants_1 request phone_number ?", "user-restaurants_1-request-price_range-?": "user act restaurants_1 request price_range ?", "user-restaurants_1-request-serves_alcohol-?": "user act restaurants_1 request serves_alcohol ?", "user-restaurants_1-request-street_address-?": "user act restaurants_1 request street_address ?", "user-restaurants_1-request_alts-none-none": "user act restaurants_1 request_alts none none", "user-restaurants_1-select-none-none": "user act restaurants_1 select none none", "user-restaurants_2-affirm_intent-none-none": "user act restaurants_2 affirm_intent none none", "user-restaurants_2-inform-category-1": "user act restaurants_2 inform category 1", "user-restaurants_2-inform-date-1": "user act restaurants_2 inform date 1", "user-restaurants_2-inform-has_vegetarian_options-1": "user act restaurants_2 inform has_vegetarian_options 1", "user-restaurants_2-inform-location-1": "user act restaurants_2 inform location 1", "user-restaurants_2-inform-number_of_seats-1": "user act restaurants_2 inform number_of_seats 1", "user-restaurants_2-inform-price_range-1": "user act restaurants_2 inform price_range 1", "user-restaurants_2-inform-restaurant_name-1": "user act restaurants_2 inform restaurant_name 1", "user-restaurants_2-inform-time-1": "user act restaurants_2 inform time 1", "user-restaurants_2-inform_intent-findrestaurants-1": "user act restaurants_2 inform_intent findrestaurants 1", "user-restaurants_2-inform_intent-reserverestaurant-1": "user act restaurants_2 inform_intent reserverestaurant 1", "user-restaurants_2-request-address-?": "user act restaurants_2 request address ?", "user-restaurants_2-request-has_seating_outdoors-?": "user act restaurants_2 request has_seating_outdoors ?", "user-restaurants_2-request-has_vegetarian_options-?": "user act restaurants_2 request has_vegetarian_options ?", "user-restaurants_2-request-phone_number-?": "user act restaurants_2 request phone_number ?", "user-restaurants_2-request-price_range-?": "user act restaurants_2 request price_range ?", "user-restaurants_2-request-rating-?": "user act restaurants_2 request rating ?", "user-restaurants_2-request_alts-none-none": "user act restaurants_2 request_alts none none", "user-restaurants_2-select-none-none": "user act restaurants_2 select none none", "user-ridesharing_1-inform-destination-1": "user act ridesharing_1 inform destination 1", "user-ridesharing_1-inform-number_of_riders-1": "user act ridesharing_1 inform number_of_riders 1", "user-ridesharing_1-inform-shared_ride-1": "user act ridesharing_1 inform shared_ride 1", "user-ridesharing_1-inform_intent-getride-1": "user act ridesharing_1 inform_intent getride 1", "user-ridesharing_1-request-approximate_ride_duration-?": "user act ridesharing_1 request approximate_ride_duration ?", "user-ridesharing_1-request-ride_fare-?": "user act ridesharing_1 request ride_fare ?", "user-ridesharing_2-inform-destination-1": "user act ridesharing_2 inform destination 1", "user-ridesharing_2-inform-number_of_seats-1": "user act ridesharing_2 inform number_of_seats 1", "user-ridesharing_2-inform-ride_type-1": "user act ridesharing_2 inform ride_type 1", "user-ridesharing_2-inform_intent-getride-1": "user act ridesharing_2 inform_intent getride 1", "user-ridesharing_2-request-ride_fare-?": "user act ridesharing_2 request ride_fare ?", "user-ridesharing_2-request-wait_time-?": "user act ridesharing_2 request wait_time ?", "user-services_1-affirm_intent-none-none": "user act services_1 affirm_intent none none", "user-services_1-inform-appointment_date-1": "user act services_1 inform appointment_date 1", "user-services_1-inform-appointment_time-1": "user act services_1 inform appointment_time 1", "user-services_1-inform-city-1": "user act services_1 inform city 1", "user-services_1-inform-is_unisex-1": "user act services_1 inform is_unisex 1", "user-services_1-inform-stylist_name-1": "user act services_1 inform stylist_name 1", "user-services_1-inform_intent-bookappointment-1": "user act services_1 inform_intent bookappointment 1", "user-services_1-inform_intent-findprovider-1": "user act services_1 inform_intent findprovider 1", "user-services_1-negate_intent-none-none": "user act services_1 negate_intent none none", "user-services_1-request-average_rating-?": "user act services_1 request average_rating ?", "user-services_1-request-is_unisex-?": "user act services_1 request is_unisex ?", "user-services_1-request-phone_number-?": "user act services_1 request phone_number ?", "user-services_1-request-street_address-?": "user act services_1 request street_address ?", "user-services_1-request_alts-none-none": "user act services_1 request_alts none none", "user-services_1-select-none-none": "user act services_1 select none none", "user-services_2-affirm_intent-none-none": "user act services_2 affirm_intent none none", "user-services_2-inform-appointment_date-1": "user act services_2 inform appointment_date 1", "user-services_2-inform-appointment_time-1": "user act services_2 inform appointment_time 1", "user-services_2-inform-city-1": "user act services_2 inform city 1", "user-services_2-inform-dentist_name-1": "user act services_2 inform dentist_name 1", "user-services_2-inform_intent-bookappointment-1": "user act services_2 inform_intent bookappointment 1", "user-services_2-inform_intent-findprovider-1": "user act services_2 inform_intent findprovider 1", "user-services_2-negate_intent-none-none": "user act services_2 negate_intent none none", "user-services_2-request-address-?": "user act services_2 request address ?", "user-services_2-request-offers_cosmetic_services-?": "user act services_2 request offers_cosmetic_services ?", "user-services_2-request-phone_number-?": "user act services_2 request phone_number ?", "user-services_2-request_alts-none-none": "user act services_2 request_alts none none", "user-services_2-select-none-none": "user act services_2 select none none", "user-services_3-affirm_intent-none-none": "user act services_3 affirm_intent none none", "user-services_3-inform-appointment_date-1": "user act services_3 inform appointment_date 1", "user-services_3-inform-appointment_time-1": "user act services_3 inform appointment_time 1", "user-services_3-inform-city-1": "user act services_3 inform city 1", "user-services_3-inform-doctor_name-1": "user act services_3 inform doctor_name 1", "user-services_3-inform-type-1": "user act services_3 inform type 1", "user-services_3-inform_intent-bookappointment-1": "user act services_3 inform_intent bookappointment 1", "user-services_3-inform_intent-findprovider-1": "user act services_3 inform_intent findprovider 1", "user-services_3-negate_intent-none-none": "user act services_3 negate_intent none none", "user-services_3-request-average_rating-?": "user act services_3 request average_rating ?", "user-services_3-request-phone_number-?": "user act services_3 request phone_number ?", "user-services_3-request-street_address-?": "user act services_3 request street_address ?", "user-services_3-request_alts-none-none": "user act services_3 request_alts none none", "user-services_3-select-none-none": "user act services_3 select none none", "user-services_4-affirm_intent-none-none": "user act services_4 affirm_intent none none", "user-services_4-inform-appointment_date-1": "user act services_4 inform appointment_date 1", "user-services_4-inform-appointment_time-1": "user act services_4 inform appointment_time 1", "user-services_4-inform-city-1": "user act services_4 inform city 1", "user-services_4-inform-type-1": "user act services_4 inform type 1", "user-services_4-inform_intent-bookappointment-1": "user act services_4 inform_intent bookappointment 1", "user-services_4-inform_intent-findprovider-1": "user act services_4 inform_intent findprovider 1", "user-services_4-negate_intent-none-none": "user act services_4 negate_intent none none", "user-services_4-request-address-?": "user act services_4 request address ?", "user-services_4-request-phone_number-?": "user act services_4 request phone_number ?", "user-services_4-request_alts-none-none": "user act services_4 request_alts none none", "user-services_4-select-none-none": "user act services_4 select none none", "user-trains_1-affirm_intent-none-none": "user act trains_1 affirm_intent none none", "user-trains_1-inform-class-1": "user act trains_1 inform class 1", "user-trains_1-inform-date_of_journey-1": "user act trains_1 inform date_of_journey 1", "user-trains_1-inform-from-1": "user act trains_1 inform from 1", "user-trains_1-inform-number_of_adults-1": "user act trains_1 inform number_of_adults 1", "user-trains_1-inform-to-1": "user act trains_1 inform to 1", "user-trains_1-inform-trip_protection-1": "user act trains_1 inform trip_protection 1", "user-trains_1-inform_intent-findtrains-1": "user act trains_1 inform_intent findtrains 1", "user-trains_1-inform_intent-gettraintickets-1": "user act trains_1 inform_intent gettraintickets 1", "user-trains_1-negate_intent-none-none": "user act trains_1 negate_intent none none", "user-trains_1-request-from_station-?": "user act trains_1 request from_station ?", "user-trains_1-request-to_station-?": "user act trains_1 request to_station ?", "user-trains_1-request_alts-none-none": "user act trains_1 request_alts none none", "user-trains_1-select-none-none": "user act trains_1 select none none", "user-travel_1-inform-category-1": "user act travel_1 inform category 1", "user-travel_1-inform-free_entry-1": "user act travel_1 inform free_entry 1", "user-travel_1-inform-good_for_kids-1": "user act travel_1 inform good_for_kids 1", "user-travel_1-inform-location-1": "user act travel_1 inform location 1", "user-travel_1-inform_intent-findattractions-1": "user act travel_1 inform_intent findattractions 1", "user-travel_1-request-free_entry-?": "user act travel_1 request free_entry ?", "user-travel_1-request-good_for_kids-?": "user act travel_1 request good_for_kids ?", "user-travel_1-request-phone_number-?": "user act travel_1 request phone_number ?", "user-travel_1-request_alts-none-none": "user act travel_1 request_alts none none", "user-travel_1-select-none-none": "user act travel_1 select none none", "user-weather_1-inform-city-1": "user act weather_1 inform city 1", "user-weather_1-inform-date-1": "user act weather_1 inform date 1", "user-weather_1-inform_intent-getweather-1": "user act weather_1 inform_intent getweather 1", "user-weather_1-request-humidity-?": "user act weather_1 request humidity ?", "user-weather_1-request-wind-?": "user act weather_1 request wind ?", "user-weather_1-request_alts-none-none": "user act weather_1 request_alts none none", "user-weather_1-select-none-none": "user act weather_1 select none none"} \ No newline at end of file diff --git a/convlab/policy/vtrace_DPT/ocl/budgets/budget_balanced.json b/convlab/policy/vtrace_DPT/ocl/budgets/budget_balanced.json deleted file mode 100644 index fae82f7a629bd2e7598a7a36e27f689bad7bdd2c..0000000000000000000000000000000000000000 --- a/convlab/policy/vtrace_DPT/ocl/budgets/budget_balanced.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "timeline": {"restaurant": 0, "hotel": 500, "train": 1500, "attraction": 3000, "taxi": 5500, - "police": 8500, "hospital": 12000}, - "budget": [["restaurant", 1000], ["hotel", 1000], ["train", 1000], ["taxi", 500], ["attraction", 1000], - ["police", 500], ["hospital", 500], - ["restaurant-hotel", 1000], ["restaurant-train", 1000], ["restaurant-attraction", 1000], - ["restaurant-taxi", 1000], ["hotel-train", 1000], ["hotel-attraction", 1000], - ["hotel-taxi", 1000], ["train-attraction", 1000], ["train-taxi", 1000], ["attraction-taxi", 1000]] -} \ No newline at end of file diff --git a/convlab/policy/vtrace_DPT/ocl/budgets/budget_multiwoz.json b/convlab/policy/vtrace_DPT/ocl/budgets/budget_multiwoz.json deleted file mode 100644 index 91af1353bd87d20e3baffe53bc69a8f9188758bf..0000000000000000000000000000000000000000 --- a/convlab/policy/vtrace_DPT/ocl/budgets/budget_multiwoz.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "timeline": {"restaurant": 0, "hotel": 1000, "train": 2000, "attraction": 3000, "taxi": 4000, - "police": 5000, "hospital": 6000}, - "budget": [["hotel-train", 1370], ["restaurant", 1311], ["attraction-train", 1194], ["restaurant-train", 1187], - ["hotel", 636], ["hotel-restaurant", 570], ["attraction-restaurant", 544], ["attraction-restaurant-taxi", 543], - ["attraction-hotel-taxi", 542], ["attraction-hotel", 542], ["hotel-restaurant-taxi", 537], ["taxi", 435], - ["train", 345], ["hospital", 287], ["police", 245], ["attraction", 150]] -} \ No newline at end of file diff --git a/convlab/policy/vtrace_DPT/ocl/budgets/budget_old.json b/convlab/policy/vtrace_DPT/ocl/budgets/budget_old.json deleted file mode 100644 index b86af8ebc1885b0b19101f78c87bc153e76bc26a..0000000000000000000000000000000000000000 --- a/convlab/policy/vtrace_DPT/ocl/budgets/budget_old.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "timeline": {"restaurant": 0, "hotel": 2000, "train": 4000, "attraction": 6000, "taxi": 7000, - "police": 8000, "hospital": 8500}, - "budget": [["restaurant", 2000], ["hotel", 2000], ["train", 2000], ["taxi", 1000], ["attraction", 1000], - ["police", 500], ["hospital", 500]] -} - diff --git a/convlab/policy/vtrace_DPT/ocl/ocl_helper.py b/convlab/policy/vtrace_DPT/ocl/ocl_helper.py deleted file mode 100644 index 49ed0fa02fb5c54ae4ddb5645668e577615e8513..0000000000000000000000000000000000000000 --- a/convlab/policy/vtrace_DPT/ocl/ocl_helper.py +++ /dev/null @@ -1,104 +0,0 @@ -import logging -import json -import os -import numpy as np -import matplotlib.pyplot as plt - -from queue import Queue - -from convlab2.util.custom_util import build_domains_goal - - -def log_used_budget(start_budget, budget): - used_budget = [] - - for start, remaining in zip(start_budget, budget): - used = start[1] - remaining[1] - if used > 0: - used_budget.append([start[0], used]) - - logging.info(f"Used budget: {used_budget}") - - -def load_budget(load_path): - - with open(load_path, "r") as f: - budget_info = json.load(f) - timeline = budget_info['timeline'] - budget = budget_info['budget'] - timeline['end'] = sum([pair[1] for pair in budget]) - name = load_path.split("/")[-1].split(".")[0] - save_budget_plot(budget, os.path.join(os.path.dirname(load_path), f"{name}.pdf")) - return timeline, budget - - -def get_goals(goal_generator, allowed_domains, budget, num_goals): - - collected_goals = [] - while len(collected_goals) != num_goals: - domains, budget = sample_domains(allowed_domains, budget) - domains = set(domains.split("-")) - goal = build_domains_goal(goal_generator, domains) - collected_goals.append(goal) - - return collected_goals, budget - - -def check_setup(timeline, budget): - # We need to check whether there is enough budget for the timeline used. - # For instance, if we only have 800 restaurants but only introduce the next domain after 1000 dialogues according - # to the timeline, this will not work - timeline_ = list(timeline.items()) - timeline_.sort(key=lambda x: x[1]) - allowed_domains = [] - - for i, domain_start in enumerate(timeline_[:-1]): - domain = domain_start[0] - allowed_domains.append(domain) - necessary_budget = timeline_[i+1][1] - real_budget = 0 - for pair in budget: - domain_combi = set(pair[0].split("-")) - if domain_combi.issubset(set(allowed_domains)): - real_budget += pair[1] - assert necessary_budget <= real_budget, "The necessary budget is higher than the real budget!" - - -def save_budget_plot(budget, save_path): - x = [pair[0] for pair in budget] - y = [pair[1] for pair in budget] - plt.style.use('ggplot') - x_pos = [i for i, _ in enumerate(x)] - plt.bar(x_pos, y, color='green') - plt.xlabel("Domain combinations") - plt.ylabel("Budget per combination") - plt.title("Budget for every domain combination") - plt.xticks(x_pos, x, rotation='vertical') - plt.tight_layout() - plt.savefig(save_path) - - -def update_online_metrics(online_metrics, metrics, log_save_path, tb_writer): - for metric in metrics: - for key in online_metrics: - online_metrics[key].append(metric[key]) - if key != "goal": - tb_writer.add_scalar(f"lifetime_{key}", np.mean(online_metrics[key]), len(online_metrics[key])) - json.dump(online_metrics, open(os.path.join(log_save_path, 'online_metrics.json'), 'w')) - - -def sample_domains(allowed_domains, budget): - - distribution = [] - for pair in budget: - domain_combi = set(pair[0].split("-")) - if domain_combi.issubset(set(allowed_domains)): - distribution.append(pair[1]) - else: - distribution.append(0) - - distribution = np.array(distribution, dtype=np.float32) - distribution /= sum(distribution) - domain_idx = np.random.choice(np.arange(0, len(budget)), p=distribution) - budget[domain_idx][1] -= 1 - return budget[domain_idx][0], budget \ No newline at end of file diff --git a/convlab/policy/vtrace_DPT/ocl/plot_ocl.py b/convlab/policy/vtrace_DPT/ocl/plot_ocl.py deleted file mode 100644 index b98c6693b304461658a858acaa848149dc7233f8..0000000000000000000000000000000000000000 --- a/convlab/policy/vtrace_DPT/ocl/plot_ocl.py +++ /dev/null @@ -1,197 +0,0 @@ -import numpy as np -import json -import matplotlib.pyplot as plt -import os -import seaborn as sns -import argparse -import sys - - -def arg_parser(): - parser = argparse.ArgumentParser() - parser.add_argument('algs', metavar='N', type=str, - nargs='+', help='all sub_folders') - parser.add_argument('--dir_path', default='') - parser.add_argument('--timeline', default='') - - args = parser.parse_args() - return args - - -def load_json(load_path): - - with open(load_path, "r") as f: - output = json.load(f) - return output - - -def plot_bars(pair_list, save_path, x_label, y_label, title): - - x = [pair[0] for pair in pair_list] - y = [pair[1] for pair in pair_list] - plt.clf() - plt.style.use('ggplot') - x_pos = [i for i, _ in enumerate(x)] - plt.bar(x_pos, y, color='green') - plt.xlabel(x_label) - plt.ylabel(y_label) - plt.title(title) - plt.xticks(x_pos, x, rotation='vertical') - plt.tight_layout() - plt.savefig(save_path) - - -def plot_goal_success_bars(dir_path, alg_names): - - alg_paths = [os.path.join(dir_path, algorithm_name) for algorithm_name in alg_names] - - for alg_path in alg_paths: - metrics_list = read_dir(alg_path) - goal_list = [] - success_list = [] - for online_metric in metrics_list: - if "goal" in online_metric: - goals = online_metric['goal'] - success = online_metric['success'] - # for every domain-combination, collect all success and fails - for goal, succ in zip(goals, success): - if set(goal) in goal_list: - goal_index = goal_list.index(set(goal)) - success_list[goal_index].append(succ) - else: - goal_list.append(set(goal)) - success_list.append([succ]) - - x = ["-".join(list(goal)) for goal in goal_list] - y = [np.mean(el) for el in success_list] - - save_path = os.path.join(alg_path, "success_bar_plot.pdf") - plot_bars(list(zip(x, y)), save_path, "Domain combinations", "Average success", "Success per domain combination") - - -def read_dir(algorithm_dir_path): - - seed_dir_paths = [f.path for f in os.scandir(algorithm_dir_path) if f.is_dir()] - seed_dir_names = [f.name for f in os.scandir(algorithm_dir_path) if f.is_dir()] - - metrics_list = [] - - for seed_dir_name, seed_dir_path in zip(seed_dir_names, seed_dir_paths): - if seed_dir_name != "plots": - metrics = load_json(os.path.join(seed_dir_path, 'logs', 'online_metrics.json')) - metrics_list.append(metrics) - - return metrics_list - - -def aggregate_across_seeds(algorithm_dir_path): - - metrics_per_seed = read_dir(algorithm_dir_path) - metrics_aggregated = metrics_per_seed[0] - if "goal" in metrics_aggregated: - del metrics_aggregated["goal"] - for key in metrics_aggregated: - for seed_metric in metrics_per_seed[1:]: - metrics_aggregated[key] = np.concatenate([metrics_aggregated[key], seed_metric[key]]) - for key in metrics_aggregated: - if len(metrics_per_seed) > 1: - metrics_aggregated[key] = metrics_aggregated[key].reshape(len(metrics_per_seed), -1) - else: - # if we only have one seed, just cast to numpy array - metrics_aggregated[key] = np.array(metrics_aggregated[key]).reshape(1, -1) - - return metrics_aggregated - - -def get_metrics(algorithm_dir_path): - metrics_aggregated = aggregate_across_seeds(algorithm_dir_path) - - performance_dict = {} - for key in metrics_aggregated: - - lifetime = [lifetime_progress(output) for output in metrics_aggregated[key]] - lifetime_mean, lifetime_std = np.mean(lifetime, axis=0), np.std(lifetime, axis=0) - performance_dict[f"lifetime_{key}"] = {"mean": lifetime_mean, "std": lifetime_std} - - average_per_step = [metric_per_step(output) for output in metrics_aggregated[key]] - step_mean, step_std = np.mean(average_per_step, axis=0), np.std(average_per_step, axis=0) - performance_dict[f"local_{key}"] = {"mean": step_mean, "std": step_std} - - return performance_dict - - -def plot_algorithms(dir_path, alg_names, timeline_path=""): - clrs = sns.color_palette("husl", 5) - window_size = 500 - - plot_path = os.path.join(dir_path, 'plots') - os.makedirs(plot_path, exist_ok=True) - - alg_paths = [os.path.join(dir_path, algorithm_name) for algorithm_name in alg_names] - performances = [get_metrics(alg_path) for alg_path in alg_paths] - - for key in performances[0].keys(): - max_value = -sys.maxsize - plt.clf() - for i, performance in enumerate(performances): - with sns.axes_style("darkgrid"): - - mean = performance[key]['mean'] - std = performance[key]['std'] - steps = np.arange(0, len(mean)) if 'lifetime' in key \ - else np.arange(int(window_size / 2), len(mean) + int(window_size / 2)) - plt.plot(steps, mean, c=clrs[i], label=f"{alg_names[i]}") - plt.fill_between(steps, mean - std, mean + std, alpha=0.3, facecolor=clrs[i]) - plt.xticks(np.arange(0, (1 + int(len(mean)) / 1000) * 1000, 1000)) - plt.xticks(fontsize=7, rotation=0) - - max_value = max_value if np.max(mean) < max_value else np.max(mean) - - if 'local' in key and timeline_path: - add_timeline(max_value, timeline_path) - - plt.xlabel('Training dialogues') - plt.ylabel(key) - plt.title(f"{key}") - plt.legend(fancybox=True, shadow=False, ncol=1, loc='lower left') - plt.savefig(plot_path + f'/{key}.pdf', bbox_inches='tight') - - -def add_timeline(max_value, timeline_path): - timeline = load_json(timeline_path)['timeline'] - for domain, time in timeline.items(): - plt.axvline(x=time, color='k', linestyle='dashed', linewidth=0.5) - plt.text(time, max_value, domain, rotation=90, verticalalignment='center') - - -def metric_per_step(metric, window_size=500): - - kernel = np.ones(window_size) / window_size - average_per_step = np.convolve(metric, kernel, mode='valid') - adaptation_rate = average_per_step[1:] - average_per_step[:-1] - average_adaptation_rate = np.convolve(adaptation_rate, kernel, mode='valid') - - return average_per_step - - -def lifetime_progress(metric): - - progress = [] - for i in range(1, len(metric) + 1): - number = metric[i - 1] - if i == 1: - mean = number - else: - mean = mean + 1/i * (number - mean) - if i > 50: - progress.append(mean) - return progress - - -if __name__ == '__main__': - - args = arg_parser() - print("Algorithms compared:", args.algs) - - plot_algorithms(args.dir_path, args.algs, args.timeline) - plot_goal_success_bars(args.dir_path, args.algs) diff --git a/convlab/policy/vtrace_DPT/ocl/semantic_level_config_ocl.json b/convlab/policy/vtrace_DPT/ocl/semantic_level_config_ocl.json deleted file mode 100644 index 4844dbd5a0bef9a388c696b65181aad218a84e59..0000000000000000000000000000000000000000 --- a/convlab/policy/vtrace_DPT/ocl/semantic_level_config_ocl.json +++ /dev/null @@ -1,46 +0,0 @@ -{"goals": {"single_domains": false, "allowed_domains": null}, - "model": { - "budget_path": "convlab/policy/vtrace_DPT/ocl/budgets/budget_balanced.json", - "load_path": "", - "use_pretrained_initialisation": false, - "pretrained_load_path": "", - "seed": 0, - "process_num": 2, - "eval_frequency": 1000, - "num_eval_dialogues": 500, - "process_num_train": 2, - "total_dialogues": 10000, - "update_rounds": 1, - "new_dialogues": 2, - "sys_semantic_to_usr": false - }, - "vectorizer_sys": { - "uncertainty_vector_mul": { - "class_path": "convlab.policy.vector.vector_nodes.VectorNodes", - "ini_params": { - "use_masking": true, - "manually_add_entity_names": true, - "seed": 0 - } - } - }, - "nlu_sys": {}, - "dst_sys": { - "RuleDST": { - "class_path": "convlab.dst.rule.multiwoz.dst.RuleDST", - "ini_params": {} - } - }, - "sys_nlg": {}, - "nlu_usr": {}, - "dst_usr": {}, - "policy_usr": { - "RulePolicy": { - "class_path": "convlab.policy.rule.multiwoz.RulePolicy", - "ini_params": { - "character": "usr" - } - } - }, - "usr_nlg": {} -} \ No newline at end of file diff --git a/convlab/policy/vtrace_DPT/semantic_level_config.json b/convlab/policy/vtrace_DPT/semantic_level_config.json index 8b1f586383b4ba4a6cd5485c62c2fbc744553c16..7f6d266751a32642c9842af3bca0d80a33a97beb 100644 --- a/convlab/policy/vtrace_DPT/semantic_level_config.json +++ b/convlab/policy/vtrace_DPT/semantic_level_config.json @@ -20,7 +20,9 @@ "ini_params": { "use_masking": true, "manually_add_entity_names": true, - "seed": 0 + "seed": 0, + "dataset_name": "multiwoz21", + "filter_state": true } } }, diff --git a/convlab/policy/vtrace_DPT/supervised/loader.py b/convlab/policy/vtrace_DPT/supervised/loader.py new file mode 100644 index 0000000000000000000000000000000000000000..8dbbcaafb95672711d17715294c6dcd9ded9326b --- /dev/null +++ b/convlab/policy/vtrace_DPT/supervised/loader.py @@ -0,0 +1,169 @@ +import os +import pickle +import torch +import time +import torch.utils.data as data + +from convlab.policy.vector.vector_binary import VectorBinary +from convlab.util import load_policy_data, load_dataset +from convlab.util.custom_util import flatten_acts +from convlab.util.multiwoz.state import default_state +from convlab.policy.vector.dataset import ActDatasetKG +from tqdm import tqdm + +mwoz_domains = ['restaurant', 'hotel', 'train', 'taxi', 'attraction'] + + +class PolicyDataVectorizer: + + def __init__(self, dataset_name='multiwoz21', vector=None, percentage=1.0, dialogue_order=0): + self.dataset_name = dataset_name + self.percentage = percentage + self.dialogue_order = dialogue_order + if vector is None: + self.vector = VectorBinary(dataset_name) + else: + self.vector = vector + self.process_data() + + def process_data(self): + + processed_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), + f'processed_data/{self.dataset_name}_{type(self.vector).__name__}') + if self.percentage != 1.0: + processed_dir += f"_{self.percentage}_{self.dialogue_order}" + if os.path.exists(processed_dir): + print('Load processed data file') + self._load_data(processed_dir) + else: + print('Start preprocessing the dataset, this can take a while..') + self._build_data(processed_dir) + + def _build_data(self, processed_dir): + self.data = {} + + os.makedirs(processed_dir, exist_ok=True) + dataset = load_dataset(self.dataset_name, dial_ids_order=self.dialogue_order, + split2ratio={'train': self.percentage, 'validation': self.percentage, + 'test': self.percentage}) + data_split = load_policy_data(dataset, context_window_size=2) + + for split in data_split: + self.data[split] = [] + raw_data = data_split[split] + + for data_point in tqdm(raw_data): + state = default_state() + + state['belief_state'] = data_point['context'][-1]['state'] + state['user_action'] = flatten_acts(data_point['context'][-1]['dialogue_acts']) + last_system_act = data_point['context'][-2]['dialogue_acts'] \ + if len(data_point['context']) > 1 else {} + state['system_action'] = flatten_acts(last_system_act) + state['terminated'] = data_point['terminated'] + if 'booked' in data_point: + state['booked'] = data_point['booked'] + dialogue_act = flatten_acts(data_point['dialogue_acts']) + + vectorized_state, mask = self.vector.state_vectorize(state) + vectorized_action = self.vector.action_vectorize(dialogue_act) + self.data[split].append({"state": self.vector.kg_info, "action": vectorized_action, "mask": mask, + "terminated": state['terminated']}) + + with open(os.path.join(processed_dir, '{}.pkl'.format(split)), 'wb') as f: + pickle.dump(self.data[split], f) + + print("Data processing done.") + + def _load_data(self, processed_dir): + self.data = {} + for part in ['train', 'validation', 'test']: + with open(os.path.join(processed_dir, '{}.pkl'.format(part)), 'rb') as f: + self.data[part] = pickle.load(f) + + def is_multiwoz_like(self, item): + + state = item['state'] + is_like = False + for node in state: + domain = node['domain'].lower() + for mw_domain in mwoz_domains: + # we check if the mw_domain as a string is contained in the domain of the node + if mw_domain in domain: + is_like = True + break + if is_like: + break + return is_like + + def create_dataset(self, part, batchsz, policy, multiwoz_like=False): + print('Start creating {} dataset'.format(part)) + time_now = time.time() + + root_dir = os.path.dirname(os.path.abspath(__file__)) + data_dir = os.path.join(root_dir, "data", self.dataset_name) + os.makedirs(data_dir, exist_ok=True) + file_path = os.path.join(data_dir, part) + if multiwoz_like: + file_path += "mw" + + if self.percentage != 1.0: + file_path += f"_{self.percentage}_{self.dialogue_order}" + + if os.path.exists(file_path): + action_batch, a_masks, max_length, small_act_batch, \ + current_domain_mask_batch, non_current_domain_mask_batch, \ + description_batch, value_batch, kg_list = torch.load(file_path) + print(f"Loaded data from {file_path}") + else: + print("Creating data from scratch.") + + action_batch, small_act_batch, \ + current_domain_mask_batch, non_current_domain_mask_batch, \ + description_batch, value_batch = [], [], [], [], [], [] + kg_list = [] + + for num, item in tqdm(enumerate(self.data[part])): + + if item['action'].sum() == 0 or len(item['state']) == 0: + continue + if multiwoz_like: + if not self.is_multiwoz_like(item): + continue + action_batch.append(torch.Tensor(item['action'])) + + kg = [item['state']] + kg_list.append(item['state']) + + description_idx_list, value_list = policy.get_descriptions_and_values(kg) + description_batch.append(description_idx_list) + value_batch.append(value_list) + + current_domains = policy.get_current_domains(kg) + current_domain_mask = policy.action_embedder.get_current_domain_mask(current_domains[0], current=True) + non_current_domain_mask = policy.action_embedder.get_current_domain_mask(current_domains[0], current=False) + current_domain_mask_batch.append(current_domain_mask) + non_current_domain_mask_batch.append(non_current_domain_mask) + + small_act_batch.append(torch.Tensor(policy.action_embedder.real_action_to_small_action_list(torch.Tensor(item['action'])))) + + print("Creating action masks..") + a_masks, max_length = policy.get_action_masks(action_batch) + action_batch = torch.stack(action_batch) + current_domain_mask_batch = torch.stack(current_domain_mask_batch) + non_current_domain_mask_batch = torch.stack(non_current_domain_mask_batch) + + print(f"Finished data set, time spent: {time.time() - time_now}") + + torch.save([action_batch, a_masks, max_length, small_act_batch, + current_domain_mask_batch, non_current_domain_mask_batch, + description_batch, value_batch, kg_list], file_path) + + dataset = ActDatasetKG(action_batch, a_masks, current_domain_mask_batch, non_current_domain_mask_batch) + dataloader = data.DataLoader(dataset, batchsz, True) + print("NUMBER OF EXAMPLES:", len(current_domain_mask_batch)) + return dataloader, max_length, small_act_batch, description_batch, value_batch, kg_list + + +if __name__ == '__main__': + data_loader = PolicyDataVectorizer() diff --git a/convlab/policy/vtrace_DPT/supervised/train_supervised.py b/convlab/policy/vtrace_DPT/supervised/train_supervised.py new file mode 100644 index 0000000000000000000000000000000000000000..1807a671da7e2938173a18277cd21980ee577a11 --- /dev/null +++ b/convlab/policy/vtrace_DPT/supervised/train_supervised.py @@ -0,0 +1,245 @@ +import argparse +import os +import torch +import logging +import json +import sys + +from torch import optim +from copy import deepcopy +from convlab.policy.vtrace_DPT.supervised.loader import PolicyDataVectorizer +from convlab.util.custom_util import set_seed, init_logging, save_config +from convlab.util.train_util import to_device +from convlab.policy.vtrace_DPT.transformer_model.EncoderDecoder import EncoderDecoder +from convlab.policy.vector.vector_nodes import VectorNodes + +root_dir = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +sys.path.append(root_dir) + +DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +class MLE_Trainer: + def __init__(self, manager, cfg, policy): + self.start_policy = deepcopy(policy) + self.policy = policy + self.policy_optim = optim.Adam(list(self.policy.parameters()), lr=cfg['supervised_lr']) + self.entropy_weight = cfg['entropy_weight'] + self.regularization_weight = cfg['regularization_weight'] + self._init_data(manager, cfg) + + def _init_data(self, manager, cfg): + multiwoz_like = cfg['multiwoz_like'] + self.data_train, self.max_length_train, self.small_act_train, self.descriptions_train, self.values_train, \ + self.kg_train = manager.create_dataset('train', cfg['batchsz'], self.policy, multiwoz_like) + self.data_valid, self.max_length_valid, self.small_act_valid, self.descriptions_valid, self.values_valid, \ + self.kg_valid = manager.create_dataset('validation', cfg['batchsz'], self.policy, multiwoz_like) + self.data_test, self.max_length_test, self.small_act_test, self.descriptions_test, self.values_test, \ + self.kg_test = manager.create_dataset('test', cfg['batchsz'], self.policy, multiwoz_like) + self.save_dir = cfg['save_dir'] + + def policy_loop(self, data): + + actions, action_masks, current_domain_mask, non_current_domain_mask, indices = to_device(data) + + small_act_batch = [self.small_act_train[i].to(DEVICE) for i in indices] + description_batch = [self.descriptions_train[i].to(DEVICE) for i in indices] + value_batch = [self.values_train[i].to(DEVICE) for i in indices] + + log_prob, entropy = self.policy.get_log_prob(actions, action_masks, self.max_length_train, small_act_batch, + current_domain_mask, non_current_domain_mask, + description_batch, value_batch) + loss_a = -1 * log_prob.mean() + + weight_loss = self.weight_loss() + + return loss_a, -entropy, weight_loss + + def weight_loss(self): + + loss = 0 + num_params = sum(p.numel() for p in self.policy.parameters() if p.requires_grad) + for paramA, paramB in zip(self.policy.parameters(), self.start_policy.parameters()): + loss += torch.sum(torch.abs(paramA - paramB.detach())) + return loss / num_params + + def imitating(self): + """ + pretrain the policy by simple imitation learning (behavioral cloning) + """ + self.policy.train() + a_loss = 0. + for i, data in enumerate(self.data_train): + self.policy_optim.zero_grad() + loss_a, entropy_loss, weight_loss = self.policy_loop(data) + a_loss += loss_a.item() + loss_a = loss_a + self.entropy_weight * entropy_loss + self.regularization_weight * weight_loss + + if i % 20 == 0 and i != 0: + print("LOSS:", a_loss / 20.0) + a_loss = 0 + loss_a.backward() + for p in self.policy.parameters(): + if p.grad is not None: + p.grad[p.grad != p.grad] = 0.0 + self.policy_optim.step() + + self.policy.eval() + + def validate(self): + def f1(a, target): + TP, FP, FN = 0, 0, 0 + real = target.nonzero().tolist() + predict = a.nonzero().tolist() + for item in real: + if item in predict: + TP += 1 + else: + FN += 1 + for item in predict: + if item not in real: + FP += 1 + return TP, FP, FN + + average_actions, average_target_actions, counter = 0, 0, 0 + a_TP, a_FP, a_FN = 0, 0, 0 + for i, data in enumerate(self.data_valid): + counter += 1 + target_a, action_masks, current_domain_mask, non_current_domain_mask, indices = to_device(data) + + kg_batch = [self.kg_valid[i] for i in indices] + a = torch.stack([self.policy.select_action([kg]) for kg in kg_batch]) + + TP, FP, FN = f1(a, target_a) + a_TP += TP + a_FP += FP + a_FN += FN + + average_actions += a.float().sum(dim=-1).mean() + average_target_actions += target_a.float().sum(dim=-1).mean() + + logging.info(f"Average actions: {average_actions / counter}") + logging.info(f"Average target actions: {average_target_actions / counter}") + prec = a_TP / (a_TP + a_FP) + rec = a_TP / (a_TP + a_FN) + F1 = 2 * prec * rec / (prec + rec) + return prec, rec, F1 + + def test(self): + def f1(a, target): + TP, FP, FN = 0, 0, 0 + real = target.nonzero().tolist() + predict = a.nonzero().tolist() + for item in real: + if item in predict: + TP += 1 + else: + FN += 1 + for item in predict: + if item not in real: + FP += 1 + return TP, FP, FN + + a_TP, a_FP, a_FN = 0, 0, 0 + for i, data in enumerate(self.data_test): + s, target_a = to_device(data) + a_weights = self.policy(s) + a = a_weights.ge(0) + TP, FP, FN = f1(a, target_a) + a_TP += TP + a_FP += FP + a_FN += FN + + prec = a_TP / (a_TP + a_FP) + rec = a_TP / (a_TP + a_FN) + F1 = 2 * prec * rec / (prec + rec) + print(a_TP, a_FP, a_FN, F1) + + def save(self, directory, epoch): + if not os.path.exists(directory): + os.makedirs(directory) + + torch.save(self.policy.state_dict(), directory + '/supervised.pol.mdl') + + logging.info('<<dialog policy>> epoch {}: saved network to mdl'.format(epoch)) + + +def arg_parser(): + parser = argparse.ArgumentParser() + + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--eval_freq", type=int, default=1) + parser.add_argument("--dataset_name", type=str, default="multiwoz21") + parser.add_argument("--model_path", type=str, default="") + + args = parser.parse_args() + return args + + +if __name__ == '__main__': + + args = arg_parser() + + root_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + with open(os.path.join(root_directory, 'config.json'), 'r') as f: + cfg = json.load(f) + + cfg['dataset_name'] = args.dataset_name + + logger, tb_writer, current_time, save_path, config_save_path, dir_path, log_save_path = \ + init_logging(os.path.dirname(os.path.abspath(__file__)), "info") + save_config(vars(args), cfg, config_save_path) + + set_seed(args.seed) + logging.info(f"Seed used: {args.seed}") + logging.info(f"Batch size: {cfg['batchsz']}") + logging.info(f"Epochs: {cfg['epoch']}") + logging.info(f"Learning rate: {cfg['supervised_lr']}") + logging.info(f"Entropy weight: {cfg['entropy_weight']}") + logging.info(f"Regularization weight: {cfg['regularization_weight']}") + logging.info(f"Only use multiwoz like domains: {cfg['multiwoz_like']}") + logging.info(f"We use: {cfg['data_percentage']*100}% of the data") + logging.info(f"Dialogue order used: {cfg['dialogue_order']}") + + vector = VectorNodes(dataset_name=args.dataset_name, use_masking=False, filter_state=True) + manager = PolicyDataVectorizer(dataset_name=args.dataset_name, vector=vector, + percentage=cfg['data_percentage'], dialogue_order=cfg["dialogue_order"]) + policy = EncoderDecoder(**cfg, action_dict=vector.act2vec).to(device=DEVICE) + try: + policy.load_state_dict(torch.load(args.model_path, map_location=DEVICE)) + logging.info(f"Loaded model from {args.model_path}") + except: + logging.info("Didnt load a model") + agent = MLE_Trainer(manager, cfg, policy) + + logging.info('Start training') + + best_recall = 0.0 + best_precision = 0.0 + best_f1 = 0.0 + precision = 0 + recall = 0 + f1 = 0 + + for e in range(cfg['epoch']): + agent.imitating() + logging.info(f"Epoch: {e}") + + if e % args.eval_freq == 0: + precision, recall, f1 = agent.validate() + + logging.info(f"Precision: {precision}") + logging.info(f"Recall: {recall}") + logging.info(f"F1: {f1}") + + if precision > best_precision: + best_precision = precision + if recall > best_recall: + best_recall = recall + if f1 > best_f1: + best_f1 = f1 + agent.save(save_path, e) + logging.info(f"Best Precision: {best_precision}") + logging.info(f"Best Recall: {best_recall}") + logging.info(f"Best F1: {best_f1}") diff --git a/convlab/policy/vtrace_DPT/train.py b/convlab/policy/vtrace_DPT/train.py index e20281f4b1c9de7ee1aaf9f9b43e02909dca92e4..f441da29cf44fb366daf9f5afd3f2f21d1bd420d 100644 --- a/convlab/policy/vtrace_DPT/train.py +++ b/convlab/policy/vtrace_DPT/train.py @@ -103,7 +103,7 @@ if __name__ == '__main__': parser = ArgumentParser() parser.add_argument("--path", type=str, default='convlab/policy/vtrace_DPT/semantic_level_config.json', help="Load path for config file") - parser.add_argument("--seed", type=int, default=0, + parser.add_argument("--seed", type=int, default=None, help="Seed for the policy parameter initialization") parser.add_argument("--mode", type=str, default='info', help="Set level for logger") @@ -118,10 +118,9 @@ if __name__ == '__main__': logger, tb_writer, current_time, save_path, config_save_path, dir_path, log_save_path = \ init_logging(os.path.dirname(os.path.abspath(__file__)), mode) - args = [('model', 'seed', seed)] + args = [('model', 'seed', seed)] if seed is not None else list() environment_config = load_config_file(path) - save_config(vars(parser.parse_args()), environment_config, config_save_path) conf = get_config(path, args) seed = conf['model']['seed'] @@ -135,6 +134,8 @@ if __name__ == '__main__': policy_sys.log_dir = config_save_path.replace('configs', 'logs') policy_sys.save_dir = save_path + save_config(vars(parser.parse_args()), environment_config, config_save_path, policy_config=policy_sys.cfg) + env, sess = env_config(conf, policy_sys) # Setup uncertainty thresholding diff --git a/convlab/policy/vtrace_DPT/train_ocl.py b/convlab/policy/vtrace_DPT/train_ocl.py deleted file mode 100644 index 749963c7ef1dfda1cdf39766088c816eae69c6f7..0000000000000000000000000000000000000000 --- a/convlab/policy/vtrace_DPT/train_ocl.py +++ /dev/null @@ -1,263 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Created on Sun Jul 14 16:14:07 2019 -@author: Chris Geishauser -""" - -import sys -import os -import logging -import time -import torch -import numpy as np -import json - -from copy import deepcopy -from torch import multiprocessing as mp -from argparse import ArgumentParser -from convlab.policy.vtrace_DPT import VTRACE -from convlab.policy.vtrace_DPT.memory import Memory -from convlab.policy.vtrace_DPT.multiprocessing_helper import get_queues, start_processes, submit_jobs, \ - terminate_processes -from convlab.policy.vtrace_DPT.ocl.ocl_helper import load_budget, check_setup, log_used_budget, get_goals, \ - update_online_metrics -from convlab.task.multiwoz.goal_generator import GoalGenerator -from convlab.util.custom_util import set_seed, init_logging, save_config, move_finished_training, env_config, \ - eval_policy, load_config_file, get_config -from datetime import datetime - -sys.path.append(os.path.dirname(os.path.dirname( - os.path.dirname(os.path.abspath(__file__))))) - -DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") -device = DEVICE - -try: - mp.set_start_method('spawn', force=True) - mp = mp.get_context('spawn') -except RuntimeError: - pass - - -def create_episodes(environment, policy, num_episodes, memory, goals): - sampled_num = 0 - traj_len = 40 - metrics = [] - - while sampled_num < num_episodes and goals: - goal = goals.pop() - s = environment.reset(goal) - rl_return = 0 - - user_act_list, sys_act_list, s_vec_list, action_list, reward_list, small_act_list, action_mask_list, mu_list, \ - trajectory_list, vector_mask_list, critic_value_list, description_idx_list, value_list, current_domain_mask, \ - non_current_domain_mask = \ - [], [], [], [], [], [], [], [], [], [], [], [], [], [], [] - - for t in range(traj_len): - - s_vec, mask = policy.vector.state_vectorize(s) - with torch.no_grad(): - a = policy.predict(s) - - # s_vec_list.append(policy.info_dict['kg']) - action_list.append(policy.info_dict['big_act'].detach()) - small_act_list.append(policy.info_dict['small_act']) - action_mask_list.append(policy.info_dict['action_mask']) - mu_list.append(policy.info_dict['a_prob'].detach()) - critic_value_list.append(policy.info_dict['critic_value']) - vector_mask_list.append(torch.Tensor(mask)) - description_idx_list.append(policy.info_dict["description_idx_list"]) - value_list.append(policy.info_dict["value_list"]) - current_domain_mask.append(policy.info_dict["current_domain_mask"]) - non_current_domain_mask.append(policy.info_dict["non_current_domain_mask"]) - - sys_act_list.append(policy.vector.action_vectorize(a)) - trajectory_list.extend([s['user_action'], a]) - - # interact with env - next_s, r, done = environment.step(a) - rl_return += r - reward_list.append(torch.Tensor([r])) - - next_s_vec, next_mask = policy.vector.state_vectorize(next_s) - - # update per step - s = next_s - - if done: - metrics.append({"success": environment.evaluator.success_strict, "return": rl_return, - "avg_actions": torch.stack(action_list).sum(dim=-1).mean().item(), - "turns": t, "goal": goal.domain_goals}) - memory.update_episode(description_idx_list, action_list, reward_list, small_act_list, mu_list, - action_mask_list, critic_value_list, description_idx_list, value_list, - current_domain_mask, non_current_domain_mask) - break - - sampled_num += 1 - return metrics - - -def log_train_configs(): - logging.info('Train seed is ' + str(seed)) - logging.info("Start of Training: " + - time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())) - logging.info(f"Number of processes for training: {train_processes}") - - -if __name__ == '__main__': - - time_now = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) - - begin_time = datetime.now() - parser = ArgumentParser() - parser.add_argument("--path", type=str, default='convlab/policy/vtrace_DPT/ocl/semantic_level_config_ocl.json', - help="Load path for config file") - parser.add_argument("--seed", type=int, default=0, - help="Seed for the policy parameter initialization") - parser.add_argument("--mode", type=str, default='info', - help="Set level for logger") - parser.add_argument("--save_eval_dials", type=bool, default=False, - help="Flag for saving dialogue_info during evaluation") - - path = parser.parse_args().path - seed = parser.parse_args().seed - mode = parser.parse_args().mode - save_eval = parser.parse_args().save_eval_dials - - logger, tb_writer, current_time, save_path, config_save_path, dir_path, log_save_path = \ - init_logging(os.path.dirname(os.path.abspath(__file__)), mode) - - args = [('model', 'seed', seed)] - - environment_config = load_config_file(path) - save_config(vars(parser.parse_args()), environment_config, config_save_path, - json.load(open(os.path.dirname(__file__) + "/config.json", "r"))) - - conf = get_config(path, args) - seed = conf['model']['seed'] - set_seed(seed) - - policy_sys = VTRACE(is_train=True, seed=seed, vectorizer=conf['vectorizer_sys_activated'], - load_path=conf['model']['load_path']) - policy_sys.share_memory() - memory = Memory(seed=seed) - policy_sys.current_time = current_time - policy_sys.log_dir = config_save_path.replace('configs', 'logs') - policy_sys.save_dir = save_path - - env, sess = env_config(conf, policy_sys) - - # Setup uncertainty thresholding - if env.sys_dst: - try: - if env.sys_dst.use_confidence_scores: - policy_sys.vector.setup_uncertain_query(env.sys_dst.thresholds) - except: - logging.info('Uncertainty threshold not set.') - - # the timeline will decide how many dialogues are needed until a domain appears - timeline, budget = load_budget(conf['model']['budget_path']) - start_budget = deepcopy(budget) - logging.info(f"Timeline: {timeline}") - logging.info(f"Budget: {budget}") - check_setup(timeline, budget) - - logging.info(f"Evaluating at start - {time_now}" + '-'*60) - time_now = time.time() - policy_sys.policy.action_embedder.forbidden_domains = [] - eval_dict = eval_policy(conf, policy_sys, env, sess, save_eval, log_save_path) - logging.info(f"Finished evaluating, time spent: {time.time() - time_now}") - for key in eval_dict: - tb_writer.add_scalar(key, eval_dict[key], 0) - - train_processes = conf['model']["process_num_train"] - - if train_processes > 1: - # We use multiprocessing - queues, episode_queues = get_queues(train_processes) - online_metric_queue = mp.SimpleQueue() - - metric_keys = ["success", "return", "avg_actions", "turns", "goal"] - online_metrics = {key: [] for key in metric_keys} - num_dialogues = 0 - new_dialogues = conf['model']["new_dialogues"] - training_done = False - - log_train_configs() - goal_generator = GoalGenerator(domain_ordering_dist=dict((tuple(pair[0].split("-")), pair[1] / timeline['end']) - for pair in start_budget)) - - while not training_done: - allowed_domains = [key for key, value in timeline.items() if value <= num_dialogues] - forbidden_domains = [domain for domain in list(timeline.keys()) if domain not in allowed_domains] - new_domain_introduced = len(allowed_domains) > len([key for key, value in timeline.items() - if value <= num_dialogues - new_dialogues]) - - # we disable regularization for the first domain we see - if len(allowed_domains) == 1: - policy_sys.use_regularization = False - else: - policy_sys.use_regularization = True - policy_sys.policy.action_embedder.forbidden_domains = forbidden_domains - - policy_sys.is_train = True - if new_domain_introduced: - # we sample a batch of goals until the next domain is introduced - number_goals_required = min([value - num_dialogues for key, value in timeline.items() - if value - num_dialogues > 0]) - - logging.info(f"Creating {number_goals_required} goals..") - goals, budget = get_goals(goal_generator, allowed_domains, budget, number_goals_required) - logging.info("Goals created.") - if train_processes > 1: - # this means a new domain has appeared and the policy in the processes should be updated - if len(allowed_domains) > 1: - # a new domain is introduced, first kill old processes before starting new - terminate_processes(processes, queues) - - processes = start_processes(train_processes, queues, episode_queues, env, policy_sys, seed, - online_metric_queue) - - new_dialogues = conf['model']["new_dialogues"] if len(goals) > conf['model']["new_dialogues"] - 1 else len(goals) - if train_processes == 1: - metrics = create_episodes(env, policy_sys, new_dialogues, memory, goals) - else: - # create dialogues using the spawned processes - time_now, metrics = submit_jobs(new_dialogues, queues, episode_queues, train_processes, memory, goals, - online_metric_queue) - num_dialogues += new_dialogues - update_online_metrics(online_metrics, metrics, log_save_path, tb_writer) - - for r in range(conf['model']['update_rounds']): - if num_dialogues > 50: - policy_sys.update(memory) - torch.cuda.empty_cache() - - if num_dialogues % 1000 == 0: - logging.info(f"Online Metric" + '-' * 15 + f'Dialogues done: {num_dialogues}' + 15 * '-') - for key in online_metrics: - if key == "goal": - continue - logging.info(f"{key}: {np.mean(online_metrics[key])}") - log_used_budget(start_budget, budget) - - if num_dialogues % conf['model']['eval_frequency'] == 0: - - # run evaluation - logging.info(f"Evaluating - " + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + '-' * 60) - policy_sys.is_train = False - policy_sys.policy.action_embedder.forbidden_domains = [] - eval_dict = eval_policy(conf, policy_sys, env, sess, save_eval, log_save_path) - for key in eval_dict: - tb_writer.add_scalar(key, eval_dict[key], num_dialogues) - policy_sys.policy.action_embedder.forbidden_domains = forbidden_domains - - # if budget is empty and goals are used, training stops - if sum([pair[1] for pair in budget]) == 0 and not goals: - training_done = True - - policy_sys.save(save_path, f"end") - - logging.info("End of Training: " + time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())) - move_finished_training(dir_path, os.path.join(os.path.dirname(os.path.abspath(__file__)), "finished_experiments")) diff --git a/convlab/policy/vtrace_DPT/transformer_model/EncoderDecoder.py b/convlab/policy/vtrace_DPT/transformer_model/EncoderDecoder.py index 4b4b024bad2890d0ccb58056755d09fdf4298bb7..cc2065f57dac4f17108f4db219368b4bc592dbee 100644 --- a/convlab/policy/vtrace_DPT/transformer_model/EncoderDecoder.py +++ b/convlab/policy/vtrace_DPT/transformer_model/EncoderDecoder.py @@ -4,6 +4,7 @@ from .transformer import TransformerModelEncoder, TransformerModelDecoder from .action_embedder import ActionEmbedder from torch.distributions.categorical import Categorical from .noisy_linear import NoisyLinear +from tqdm import tqdm import torch import torch.nn as nn @@ -23,11 +24,12 @@ class EncoderDecoder(nn.Module): node_embedding_dim, roberta_path="", node_attention=True, max_length=25, semantic_descriptions=True, freeze_roberta=True, use_pooled=False, verbose=False, mean=False, ignore_features=None, only_active_values=False, roberta_actions=False, independent_descriptions=False, need_weights=True, - random_matrix=False, distance_metric=False, noisy_linear=False, **kwargs): + random_matrix=False, distance_metric=False, noisy_linear=False, dataset_name='multiwoz21', **kwargs): super(EncoderDecoder, self).__init__() self.node_embedder = NodeEmbedderRoberta(node_embedding_dim, freeze_roberta=freeze_roberta, use_pooled=use_pooled, roberta_path=roberta_path, - semantic_descriptions=semantic_descriptions, mean=mean).to(DEVICE) + semantic_descriptions=semantic_descriptions, mean=mean, + dataset_name=dataset_name).to(DEVICE) #TODO: Encoder input dim should be same as projection dim or use another linear layer? self.encoder = TransformerModelEncoder(enc_input_dim, enc_nhead, enc_d_hid, enc_nlayers, enc_dropout, need_weights).to(DEVICE) self.decoder = TransformerModelDecoder(action_embedding_dim, dec_nhead, dec_d_hid, dec_nlayers, dec_dropout, need_weights).to(DEVICE) @@ -246,7 +248,7 @@ class EncoderDecoder(nn.Module): return self.action_embedder.small_action_list_to_real_actions(action_list) def get_log_prob(self, actions, action_mask_list, max_length, action_targets, - current_domain_mask, non_current_domain_mask, descriptions_list, value_list): + current_domain_mask, non_current_domain_mask, descriptions_list, value_list, no_slots=False): action_probs, entropy_probs = self.get_prob(actions, action_mask_list, max_length, action_targets, current_domain_mask, non_current_domain_mask, descriptions_list, value_list) @@ -259,6 +261,12 @@ class EncoderDecoder(nn.Module): # sometimes a domain will be masked because it is inactive due to labelling error. Will ignore these cases. log_probs[log_probs == -float("Inf")] = 0 + if no_slots: + time_steps = torch.arange(0, max_length) + slot_steps = torch.where(time_steps % 3 == 2, torch.zeros(max_length), torch.ones(max_length))\ + .view(1, -1).to(DEVICE) + log_probs *= slot_steps + return log_probs.sum(-1), entropy def get_prob(self, actions, action_mask_list, max_length, action_targets, @@ -322,7 +330,7 @@ class EncoderDecoder(nn.Module): # set padded time-steps to probability 1, so that log will be 0 action_probs = action_probs + action_prob_helper_rev - entropy_probs = action_distribution * action_prob_helper.unsqueeze(-1) + action_prob_helper_rev.unsqueeze(-1) + entropy_probs = action_distribution_general * action_prob_helper.unsqueeze(-1) + action_prob_helper_rev.unsqueeze(-1) #entropy_probs = entropy_probs + domain_steps return action_probs, entropy_probs @@ -342,11 +350,11 @@ class EncoderDecoder(nn.Module): # Map the actions to action embeddings that are fed as input to decoder model # pad input and remove "eos" token padded_decoder_input = torch.stack( - [torch.cat([act[:-1], torch.zeros(max_length - len(act))], dim=-1) for act in action_targets], dim=0) \ + [torch.cat([act[:-1].to(DEVICE), torch.zeros(max_length - len(act)).to(DEVICE)], dim=-1) for act in action_targets], dim=0) \ .to(DEVICE).long() padded_action_targets = torch.stack( - [torch.cat([act, torch.zeros(max_length - len(act))], dim=-1) for act in action_targets], dim=0) \ + [torch.cat([act.to(DEVICE), torch.zeros(max_length - len(act)).to(DEVICE)], dim=-1) for act in action_targets], dim=0) \ .to(DEVICE) decoder_input = self.action_embedder.action_embeddings[padded_decoder_input] @@ -440,7 +448,7 @@ class EncoderDecoder(nn.Module): semantic_acts = [self.action_embedder.real_action_to_small_action_list(act, semantic=True) for act in actions] action_mask_list = [] decoder_encoder_mask_list = [] - for i, act_sequence in enumerate(semantic_acts): + for i, act_sequence in tqdm(enumerate(semantic_acts)): action_mask = [self.action_embedder.get_action_mask(start=True)] for t, act in enumerate(act_sequence): diff --git a/convlab/policy/vtrace_DPT/transformer_model/action_embedder.py b/convlab/policy/vtrace_DPT/transformer_model/action_embedder.py index 91b07682d6ff6752d586c2ff32157e732c1afe22..283443eb2daf55c56427e6de01f9cf6a2b4049cf 100644 --- a/convlab/policy/vtrace_DPT/transformer_model/action_embedder.py +++ b/convlab/policy/vtrace_DPT/transformer_model/action_embedder.py @@ -5,6 +5,8 @@ import logging import json from copy import deepcopy +from convlab.policy.vtrace_DPT.transformer_model.noisy_linear import NoisyLinear + DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -40,6 +42,7 @@ class ActionEmbedder(nn.Module): self.action_embeddings = nn.Parameter(self.action_embeddings) else: logging.info("We use Roberta to embed actions.") + self.dataset_name = node_embedder.dataset_name self.create_action_embeddings_roberta(node_embedder) self.action_embeddings.requires_grad = False embedding_dim = 768 @@ -49,6 +52,7 @@ class ActionEmbedder(nn.Module): self.small_action_dict_reversed = dict((value, key) for key, value in self.small_action_dict.items()) self.linear = torch.nn.Linear(embedding_dim, action_embedding_dim).to(DEVICE) + #self.linear = NoisyLinear(embedding_dim, action_embedding_dim).to(DEVICE) self.random_matrix = torch.randn(embedding_dim, action_embedding_dim).to(DEVICE) / \ torch.sqrt(torch.Tensor([768])).to(DEVICE) @@ -102,15 +106,15 @@ class ActionEmbedder(nn.Module): return action_mask.to(DEVICE) - def get_action_mask(self, domain="", intent="", start=False): + def get_action_mask(self, domain=None, intent="", start=False): action_mask = torch.ones(len(self.small_action_dict)) # This is for predicting end of sequence token <eos> - if not start and not domain: + if not start and domain is None: action_mask[self.small_action_dict['eos']] = 0 - if not domain: + if domain is None: #TODO: I allow all domains now for checking supervised training for domain in self.domain_dict: if domain not in self.forbidden_domains: @@ -125,7 +129,7 @@ class ActionEmbedder(nn.Module): # Domain was selected, need intent now for intent in self.intent_dict: domain_intent = f"{domain}-{intent}" - valid = self.is_valid(domain_intent) + valid = self.is_valid(domain_intent + "-") if valid: action_mask[self.small_action_dict[intent]] = 0 else: @@ -136,6 +140,8 @@ class ActionEmbedder(nn.Module): if valid: action_mask[self.small_action_dict[slot_value]] = 0 + assert not torch.equal(action_mask, torch.ones(len(self.small_action_dict))) + return action_mask.to(DEVICE) def get_current_domain_mask(self, current_domains, current=True): @@ -154,7 +160,7 @@ class ActionEmbedder(nn.Module): def is_valid(self, part_action): for act in self.action_dict: - if part_action in act: + if act.startswith(part_action): return True return False @@ -202,8 +208,10 @@ class ActionEmbedder(nn.Module): action_embeddings.append("pad") #add the PAD token small_action_dict['pad'] = len(small_action_dict) - action_embeddings_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'action_embeddings.pt') - small_action_dict_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'small_action_dict.json') + action_embeddings_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), + f'action_embeddings_{self.dataset_name}.pt') + small_action_dict_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), + f'small_action_dict_{self.dataset_name}.json') if os.path.exists(action_embeddings_path): self.action_embeddings = torch.load(action_embeddings_path).to(DEVICE) diff --git a/convlab/policy/vtrace_DPT/transformer_model/action_embeddings_multiwoz21.pt b/convlab/policy/vtrace_DPT/transformer_model/action_embeddings_multiwoz21.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f7e5e9cdf949b082b9ae752c09d2eafeb322bfb Binary files /dev/null and b/convlab/policy/vtrace_DPT/transformer_model/action_embeddings_multiwoz21.pt differ diff --git a/convlab/policy/vtrace_DPT/transformer_model/action_embeddings_sgd.pt b/convlab/policy/vtrace_DPT/transformer_model/action_embeddings_sgd.pt new file mode 100644 index 0000000000000000000000000000000000000000..67e557ce5ef7a0bfd40c60ae5a03937b47b92de2 Binary files /dev/null and b/convlab/policy/vtrace_DPT/transformer_model/action_embeddings_sgd.pt differ diff --git a/convlab/policy/vtrace_DPT/transformer_model/embedded_descriptions_base_multiwoz21.pt b/convlab/policy/vtrace_DPT/transformer_model/embedded_descriptions_base_multiwoz21.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c6193163b6a1fc788b959eff2d0153b3a4bf7cb Binary files /dev/null and b/convlab/policy/vtrace_DPT/transformer_model/embedded_descriptions_base_multiwoz21.pt differ diff --git a/convlab/policy/vtrace_DPT/transformer_model/embedded_descriptions_base_sgd.pt b/convlab/policy/vtrace_DPT/transformer_model/embedded_descriptions_base_sgd.pt new file mode 100644 index 0000000000000000000000000000000000000000..619824588654a36cab3bf795e6fe94527b04ba68 Binary files /dev/null and b/convlab/policy/vtrace_DPT/transformer_model/embedded_descriptions_base_sgd.pt differ diff --git a/convlab/policy/vtrace_DPT/transformer_model/node_embedder.py b/convlab/policy/vtrace_DPT/transformer_model/node_embedder.py index 432e58f12c244a6b72c77b8965e31a8cc71ad8c4..ab57df122369e9457c78e04fe2e64eab389d5abd 100644 --- a/convlab/policy/vtrace_DPT/transformer_model/node_embedder.py +++ b/convlab/policy/vtrace_DPT/transformer_model/node_embedder.py @@ -3,7 +3,7 @@ import torch import torch.nn as nn from transformers import RobertaTokenizer, RobertaModel - +from convlab.policy.vtrace_DPT.transformer_model.noisy_linear import NoisyLinear from convlab.policy.vtrace_DPT.create_descriptions import create_description_dicts DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -16,13 +16,15 @@ class NodeEmbedderRoberta(nn.Module): ''' def __init__(self, projection_dim, freeze_roberta=True, use_pooled=False, max_length=25, roberta_path="", - description_dict=None, semantic_descriptions=True, mean=False): + description_dict=None, semantic_descriptions=True, mean=False, dataset_name="multiwoz21"): super(NodeEmbedderRoberta, self).__init__() + self.dataset_name = dataset_name self.max_length = max_length self.description_size = 768 self.projection_dim = projection_dim self.feature_projection = torch.nn.Linear(2 * self.description_size, projection_dim).to(DEVICE) + #self.feature_projection = NoisyLinear(2 * self.description_size, projection_dim).to(DEVICE) self.value_embedding = torch.nn.Linear(1, self.description_size).to(DEVICE) self.semantic_descriptions = semantic_descriptions @@ -36,7 +38,7 @@ class NodeEmbedderRoberta(nn.Module): if roberta_path: embedded_descriptions_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'embedded_descriptions.pt') + f'embedded_descriptions_{self.dataset_name}.pt') if os.path.exists(embedded_descriptions_path): self.embedded_descriptions = torch.load(embedded_descriptions_path).to(DEVICE) else: @@ -46,7 +48,7 @@ class NodeEmbedderRoberta(nn.Module): else: embedded_descriptions_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'embedded_descriptions_base.pt') + f'embedded_descriptions_base_{self.dataset_name}.pt') if os.path.exists(embedded_descriptions_path): self.embedded_descriptions = torch.load(embedded_descriptions_path).to(DEVICE) else: @@ -63,6 +65,7 @@ class NodeEmbedderRoberta(nn.Module): logging.info(f"Embedding semantic descriptions: {semantic_descriptions}") logging.info(f"Embedded descriptions successfully. Size: {self.embedded_descriptions.size()}") + logging.info(f"Data set used for descriptions: {dataset_name}") def form_embedded_descriptions(self): @@ -109,10 +112,10 @@ class NodeEmbedderRoberta(nn.Module): def init_description_dict(self): - create_description_dicts() + create_description_dicts(self.dataset_name) root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if self.semantic_descriptions: - path = os.path.join(root_dir, 'semantic_information_descriptions.json') + path = os.path.join(root_dir, f'descriptions/semantic_information_descriptions_{self.dataset_name}.json') else: path = os.path.join(root_dir, 'information_descriptions.json') with open(path, "r") as f: diff --git a/convlab/policy/vtrace_DPT/transformer_model/small_action_dict_multiwoz21.json b/convlab/policy/vtrace_DPT/transformer_model/small_action_dict_multiwoz21.json new file mode 100644 index 0000000000000000000000000000000000000000..0d5bd2002fa7d0082e7589b80ae3664781732ece --- /dev/null +++ b/convlab/policy/vtrace_DPT/transformer_model/small_action_dict_multiwoz21.json @@ -0,0 +1 @@ +{"attraction": 0, "general": 1, "hospital": 2, "hotel": 3, "police": 4, "restaurant": 5, "taxi": 6, "train": 7, "eos": 8, "inform": 9, "nooffer": 10, "recommend": 11, "request": 12, "select": 13, "bye": 14, "greet": 15, "reqmore": 16, "welcome": 17, "book": 18, "offerbook": 19, "nobook": 20, "address-1": 21, "address-2": 22, "address-3": 23, "area-1": 24, "area-2": 25, "area-3": 26, "choice-1": 27, "choice-2": 28, "choice-3": 29, "entrance fee-1": 30, "entrance fee-2": 31, "name-1": 32, "name-2": 33, "name-3": 34, "name-4": 35, "phone-1": 36, "postcode-1": 37, "type-1": 38, "type-2": 39, "type-3": 40, "type-4": 41, "type-5": 42, "none-none": 43, "area-?": 44, "entrance fee-?": 45, "name-?": 46, "type-?": 47, "department-1": 48, "department-?": 49, "book day-1": 50, "book people-1": 51, "book stay-1": 52, "internet-1": 53, "parking-1": 54, "price range-1": 55, "price range-2": 56, "ref-1": 57, "stars-1": 58, "stars-2": 59, "book day-?": 60, "book people-?": 61, "book stay-?": 62, "internet-?": 63, "parking-?": 64, "price range-?": 65, "stars-?": 66, "book time-1": 67, "food-1": 68, "food-2": 69, "food-3": 70, "food-4": 71, "postcode-2": 72, "book time-?": 73, "food-?": 74, "arrive by-1": 75, "departure-1": 76, "destination-1": 77, "leave at-1": 78, "arrive by-?": 79, "departure-?": 80, "destination-?": 81, "leave at-?": 82, "arrive by-2": 83, "day-1": 84, "duration-1": 85, "leave at-2": 86, "leave at-3": 87, "price-1": 88, "train id-1": 89, "day-?": 90, "pad": 91} \ No newline at end of file diff --git a/convlab/policy/vtrace_DPT/transformer_model/small_action_dict_sgd.json b/convlab/policy/vtrace_DPT/transformer_model/small_action_dict_sgd.json new file mode 100644 index 0000000000000000000000000000000000000000..c48573262f50cabe1d4cd3811638ebf4c046a7e0 --- /dev/null +++ b/convlab/policy/vtrace_DPT/transformer_model/small_action_dict_sgd.json @@ -0,0 +1 @@ +{"": 0, "alarm_1": 1, "banks_1": 2, "banks_2": 3, "buses_1": 4, "buses_2": 5, "buses_3": 6, "calendar_1": 7, "events_1": 8, "events_2": 9, "events_3": 10, "flights_1": 11, "flights_2": 12, "flights_3": 13, "flights_4": 14, "homes_1": 15, "homes_2": 16, "hotels_1": 17, "hotels_2": 18, "hotels_3": 19, "hotels_4": 20, "media_1": 21, "media_2": 22, "media_3": 23, "messaging_1": 24, "movies_1": 25, "movies_2": 26, "movies_3": 27, "music_1": 28, "music_2": 29, "music_3": 30, "payment_1": 31, "rentalcars_1": 32, "rentalcars_2": 33, "rentalcars_3": 34, "restaurants_1": 35, "restaurants_2": 36, "ridesharing_1": 37, "ridesharing_2": 38, "services_1": 39, "services_2": 40, "services_3": 41, "services_4": 42, "trains_1": 43, "travel_1": 44, "weather_1": 45, "eos": 46, "goodbye": 47, "req_more": 48, "confirm": 49, "inform_count": 50, "notify_success": 51, "offer": 52, "offer_intent": 53, "request": 54, "inform": 55, "notify_failure": 56, "none-none": 57, "new_alarm_name-1": 58, "new_alarm_time-1": 59, "count-1": 60, "alarm_name-1": 61, "alarm_time-1": 62, "addalarm-1": 63, "new_alarm_time-?": 64, "account_type-1": 65, "amount-1": 66, "recipient_account_name-1": 67, "recipient_account_type-1": 68, "balance-1": 69, "transfermoney-1": 70, "account_type-?": 71, "amount-?": 72, "recipient_account_name-?": 73, "recipient_name-1": 74, "transfer_amount-1": 75, "transfer_time-1": 76, "account_balance-1": 77, "recipient_name-?": 78, "transfer_amount-?": 79, "from_location-1": 80, "leaving_date-1": 81, "leaving_time-1": 82, "to_location-1": 83, "travelers-1": 84, "from_station-1": 85, "to_station-1": 86, "transfers-1": 87, "fare-1": 88, "buybusticket-1": 89, "from_location-?": 90, "leaving_date-?": 91, "leaving_time-?": 92, "to_location-?": 93, "travelers-?": 94, "departure_date-1": 95, "departure_time-1": 96, "destination-1": 97, "fare_type-1": 98, "group_size-1": 99, "origin-1": 100, "destination_station_name-1": 101, "origin_station_name-1": 102, "price-1": 103, "departure_date-?": 104, "departure_time-?": 105, "destination-?": 106, "group_size-?": 107, "origin-?": 108, "additional_luggage-1": 109, "from_city-1": 110, "num_passengers-1": 111, "to_city-1": 112, "category-1": 113, "from_city-?": 114, "num_passengers-?": 115, "to_city-?": 116, "event_date-1": 117, "event_location-1": 118, "event_name-1": 119, "event_time-1": 120, "available_end_time-1": 121, "available_start_time-1": 122, "addevent-1": 123, "event_date-?": 124, "event_location-?": 125, "event_name-?": 126, "event_time-?": 127, "city_of_event-1": 128, "date-1": 129, "number_of_seats-1": 130, "address_of_location-1": 131, "subcategory-1": 132, "time-1": 133, "buyeventtickets-1": 134, "category-?": 135, "city_of_event-?": 136, "date-?": 137, "number_of_seats-?": 138, "city-1": 139, "number_of_tickets-1": 140, "venue-1": 141, "venue_address-1": 142, "city-?": 143, "event_type-?": 144, "number_of_tickets-?": 145, "price_per_ticket-1": 146, "airlines-1": 147, "destination_city-1": 148, "inbound_departure_time-1": 149, "origin_city-1": 150, "outbound_departure_time-1": 151, "passengers-1": 152, "return_date-1": 153, "seating_class-1": 154, "destination_airport-1": 155, "inbound_arrival_time-1": 156, "number_stops-1": 157, "origin_airport-1": 158, "outbound_arrival_time-1": 159, "refundable-1": 160, "reserveonewayflight-1": 161, "reserveroundtripflights-1": 162, "airlines-?": 163, "destination_city-?": 164, "inbound_departure_time-?": 165, "origin_city-?": 166, "outbound_departure_time-?": 167, "return_date-?": 168, "is_redeye-1": 169, "arrives_next_day-1": 170, "destination_airport_name-1": 171, "origin_airport_name-1": 172, "is_nonstop-1": 173, "destination_airport-?": 174, "origin_airport-?": 175, "property_name-1": 176, "visit_date-1": 177, "furnished-1": 178, "pets_allowed-1": 179, "phone_number-1": 180, "address-1": 181, "number_of_baths-1": 182, "number_of_beds-1": 183, "rent-1": 184, "schedulevisit-1": 185, "area-?": 186, "number_of_beds-?": 187, "visit_date-?": 188, "has_garage-1": 189, "in_unit_laundry-1": 190, "intent-?": 191, "number_of_baths-?": 192, "check_in_date-1": 193, "hotel_name-1": 194, "number_of_days-1": 195, "number_of_rooms-1": 196, "has_wifi-1": 197, "price_per_night-1": 198, "street_address-1": 199, "star_rating-1": 200, "reservehotel-1": 201, "check_in_date-?": 202, "hotel_name-?": 203, "number_of_days-?": 204, "check_out_date-1": 205, "number_of_adults-1": 206, "where_to-1": 207, "has_laundry_service-1": 208, "total_price-1": 209, "rating-1": 210, "bookhouse-1": 211, "check_out_date-?": 212, "number_of_adults-?": 213, "where_to-?": 214, "location-1": 215, "pets_welcome-1": 216, "average_rating-1": 217, "location-?": 218, "place_name-1": 219, "stay_length-1": 220, "smoking_allowed-1": 221, "stay_length-?": 222, "subtitles-1": 223, "title-1": 224, "directed_by-1": 225, "genre-1": 226, "title-2": 227, "title-3": 228, "playmovie-1": 229, "genre-?": 230, "title-?": 231, "movie_name-1": 232, "subtitle_language-1": 233, "movie_name-2": 234, "movie_name-3": 235, "rentmovie-1": 236, "starring-1": 237, "contact_name-1": 238, "contact_name-?": 239, "show_date-1": 240, "show_time-1": 241, "show_type-1": 242, "theater_name-1": 243, "buymovietickets-1": 244, "movie_name-?": 245, "show_date-?": 246, "show_time-?": 247, "show_type-?": 248, "aggregate_rating-1": 249, "cast-1": 250, "movie_title-1": 251, "percent_rating-1": 252, "playback_device-1": 253, "song_name-1": 254, "album-1": 255, "year-1": 256, "artist-1": 257, "playsong-1": 258, "song_name-?": 259, "playmedia-1": 260, "device-1": 261, "track-1": 262, "payment_method-1": 263, "private_visibility-1": 264, "receiver-1": 265, "payment_method-?": 266, "receiver-?": 267, "dropoff_date-1": 268, "pickup_date-1": 269, "pickup_location-1": 270, "pickup_time-1": 271, "type-1": 272, "car_name-1": 273, "reservecar-1": 274, "dropoff_date-?": 275, "pickup_city-?": 276, "pickup_date-?": 277, "pickup_location-?": 278, "pickup_time-?": 279, "type-?": 280, "car_type-1": 281, "car_type-?": 282, "add_insurance-1": 283, "end_date-1": 284, "start_date-1": 285, "price_per_day-1": 286, "add_insurance-?": 287, "end_date-?": 288, "start_date-?": 289, "party_size-1": 290, "restaurant_name-1": 291, "cuisine-1": 292, "has_live_music-1": 293, "price_range-1": 294, "serves_alcohol-1": 295, "reserverestaurant-1": 296, "cuisine-?": 297, "restaurant_name-?": 298, "time-?": 299, "has_seating_outdoors-1": 300, "has_vegetarian_options-1": 301, "number_of_riders-1": 302, "shared_ride-1": 303, "approximate_ride_duration-1": 304, "ride_fare-1": 305, "number_of_riders-?": 306, "shared_ride-?": 307, "ride_type-1": 308, "wait_time-1": 309, "ride_type-?": 310, "appointment_date-1": 311, "appointment_time-1": 312, "stylist_name-1": 313, "is_unisex-1": 314, "bookappointment-1": 315, "appointment_date-?": 316, "appointment_time-?": 317, "dentist_name-1": 318, "offers_cosmetic_services-1": 319, "doctor_name-1": 320, "therapist_name-1": 321, "class-1": 322, "date_of_journey-1": 323, "from-1": 324, "journey_start_time-1": 325, "to-1": 326, "trip_protection-1": 327, "total-1": 328, "gettraintickets-1": 329, "date_of_journey-?": 330, "from-?": 331, "to-?": 332, "trip_protection-?": 333, "free_entry-1": 334, "good_for_kids-1": 335, "attraction_name-1": 336, "humidity-1": 337, "wind-1": 338, "precipitation-1": 339, "temperature-1": 340, "pad": 341} \ No newline at end of file diff --git a/convlab/policy/vtrace_DPT/vtrace.py b/convlab/policy/vtrace_DPT/vtrace.py index bb1bb5f0b105efaaad3bf55b38dc1f835f39e2ff..b03662c60539a2e3aa80a5132618a3f7563a0f09 100644 --- a/convlab/policy/vtrace_DPT/vtrace.py +++ b/convlab/policy/vtrace_DPT/vtrace.py @@ -48,6 +48,7 @@ class VTRACE(nn.Module, Policy): self.info_dict = {} self.use_regularization = False self.supervised_weight = cfg.get('supervised_weight', 0.0) + logging.info(f"Entropy weight: {self.entropy_weight}") logging.info(f"Online-Offline-ratio: {self.online_offline_ratio}") logging.info(f"Behaviour cloning weight: {self.behaviour_cloning_weight}") @@ -69,8 +70,6 @@ class VTRACE(nn.Module, Policy): if self.cfg['independent']: self.value = EncoderCritic(self.value_helper.node_embedder, self.value_helper.encoder, **self.cfg).to( device=DEVICE) - self.smoothed_value = EncoderCritic(self.value_helper.node_embedder, self.value_helper.encoder, **self.cfg).\ - to(device=DEVICE) else: self.value = EncoderCritic(self.policy.node_embedder, self.policy.encoder, **self.cfg).to(device=DEVICE) @@ -79,10 +78,10 @@ class VTRACE(nn.Module, Policy): except Exception as e: print(f"Could not load the critic, Exception: {e}") - self.policy_optim = optim.RMSprop( - list(self.policy.parameters()), lr=cfg['policy_lr']) - self.value_optim = optim.Adam( - list(self.value.parameters()), lr=cfg['value_lr']) + self.optimizer = optim.Adam([ + {'params': self.policy.parameters(), 'lr': cfg['policy_lr'], 'betas': (0.0, 0.999)}, + {'params': self.value.parameters(), 'lr': cfg['value_lr']} + ]) try: self.load_optimizer_dicts(load_path) @@ -127,74 +126,51 @@ class VTRACE(nn.Module, Policy): return action - def update(self, memory, supervised_trainer=None): - self.total_it += 1 + def update(self, memory): + p_loss, v_loss = self.get_loss(memory) + loss = v_loss + if p_loss is not None: + loss += p_loss + + self.optimizer.zero_grad() + loss.backward() + + torch.nn.utils.clip_grad_norm_(self.value.parameters(), 40) + for p in self.policy.parameters(): + if p.grad is not None: + p.grad[p.grad != p.grad] = 0.0 + torch.nn.utils.clip_grad_norm_(self.policy.parameters(), 10) + + self.optimizer.step() + + def get_loss(self, memory): + self.is_train = True if self.is_train: + self.total_it += 1 for param in self.policy.parameters(): param.requires_grad = True for param in self.value.parameters(): param.requires_grad = True - if self.use_regularization: - batch, num_online = memory.sample(self.online_offline_ratio) - else: - batch, num_online = memory.sample(0.0) - - unflattened_states = batch['states'] - states = [kg for kg_list in unflattened_states for kg in kg_list] - - description_batch = batch['description_idx_list'] - description_batch = [descr_ for descr_episode in description_batch for descr_ in descr_episode] - - value_batch = batch['value_list'] - value_batch = [value_ for value_episode in value_batch for value_ in value_episode] - - current_domain_mask = batch['current_domain_mask'] - current_domain_mask = torch.stack([curr_mask for curr_mask_episode in current_domain_mask - for curr_mask in curr_mask_episode]).to(DEVICE) - - non_current_domain_mask = batch['non_current_domain_mask'] - non_current_domain_mask = torch.stack([non_curr_mask for non_curr_mask_episode in non_current_domain_mask - for non_curr_mask in non_curr_mask_episode]).to(DEVICE) - - actions = batch['actions'] - actions = torch.stack([act for act_list in actions for act in act_list], dim=0).to(DEVICE) - - small_actions = batch['small_actions'] - small_actions = [act for act_list in small_actions for act in act_list] - - rewards = batch['rewards'] - rewards = torch.stack([r for r_episode in rewards for r in r_episode]).to(DEVICE) - #rewards = torch.from_numpy(np.concatenate(np.array(rewards), axis=0)).to(DEVICE) + batch, num_online = self.get_batch(memory) - mu = batch['mu'] - mu = torch.stack([mu_ for mu_list in mu for mu_ in mu_list], dim=0).to(DEVICE) - - critic_v = batch['critic_value'] - critic_v = torch.stack([v for v_list in critic_v for v in v_list]).to(DEVICE) - - max_length = max(len(act) for act in small_actions) - - action_masks = batch['action_masks'] - action_mask_list = [mask for mask_list in action_masks for mask in mask_list] - action_masks = torch.stack([torch.cat([ - action_mask.to(DEVICE), - torch.zeros(max_length - len(action_mask), len(self.policy.action_embedder.small_action_dict)).to( - DEVICE)], - dim=0) for action_mask in action_mask_list]).to(DEVICE) + action_masks, actions, critic_v, current_domain_mask, description_batch, max_length, mu, \ + non_current_domain_mask, rewards, small_actions, unflattened_states, value_batch \ + = self.prepare_batch(batch) with torch.no_grad(): values = self.value(description_batch, value_batch).squeeze(-1) + pi_prob, _ = self.policy.get_prob(actions, action_masks, max_length, small_actions, current_domain_mask, non_current_domain_mask, description_batch, value_batch) pi_prob = pi_prob.prod(dim=-1) - rho = torch.min(torch.Tensor([self.rho_bar]).to(DEVICE), pi_prob / mu).tolist() - cs = torch.min(torch.Tensor([self.c]).to(DEVICE), pi_prob / mu).tolist() + rho = torch.min(torch.Tensor([self.rho_bar]).to(DEVICE), pi_prob / mu) + cs = torch.min(torch.Tensor([self.c]).to(DEVICE), pi_prob / mu) vtrace_target, advantages = self.compute_vtrace_advantage(unflattened_states, rewards, rho, cs, values) @@ -202,20 +178,13 @@ class VTRACE(nn.Module, Policy): current_v = self.value(description_batch, value_batch).to(DEVICE) critic_loss = torch.square(vtrace_target.unsqueeze(-1).to(DEVICE) - current_v).mean() - # do behaviour cloning on the buffer data - num_online = sum([len(reward_list) for reward_list in batch['rewards'][:num_online]]) - - behaviour_loss_critic = torch.square( - critic_v[num_online:].unsqueeze(-1).to(DEVICE) - current_v[num_online:]).mean() - if self.use_regularization: - critic_loss += self.behaviour_cloning_weight * behaviour_loss_critic + # do behaviour cloning on the buffer data + num_online = sum([len(reward_list) for reward_list in batch['rewards'][:num_online]]) - # Optimize the critic - self.value_optim.zero_grad() - critic_loss.backward() - torch.nn.utils.clip_grad_norm_(self.value.parameters(), 40) - self.value_optim.step() + behaviour_loss_critic = torch.square( + critic_v[num_online:].unsqueeze(-1).to(DEVICE) - current_v[num_online:]).mean() + critic_loss += self.behaviour_cloning_weight * behaviour_loss_critic actor_loss = None @@ -226,38 +195,67 @@ class VTRACE(nn.Module, Policy): current_domain_mask, non_current_domain_mask, description_batch, value_batch) actor_loss = -1 * actor_loss - actor_loss = actor_loss * (advantages * torch.Tensor(rho)).to(DEVICE) + actor_loss = actor_loss * (advantages.to(DEVICE) * rho) actor_loss = actor_loss.mean() - entropy * self.entropy_weight - log_prob, entropy = self.policy.get_log_prob(actions[num_online:], action_masks[num_online:], - max_length, small_actions[num_online:], - current_domain_mask[num_online:], - non_current_domain_mask[num_online:], - description_batch[num_online:], - value_batch[num_online:]) - if self.use_regularization: + log_prob, entropy = self.policy.get_log_prob(actions[num_online:], action_masks[num_online:], + max_length, small_actions[num_online:], + current_domain_mask[num_online:], + non_current_domain_mask[num_online:], + description_batch[num_online:], + value_batch[num_online:]) actor_loss = actor_loss - log_prob.mean() * self.behaviour_cloning_weight - if supervised_trainer is not None: - supervised_loss = supervised_trainer.supervised_loss(self.policy) - actor_loss = actor_loss + self.supervised_weight * supervised_loss - # Optimize the actor - self.policy_optim.zero_grad() - actor_loss.backward() - - for p in self.policy.parameters(): - if p.grad is not None: - p.grad[p.grad != p.grad] = 0.0 - - torch.nn.utils.clip_grad_norm_(self.policy.parameters(), 10) - self.policy_optim.step() - return actor_loss, critic_loss else: return np.nan + def get_batch(self, memory): + + if self.use_regularization or self.online_offline_ratio == 1.0: + batch, num_online = memory.sample(self.online_offline_ratio) + else: + batch, num_online = memory.sample(0.0) + return batch, num_online + + def prepare_batch(self, batch): + unflattened_states = batch['states'] + states = [kg for kg_list in unflattened_states for kg in kg_list] + description_batch = batch['description_idx_list'] + description_batch = [descr_ for descr_episode in description_batch for descr_ in descr_episode] + value_batch = batch['value_list'] + value_batch = [value_ for value_episode in value_batch for value_ in value_episode] + + current_domain_mask = batch['current_domain_mask'] + current_domain_mask = torch.stack([curr_mask for curr_mask_episode in current_domain_mask + for curr_mask in curr_mask_episode]).to(DEVICE) + non_current_domain_mask = batch['non_current_domain_mask'] + non_current_domain_mask = torch.stack([non_curr_mask for non_curr_mask_episode in non_current_domain_mask + for non_curr_mask in non_curr_mask_episode]).to(DEVICE) + actions = batch['actions'] + actions = torch.stack([act for act_list in actions for act in act_list], dim=0).to(DEVICE) + small_actions = batch['small_actions'] + small_actions = [act for act_list in small_actions for act in act_list] + rewards = batch['rewards'] + rewards = torch.stack([r for r_episode in rewards for r in r_episode]).to(DEVICE) + # rewards = torch.from_numpy(np.concatenate(np.array(rewards), axis=0)).to(DEVICE) + mu = batch['mu'] + mu = torch.stack([mu_ for mu_list in mu for mu_ in mu_list], dim=0).to(DEVICE) + critic_v = batch['critic_value'] + critic_v = torch.stack([v for v_list in critic_v for v in v_list]).to(DEVICE) + max_length = max(len(act) for act in small_actions) + action_masks = batch['action_masks'] + action_mask_list = [mask for mask_list in action_masks for mask in mask_list] + action_masks = torch.stack([torch.cat([ + action_mask.to(DEVICE), + torch.zeros(max_length - len(action_mask), len(self.policy.action_embedder.small_action_dict)).to( + DEVICE)], + dim=0) for action_mask in action_mask_list]).to(DEVICE) + return action_masks, actions, critic_v, current_domain_mask, description_batch, max_length, mu, \ + non_current_domain_mask, rewards, small_actions, unflattened_states, value_batch + def compute_vtrace_advantage(self, states, rewards, rho, cs, values): vtraces, advantages, offset = [], [], 0 @@ -289,10 +287,9 @@ class VTRACE(nn.Module, Policy): torch.save(self.value.state_dict(), directory + f'/{addition}_vtrace.val.mdl') torch.save(self.policy.state_dict(), directory + f'/{addition}_vtrace.pol.mdl') - torch.save(self.policy_optim.state_dict(), directory + f'/{addition}_vtrace.policy_optim') - torch.save(self.value_optim.state_dict(), directory + f'/{addition}_vtrace.value_optim') + torch.save(self.optimizer.state_dict(), directory + f'/{addition}_vtrace.optimizer') - logging.info(f"Saved policy, critic and memory.") + logging.info(f"Saved policy, critic and optimizer.") def load(self, filename): @@ -361,8 +358,7 @@ class VTRACE(nn.Module, Policy): break def load_optimizer_dicts(self, filename): - self.policy_optim.load_state_dict(torch.load(filename + f".policy_optim")) - self.value_optim.load_state_dict(torch.load(filename + f".value_optim")) + self.optimizer.load_state_dict(torch.load(filename + f".optimizer", map_location=DEVICE)) logging.info('<<dialog policy>> loaded optimisers from file: {}'.format(filename)) def from_pretrained(self): diff --git a/convlab/util/custom_util.py b/convlab/util/custom_util.py index c79c6f0d06e26f916f06a926c58783ed367e8828..38e1201479df57da24578dd85d5568a5ed91a5cd 100644 --- a/convlab/util/custom_util.py +++ b/convlab/util/custom_util.py @@ -21,6 +21,7 @@ from convlab.evaluator.multiwoz_eval import MultiWozEvaluator from convlab.util import load_dataset import shutil +import signal slot_mapping = {"pricerange": "price range", "post": "postcode", "arriveBy": "arrive by", "leaveAt": "leave at", @@ -34,6 +35,22 @@ DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") device = DEVICE +class timeout: + def __init__(self, seconds=10, error_message='Timeout'): + self.seconds = seconds + self.error_message = error_message + + def handle_timeout(self, signum, frame): + raise TimeoutError(self.error_message) + + def __enter__(self): + signal.signal(signal.SIGALRM, self.handle_timeout) + signal.alarm(self.seconds) + + def __exit__(self, type, value, traceback): + signal.alarm(0) + + class NumpyEncoder(json.JSONEncoder): """ Special json encoder for numpy types """ @@ -154,20 +171,20 @@ def eval_policy(conf, policy_sys, env, sess, save_eval, log_save_path, single_do if conf['model']['process_num'] == 1: complete_rate, success_rate, success_rate_strict, avg_return, turns, \ avg_actions, task_success, book_acts, inform_acts, request_acts, \ - select_acts, offer_acts = evaluate(sess, + select_acts, offer_acts, recommend_acts = evaluate(sess, num_dialogues=conf['model']['num_eval_dialogues'], sys_semantic_to_usr=conf['model'][ 'sys_semantic_to_usr'], save_flag=save_eval, save_path=log_save_path, goals=goals) - total_acts = book_acts + inform_acts + request_acts + select_acts + offer_acts + total_acts = book_acts + inform_acts + request_acts + select_acts + offer_acts + recommend_acts else: complete_rate, success_rate, success_rate_strict, avg_return, turns, \ avg_actions, task_success, book_acts, inform_acts, request_acts, \ - select_acts, offer_acts = \ + select_acts, offer_acts, recommend_acts = \ evaluate_distributed(sess, list(range(1000, 1000 + conf['model']['num_eval_dialogues'])), conf['model']['process_num'], goals) - total_acts = book_acts + inform_acts + request_acts + select_acts + offer_acts + total_acts = book_acts + inform_acts + request_acts + select_acts + offer_acts + recommend_acts task_success_gathered = {} for task_dict in task_success: @@ -182,7 +199,7 @@ def eval_policy(conf, policy_sys, env, sess, save_eval, log_save_path, single_do f"Average Return: {avg_return}, Turns: {turns}, Average Actions: {avg_actions}, " f"Book Actions: {book_acts/total_acts}, Inform Actions: {inform_acts/total_acts}, " f"Request Actions: {request_acts/total_acts}, Select Actions: {select_acts/total_acts}, " - f"Offer Actions: {offer_acts/total_acts}") + f"Offer Actions: {offer_acts/total_acts}, Recommend Actions: {recommend_acts/total_acts}") for key in task_success: logging.info( @@ -294,7 +311,7 @@ def evaluate(sess, num_dialogues=400, sys_semantic_to_usr=False, save_flag=False task_success = {'All_user_sim': [], 'All_evaluator': [], "All_evaluator_strict": [], 'total_return': [], 'turns': [], 'avg_actions': [], 'total_booking_acts': [], 'total_inform_acts': [], 'total_request_acts': [], - 'total_select_acts': [], 'total_offer_acts': []} + 'total_select_acts': [], 'total_offer_acts': [], 'total_recommend_acts': []} dial_count = 0 for seed in range(1000, 1000 + num_dialogues): set_seed(seed) @@ -310,6 +327,7 @@ def evaluate(sess, num_dialogues=400, sys_semantic_to_usr=False, save_flag=False request = 0 select = 0 offer = 0 + recommend = 0 # this 40 represents the max turn of dialogue for i in range(40): sys_response, user_response, session_over, reward = sess.next_turn( @@ -332,6 +350,8 @@ def evaluate(sess, num_dialogues=400, sys_semantic_to_usr=False, save_flag=False select += 1 if intent.lower() == 'offerbook': offer += 1 + if intent.lower() == 'recommend': + recommend += 1 avg_actions += len(acts) turn_counter += 1 turns += 1 @@ -368,6 +388,8 @@ def evaluate(sess, num_dialogues=400, sys_semantic_to_usr=False, save_flag=False task_success['total_request_acts'].append(request) task_success['total_select_acts'].append(select) task_success['total_offer_acts'].append(offer) + task_success['total_offer_acts'].append(offer) + task_success['total_recommend_acts'].append(recommend) # print(agent_sys.agent_saves) eval_save['Conversation {}'.format(str(dial_count))] = [ @@ -388,7 +410,7 @@ def evaluate(sess, num_dialogues=400, sys_semantic_to_usr=False, save_flag=False np.average(task_success['turns']), np.average(task_success['avg_actions']), task_success, \ np.average(task_success['total_booking_acts']), np.average(task_success['total_inform_acts']), \ np.average(task_success['total_request_acts']), np.average(task_success['total_select_acts']), \ - np.average(task_success['total_offer_acts']) + np.average(task_success['total_offer_acts']), np.average(task_success['total_recommend_acts']) def model_downloader(download_dir, model_path):