From 6684b67ac9d05d3b5452484d7445ba455a90d5fe Mon Sep 17 00:00:00 2001
From: Jinchao Li <38700695+jincli@users.noreply.github.com>
Date: Sat, 26 Sep 2020 18:57:02 -0700
Subject: [PATCH] Human (#131)
* change task config
* add final goal logging
* encapsule PipelineAgent internal state interface for return and
replacement
---
convlab2/dialog_agent/agent.py | 20 +++++++++++
convlab2/human_eval/run.py | 2 +-
convlab2/human_eval/run_agent.py | 19 ++++++----
convlab2/human_eval/task_config.py | 58 ++++++++++++++++++++++--------
convlab2/human_eval/worlds.py | 5 ++-
5 files changed, 79 insertions(+), 25 deletions(-)
diff --git a/convlab2/dialog_agent/agent.py b/convlab2/dialog_agent/agent.py
index adb9d33..b408303 100755
--- a/convlab2/dialog_agent/agent.py
+++ b/convlab2/dialog_agent/agent.py
@@ -91,6 +91,26 @@ class PipelineAgent(Agent):
self.init_session()
self.history = []
+ def state_replace(self, agent_state):
+ """
+ this interface is reserved to replace all interal states of agent
+ the code snippet example below is for the scenario when the agent state only depends on self.history and self.dst.state
+ """
+ self.history = deepcopy(agent_state['history'])
+ self.dst.state = deepcopy(agent_state['dst_state'])
+
+ def state_return(self):
+ """
+ this interface is reserved to return all interal states of agent
+ the code snippet example below is for the scenario when the agent state only depends on self.history and self.dst.state
+ """
+ agent_state = {}
+ agent_state['history'] = deepcopy(self.history)
+ agent_state['dst_state'] = deepcopy(self.dst.state)
+
+ return agent_state
+
+
def response(self, observation):
"""Generate agent response using the agent modules."""
# Note: If you modify the logic of this function, please ensure that it is consistent with deploy.server.ServerCtrl._turn()
diff --git a/convlab2/human_eval/run.py b/convlab2/human_eval/run.py
index c556cc9..d594534 100755
--- a/convlab2/human_eval/run.py
+++ b/convlab2/human_eval/run.py
@@ -23,7 +23,7 @@ MASTER_QUALIF_SDBOX = {
LOCALE_QUALIF_SDBOX = {
'QualificationTypeId': '00000000000000000071',
"Comparator": "In",
- 'LocaleValues': [{'Country': "HK"}, {'Country': "US"}, {'Country': "CN"}]
+ 'LocaleValues': [{'Country': "US"}, {'Country': 'HK'}, {'Country': 'IN'} ]
}
diff --git a/convlab2/human_eval/run_agent.py b/convlab2/human_eval/run_agent.py
index 0ae7aae..f848674 100755
--- a/convlab2/human_eval/run_agent.py
+++ b/convlab2/human_eval/run_agent.py
@@ -11,6 +11,7 @@ from threading import Thread
# Agent
from convlab2.dialog_agent import PipelineAgent, BiSession
from convlab2.nlu.milu.multiwoz import MILU
+from convlab2.nlu.jointBERT.multiwoz import BERTNLU
from convlab2.dst.rule.multiwoz import RuleDST
from convlab2.policy.rule.multiwoz import RulePolicy
from convlab2.nlg.template.multiwoz import TemplateNLG
@@ -24,6 +25,7 @@ rgo_queue = PriorityQueue(maxsize=0)
app = Flask(__name__)
+# sys_nlu = BERTNLU()
sys_nlu = MILU()
sys_dst = RuleDST()
sys_policy = RulePolicy(character='sys')
@@ -53,19 +55,21 @@ def process():
def generate_response(in_queue, out_queue):
while True:
# pop input
- last_action = 'null'
+ # last_action = 'null'
in_request = in_queue.get()
obs = in_request['input']
if in_request['agent_state'] == {}:
agent.init_session()
else:
- encoded_state, dst_state, last_action = in_request['agent_state']
- agent.dst.state = copy.deepcopy(dst_state)
+ # encoded_state, dst_state, last_action = in_request['agent_state']
+ # agent.dst.state = copy.deepcopy(dst_state)
+ agent.state_replace(in_request['agent_state'])
try:
action = agent.response(obs)
print(f'obs:{obs}; action:{action}')
- dst_state = copy.deepcopy(agent.dst.state)
- encoded_state = None
+ # dst_state = copy.deepcopy(agent.dst.state)
+ # encoded_state = None
+ in_request['agent_state'] = agent.state_return()
except Exception as e:
print('agent error', e)
@@ -78,8 +82,9 @@ def generate_response(in_queue, out_queue):
print('Response generation error', e)
response = 'What did you say?'
- last_action = action
- out_queue.put({'response': response, 'agent_state': (encoded_state, dst_state, last_action)})
+ # last_action = action
+ # out_queue.put({'response': response, 'agent_state': (encoded_state, dst_state, last_action)})
+ out_queue.put({'response': response, 'agent_state': in_request['agent_state']})
in_queue.task_done()
out_queue.join()
diff --git a/convlab2/human_eval/task_config.py b/convlab2/human_eval/task_config.py
index aee64f5..82f226c 100755
--- a/convlab2/human_eval/task_config.py
+++ b/convlab2/human_eval/task_config.py
@@ -7,18 +7,18 @@ task_config = {}
On the Amazon Mechanical Turk web site, the HIT title appears in search results,
and everywhere the HIT is mentioned.
"""
-task_config['hit_title'] = 'Chat and evaluate bot!'
+task_config['hit_title'] = 'Chat and evaluate bot with a specified goal'
"""A description includes detailed information about the kind of task the HIT contains.
On the Amazon Mechanical Turk web site, the HIT description appears in the expanded
view of search results, and in the HIT and assignment screens.
"""
-task_config['hit_description'] = 'You will chat to a tour information bot and then evaluate that bot.'
+task_config['hit_description'] = 'You will chat to a tour information bot and then evaluate that bot, type "success" or "fail" once finished'
"""One or more words or phrases that describe the HIT, separated by commas.
On MTurk website, these words are used in searches to find HITs.
"""
-task_config['hit_keywords'] = 'chat,dialog'
+task_config['hit_keywords'] = 'chat,dialog, dialogue, evaluation'
"""A detailed task description that will be shown on the HIT task preview page
and on the left side of the chat page. Supports HTML formatting.
@@ -27,26 +27,56 @@ task_config['task_description'] = \
"""
(You can keep accepting new HITs after you finish your current one, so keep working on it if you like the task!)
<br>
-
- <span id="user-goal" style="font-size: 16px;">
- </span>
-
- <br><br>
- Chat with the bot naturally and stick to your own goal but <b>do not trivially copy the goal descriptions into the message.</b>
<br>
- Once the conversation is done, you will be asked to rate the bot on metrics like <b>goal accomplishment, language understanding, and response naturalness</b>.
+ Chat with the bot naturally and stick to your own goal but do not trivially copy the goal descriptions into the message.
+ <br>
+ <br>
+ You can start with the conversation with sentences like <b>'I am looking for / I need to/ etc.'</b> and terminate the dialog session by typing <b> 'success' or 'fail'</b>. Once the conversation is done, you will be asked to rate the bot on metrics like goal accomplishment, language understanding, and response naturalness.
<br>
- There is a <b>2 min</b> time limit for each turn.
+ There is a <b>3 min</b> time limit for each turn.
<br>
<br>
+ In this task you will chat with an information desk clerk bot to plan your tour according to a given goal.
+ <span id="user-goal" style="font-size: 16px;">
+ </span>
+
+ <br>
+ For example, your given goal and expected conversation could be: <br><br>
+ <table border="1" cellpadding="10">
+ <tr><th>Your goal</th><th>Expected conversation</th></tr>
+ <tr><td>
+ <ul>
+ <li>You are looking for a <b>place to stay</b>. The hotel should be in the <b>cheap</b> price range and should be in the type of <b>hotel</b></li>
+ <li>The hotel should include <b>free parking</b> and should include <b>free wifi</b></li>
+ <li>Once you find the hotel, you want to book it for <b>6</b> people and <b>3</b> nights</b> starting from <b>tuesday</b></li>
+ <li>If the booking fails how about <b>2</b> nights</li>
+ <li>Make sure you get the <b>reference number</b></li>
+ </ul>
+ </td>
+ <td>
+ <b>You: </b>I am looking for a place to to stay that has cheap price range it should be in a type of hotel<br>
+ <b>Info desk: </b>Okay, do you have a specific area you want to stay in?<br>
+ <b>You: </b>no, i just need to make sure it's cheap. oh, and i need parking<br>
+ <b>Info desk: </b>I found 1 cheap hotel for you that includes parking. Do you like me to book it?<br>
+ <b>You: </b>Yes, please. 6 people 3 nights starting on tuesday.<br>
+ <b>Info desk: </b>I am sorry but I wasn't able to book that for you for Tuesday. Is there another day you would like to stay or perhaps a shorter stay?<br>
+ <b>You: </b>how about only 2 nights.<br>
+ <b>Info desk: </b>Booking was successful.\nReference number is : 7GAWK763. Anything else I can do for you?<br>
+ <b>You: </b>No, that will be all. Good bye.<br>
+ <b>Info desk: </b>Thank you for using our services.<br>
+ <b>You: </b>Success<br>
+ </td>
+ </table>
+
+
- Do not reference the task or MTurk itself during the conversation.
<br>
<b><span style="color:red">- No racism, sexism or otherwise offensive comments, or the submission will be rejected and we will report to Amazon.</b></span>
<br>
- <br>
-
+
+
<script type="text/javascript">
-
+
function handle_new_message(new_message_id, message) {
var agent_id = message.id;
var message_text = message.text
diff --git a/convlab2/human_eval/worlds.py b/convlab2/human_eval/worlds.py
index 7aaac4c..2158aa8 100755
--- a/convlab2/human_eval/worlds.py
+++ b/convlab2/human_eval/worlds.py
@@ -202,10 +202,10 @@ class MultiWozEvalWorld(MTurkTaskWorld):
def __init__(self, opt, agent,
num_extra_trial=2,
max_turn=50,
- max_resp_time=300,
+ max_resp_time=180,
model_agent_opt=None,
world_tag='',
- agent_timeout_shutdown=300):
+ agent_timeout_shutdown=180):
self.opt = opt
self.agent = agent
self.turn_idx = 1
@@ -422,7 +422,6 @@ class MultiWozEvalWorld(MTurkTaskWorld):
if 'text' in acts[idx] and \
acts[idx]['text'] != '':
self.final_goal[domain]['reqt'][slot] = acts[idx]['text']
- # print(self.final_goal)
# Language Understanding Check
control_msg['text'] = UNDERSTANDING_MSG
--
GitLab