Commit fb16cd9c authored by Carel van Niekerk's avatar Carel van Niekerk
Browse files

Init

parents
This diff is collapsed.
This diff is collapsed.
###############################################################################
# PyDial: Multi-domain Statistical Spoken Dialogue System Software
###############################################################################
#
# Copyright 2015 - 2017
# Cambridge University Engineering Department Dialogue Systems Group
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
###############################################################################
# Makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
PAPER =
BUILDDIR = _build
# User-friendly check for sphinx-build
ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
endif
# Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter
ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
# the i18n builder cannot share the environment and doctrees with the others
I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " html to make standalone HTML files"
@echo " dirhtml to make HTML files named index.html in directories"
@echo " singlehtml to make a single large HTML file"
@echo " pickle to make pickle files"
@echo " json to make JSON files"
@echo " htmlhelp to make HTML files and a HTML help project"
@echo " qthelp to make HTML files and a qthelp project"
@echo " applehelp to make an Apple Help Book"
@echo " devhelp to make HTML files and a Devhelp project"
@echo " epub to make an epub"
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
@echo " latexpdf to make LaTeX files and run them through pdflatex"
@echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
@echo " text to make text files"
@echo " man to make manual pages"
@echo " texinfo to make Texinfo files"
@echo " info to make Texinfo files and run them through makeinfo"
@echo " gettext to make PO message catalogs"
@echo " changes to make an overview of all changed/added/deprecated items"
@echo " xml to make Docutils-native XML files"
@echo " pseudoxml to make pseudoxml-XML files for display purposes"
@echo " linkcheck to check all external links for integrity"
@echo " doctest to run all doctests embedded in the documentation (if enabled)"
@echo " coverage to run coverage check of the documentation (if enabled)"
clean:
rm -rf $(BUILDDIR)/*
html:
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
dirhtml:
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
singlehtml:
$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
@echo
@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
pickle:
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
@echo
@echo "Build finished; now you can process the pickle files."
json:
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
@echo
@echo "Build finished; now you can process the JSON files."
htmlhelp:
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
@echo
@echo "Build finished; now you can run HTML Help Workshop with the" \
".hhp project file in $(BUILDDIR)/htmlhelp."
qthelp:
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
@echo
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
".qhcp project file in $(BUILDDIR)/qthelp, like this:"
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Python_DMan.qhcp"
@echo "To view the help file:"
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Python_DMan.qhc"
applehelp:
$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
@echo
@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
@echo "N.B. You won't be able to view it unless you put it in" \
"~/Library/Documentation/Help or install it in your application" \
"bundle."
devhelp:
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
@echo
@echo "Build finished."
@echo "To view the help file:"
@echo "# mkdir -p $$HOME/.local/share/devhelp/Python_DMan"
@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Python_DMan"
@echo "# devhelp"
epub:
$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
@echo
@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
latex:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
@echo "Run \`make' in that directory to run these through (pdf)latex" \
"(use \`make latexpdf' here to do that automatically)."
latexpdf:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through pdflatex..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
latexpdfja:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through platex and dvipdfmx..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
text:
$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
@echo
@echo "Build finished. The text files are in $(BUILDDIR)/text."
man:
$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
@echo
@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
texinfo:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo
@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
@echo "Run \`make' in that directory to run these through makeinfo" \
"(use \`make info' here to do that automatically)."
info:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo "Running Texinfo files through makeinfo..."
make -C $(BUILDDIR)/texinfo info
@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
gettext:
$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
@echo
@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
changes:
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
@echo
@echo "The overview file is in $(BUILDDIR)/changes."
linkcheck:
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
@echo
@echo "Link check complete; look for any errors in the above output " \
"or in $(BUILDDIR)/linkcheck/output.txt."
doctest:
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
@echo "Testing of doctests in the sources finished, look at the " \
"results in $(BUILDDIR)/doctest/output.txt."
coverage:
$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
@echo "Testing of coverage in the sources finished, look at the " \
"results in $(BUILDDIR)/coverage/python.txt."
xml:
$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
@echo
@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
pseudoxml:
$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
@echo
@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
###############################################################################
# PyDial: Multi-domain Statistical Spoken Dialogue System Software
###############################################################################
#
# Copyright 2015 - 2019
# Cambridge University Engineering Department Dialogue Systems Group
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
###############################################################################
'''
Simulate.py - semantic level user simulator system.
====================================================
Copyright CUED Dialogue Systems Group 2015 - 2017
**Basic Execution**:
>>> python Simulate.py [-h] -C CONFIG [-n -r -l -t -g -s]
Optional arguments/flags [default values]::
-n Number of dialogs [1]
-r semantic error rate [0]
-s set random seed
-g generate text prompts
-h help
**Relevant Config variables** [Default values]::
[simulate]
maxturns = 30
continuewhensuccessful = False
forcenullpositive = False
confscorer = additive
.. seealso:: CUED Imports/Dependencies:
import :mod:`utils.ContextLogger` |.|
import :mod:`utils.Settings` |.|
import :mod:`usersimulator.SimulatedUsersManager` |.|
import :mod:`ontology.FlatOntology` |.|
import :mod:`Agent` |.|
************************
'''
import os
import argparse
import Agent
from usersimulator import SimulatedUsersManager
from utils import Settings
from utils import ContextLogger
from ontology import Ontology
logger = ContextLogger.getLogger('')
__author__ = "cued_dialogue_systems_group"
__version__ = Settings.__version__
class SimulationSystem(object):
'''
Semantic level simulated dialog system
'''
def __init__(self, error_rate):
'''
:param error_rate: error rate of the simulated environment
:type error_rate: float
'''
# Dialogue Agent Factory:
#-----------------------------------------
self.agent_factory = Agent.AgentFactory(hub_id='simulate')
# NOTE - using agent factory here rather than just an agent - since for simulate I can easily envisage wanting to
# have multiple agents and looking at combining their policies etc... This is not being used now though; will just use
# a single agent in here at present.
# Simulated User.
#-----------------------------------------
self.simulator = SimulatedUsersManager.SimulatedUsersManager(error_rate)
self.traceDialog = 2
self.sim_level = 'dial_act'
self.text_sampling = 'dict'
if Settings.config.has_option("GENERAL", "tracedialog"):
self.traceDialog = Settings.config.getint("GENERAL", "tracedialog")
if Settings.config.has_option("usermodel", "simlevel"):
self.sim_level = Settings.config.get("usermodel", "simlevel")
if Settings.config.has_option("usermodel", "textsampling"):
self.text_sampling = Settings.config.get("usermodel", "textsampling")
if self.sim_level == 'text':
#Load the text generator
if self.text_sampling == 'dict':
sampling_dict = os.path.join(Settings.root, 'usersimulator/textgenerator/textgen_dict.pkl')
else:
sampling_dict = None
import usersimulator.textgenerator.textgen_toolkit.SCTranslate as SCT
self.SCT = SCT.SCTranslate(sampling_dict=sampling_dict)
elif self.sim_level == 'sys2text':
pass #load here florians model
def run_dialogs(self, numDialogs):
'''
run a loop over the run() method for the given number of dialogues.
:param numDialogs: number of dialogues to loop over.
:type numDialogs: int
:return: None
'''
for i in range(numDialogs):
logger.info('Dialogue %d' % (i+1))
self.run(session_id='simulate_dialog'+str(i), sim_level=self.sim_level)
self.agent_factory.power_down_factory() # Important! -uses FORCE_SAVE on policy- which will finalise learning and save policy.
def run(self, session_id, agent_id='Smith', sim_level='dial_act'):
'''
Runs one episode through the simulator
:param session_id: session id
:type session_id: int
:param agent_id: agent id, default = 'Smith'
:type agent_id: string
:return: None
'''
# RESET THE USER SIMULATOR:
self.simulator.restart()
for domain in self.simulator.simUserManagers:
if self.simulator.simUserManagers[domain] and self.sim_level != 'sys2text':
goal = self.simulator.simUserManagers[domain].um.goal
logger.dial('User will execute the following goal: {}'
.format(str(goal.request_type) + str(goal.constraints) + str([req for req in goal.requests])))
user_act = ''
endingDialogue = False
# SYSTEM STARTS THE CALL:
sys_act = self.agent_factory.agents[agent_id].start_call(session_id,
domainSimulatedUsers=self.simulator.simUserManagers,
maxNumTurnsScaling=self.simulator.number_domains_this_dialog)
prompt_str = sys_act.prompt
if prompt_str is not None: # if we are generating text, versus remaining only at semantic level.
if self.traceDialog > 1: print(' Prompt >', prompt_str)
logger.info('| Prompt > '+ prompt_str)
# LOOP OVER TURNS:
while not endingDialogue:
# USER ACT:
#-------------------------------------------------------------------------------------------------------------
sys_act = self.agent_factory.agents[agent_id].retrieve_last_sys_act()
if sim_level == 'sys2text':
text_user_act, user_actsDomain, _ = self.simulator.act_on(sys_act)
#user_actsDomain = 'CamRestaurants'
hyps = [(text_user_act, 1.0)]
else:
user_act, user_actsDomain, hyps = self.simulator.act_on(sys_act)
if sim_level == 'text':
#todo: convert dialact to text
#text_user_act = raw_input('Translate user act: {} > '.format(user_act))
text_user_act = self.SCT.translateUserAct(str(user_act),1)[2]
try:
text_user_act = text_user_act[0]
except:
logger.error('Wrong user act: ' + user_act, text_user_act)
hyps = [(text_user_act, 1.0)]
#actually also output user_actsDomain (the TRUE DOMAIN) here too - which can be used to avoid doing topic tracking
if self.traceDialog>1:
print(' User >', user_act)
if self.sim_level != 'sys2text':
logger.dial('| User > ' + user_act.to_string())
else:
logger.dial('| User > ' + text_user_act)
# SYSTEM ACT:
#-------------------------------------------------------------------------------------------------------------
sys_act = self.agent_factory.agents[agent_id].continue_call(asr_info = hyps,
domainString=user_actsDomain,
domainSimulatedUsers=self.simulator.simUserManagers)
prompt_str = sys_act.prompt
if prompt_str is not None: # if we are generating text, versus remaining only at semantic level.
if self.traceDialog>1: print(' Prompt >', prompt_str)
logger.info('| Prompt > ' + prompt_str)
if self.sim_level != 'sys2text':
if 'bye' == user_act.act or 'bye' == sys_act.act:
endingDialogue = True
else:
if 'bye' in text_user_act or 'bye' == sys_act.act:
endingDialogue = True
# Process ends.
for domain in self.simulator.simUserManagers:
if self.simulator.simUserManagers[domain]:
if self.sim_level != 'sys2text':
goal = self.simulator.simUserManagers[domain].um.goal
logger.dial('User goal at the end of the dialogue: {}'
.format(str(goal.request_type) + str(goal.constraints) + str([req for req in goal.requests])))
self.agent_factory.agents[agent_id].end_call(domainSimulatedUsers=self.simulator.simUserManagers)
return
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Simulate')
parser.add_argument('-C', '-c', '--config', help='set config file', required=True, type=argparse.FileType('r'))
parser.add_argument('-n', '--number', help='set the number of dialogues', type=int)
parser.add_argument('-r', '--error', help='set error rate', type=int)
parser.set_defaults(use_color=True)
parser.add_argument('--nocolor', dest='use_color',action='store_false', help='no color in logging. best to\
turn off if dumping to file. Will be overriden by [logging] config setting of "usecolor=".')
parser.add_argument('-s', '--seed', help='set random seed', type=int)
args = parser.parse_args()
if args.error is None:
args.error = 0 # default simulated error rate
if args.number is None:
args.number = 1 # default number of dialogs
seed = Settings.init(config_file=args.config.name,seed=args.seed)
ContextLogger.createLoggingHandlers(config=Settings.config, use_color=args.use_color)
logger.info("Random Seed is {}".format(seed))
Ontology.init_global_ontology()
simulator = SimulationSystem(error_rate=float(args.error)/100)
simulator.run_dialogs(args.number)
#END OF FILE
###############################################################################
# PyDial: Multi-domain Statistical Spoken Dialogue System Software
###############################################################################
#
# Copyright 2015 - 2019
# Cambridge University Engineering Department Dialogue Systems Group
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
###############################################################################
'''
Texthub.py - text level dialog system.
====================================================
Copyright CUED Dialogue Systems Group 2015 - 2017
**Basic Execution**:
>>> python texthub.py [-h] -C CONFIG [-l -r]
Optional arguments/flags [default values]::
-r semantic error rate [0]
-l set the system to use the given policy file
-h help
**Relevant config variables** (values are defaults)::
[semi_DOMAIN]
semitype = PassthroughSemI
[semo_DOMAIN]
semotype = PassthroughSemO
.. seealso:: CUED Imports/Dependencies:
import :mod:`utils.ContextLogger` |.|
import :mod:`utils.Settings` |.|
import :class:`Agent.DialogueAgent` |.|
import :mod:`ontology.Ontology`
************************
'''
import argparse, re
from Agent import DialogueAgent
from utils import ContextLogger
from utils import Settings
from ontology import Ontology
logger = ContextLogger.getLogger('')
__author__ = "cued_dialogue_systems_group"
__version__ = Settings.__version__
class ConsoleHub(object):
'''
text based dialog system
'''
def __init__(self):
# Dialogue Agent:
#-----------------------------------------
self.agent = DialogueAgent(hub_id='texthub')
def run(self):
'''
Runs one episode through Hub
:returns: None
'''
logger.warning("NOTE: texthub is not using any error simulation at present.")
sys_act = self.agent.start_call(session_id='texthub_dialog')
if isinstance(sys_act.prompt, str):
print('Prompt > ' + sys_act.prompt)
while not self.agent.ENDING_DIALOG:
# USER ACT:
obs = input('User > ')
'''
# Confused user act.
# lastHyps = self.errorSimulator.confuseAct(lastUserAct)
# print 'lastHyps >', lastHyps
# nullProb = 0.0
# for (act, prob) in lastHyps:
# if act == 'null()':
# nullProb += prob
# print 'Semi >', act, '['+str(prob)+']'
# if self.forceNullPositive and nullProb < 0.001:
# lastHyps.append(('null()',0.001))
# print 'Semi > null() [0.001]'
#--------------------------------
'''
domain = None
if "domain(" in obs:
match = re.search("(.*)(domain\()([^\)]+)(\))(.*)",obs)
if match is not None:
domain = match.group(3)
obs = match.group(1) + match.group(5)
# SYSTEM ACT:
sys_act = self.agent.continue_call(asr_info = [(obs,1.0)], domainString = domain)
if(isinstance(sys_act.prompt,str)):
print('Prompt > ' + sys_act.prompt)
# Process ends. -----------------------------------------------------
# NB: Can add evaluation things here - possibly useful to check things by hand with texthub ...
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='TextHub')
parser.add_argument('-C','-c', '--config', help='set config file', required=True, type=argparse.FileType('r'))
parser.add_argument('-s', '--seed', help='set random seed', type=int)
parser.set_defaults(use_color=True)
parser.add_argument('--nocolor', dest='use_color',action='store_false', help='no color in logging. best to\
turn off if dumping to file. Will be overriden by [logging] config setting of "usecolor=".')
args = parser.parse_args()
seed = Settings.init(config_file=args.config.name,seed=args.seed)
ContextLogger.createLoggingHandlers(config=Settings.config, use_color=args.use_color)
logger.info("Random Seed is {}".format(seed))
Ontology.init_global_ontology()
hub = ConsoleHub()
hub.run()
#END OF FILE
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Quick description of how to use the deep RL policies in PyDial\n",
"\n",
"Note that this text is not in the quality of a full tutorial nor is it meant to be. Its purpose is merely to give guidance on how to use the deep RL policies which are part of the new PyDial release.\n"