Commit fb16cd9c authored by Carel van Niekerk's avatar Carel van Niekerk
Browse files

Init

parents
This diff is collapsed.
This diff is collapsed.
###############################################################################
# PyDial: Multi-domain Statistical Spoken Dialogue System Software
###############################################################################
#
# Copyright 2015 - 2017
# Cambridge University Engineering Department Dialogue Systems Group
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
###############################################################################
# Makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
PAPER =
BUILDDIR = _build
# User-friendly check for sphinx-build
ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
endif
# Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter
ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
# the i18n builder cannot share the environment and doctrees with the others
I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " html to make standalone HTML files"
@echo " dirhtml to make HTML files named index.html in directories"
@echo " singlehtml to make a single large HTML file"
@echo " pickle to make pickle files"
@echo " json to make JSON files"
@echo " htmlhelp to make HTML files and a HTML help project"
@echo " qthelp to make HTML files and a qthelp project"
@echo " applehelp to make an Apple Help Book"
@echo " devhelp to make HTML files and a Devhelp project"
@echo " epub to make an epub"
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
@echo " latexpdf to make LaTeX files and run them through pdflatex"
@echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
@echo " text to make text files"
@echo " man to make manual pages"
@echo " texinfo to make Texinfo files"
@echo " info to make Texinfo files and run them through makeinfo"
@echo " gettext to make PO message catalogs"
@echo " changes to make an overview of all changed/added/deprecated items"
@echo " xml to make Docutils-native XML files"
@echo " pseudoxml to make pseudoxml-XML files for display purposes"
@echo " linkcheck to check all external links for integrity"
@echo " doctest to run all doctests embedded in the documentation (if enabled)"
@echo " coverage to run coverage check of the documentation (if enabled)"
clean:
rm -rf $(BUILDDIR)/*
html:
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
dirhtml:
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
singlehtml:
$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
@echo
@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
pickle:
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
@echo
@echo "Build finished; now you can process the pickle files."
json:
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
@echo
@echo "Build finished; now you can process the JSON files."
htmlhelp:
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
@echo
@echo "Build finished; now you can run HTML Help Workshop with the" \
".hhp project file in $(BUILDDIR)/htmlhelp."
qthelp:
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
@echo
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
".qhcp project file in $(BUILDDIR)/qthelp, like this:"
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Python_DMan.qhcp"
@echo "To view the help file:"
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Python_DMan.qhc"
applehelp:
$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
@echo
@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
@echo "N.B. You won't be able to view it unless you put it in" \
"~/Library/Documentation/Help or install it in your application" \
"bundle."
devhelp:
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
@echo
@echo "Build finished."
@echo "To view the help file:"
@echo "# mkdir -p $$HOME/.local/share/devhelp/Python_DMan"
@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Python_DMan"
@echo "# devhelp"
epub:
$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
@echo
@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
latex:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
@echo "Run \`make' in that directory to run these through (pdf)latex" \
"(use \`make latexpdf' here to do that automatically)."
latexpdf:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through pdflatex..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
latexpdfja:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through platex and dvipdfmx..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
text:
$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
@echo
@echo "Build finished. The text files are in $(BUILDDIR)/text."
man:
$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
@echo
@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
texinfo:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo
@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
@echo "Run \`make' in that directory to run these through makeinfo" \
"(use \`make info' here to do that automatically)."
info:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo "Running Texinfo files through makeinfo..."
make -C $(BUILDDIR)/texinfo info
@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
gettext:
$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
@echo
@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
changes:
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
@echo
@echo "The overview file is in $(BUILDDIR)/changes."
linkcheck:
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
@echo
@echo "Link check complete; look for any errors in the above output " \
"or in $(BUILDDIR)/linkcheck/output.txt."
doctest:
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
@echo "Testing of doctests in the sources finished, look at the " \
"results in $(BUILDDIR)/doctest/output.txt."
coverage:
$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
@echo "Testing of coverage in the sources finished, look at the " \
"results in $(BUILDDIR)/coverage/python.txt."
xml:
$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
@echo
@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
pseudoxml:
$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
@echo
@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."
###############################################################################
# PyDial: Multi-domain Statistical Spoken Dialogue System Software
###############################################################################
#
# Copyright 2015 - 2019
# Cambridge University Engineering Department Dialogue Systems Group
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
###############################################################################
'''
Simulate.py - semantic level user simulator system.
====================================================
Copyright CUED Dialogue Systems Group 2015 - 2017
**Basic Execution**:
>>> python Simulate.py [-h] -C CONFIG [-n -r -l -t -g -s]
Optional arguments/flags [default values]::
-n Number of dialogs [1]
-r semantic error rate [0]
-s set random seed
-g generate text prompts
-h help
**Relevant Config variables** [Default values]::
[simulate]
maxturns = 30
continuewhensuccessful = False
forcenullpositive = False
confscorer = additive
.. seealso:: CUED Imports/Dependencies:
import :mod:`utils.ContextLogger` |.|
import :mod:`utils.Settings` |.|
import :mod:`usersimulator.SimulatedUsersManager` |.|
import :mod:`ontology.FlatOntology` |.|
import :mod:`Agent` |.|
************************
'''
import os
import argparse
import Agent
from usersimulator import SimulatedUsersManager
from utils import Settings
from utils import ContextLogger
from ontology import Ontology
logger = ContextLogger.getLogger('')
__author__ = "cued_dialogue_systems_group"
__version__ = Settings.__version__
class SimulationSystem(object):
'''
Semantic level simulated dialog system
'''
def __init__(self, error_rate):
'''
:param error_rate: error rate of the simulated environment
:type error_rate: float
'''
# Dialogue Agent Factory:
#-----------------------------------------
self.agent_factory = Agent.AgentFactory(hub_id='simulate')
# NOTE - using agent factory here rather than just an agent - since for simulate I can easily envisage wanting to
# have multiple agents and looking at combining their policies etc... This is not being used now though; will just use
# a single agent in here at present.
# Simulated User.
#-----------------------------------------
self.simulator = SimulatedUsersManager.SimulatedUsersManager(error_rate)
self.traceDialog = 2
self.sim_level = 'dial_act'
self.text_sampling = 'dict'
if Settings.config.has_option("GENERAL", "tracedialog"):
self.traceDialog = Settings.config.getint("GENERAL", "tracedialog")
if Settings.config.has_option("usermodel", "simlevel"):
self.sim_level = Settings.config.get("usermodel", "simlevel")
if Settings.config.has_option("usermodel", "textsampling"):
self.text_sampling = Settings.config.get("usermodel", "textsampling")
if self.sim_level == 'text':
#Load the text generator
if self.text_sampling == 'dict':
sampling_dict = os.path.join(Settings.root, 'usersimulator/textgenerator/textgen_dict.pkl')
else:
sampling_dict = None
import usersimulator.textgenerator.textgen_toolkit.SCTranslate as SCT
self.SCT = SCT.SCTranslate(sampling_dict=sampling_dict)
elif self.sim_level == 'sys2text':
pass #load here florians model
def run_dialogs(self, numDialogs):
'''
run a loop over the run() method for the given number of dialogues.
:param numDialogs: number of dialogues to loop over.
:type numDialogs: int
:return: None
'''
for i in range(numDialogs):
logger.info('Dialogue %d' % (i+1))
self.run(session_id='simulate_dialog'+str(i), sim_level=self.sim_level)
self.agent_factory.power_down_factory() # Important! -uses FORCE_SAVE on policy- which will finalise learning and save policy.
def run(self, session_id, agent_id='Smith', sim_level='dial_act'):
'''
Runs one episode through the simulator
:param session_id: session id
:type session_id: int
:param agent_id: agent id, default = 'Smith'
:type agent_id: string
:return: None
'''
# RESET THE USER SIMULATOR:
self.simulator.restart()
for domain in self.simulator.simUserManagers:
if self.simulator.simUserManagers[domain] and self.sim_level != 'sys2text':
goal = self.simulator.simUserManagers[domain].um.goal
logger.dial('User will execute the following goal: {}'
.format(str(goal.request_type) + str(goal.constraints) + str([req for req in goal.requests])))
user_act = ''
endingDialogue = False
# SYSTEM STARTS THE CALL:
sys_act = self.agent_factory.agents[agent_id].start_call(session_id,
domainSimulatedUsers=self.simulator.simUserManagers,
maxNumTurnsScaling=self.simulator.number_domains_this_dialog)
prompt_str = sys_act.prompt
if prompt_str is not None: # if we are generating text, versus remaining only at semantic level.
if self.traceDialog > 1: print(' Prompt >', prompt_str)
logger.info('| Prompt > '+ prompt_str)
# LOOP OVER TURNS:
while not endingDialogue:
# USER ACT:
#-------------------------------------------------------------------------------------------------------------
sys_act = self.agent_factory.agents[agent_id].retrieve_last_sys_act()
if sim_level == 'sys2text':
text_user_act, user_actsDomain, _ = self.simulator.act_on(sys_act)
#user_actsDomain = 'CamRestaurants'
hyps = [(text_user_act, 1.0)]
else:
user_act, user_actsDomain, hyps = self.simulator.act_on(sys_act)
if sim_level == 'text':
#todo: convert dialact to text
#text_user_act = raw_input('Translate user act: {} > '.format(user_act))
text_user_act = self.SCT.translateUserAct(str(user_act),1)[2]
try:
text_user_act = text_user_act[0]
except:
logger.error('Wrong user act: ' + user_act, text_user_act)
hyps = [(text_user_act, 1.0)]
#actually also output user_actsDomain (the TRUE DOMAIN) here too - which can be used to avoid doing topic tracking
if self.traceDialog>1:
print(' User >', user_act)
if self.sim_level != 'sys2text':
logger.dial('| User > ' + user_act.to_string())
else:
logger.dial('| User > ' + text_user_act)
# SYSTEM ACT:
#-------------------------------------------------------------------------------------------------------------
sys_act = self.agent_factory.agents[agent_id].continue_call(asr_info = hyps,
domainString=user_actsDomain,
domainSimulatedUsers=self.simulator.simUserManagers)
prompt_str = sys_act.prompt
if prompt_str is not None: # if we are generating text, versus remaining only at semantic level.
if self.traceDialog>1: print(' Prompt >', prompt_str)
logger.info('| Prompt > ' + prompt_str)
if self.sim_level != 'sys2text':
if 'bye' == user_act.act or 'bye' == sys_act.act:
endingDialogue = True
else:
if 'bye' in text_user_act or 'bye' == sys_act.act:
endingDialogue = True
# Process ends.
for domain in self.simulator.simUserManagers:
if self.simulator.simUserManagers[domain]:
if self.sim_level != 'sys2text':
goal = self.simulator.simUserManagers[domain].um.goal
logger.dial('User goal at the end of the dialogue: {}'
.format(str(goal.request_type) + str(goal.constraints) + str([req for req in goal.requests])))
self.agent_factory.agents[agent_id].end_call(domainSimulatedUsers=self.simulator.simUserManagers)
return
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Simulate')
parser.add_argument('-C', '-c', '--config', help='set config file', required=True, type=argparse.FileType('r'))
parser.add_argument('-n', '--number', help='set the number of dialogues', type=int)
parser.add_argument('-r', '--error', help='set error rate', type=int)
parser.set_defaults(use_color=True)
parser.add_argument('--nocolor', dest='use_color',action='store_false', help='no color in logging. best to\
turn off if dumping to file. Will be overriden by [logging] config setting of "usecolor=".')
parser.add_argument('-s', '--seed', help='set random seed', type=int)
args = parser.parse_args()
if args.error is None:
args.error = 0 # default simulated error rate
if args.number is None:
args.number = 1 # default number of dialogs
seed = Settings.init(config_file=args.config.name,seed=args.seed)
ContextLogger.createLoggingHandlers(config=Settings.config, use_color=args.use_color)
logger.info("Random Seed is {}".format(seed))
Ontology.init_global_ontology()
simulator = SimulationSystem(error_rate=float(args.error)/100)
simulator.run_dialogs(args.number)
#END OF FILE
###############################################################################
# PyDial: Multi-domain Statistical Spoken Dialogue System Software
###############################################################################
#
# Copyright 2015 - 2019
# Cambridge University Engineering Department Dialogue Systems Group
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
###############################################################################
'''
Texthub.py - text level dialog system.
====================================================
Copyright CUED Dialogue Systems Group 2015 - 2017
**Basic Execution**:
>>> python texthub.py [-h] -C CONFIG [-l -r]
Optional arguments/flags [default values]::
-r semantic error rate [0]
-l set the system to use the given policy file
-h help
**Relevant config variables** (values are defaults)::
[semi_DOMAIN]
semitype = PassthroughSemI
[semo_DOMAIN]
semotype = PassthroughSemO
.. seealso:: CUED Imports/Dependencies:
import :mod:`utils.ContextLogger` |.|
import :mod:`utils.Settings` |.|
import :class:`Agent.DialogueAgent` |.|
import :mod:`ontology.Ontology`
************************
'''
import argparse, re
from Agent import DialogueAgent
from utils import ContextLogger
from utils import Settings
from ontology import Ontology
logger = ContextLogger.getLogger('')
__author__ = "cued_dialogue_systems_group"
__version__ = Settings.__version__
class ConsoleHub(object):
'''
text based dialog system
'''
def __init__(self):
# Dialogue Agent:
#-----------------------------------------
self.agent = DialogueAgent(hub_id='texthub')
def run(self):
'''
Runs one episode through Hub
:returns: None
'''
logger.warning("NOTE: texthub is not using any error simulation at present.")
sys_act = self.agent.start_call(session_id='texthub_dialog')
if isinstance(sys_act.prompt, str):
print('Prompt > ' + sys_act.prompt)
while not self.agent.ENDING_DIALOG:
# USER ACT:
obs = input('User > ')
'''
# Confused user act.
# lastHyps = self.errorSimulator.confuseAct(lastUserAct)
# print 'lastHyps >', lastHyps
# nullProb = 0.0
# for (act, prob) in lastHyps:
# if act == 'null()':
# nullProb += prob
# print 'Semi >', act, '['+str(prob)+']'
# if self.forceNullPositive and nullProb < 0.001:
# lastHyps.append(('null()',0.001))
# print 'Semi > null() [0.001]'
#--------------------------------
'''
domain = None
if "domain(" in obs:
match = re.search("(.*)(domain\()([^\)]+)(\))(.*)",obs)
if match is not None:
domain = match.group(3)
obs = match.group(1) + match.group(5)
# SYSTEM ACT:
sys_act = self.agent.continue_call(asr_info = [(obs,1.0)], domainString = domain)
if(isinstance(sys_act.prompt,str)):
print('Prompt > ' + sys_act.prompt)
# Process ends. -----------------------------------------------------
# NB: Can add evaluation things here - possibly useful to check things by hand with texthub ...
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='TextHub')
parser.add_argument('-C','-c', '--config', help='set config file', required=True, type=argparse.FileType('r'))
parser.add_argument('-s', '--seed', help='set random seed', type=int)
parser.set_defaults(use_color=True)
parser.add_argument('--nocolor', dest='use_color',action='store_false', help='no color in logging. best to\
turn off if dumping to file. Will be overriden by [logging] config setting of "usecolor=".')
args = parser.parse_args()
seed = Settings.init(config_file=args.config.name,seed=args.seed)
ContextLogger.createLoggingHandlers(config=Settings.config, use_color=args.use_color)
logger.info("Random Seed is {}".format(seed))
Ontology.init_global_ontology()
hub = ConsoleHub()
hub.run()
#END OF FILE
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Quick description of how to use the deep RL policies in PyDial\n",