diff --git a/data/unified_datasets/dailydialog/README.md b/data/unified_datasets/dailydialog/README.md
index 57a65ee94fa9dcb7c891e798dbaae28952706dd1..69e627cf2f5fbd7c4f5b10b3450a659afd6822d5 100644
--- a/data/unified_datasets/dailydialog/README.md
+++ b/data/unified_datasets/dailydialog/README.md
@@ -14,7 +14,10 @@ DailyDialog is a high-quality multi-turn dialog dataset. It is intriguing in sev
- Run `python preprocess.py` in the current directory.
- **Main changes of the transformation:**
- Use `topic` annotation as `domain`. If duplicated dialogs are annotated with different topics, use the most frequent one.
- - Combine `intent` and `domain` annotation as `binary` dialogue acts.
+ - Use `intent` annotation as `binary` dialogue act.
+ - Retain emotion annotation in the `emotion` field of each turn.
+ - Use nltk to remove space before punctuation: `utt = ' '.join([detokenizer.detokenize(word_tokenize(s)) for s in sent_tokenize(utt)])`.
+ - Replace `" ’ "` with `"'"`: `utt = utt.replace(' ’ ', "'")`.
- **Annotations:**
- intent, emotion
@@ -30,10 +33,10 @@ English
| split | dialogues | utterances | avg_utt | avg_tokens | avg_domains | cat slot match(state) | cat slot match(goal) | cat slot match(dialogue act) | non-cat slot span(dialogue act) |
|------------|-------------|--------------|-----------|--------------|---------------|-------------------------|------------------------|--------------------------------|-----------------------------------|
-| train | 11118 | 87170 | 7.84 | 13.61 | 1 | - | - | - | - |
-| validation | 1000 | 8069 | 8.07 | 13.5 | 1 | - | - | - | - |
-| test | 1000 | 7740 | 7.74 | 13.78 | 1 | - | - | - | - |
-| all | 13118 | 102979 | 7.85 | 13.61 | 1 | - | - | - | - |
+| train | 11118 | 87170 | 7.84 | 11.18 | 1 | - | - | - | - |
+| validation | 1000 | 8069 | 8.07 | 11.14 | 1 | - | - | - | - |
+| test | 1000 | 7740 | 7.74 | 11.33 | 1 | - | - | - | - |
+| all | 13118 | 102979 | 7.85 | 11.19 | 1 | - | - | - | - |
10 domains: ['Ordinary Life', 'School Life', 'Culture & Education', 'Attitude & Emotion', 'Relationship', 'Tourism', 'Health', 'Work', 'Politics', 'Finance']
- **cat slot match**: how many values of categorical slots are in the possible values of ontology in percentage.
diff --git a/data/unified_datasets/dailydialog/data.zip b/data/unified_datasets/dailydialog/data.zip
index 4909a3ff5f638a581a57f315141ff5459e28dc9e..6d5733d518fdf52b2607e175f6cb08da2042943a 100644
Binary files a/data/unified_datasets/dailydialog/data.zip and b/data/unified_datasets/dailydialog/data.zip differ
diff --git a/data/unified_datasets/dailydialog/dummy_data.json b/data/unified_datasets/dailydialog/dummy_data.json
index c64ad063d1dc96a09c601eea07b87ad9d334e881..3e097cd30aa1188085575276bcfe835cbc8bb2d5 100644
--- a/data/unified_datasets/dailydialog/dummy_data.json
+++ b/data/unified_datasets/dailydialog/dummy_data.json
@@ -15,13 +15,13 @@
"turns": [
{
"speaker": "user",
- "utterance": "Say , Jim , how about going for a few beers after dinner ?",
+ "utterance": "Say, Jim, how about going for a few beers after dinner?",
"utt_idx": 0,
"dialogue_acts": {
"binary": [
{
"intent": "directive",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -33,13 +33,13 @@
},
{
"speaker": "system",
- "utterance": "You know that is tempting but is really not good for our fitness .",
+ "utterance": "You know that is tempting but is really not good for our fitness.",
"utt_idx": 1,
"dialogue_acts": {
"binary": [
{
"intent": "commissive",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -51,13 +51,13 @@
},
{
"speaker": "user",
- "utterance": "What do you mean ? It will help us to relax .",
+ "utterance": "What do you mean? It will help us to relax.",
"utt_idx": 2,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -69,13 +69,13 @@
},
{
"speaker": "system",
- "utterance": "Do you really think so ? I don't . It will just make us fat and act silly . Remember last time ?",
+ "utterance": "Do you really think so? I don't. It will just make us fat and act silly. Remember last time?",
"utt_idx": 3,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -87,13 +87,13 @@
},
{
"speaker": "user",
- "utterance": "I guess you are right.But what shall we do ? I don't feel like sitting at home .",
+ "utterance": "I guess you are right.But what shall we do? I don't feel like sitting at home.",
"utt_idx": 4,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -105,13 +105,13 @@
},
{
"speaker": "system",
- "utterance": "I suggest a walk over to the gym where we can play singsong and meet some of our friends .",
+ "utterance": "I suggest a walk over to the gym where we can play singsong and meet some of our friends.",
"utt_idx": 5,
"dialogue_acts": {
"binary": [
{
"intent": "directive",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -123,13 +123,13 @@
},
{
"speaker": "user",
- "utterance": "That's a good idea . I hear Mary and Sally often go there to play pingpong.Perhaps we can make a foursome with them .",
+ "utterance": "That's a good idea. I hear Mary and Sally often go there to play pingpong.Perhaps we can make a foursome with them.",
"utt_idx": 6,
"dialogue_acts": {
"binary": [
{
"intent": "commissive",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -141,13 +141,13 @@
},
{
"speaker": "system",
- "utterance": "Sounds great to me ! If they are willing , we could ask them to go dancing with us.That is excellent exercise and fun , too .",
+ "utterance": "Sounds great to me! If they are willing, we could ask them to go dancing with us.That is excellent exercise and fun, too.",
"utt_idx": 7,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -159,13 +159,13 @@
},
{
"speaker": "user",
- "utterance": "Good.Let ' s go now .",
+ "utterance": "Good.Let' s go now.",
"utt_idx": 8,
"dialogue_acts": {
"binary": [
{
"intent": "directive",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -177,13 +177,13 @@
},
{
"speaker": "system",
- "utterance": "All right .",
+ "utterance": "All right.",
"utt_idx": 9,
"dialogue_acts": {
"binary": [
{
"intent": "commissive",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -211,13 +211,13 @@
"turns": [
{
"speaker": "user",
- "utterance": "Can you do push-ups ?",
+ "utterance": "Can you do push-ups?",
"utt_idx": 0,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -229,13 +229,13 @@
},
{
"speaker": "system",
- "utterance": "Of course I can . It's a piece of cake ! Believe it or not , I can do 30 push-ups a minute .",
+ "utterance": "Of course I can. It's a piece of cake! Believe it or not, I can do 30 push-ups a minute.",
"utt_idx": 1,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -247,13 +247,13 @@
},
{
"speaker": "user",
- "utterance": "Really ? I think that's impossible !",
+ "utterance": "Really? I think that's impossible!",
"utt_idx": 2,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -265,13 +265,13 @@
},
{
"speaker": "system",
- "utterance": "You mean 30 push-ups ?",
+ "utterance": "You mean 30 push-ups?",
"utt_idx": 3,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -283,13 +283,13 @@
},
{
"speaker": "user",
- "utterance": "Yeah !",
+ "utterance": "Yeah!",
"utt_idx": 4,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -301,13 +301,13 @@
},
{
"speaker": "system",
- "utterance": "It's easy . If you do exercise everyday , you can make it , too .",
+ "utterance": "It's easy. If you do exercise everyday, you can make it, too.",
"utt_idx": 5,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -335,13 +335,13 @@
"turns": [
{
"speaker": "user",
- "utterance": "Can you study with the radio on ?",
+ "utterance": "Can you study with the radio on?",
"utt_idx": 0,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -353,13 +353,13 @@
},
{
"speaker": "system",
- "utterance": "No , I listen to background music .",
+ "utterance": "No, I listen to background music.",
"utt_idx": 1,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -371,13 +371,13 @@
},
{
"speaker": "user",
- "utterance": "What is the difference ?",
+ "utterance": "What is the difference?",
"utt_idx": 2,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -389,13 +389,13 @@
},
{
"speaker": "system",
- "utterance": "The radio has too many comerials .",
+ "utterance": "The radio has too many comerials.",
"utt_idx": 3,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -407,13 +407,13 @@
},
{
"speaker": "user",
- "utterance": "That's true , but then you have to buy a record player .",
+ "utterance": "That's true, but then you have to buy a record player.",
"utt_idx": 4,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -441,13 +441,13 @@
"turns": [
{
"speaker": "user",
- "utterance": "Are you all right ?",
+ "utterance": "Are you all right?",
"utt_idx": 0,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -459,13 +459,13 @@
},
{
"speaker": "system",
- "utterance": "I will be all right soon . I was terrified when I watched them fall from the wire .",
+ "utterance": "I will be all right soon. I was terrified when I watched them fall from the wire.",
"utt_idx": 1,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -483,7 +483,7 @@
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -495,13 +495,13 @@
},
{
"speaker": "system",
- "utterance": "I see .",
+ "utterance": "I see.",
"utt_idx": 3,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -529,13 +529,13 @@
"turns": [
{
"speaker": "user",
- "utterance": "Hey John , nice skates . Are they new ?",
+ "utterance": "Hey John, nice skates. Are they new?",
"utt_idx": 0,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -547,13 +547,13 @@
},
{
"speaker": "system",
- "utterance": "Yeah , I just got them . I started playing ice hockey in a community league . So , I finally got myself new skates .",
+ "utterance": "Yeah, I just got them. I started playing ice hockey in a community league. So, I finally got myself new skates.",
"utt_idx": 1,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -565,13 +565,13 @@
},
{
"speaker": "user",
- "utterance": "What position do you play ?",
+ "utterance": "What position do you play?",
"utt_idx": 2,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -583,13 +583,13 @@
},
{
"speaker": "system",
- "utterance": "I ’ m a defender . It ’ s a lot of fun . You don ’ t have to be able to skate as fast on defense .",
+ "utterance": "I'm a defender. It's a lot of fun. You don't have to be able to skate as fast on defense.",
"utt_idx": 3,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -601,13 +601,13 @@
},
{
"speaker": "user",
- "utterance": "Yeah , you ’ re a pretty big guy . I play goalie , myself .",
+ "utterance": "Yeah, you're a pretty big guy. I play goalie, myself.",
"utt_idx": 4,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -619,13 +619,13 @@
},
{
"speaker": "system",
- "utterance": "Oh , yeah ? Which team ?",
+ "utterance": "Oh, yeah? Which team?",
"utt_idx": 5,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -637,13 +637,13 @@
},
{
"speaker": "user",
- "utterance": "The Rockets .",
+ "utterance": "The Rockets.",
"utt_idx": 6,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -655,13 +655,13 @@
},
{
"speaker": "system",
- "utterance": "Really ? I think we play you guys next week . Well , I have to go to practice . See you later .",
+ "utterance": "Really? I think we play you guys next week. Well, I have to go to practice. See you later.",
"utt_idx": 7,
"dialogue_acts": {
"binary": [
{
"intent": "directive",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -673,13 +673,13 @@
},
{
"speaker": "user",
- "utterance": "All right , see you later .",
+ "utterance": "All right, see you later.",
"utt_idx": 8,
"dialogue_acts": {
"binary": [
{
"intent": "commissive",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -707,13 +707,13 @@
"turns": [
{
"speaker": "user",
- "utterance": "Hey Lydia , what are you reading ?",
+ "utterance": "Hey Lydia, what are you reading?",
"utt_idx": 0,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -725,13 +725,13 @@
},
{
"speaker": "system",
- "utterance": "I ’ m looking at my horoscope for this month ! My outlook is very positive . It says that I should take a vacation to someplace exotic , and that I will have a passionate summer fling !",
+ "utterance": "I'm looking at my horoscope for this month! My outlook is very positive. It says that I should take a vacation to someplace exotic, and that I will have a passionate summer fling!",
"utt_idx": 1,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -743,13 +743,13 @@
},
{
"speaker": "user",
- "utterance": "What are you talking about ? Let me see that ... What are horoscopes ?",
+ "utterance": "What are you talking about? Let me see that...What are horoscopes?",
"utt_idx": 2,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -761,13 +761,13 @@
},
{
"speaker": "system",
- "utterance": "It ’ s a prediction of your month , based on your zodiac sign . You have a different sign for the month and date you were born in . I was born on April 15th , so I ’ m an Aries . When were you born ?",
+ "utterance": "It's a prediction of your month, based on your zodiac sign. You have a different sign for the month and date you were born in. I was born on April 15th, so I'm an Aries. When were you born?",
"utt_idx": 3,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -779,13 +779,13 @@
},
{
"speaker": "user",
- "utterance": "January 5th .",
+ "utterance": "January 5th.",
"utt_idx": 4,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -797,13 +797,13 @@
},
{
"speaker": "system",
- "utterance": "Let ’ s see . . . you ’ re a Capricorn . It says that you will be feeling stress at work , but you could see new , exciting developments in your love life . Looks like we ’ ll both have interesting summers !",
+ "utterance": "Let's see. . . you're a Capricorn. It says that you will be feeling stress at work, but you could see new, exciting developments in your love life. Looks like we'll both have interesting summers!",
"utt_idx": 5,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -815,13 +815,13 @@
},
{
"speaker": "user",
- "utterance": "That ’ s bogus . I don't feel any stress at work , and my love life is practically nonexistent . This zodiac stuff is all a bunch of nonsense .",
+ "utterance": "That's bogus. I don't feel any stress at work, and my love life is practically nonexistent. This zodiac stuff is all a bunch of nonsense.",
"utt_idx": 6,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -833,13 +833,13 @@
},
{
"speaker": "system",
- "utterance": "No , it ’ s not , your astrology sign can tell you a lot about your personality . See ? It says that an Aries is energetic and loves to socialize .",
+ "utterance": "No, it's not, your astrology sign can tell you a lot about your personality. See? It says that an Aries is energetic and loves to socialize.",
"utt_idx": 7,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -851,13 +851,13 @@
},
{
"speaker": "user",
- "utterance": "Well , you certainly match those criteria , but they ’ re so broad they could apply to anyone . What does it say about me ?",
+ "utterance": "Well, you certainly match those criteria, but they're so broad they could apply to anyone. What does it say about me?",
"utt_idx": 8,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -869,13 +869,13 @@
},
{
"speaker": "system",
- "utterance": "A Capricorn is serious-minded and practical . She likes to do things in conventional ways . That sounds just like you !",
+ "utterance": "A Capricorn is serious-minded and practical. She likes to do things in conventional ways. That sounds just like you!",
"utt_idx": 9,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -903,13 +903,13 @@
"turns": [
{
"speaker": "user",
- "utterance": "Frank ’ s getting married , do you believe this ?",
+ "utterance": "Frank's getting married, do you believe this?",
"utt_idx": 0,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -921,13 +921,13 @@
},
{
"speaker": "system",
- "utterance": "Is he really ?",
+ "utterance": "Is he really?",
"utt_idx": 1,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -939,13 +939,13 @@
},
{
"speaker": "user",
- "utterance": "Yes , he is . He loves the girl very much .",
+ "utterance": "Yes, he is. He loves the girl very much.",
"utt_idx": 2,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -957,13 +957,13 @@
},
{
"speaker": "system",
- "utterance": "Who is he marring ?",
+ "utterance": "Who is he marring?",
"utt_idx": 3,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -975,13 +975,13 @@
},
{
"speaker": "user",
- "utterance": "A girl he met on holiday in Spain , I think .",
+ "utterance": "A girl he met on holiday in Spain, I think.",
"utt_idx": 4,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -993,13 +993,13 @@
},
{
"speaker": "system",
- "utterance": "Have they set a date for the wedding ?",
+ "utterance": "Have they set a date for the wedding?",
"utt_idx": 5,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -1011,13 +1011,13 @@
},
{
"speaker": "user",
- "utterance": "Not yet .",
+ "utterance": "Not yet.",
"utt_idx": 6,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -1045,13 +1045,13 @@
"turns": [
{
"speaker": "user",
- "utterance": "I hear you bought a new house in the northern suburbs .",
+ "utterance": "I hear you bought a new house in the northern suburbs.",
"utt_idx": 0,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -1063,13 +1063,13 @@
},
{
"speaker": "system",
- "utterance": "That ’ s right , we bought it the same day we came on the market .",
+ "utterance": "That's right, we bought it the same day we came on the market.",
"utt_idx": 1,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -1081,13 +1081,13 @@
},
{
"speaker": "user",
- "utterance": "What kind of house is it ?",
+ "utterance": "What kind of house is it?",
"utt_idx": 2,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -1099,13 +1099,13 @@
},
{
"speaker": "system",
- "utterance": "It ’ s a wonderful Spanish style .",
+ "utterance": "It's a wonderful Spanish style.",
"utt_idx": 3,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -1117,13 +1117,13 @@
},
{
"speaker": "user",
- "utterance": "Oh , I love the roof tiles on Spanish style houses .",
+ "utterance": "Oh, I love the roof tiles on Spanish style houses.",
"utt_idx": 4,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -1135,13 +1135,13 @@
},
{
"speaker": "system",
- "utterance": "And it ’ s a bargaining . A house like this in river side costs double the price .",
+ "utterance": "And it's a bargaining. A house like this in river side costs double the price.",
"utt_idx": 5,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -1153,13 +1153,13 @@
},
{
"speaker": "user",
- "utterance": "Great , is it a two bedroom house ?",
+ "utterance": "Great, is it a two bedroom house?",
"utt_idx": 6,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -1171,13 +1171,13 @@
},
{
"speaker": "system",
- "utterance": "No , it has three bedrooms and three beds , and has a living room with a twelve-foot ceiling . There ’ s a two-car garage .",
+ "utterance": "No, it has three bedrooms and three beds, and has a living room with a twelve-foot ceiling. There's a two-car garage.",
"utt_idx": 7,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -1189,13 +1189,13 @@
},
{
"speaker": "user",
- "utterance": "That ’ s a nice area too . It ’ ll be a good investment for you .",
+ "utterance": "That's a nice area too. It'll be a good investment for you.",
"utt_idx": 8,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -1207,13 +1207,13 @@
},
{
"speaker": "system",
- "utterance": "Yeas , when will you buy a house ?",
+ "utterance": "Yeas, when will you buy a house?",
"utt_idx": 9,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -1225,13 +1225,13 @@
},
{
"speaker": "user",
- "utterance": "Not untill the end of this year , you know , just before my wedding .",
+ "utterance": "Not untill the end of this year, you know, just before my wedding.",
"utt_idx": 10,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -1243,13 +1243,13 @@
},
{
"speaker": "system",
- "utterance": "Right , congratulations .",
+ "utterance": "Right, congratulations.",
"utt_idx": 11,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -1261,13 +1261,13 @@
},
{
"speaker": "user",
- "utterance": "Thank you .",
+ "utterance": "Thank you.",
"utt_idx": 12,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Relationship",
+ "domain": "",
"slot": ""
}
],
@@ -1295,13 +1295,13 @@
"turns": [
{
"speaker": "user",
- "utterance": "Hi , Becky , what's up ?",
+ "utterance": "Hi, Becky, what's up?",
"utt_idx": 0,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1313,13 +1313,13 @@
},
{
"speaker": "system",
- "utterance": "Not much , except that my mother-in-law is driving me up the wall .",
+ "utterance": "Not much, except that my mother-in-law is driving me up the wall.",
"utt_idx": 1,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1331,13 +1331,13 @@
},
{
"speaker": "user",
- "utterance": "What's the problem ?",
+ "utterance": "What's the problem?",
"utt_idx": 2,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1349,13 +1349,13 @@
},
{
"speaker": "system",
- "utterance": "She loves to nit-pick and criticizes everything that I do . I can never do anything right when she's around .",
+ "utterance": "She loves to nit-pick and criticizes everything that I do. I can never do anything right when she's around.",
"utt_idx": 3,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1367,13 +1367,13 @@
},
{
"speaker": "user",
- "utterance": "For example ?",
+ "utterance": "For example?",
"utt_idx": 4,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1385,13 +1385,13 @@
},
{
"speaker": "system",
- "utterance": "Well , last week I invited her over to dinner . My husband and I had no problem with the food , but if you listened to her , then it would seem like I fed her old meat and rotten vegetables . There's just nothing can please her .",
+ "utterance": "Well, last week I invited her over to dinner. My husband and I had no problem with the food, but if you listened to her, then it would seem like I fed her old meat and rotten vegetables. There's just nothing can please her.",
"utt_idx": 5,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1403,13 +1403,13 @@
},
{
"speaker": "user",
- "utterance": "No , I can't see that happening . I know you're a good cook and nothing like that would ever happen .",
+ "utterance": "No, I can't see that happening. I know you're a good cook and nothing like that would ever happen.",
"utt_idx": 6,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1421,13 +1421,13 @@
},
{
"speaker": "system",
- "utterance": "It's not just that . She also criticizes how we raise the kids .",
+ "utterance": "It's not just that. She also criticizes how we raise the kids.",
"utt_idx": 7,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1439,13 +1439,13 @@
},
{
"speaker": "user",
- "utterance": "My mother-in-law used to do the same thing to us . If it wasn't disciplining them enough , then we were disciplining them too much . She also complained about the food we fed them , the schools we sent them too , and everything else under the sun .",
+ "utterance": "My mother-in-law used to do the same thing to us. If it wasn't disciplining them enough, then we were disciplining them too much. She also complained about the food we fed them, the schools we sent them too, and everything else under the sun.",
"utt_idx": 8,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1457,13 +1457,13 @@
},
{
"speaker": "system",
- "utterance": "You said she used to ? How did you stop her ?",
+ "utterance": "You said she used to? How did you stop her?",
"utt_idx": 9,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1475,13 +1475,13 @@
},
{
"speaker": "user",
- "utterance": "We basically sat her down and told her how we felt about her constant criticizing , and how we welcomed her advice but hoped she'd let us do our things . She understood , and now everything is a lot more peaceful .",
+ "utterance": "We basically sat her down and told her how we felt about her constant criticizing, and how we welcomed her advice but hoped she'd let us do our things. She understood, and now everything is a lot more peaceful.",
"utt_idx": 10,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1493,13 +1493,13 @@
},
{
"speaker": "system",
- "utterance": "That sounds like a good idea . I'll have to try that .",
+ "utterance": "That sounds like a good idea. I'll have to try that.",
"utt_idx": 11,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1527,13 +1527,13 @@
"turns": [
{
"speaker": "user",
- "utterance": "How are Zina's new programmers working out ?",
+ "utterance": "How are Zina's new programmers working out?",
"utt_idx": 0,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1545,13 +1545,13 @@
},
{
"speaker": "system",
- "utterance": "I hate to admit it , but they're good . And fast . The Filipino kid is a genius .",
+ "utterance": "I hate to admit it, but they're good. And fast. The Filipino kid is a genius.",
"utt_idx": 1,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1563,13 +1563,13 @@
},
{
"speaker": "user",
- "utterance": "So you'll make the Stars.com deadline , and have us up and running next week ?",
+ "utterance": "So you'll make the Stars.com deadline, and have us up and running next week?",
"utt_idx": 2,
"dialogue_acts": {
"binary": [
{
"intent": "question",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1581,13 +1581,13 @@
},
{
"speaker": "system",
- "utterance": "It'll be close , but we'll make it .",
+ "utterance": "It'll be close, but we'll make it.",
"utt_idx": 3,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1599,13 +1599,13 @@
},
{
"speaker": "user",
- "utterance": "Good . After Stars.com starts paying us , we won't need Vikam's cash anymore .",
+ "utterance": "Good. After Stars.com starts paying us, we won't need Vikam's cash anymore.",
"utt_idx": 4,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
@@ -1617,13 +1617,13 @@
},
{
"speaker": "system",
- "utterance": "And if we don't need them , we won't need Zina , either .",
+ "utterance": "And if we don't need them, we won't need Zina, either.",
"utt_idx": 5,
"dialogue_acts": {
"binary": [
{
"intent": "inform",
- "domain": "Attitude & Emotion",
+ "domain": "",
"slot": ""
}
],
diff --git a/data/unified_datasets/dailydialog/preprocess.py b/data/unified_datasets/dailydialog/preprocess.py
index 8a04aae5d620c813ed9be7c72b55d6d2c718fcdb..d082a2d2d48291b66f670432336c2aaff673c2fd 100644
--- a/data/unified_datasets/dailydialog/preprocess.py
+++ b/data/unified_datasets/dailydialog/preprocess.py
@@ -1,13 +1,12 @@
-import copy
-import re
from zipfile import ZipFile, ZIP_DEFLATED
-from shutil import copy2, rmtree
+from shutil import rmtree
import json
import os
from tqdm import tqdm
from collections import Counter
from pprint import pprint
-from datasets import load_dataset
+from nltk.tokenize import sent_tokenize, word_tokenize
+from nltk.tokenize.treebank import TreebankWordDetokenizer
topic_map = {
1: "Ordinary Life",
@@ -78,12 +77,14 @@ def preprocess():
"binary": {}
}}
+ detokenizer = TreebankWordDetokenizer()
+
for data_split in splits:
archive = ZipFile(os.path.join(original_data_dir, f'{data_split}.zip'))
with archive.open(f'{data_split}/dialogues_{data_split}.txt') as dialog_file, \
archive.open(f'{data_split}/dialogues_act_{data_split}.txt') as act_file, \
archive.open(f'{data_split}/dialogues_emotion_{data_split}.txt') as emotion_file:
- for dialog_line, act_line, emotion_line in zip(dialog_file, act_file, emotion_file):
+ for dialog_line, act_line, emotion_line in tqdm(zip(dialog_file, act_file, emotion_file)):
if not dialog_line.strip():
break
utts = dialog_line.decode().split("__eou__")[:-1]
@@ -114,6 +115,9 @@ def preprocess():
speaker = 'user' if len(dialogue['turns']) % 2 == 0 else 'system'
intent = act_map[int(act)]
emotion = emotion_map[int(emotion)]
+ utt = ' '.join([detokenizer.detokenize(word_tokenize(s)) for s in sent_tokenize(utt)])
+ utt = utt.replace(' ’ ', "'")
+
dialogue['turns'].append({
'speaker': speaker,
'utterance': utt.strip(),
@@ -121,7 +125,7 @@ def preprocess():
'dialogue_acts': {
'binary': [{
'intent': intent,
- 'domain': domain,
+ 'domain': '',
'slot': ''
}],
'categorical': [],
@@ -134,8 +138,8 @@ def preprocess():
else:
dialogue['turns'][-1]['state'] = {}
- ontology["dialogue_acts"]['binary'].setdefault((intent, domain, ''), {})
- ontology["dialogue_acts"]['binary'][(intent, domain, '')][speaker] = True
+ ontology["dialogue_acts"]['binary'].setdefault((intent, '', ''), {})
+ ontology["dialogue_acts"]['binary'][(intent, '', '')][speaker] = True
dialogues_by_split[data_split].append(dialogue)