make nlu,dst serialization more compact

convlab2/base_models/t5/create_data.py

@@ -80,9 +80,9 @@ def create_nlg_data(dataset, data_dir, args):
             dialogue_acts_seq = serialize_dialogue_acts(sample['dialogue_acts'])
             if args.context_window_size>0:
                 context = '\n'.join([f"{turn['speaker']}: {turn['utterance']}" for turn in sample['context']]+[f'{sample["speaker"]}: '])
-                context = f'{dialogue_acts_seq}\n\n{context}'
+                context = f'{dialogue_acts_seq}\n{context}'
             else:
-                context = f'{dialogue_acts_seq}\n\n{sample["speaker"]}: '
+                context = f'{dialogue_acts_seq}\n{sample["speaker"]}: '
             assert equal_da_seq(sample['dialogue_acts'], dialogue_acts_seq), print(sample['dialogue_acts'], dialogue_acts_seq, deserialize_dialogue_acts(dialogue_acts_seq))
             data.append(json.dumps({'context+da': context, 'response': sample['utterance']}, ensure_ascii=False)+'\n')
 

convlab2/base_models/t5/dst/serialization.py

 def serialize_dialogue_state(state):
-    state_seqs = []
+    state_dict = {}
     for domain in state:
-        for slot, value in state[domain].items():
+        for slot, value in sorted(state[domain].items()):
             if len(value) > 0:
-                state_seqs.append(f'[{domain}][{slot}][{value}]')
-    
-    return ';'.join(state_seqs)
+                state_dict.setdefault(f'[{domain}]', [])
+                state_dict[f'[{domain}]'].append(f'[{slot}][{value}]')
+    return ';'.join([domain+'{'+','.join(slot_values)+'}' for domain, slot_values in state_dict.items()])
 
 def deserialize_dialogue_state(state_seq):
     state = {}
     if len(state_seq) == 0:
         return state
-    state_seqs = state_seq.split('];[')
+    state_seqs = state_seq.split(']};[')  # will consume "]}" and "["
     for i, state_seq in enumerate(state_seqs):
-        if len(state_seq) == 0:
+        if len(state_seq) == 0 or len(state_seq.split(']{[')) != 2:
             continue
         if i == 0:
             if state_seq[0] == '[':
                 state_seq = state_seq[1:]
         if i == len(state_seqs) - 1:
-            if state_seq[-1] == ']':
-                state_seq = state_seq[:-1]
-        s = state_seq.split('][')
-        if len(s) != 3:
-            continue
-        domain, slot, value = s
+            if state_seq[-2:] == ']}':
+                state_seq = state_seq[:-2]
+        
+        domain, slot_values = state_seq.split(']{[')
+        for slot_value in slot_values.split('],['):
+            slot, value = slot_value.split('][')
             state.setdefault(domain, {})
             state[domain][slot] = value
     return state

convlab2/base_models/t5/nlu/serialization.py

 def serialize_dialogue_acts(dialogue_acts):
-    da_seqs = []
+    da_dict = {}
     for da_type in dialogue_acts:
         for da in dialogue_acts[da_type]:
-            intent, domain, slot = da['intent'], da['domain'], da['slot']
-            if da_type == 'binary':
-                da_seq = f'[{da_type}][{intent}][{domain}][{slot}]'
-            else:
-                value = da['value']
-                da_seq = f'[{da_type}][{intent}][{domain}][{slot}][{value}]'
-            da_seqs.append(da_seq)
-    return ';'.join(da_seqs)
+            intent, domain, slot, value = da['intent'], da['domain'], da['slot'], da.get('value', '')
+            intent_domain = f'[{intent}][{domain}]'
+            da_dict.setdefault(intent_domain, [])
+            da_dict[intent_domain].append(f'[{slot}][{value}]')
+    return ';'.join([intent_domain+'{'+','.join(slot_values)+'}' for intent_domain, slot_values in da_dict.items()])
 
 def deserialize_dialogue_acts(das_seq):
-    dialogue_acts = {'binary': [], 'categorical': [], 'non-categorical': []}
+    dialogue_acts = []
     if len(das_seq) == 0:
         return dialogue_acts
-    da_seqs = das_seq.split('];[')
+    da_seqs = das_seq.split(']};[')  # will consume "]}" and "["
     for i, da_seq in enumerate(da_seqs):
-        if len(da_seq) == 0:
+        if len(da_seq) == 0 or len(da_seq.split(']{[')) != 2:
             continue
         if i == 0:
             if da_seq[0] == '[':
                 da_seq = da_seq[1:]
         if i == len(da_seqs) - 1:
-            if da_seq[-1] == ']':
-                da_seq = da_seq[:-1]
-        da = da_seq.split('][')
-        if len(da) == 0:
-            continue
-        da_type = da[0]
-        if len(da) == 5 and da_type in ['categorical', 'non-categorical']:
-            dialogue_acts[da_type].append({'intent': da[1], 'domain': da[2], 'slot': da[3], 'value': da[4]})
-        elif len(da) == 4 and da_type == 'binary':
-            dialogue_acts[da_type].append({'intent': da[1], 'domain': da[2], 'slot': da[3]})
-        else:
-            # invalid da format, skip
-            # print(das_seq)
-            # print(da_seq)
-            # print()
-            pass
+            if da_seq[-2:] == ']}':
+                da_seq = da_seq[:-2]
+        
+        intent_domain, slot_values = da_seq.split(']{[')
+        intent, domain = intent_domain.split('][')
+        for slot_value in slot_values.split('],['):
+            slot, value = slot_value.split('][')
+            dialogue_acts.append({'intent': intent, 'domain': domain, 'slot': slot, 'value': value})
     return dialogue_acts
 
 def equal_da_seq(dialogue_acts, das_seq):
     predict_dialogue_acts = deserialize_dialogue_acts(das_seq)
-    for da_type in ['binary', 'categorical', 'non-categorical']:
-        das = sorted([(da['intent'], da['domain'], da['slot'], da.get('value', '')) for da in dialogue_acts[da_type]])
-        predict_das = sorted([(da['intent'], da['domain'], da['slot'], da.get('value', '')) for da in predict_dialogue_acts[da_type]])
+    das = sorted([(da['intent'], da['domain'], da['slot'], da.get('value', '')) for da_type in ['binary', 'categorical', 'non-categorical'] for da in dialogue_acts[da_type]])
+    predict_das = sorted([(da['intent'], da['domain'], da['slot'], da.get('value', '')) for da in predict_dialogue_acts])
     if das != predict_das:
         return False
     return True