Updated sbert embeddings script; added .pkl file with neighborhoods

tda/sbert_create_static_embeddings.py

@@ -69,7 +69,11 @@ LOAD_EMBEDDINGS = False
 data_desc_list = ['paraphrase-MiniLM-L6-v2']
 
 DEBUG_DATA_DESC = 'paraphrase-MiniLM-L6-v2'
-VOCAB_DESC = 'pretrained_cc_en'
+
+VOCAB_PATH = embeddings_config['data']['multiwoz_and_sgd_joint_vocabulary_path']
+VOCAB_DESC = 'multiwoz_and_sgd'
+# VOCAB_PATH = embeddings_config['data']['pretrained_cc_en_vocabulary_path']
+# VOCAB_DESC = 'pretrained_cc_en'
 
 # Set up logging
 logger = logging.getLogger()
@@ -109,10 +113,8 @@ models['paraphrase-MiniLM-L6-v2'] = SentenceTransformer('paraphrase-MiniLM-L6-v2
 
 logging.info('Loading data ...')
 
-with open(embeddings_config['data']['multiwoz_and_sgd_joint_vocabulary_path'], "r") as file:
+with open(VOCAB_PATH, "r") as file:
     vocabulary = json.load(file)
-# with open(embeddings_config['data']['pretrained_cc_en_vocabulary_path'], "r") as file:
-#     vocabulary = json.load(file)
 
 # Testing tokenization of certain sentences
 # sentences = \

tda/sbert_static_embeddings_config_50_0.yaml

 data:
-  data_folder_path: '../../data'
-  multiwoz_and_sgd_joint_vocabulary_path: '../multiwoz_and_sgd_joint_vocabulary.json'
-  pretrained_cc_en_vocabulary_path: '../pretrained_cc_en_vocabulary.json'
+  data_folder_path: '../data'
+  multiwoz_and_sgd_joint_vocabulary_path: '../data/multiwoz_and_sgd_joint_vocabulary.json'
+  pretrained_cc_en_vocabulary_path: '../data/pretrained_cc_en_vocabulary.json'
 embeddings:
-  embeddings_dict_path: '../../embeddings'
-  embeddings_dataframes_path: '../../embeddings'
+  embeddings_dict_path: '../data'
+  embeddings_dataframes_path: '../data'
   context: 'word'
   pooling_method: 'mean'
   special_tokens: 'ignore'
 neighborhoods:
   nbhd_size: 50
   nbhd_remove: 0
-  neighborhoods_path: '../../neighborhoods'
-  persistence_features_path: '../../persistence_features/sbert_persistence'
+  neighborhoods_path: '../data/neighborhoods'
+  persistence_features_path: '../data'
   normalize: False