diff --git a/data/neighborhoods/neighborhoods_ambient_static_50000_sbert_size_50_remove_0_normalize_False.pkl b/data/neighborhoods/neighborhoods_ambient_static_50000_sbert_size_50_remove_0_normalize_False.pkl new file mode 100644 index 0000000000000000000000000000000000000000..7073b95648d85e29f3614d75c23b0a25447180ca --- /dev/null +++ b/data/neighborhoods/neighborhoods_ambient_static_50000_sbert_size_50_remove_0_normalize_False.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9537e717977c4f26569228852441deec160d9156084635453825650e3d49c90b +size 2591340055 diff --git a/tda/sbert_create_static_embeddings.py b/tda/sbert_create_static_embeddings.py index 9e22d0a71dd51f361ef31b8510cba45979acecc1..2cb2873844f3d769dbcaac46b87a3e9314c5d055 100644 --- a/tda/sbert_create_static_embeddings.py +++ b/tda/sbert_create_static_embeddings.py @@ -69,7 +69,11 @@ LOAD_EMBEDDINGS = False data_desc_list = ['paraphrase-MiniLM-L6-v2'] DEBUG_DATA_DESC = 'paraphrase-MiniLM-L6-v2' -VOCAB_DESC = 'pretrained_cc_en' + +VOCAB_PATH = embeddings_config['data']['multiwoz_and_sgd_joint_vocabulary_path'] +VOCAB_DESC = 'multiwoz_and_sgd' +# VOCAB_PATH = embeddings_config['data']['pretrained_cc_en_vocabulary_path'] +# VOCAB_DESC = 'pretrained_cc_en' # Set up logging logger = logging.getLogger() @@ -109,10 +113,8 @@ models['paraphrase-MiniLM-L6-v2'] = SentenceTransformer('paraphrase-MiniLM-L6-v2 logging.info('Loading data ...') -with open(embeddings_config['data']['multiwoz_and_sgd_joint_vocabulary_path'], "r") as file: +with open(VOCAB_PATH, "r") as file: vocabulary = json.load(file) -# with open(embeddings_config['data']['pretrained_cc_en_vocabulary_path'], "r") as file: -# vocabulary = json.load(file) # Testing tokenization of certain sentences # sentences = \ diff --git a/tda/sbert_static_embeddings_config_50_0.yaml b/tda/sbert_static_embeddings_config_50_0.yaml index 4eceb42e81086e04c732dd3626a71c43aa44200b..ba3965618a9967aa77f95b76e651630be7d82e21 100644 --- a/tda/sbert_static_embeddings_config_50_0.yaml +++ b/tda/sbert_static_embeddings_config_50_0.yaml @@ -1,16 +1,16 @@ data: - data_folder_path: '../../data' - multiwoz_and_sgd_joint_vocabulary_path: '../multiwoz_and_sgd_joint_vocabulary.json' - pretrained_cc_en_vocabulary_path: '../pretrained_cc_en_vocabulary.json' + data_folder_path: '../data' + multiwoz_and_sgd_joint_vocabulary_path: '../data/multiwoz_and_sgd_joint_vocabulary.json' + pretrained_cc_en_vocabulary_path: '../data/pretrained_cc_en_vocabulary.json' embeddings: - embeddings_dict_path: '../../embeddings' - embeddings_dataframes_path: '../../embeddings' + embeddings_dict_path: '../data' + embeddings_dataframes_path: '../data' context: 'word' pooling_method: 'mean' special_tokens: 'ignore' neighborhoods: nbhd_size: 50 nbhd_remove: 0 - neighborhoods_path: '../../neighborhoods' - persistence_features_path: '../../persistence_features/sbert_persistence' + neighborhoods_path: '../data/neighborhoods' + persistence_features_path: '../data' normalize: False