importsysfromtextmatch.models.text_embedding.model_factory_sklearnimportModelFactoryif__name__=='__main__': # docdoc_dict={"0":"我去玉龙雪山并且喜欢玉龙雪山玉龙雪山", "1":"我在玉龙雪山并且喜欢玉龙雪山", "2":"我在九寨沟", "3":"你好"} # queryquery="我在九寨沟,很喜欢"# 模型工厂，选择需要的模型加到列表中: 'bow', 'tfidf', 'ngram_tfidf', 'bert', 'albert', 'w2v'mf=ModelFactory( match_models=['bow', 'tfidf', 'ngram_tfidf'] ) # 模型处理初始化mf.init(words_dict=doc_dict, update=True) # query 与 doc的相似度search_res=mf.predict(query) print ('search_res>>>>>', search_res) # search_res>>>>>{'bow': [('0', 0.2773500981126146), ('1', 0.5303300858899106), ('2', 0.8660254037844388), ('3', 0.0)], 'tfidf': [('0', 0.2201159065358879), ('1', 0.46476266418455736), ('2', 0.8749225357988296), ('3', 0.0)], 'ngram_tfidf': [('0', 0.035719486884261346), ('1', 0.09654705406841395), ('2', 0.9561288696241232), ('3', 0.0)]}# query的embeddingquery_emb=mf.predict_emb(query) print ('query_emb>>>>>', query_emb) ''' pre_emb>>>>>{'bow': array([1., 0., 0., 1., 1., 0., 1., 0.]), 'tfidf': array([0.61422608, 0. , 0. , 0.4842629 , 0.4842629 , 0. , 0.39205255, 0. ]), 'ngram_tfidf': array([0. , 0. , 0.37156534, 0.37156534, 0. , 0. , 0. , 0.29294639, 0. , 0.37156534, 0.37156534, 0. , 0. , 0.37156534, 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0.29294639, 0.37156534, 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ])} '''

importsysimportjsonimporttimeimportfaissimportnumpyasnpfromfaissimportnormalize_L2fromtextmatch.config.constantimportConstantasconstfromtextmatch.core.text_embeddingimportTextEmbeddingfromtextmatch.tools.decomposition.pcaimportPCADecompositionfromtextmatch.tools.faiss.faissimportFaissSearchtest_dict={"id0": "其实事物发展有自己的潮流和规律", "id1": "当你身处潮流之中的时候，要紧紧抓住潮流的机会", "id2": "想办法脱颖而出，即使没有成功，也会更加洞悉时代的脉搏", "id3": "收获珍贵的知识和经验。而如果潮流已经退去", "id4": "这个时候再去往这个方向上努力，只会收获迷茫与压抑", "id5": "对时代、对自己都没有什么帮助", "id6": "但是时代的浪潮犹如海滩上的浪花，总是一浪接着一浪，只要你站在海边，身处这个行业之中，下一个浪潮很快又会到来。你需要敏感而又深刻地去观察，略去那些浮躁的泡沫，抓住真正潮流的机会，奋力一搏，不管成败，都不会遗憾。", "id7": "其实事物发展有自己的潮流和规律", "id8": "当你身处潮流之中的时候，要紧紧抓住潮流的机会" } if__name__=='__main__': # ['bow', 'tfidf', 'ngram_tfidf', 'bert']# ['bow', 'tfidf', 'ngram_tfidf', 'bert', 'w2v']# text_embedding = TextEmbedding( match_models=['bow', 'tfidf', 'ngram_tfidf', 'w2v'], words_dict=test_dict ) text_embedding=TextEmbedding( match_models=['bow', 'tfidf', 'ngram_tfidf', 'w2v'], words_dict=None, update=False ) feature_list= [] forsentenceintest_dict.values(): pre=text_embedding.predict(sentence) feature=np.concatenate([pre[model] formodelin ['bow', 'tfidf', 'ngram_tfidf', 'w2v']], axis=0) feature_list.append(feature) pca=PCADecomposition(n_components=8) data=np.array( feature_list ) pca.fit( data ) res=pca.transform( data ) print('res>>', res) pre=text_embedding.predict("潮流和规律") feature=np.concatenate([pre[model] formodelin ['bow', 'tfidf', 'ngram_tfidf', 'w2v']], axis=0) test=pca.transform( [feature] ) faiss_search=FaissSearch( res, sport_mode=False ) faiss_res=faiss_search.predict( test ) print( "faiss_res:", faiss_res ) ''' faiss kmeans result times 8.0108642578125e-05 faiss_res: [{0: 0.7833399, 7: 0.7833399, 3: 0.63782495}] '''faiss_search=FaissSearch( res, sport_mode=True ) faiss_res=faiss_search.predict( test ) print( "faiss_res:", faiss_res ) ''' faiss kmeans result times 3.266334533691406e-05 faiss_res: [{0: 0.7833399, 7: 0.7833399, 3: 0.63782495}] '''