from?sklearn.feature_extraction.text?import?TfidfVectorizer
vectorizer?=?TfidfVectorizer()
X?=?vectorizer.fit_transform(corpus)
print(X.toarray())

import?spacy
import?nltk
from?nltk.tokenize?import?word_tokenize
from?nltk.corpus?import?stopwords
import?regex?as?re
import?string
import?pandas?as?pd
import?numpy?as?np
import?nltk.data
import?re
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')
nltk.download('averaged_perceptron_tagger')
from?nltk.stem?import?WordNetLemmatizer
from?nltk?import?word_tokenize,?sent_tokenize,?pos_tag

import?os????????
path?=?"./data/theses100/"?????
all_files?=?os.listdir(path?+?"docsutf8")?
all_keys?=?os.listdir(path?+?"keys")?
print(len(all_files),"?files?n",all_files,
??????"n",?all_keys)?#?不一定要排序

all_documents?=[]?
all_keys?=?[]?
all_files_names?=?[]?
for?i,?fname?in?enumerate(all_files):?
??with?open(path+'docsutf8/'+fname)?as?f:?
??????lines?=?f.readlines()?
??key_name=?fname[:-4?]?
??with?open(path+'keys/'+key_name+'.key')?as?f:?
??????k?=?f.readlines()?
??all_text?=?'?'.join(lines)?
??keyss?=?'?'.join(k)?
??all_documents.append(all_text)?
??all_keys.append(keyss.split("n"))?
??all_files_names.append(key_name)
??
import?pandas?as?pd
dtf?=?pd.DataFrame({'goldkeys':?all_keys,
????????????????????'text':?all_documents})
dtf.head()

dtf['cleaned_text']?=?dtf.text.apply(lambda?x:?'?'.join(preprocess_text(x)))
dtf.head()

#?清理基本關(guān)鍵字，刪除空格和噪音
def?clean_orginal_kw(orginal_kw):
??orginal_kw_clean?=[]
??for?doc_kw?in?orginal_kw:
????temp?=[]
????for?t?in?doc_kw:
??????tt?=?'?'.join(preprocess_text(t))
??????if?len(tt.split())>0:
????????temp.append(tt)
????orginal_kw_clean.append(temp)
??return?orginal_kw_clean

orginal_kw=?clean_orginal_kw(dtf['goldkeys'])
orginal_kw[0:1]

from?sklearn.feature_extraction.text?import?TfidfVectorizer
vectorizer?=?TfidfVectorizer(use_idf=True,?
?????????????????????????????max_df=0.5,?min_df=1,?
?????????????????????????????ngram_range=(1,3))
vectors?=?vectorizer.fit_transform(dtf['cleaned_text'])

dict_of_tokens={i[1]:i[0]?for?i?in?vectorizer.vocabulary_.items()}

tfidf_vectors?=?[]??#?all?deoc?vectors?by?tfidf
for?row?in?vectors:
????tfidf_vectors.append({dict_of_tokens[column]:value?for
??????????????????????????(column,value)?in?
??????????????????????????zip(row.indices,row.data)})

print("The?number?of?document?vectors?=?",?len(tfidf_vectors)?,?
??????"\nThe?dictionary?of?document[0]?:",?tfidf_vectors[0])

doc_sorted_tfidfs?=[]??#?帶有tfidf權(quán)重的文檔特征列表
#?對(duì)文檔的每個(gè)字典進(jìn)行排序
for?dn?in?tfidf_vectors:
????newD?=?sorted(dn.items(),?key=lambda?x:?x[1],?reverse=True)
????newD?=?dict(newD)
????doc_sorted_tfidfs.append(newD)

tfidf_kw?=?[]
for?doc_tfidf?in?doc_sorted_tfidfs:
????ll?=?list(doc_tfidf.keys())
????tfidf_kw.append(ll)

TopN=?5
print(tfidf_kw[0][0:TopN])

['cone', 'cone tree', 
'dimensional', 'shadow',
'visualization']

def?get_exact_intersect(doc_orginal_kw,?doc_my_kw):
????general?=?[]
????for?kw?in?doc_my_kw:
????????for?kww?in?doc_orginal_kw:
????????????l_my?=?len(kw.split())
????????????l_org?=?len(kww.split())
????????????if?(kw?==?kww):
#?????????????????print("exact?matching?========",?kw,?kww)
????????????????if?kww?not?in?general:
????????????????????general.append(kww)?
????return?general

get_exact_intersect(orginal_kw[0],?tfidf_kw[0])

['visualization',
 'animation',
 'unix',
 'dimension',
 'cod',
 'icon',
 'shape',
 'fisheye lens',
 'rapid prototyping',
 'script language',
 'tree structure',
 'programming language']

def?apk(kw_actual,?kw_predicted,?k=10):
????if?len(kw_predicted)>k:
????????kw_predicted?=?kw_predicted[:k]
????score?=?0.0
????num_hits?=?0.0
????for?i,p?in?enumerate(kw_predicted):
????????if?p?in?kw_actual?and?p?not?in?kw_predicted[:i]:
????????????num_hits?+=?1.0
????????????score?+=?num_hits?/?(i+1.0)
????if?not?kw_actual:
????????return?0.0
????return?score?/?min(len(kw_actual),?k)

def?mapk(kw_actual,?kw_predicted,?k=10):
????return?np.mean([apk(a,p,k)?for?a,p?in?zip(kw_actual,?kw_predicted)])

for?k?in?[5,?10,20,40]:
????mpak=?mapk(orginal_kw,?tfidf_kw,?k)
????print("mean?average?precession??@",k,
??????????'=??{0:.4g}'.format(mpak))

mean average precession  @ 5 =  0.2037
mean average precession  @ 10 =  0.1379
mean average precession  @ 20 =  0.08026
mean average precession  @ 40 =  0.05371

def?preprocess_text(text):
????#?1.?將其標(biāo)記為字母符號(hào)
????text?=?remove_numbers(text)
????text?=?remove_http(text)
????text?=?remove_punctuation(text)
????text?=?convert_to_lower(text)
????text?=?remove_white_space(text)
????text?=?remove_short_words(text)
????tokens?=?toknizing(text)
????#?2.?POS?tagging
????pos_map?=?{'J':?'a',?'N':?'n',?'R':?'r',?'V':?'v'}
????pos_tags_list?=?pos_tag(tokens)
????#?print(pos_tags)
????#?3.?小寫變換和詞形還原?
????lemmatiser?=?WordNetLemmatizer()
????tokens?=?[lemmatiser.lemmatize(w.lower(),?
???????????????????????????????????pos=pos_map.get(p[0],?'v'))?
??????????????for?w,?p?in?pos_tags_list]
????return?tokens

def?convert_to_lower(text):
????#?小寫轉(zhuǎn)換
????return?text.lower()

def?remove_numbers(text):
????#?除去數(shù)字
????text?=?re.sub(r'\d+'?,?'',?text)
????return?text

def?remove_http(text):
????#?除去網(wǎng)址
????text?=?re.sub("https?:\/\/t.co\/[A-Za-z0-9]*",?'?',?text)
????return?text

def?remove_short_words(text):
????#?去除短于三個(gè)字母的單詞
????text?=?re.sub(r'\b\w{1,2}\b',?'',?text)
????return?text

def?remove_punctuation(text):
????#?去除符號(hào)和標(biāo)點(diǎn)符號(hào)
????punctuations?=?'''!()[]{};?№?:'"\,`<>./?@=#$-(%^)+&[*_]~'''
????no_punct?=?""
?????
????for?char?in?text:
????????if?char?not?in?punctuations:
????????????no_punct?=?no_punct?+?char
????return?no_punct

def?remove_white_space(text):
????#?去除空格
????text?=?text.strip()
????return?text

def?toknizing(text):
????stp?=?my_stopwords
????#stops?=?set(stopwords.words('english'))
????stop_words?=?set(stp)
????tokens?=?word_tokenize(text)
????##?從tokens中去除停用詞
????result?=?[i?for?i?in?tokens?if?not?i?in?stop_words]
????return?result