# -*- coding: utf-8 -*-
from collections import defaultdict
import math
import operator

"""
函數(shù)說(shuō)明:創(chuàng)建數(shù)據(jù)樣本
Returns:
    dataset - 實(shí)驗(yàn)樣本切分的詞條
    classVec - 類別標(biāo)簽向量
"""


def loadDataSet():
    dataset = [['my', 'dog', 'has', 'flea', 'problems', 'help', 'please'],  # 切分的詞條
               ['maybe', 'not', 'take', 'him', 'to', 'dog', 'park', 'stupid'],
               ['my', 'dalmation', 'is', 'so', 'cute', 'I', 'love', 'my'],
               ['stop', 'posting', 'stupid', 'worthless', 'garbage'],
               ['mr', 'licks', 'ate', 'my', 'steak', 'how', 'to', 'stop', 'him'],
               ['quit', 'buying', 'worthless', 'dog', 'food', 'stupid']]
    classVec = [0, 1, 0, 1, 0, 1]  # 類別標(biāo)簽向量，1代表好，0代表不好
    return dataset, classVec


"""
函數(shù)說(shuō)明：特征選擇TF-IDF算法
Parameters:
     list_words:詞列表
Returns:
     dict_feature_select:特征選擇詞字典
"""
#dataset:文件夾，word_list:某一個(gè)文件，word某個(gè)詞

def feature_select(dataset):
    # 總詞頻統(tǒng)計(jì)
    doc_frequency = defaultdict(int) #記錄每個(gè)詞出現(xiàn)的次數(shù)，可以把它理解成一個(gè)可變長(zhǎng)度的list，只要你索引它，它就自動(dòng)擴(kuò)列
    for file in dataset:
        for word in file:
            doc_frequency[word] += 1
    # 計(jì)算每個(gè)詞的TF值
    word_tf = {}  # 存儲(chǔ)沒個(gè)詞的tf值
    for i in doc_frequency:
        word_tf[i] = doc_frequency[i] / sum(doc_frequency.values()) #sum(doc.frequency.values)

    # 計(jì)算每個(gè)詞的IDF值
    doc_num = len(dataset)
    word_idf = {}  # 存儲(chǔ)每個(gè)詞的idf值
    word_doc = defaultdict(int)  # 存儲(chǔ)包含該詞的文檔數(shù)
    for word in doc_frequency:
        for file in dataset:
            if word in file:
                word_doc[word] += 1
    #word_doc和doc_frequency的區(qū)別是word_doc存儲(chǔ)的是包含這個(gè)詞的文檔數(shù)，即如果一個(gè)文檔里有重復(fù)出現(xiàn)一個(gè)詞則word_doc < doc_frequency
    for word in doc_frequency:
        word_idf[word] = math.log(doc_num / (word_doc[word] + 1))

    # 計(jì)算每個(gè)詞的TF*IDF的值
    word_tf_idf = {}
    for word in doc_frequency:
        word_tf_idf[word] = word_tf[word] * word_idf[word]

    # 對(duì)字典按值由大到小排序
    dict_feature_select = sorted(word_tf_idf.items(), key=operator.itemgetter(1), reverse=True)
    return dict_feature_select


if __name__ == '__main__':
    data_list, label_list = loadDataSet()  # 加載數(shù)據(jù)
    features = feature_select(data_list)  # 所有詞的TF-IDF值
    print(features)

from nltk.text import TextCollection
from nltk.tokenize import word_tokenize
 
#首先，構(gòu)建語(yǔ)料庫(kù)corpus
sents=['this is sentence one','this is sentence two','this is sentence three']
sents=[word_tokenize(sent) for sent in sents] #對(duì)每個(gè)句子進(jìn)行分詞
print(sents)  #輸出分詞后的結(jié)果
corpus=TextCollection(sents)  #構(gòu)建語(yǔ)料庫(kù)
print(corpus)  #輸出語(yǔ)料庫(kù)
 
#計(jì)算語(yǔ)料庫(kù)中"one"的tf值
tf=corpus.tf('one',corpus)    # 1/12
print(tf)
 
#計(jì)算語(yǔ)料庫(kù)中"one"的idf值
idf=corpus.idf('one')      #log(3/1)
print(idf)
 
#計(jì)算語(yǔ)料庫(kù)中"one"的tf-idf值
tf_idf=corpus.tf_idf('one',corpus)
print(tf_idf)

import sklearn
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer

x_train = ['TF-IDF 主要 思想 是', '算法 一個(gè) 重要 特點(diǎn) 可以 脫離 語(yǔ)料庫(kù) 背景',
           '如果 一個(gè) 網(wǎng)頁(yè) 被 很多 其他 網(wǎng)頁(yè) 鏈接 說(shuō)明 網(wǎng)頁(yè) 重要']
x_test = ['原始 文本 進(jìn)行 標(biāo)記', '主要 思想']

# 該類會(huì)將文本中的詞語(yǔ)轉(zhuǎn)換為詞頻矩陣，矩陣元素a[i][j] 表示j詞在i類文本下的詞頻
vectorizer = CountVectorizer(max_features=10) #列數(shù)為10
# 該類會(huì)統(tǒng)計(jì)每個(gè)詞語(yǔ)的tf-idf權(quán)值
tf_idf_transformer = TfidfTransformer()
# 將文本轉(zhuǎn)為詞頻矩陣并計(jì)算tf-idf
tf_idf = tf_idf_transformer.fit_transform(vectorizer.fit_transform(x_train))
# 將tf-idf矩陣抽取出來(lái)，元素a[i][j]表示j詞在i類文本中的tf-idf權(quán)重
x_train_weight = tf_idf.toarray()

# 對(duì)測(cè)試集進(jìn)行tf-idf權(quán)重計(jì)算
tf_idf = tf_idf_transformer.transform(vectorizer.transform(x_test))
x_test_weight = tf_idf.toarray()  # 測(cè)試集TF-IDF權(quán)重矩陣

print('vectorizer.fit_transform(x_train) : ')
print(vectorizer.fit_transform(x_train))
print('輸出x_train文本向量：')
print(x_train_weight)
print('輸出x_test文本向量：')
print(x_test_weight)

import jieba.analyse
 
text='關(guān)鍵詞是能夠表達(dá)文檔中心內(nèi)容的詞語(yǔ)，常用于計(jì)算機(jī)系統(tǒng)標(biāo)引論文內(nèi)容特征、
信息檢索、系統(tǒng)匯集以供讀者檢閱。關(guān)鍵詞提取是文本挖掘領(lǐng)域的一個(gè)分支，是文本檢索、
文檔比較、摘要生成、文檔分類和聚類等文本挖掘研究的基礎(chǔ)性工作'
 
keywords=jieba.analyse.extract_tags(text, topK=5, withWeight=False, allowPOS=())
print(keywords)

jieba.analyse.extract_tags(sentence, topK=20, withWeight=False, allowPOS=())