from collections import Counter
import pandas as pd
from tqdm import tqdm
import numpy as np

def process_line(line_vocab, len_size):
    """
    處理單行數(shù)據(jù)，構(gòu)建局部詞匯表
    
    參數(shù):
        line_vocab (str): 輸入文本行
        len_size (int): n-gram長度
        
    返回:
        Counter: 詞頻統(tǒng)計(jì)對(duì)象
    """
    local_vocab = Counter()
    
    # 單字統(tǒng)計(jì)
    if len_size == 1:
        local_vocab.update(Counter(list(line_vocab)))
    
    # 雙字組合統(tǒng)計(jì)    
    elif len_size == 2:
        vocab_data = np.array([list(line_vocab[:-1]), list(line_vocab[1:])])
        vocab_data = vocab_data[0,:] + vocab_data[1,:]
        local_vocab.update(Counter(vocab_data.tolist()))
    
    # 三字組合統(tǒng)計(jì)    
    elif len_size == 3:
        vocab_data = np.array([list(line_vocab[:-2]), 
                              list(line_vocab[1:-1]), 
                              list(line_vocab[2:])])
        vocab_data = vocab_data[0,:] + vocab_data[1,:] + vocab_data[2,:]
        local_vocab.update(Counter(vocab_data.tolist()))
    
    # 四字組合統(tǒng)計(jì)    
    elif len_size == 4:
        vocab_data = np.array([list(line_vocab[:-3]), 
                              list(line_vocab[1:-2]), 
                              list(line_vocab[2:-1]), 
                              list(line_vocab[3:])])
        vocab_data = vocab_data[0,:] + vocab_data[1,:] + vocab_data[2,:] + vocab_data[3,:]
        local_vocab.update(Counter(vocab_data.tolist()))
    
    # 五字組合統(tǒng)計(jì)    
    elif len_size == 5:
        vocab_data = np.array([list(line_vocab[:-4]), 
                              list(line_vocab[1:-3]), 
                              list(line_vocab[2:-2]), 
                              list(line_vocab[3:-1]), 
                              list(line_vocab[4:])])
        vocab_data = vocab_data[0,:] + vocab_data[1,:] + vocab_data[2,:] + vocab_data[3,:] + vocab_data[4,:]
        local_vocab.update(Counter(vocab_data.tolist()))

    del line_vocab  # 顯式釋放內(nèi)存
    return local_vocab

# 加載預(yù)處理數(shù)據(jù)
lines = pd.read_pickle("pretrain_hq.pkl")

# 初始化全局詞表
global_vocab = Counter()

# 逐行處理
for line in tqdm(lines):
    global_vocab.update(process_line(line, 1))

# 保存結(jié)果
pd.to_pickle(global_vocab, "vocab_{}.pkl".format(1))

from multiprocessing import Pool

def parallel_process(lines, len_size):
    with Pool() as pool:
        results = pool.starmap(process_line, [(line, len_size) for line in lines])
    return sum(results, Counter())

from flask import Flask, request
import pandas as pd

app = Flask(__name__)
vocab = pd.read_pickle("vocab_1.pkl")

@app.route('/analyze', methods=['POST'])
def analyze():
    text = request.json['text']
    result = {word: vocab[word] for word in text.split()}
    return jsonify(result)