import os
import json
import time
from tqdm import tqdm  # 需要先安裝：pip install tqdm
 
 
def process_files():
    # 設(shè)置目錄路徑
    dir_path = r"D:\daku\關(guān)鍵詞識(shí)別\1623-0000001\zh"
 
    # 獲取并排序文件列表
    file_list = sorted([f for f in os.listdir(dir_path) if f.lower().endswith('.jsonl')],
                       key=lambda x: os.path.getsize(os.path.join(dir_path, x)),
                       reverse=True)  # 按文件大小降序排列
 
    # 進(jìn)度統(tǒng)計(jì)
    total_files = len(file_list)
    processed_files = 0
    total_lines = sum(1 for f in file_list for _ in open(os.path.join(dir_path, f), 'r', encoding='utf-8'))
    processed_lines = 0
    start_time = time.time()
 
    # 輸出文件設(shè)置
    output_file = os.path.join(dir_path, "combined_contents.txt")
 
    with open(output_file, "w", encoding="utf-8") as outfile:
        with tqdm(total=total_lines, desc="合并進(jìn)度", unit="line") as pbar:
            for filename in file_list:
                file_path = os.path.join(dir_path, filename)
                try:
                    with open(file_path, "r", encoding="utf-8") as infile:
                        file_size = os.path.getsize(file_path)
                        chunk_size = max(1024 * 1024, file_size // 100)  # 動(dòng)態(tài)調(diào)整讀取塊大小
 
                        while True:
                            lines = infile.readlines(chunk_size)
                            if not lines:
                                break
 
                            for line_num, line in enumerate(lines, 1):
                                line = line.strip()
                                if not line:
                                    continue
 
                                try:
                                    data = json.loads(line)
                                    content = data.get("content", "").replace("\n", " ")  # 清除內(nèi)容中的換行符
                                    outfile.write(content + "\n\n")  # 用雙換行分隔記錄
                                    processed_lines += 1
                                except json.JSONDecodeError:
                                    print(f"\nJSON解析失敗: {filename} 第{processed_lines + 1}行")
                                except Exception as e:
                                    print(f"\n處理異常: {filename} 第{processed_lines + 1}行 - {str(e)}")
 
                                # 進(jìn)度更新
                                pbar.update(1)
                                if processed_lines % 1000 == 0:
                                    elapsed = time.time() - start_time
                                    speed = processed_lines / (elapsed + 1e-5)
                                    remaining = (total_lines - processed_lines) / (speed + 1e-5)
                                    pbar.set_postfix({
                                        '速度': f"{speed:.1f} lines/s",
                                        '剩余時(shí)間': f"{remaining // 3600:.0f}h {remaining % 3600 // 60:.0f}m"
                                    })
 
                    processed_files += 1
                except Exception as e:
                    print(f"\n無(wú)法讀取文件 {filename}: {str(e)}")
 
    # 生成統(tǒng)計(jì)報(bào)告
    end_time = time.time()
    print(f"\n合并完成！共處理 {processed_files}/{total_files} 個(gè)文件")
    print(f"總記錄數(shù): {processed_lines:,} 條")
    print(f"耗時(shí): {end_time - start_time:.2f} 秒")
    print(f"輸出文件路徑: {output_file}")
 
 
if __name__ == "__main__":
    process_files()

[{"name": "John", "age": 30},
{"name": "Jane", "age": 25},
{"name": "Bob", "age": 40}]

{"name": "John", "age": 30}
{"name": "Jane", "age": 25}
{"name": "Bob", "age": 40}

import os
from chardet import detect
 
 
def get_safe_encoding(encoding):
    """將檢測(cè)到的編碼轉(zhuǎn)換為更安全的兼容編碼"""
    encoding = encoding.lower()
    if encoding in ['gb2312', 'gbk']:
        return 'gb18030'  # 最全面的中文編碼
    return encoding
 
 
def get_file_encoding(file_path):
    """獲取文件編碼并自動(dòng)升級(jí)到更安全的版本"""
    with open(file_path, 'rb') as f:
        raw_data = f.read(10000)
    result = detect(raw_data)
    # 過(guò)濾低置信度檢測(cè)（confidence<0.8則視為不可信）
    if result['confidence'] < 0.8:
        return 'gb18030'
    return get_safe_encoding(result['encoding'])
 
 
def merge_files(directory, output_filename='merged.txt'):
    seen_lines = set()
    output_path = os.path.join(directory, output_filename)
 
    txt_files = [os.path.join(directory, f) for f in os.listdir(directory) if f.endswith('.txt')]
 
    with open(output_path, 'w', encoding='utf-8', errors='ignore') as outfile:
        for file_path in txt_files:
            try:
                # 獲取安全編碼并添加錯(cuò)誤處理
                file_enc = get_file_encoding(file_path)
                with open(file_path, 'r',
                          encoding=file_enc,
                          errors='backslashreplace') as infile:  # 保留無(wú)法解碼字符
                    for line_idx, line in enumerate(infile, 1):
                        try:
                            stripped_line = line.rstrip('\n')
                            if stripped_line not in seen_lines:
                                outfile.write(line)
                                seen_lines.add(stripped_line)
                        except Exception as line_err:
                            print(f"文件 {os.path.basename(file_path)} 第 {line_idx} 行處理異常: {str(line_err)}")
                            continue
            except Exception as file_err:
                print(f"文件 {os.path.basename(file_path)} 讀取失敗: {str(file_err)}")
                continue
 
 
if __name__ == '__main__':
    target_directory = r'D:\daku\關(guān)鍵詞識(shí)別\stop6931'
    merge_files(target_directory)
    print(f'合并完成，輸出文件：{os.path.join(target_directory, "merged.txt")}')