brew install ghostscript

sudo apt-get install ghostscript

pip install camelot-py[cv]

python -m venv camelot_env
source camelot_env/bin/activate  # Windows用 camelot_env\Scripts\activate
pip install camelot-py[cv]

import camelot
print(camelot.__version__)

pip install opencv-python

pip install tabula-py

pip install tabula-py[jpype]

import camelot
# Lattice模式是默認(rèn)模式，直接使用即可
tables = camelot.read_pdf('financial_report.pdf')
# 也可以顯式指定flavor參數(shù)
tables = camelot.read_pdf('financial_report.pdf', flavor='lattice')
# 查看提取結(jié)果
print(f"共提取到 {len(tables)} 個(gè)表格")
print(tables[0].df)

# 當(dāng)PDF中存在背景線條可能干擾識別時(shí)
tables = camelot.read_pdf(
    'report_with_background.pdf',
    flavor='lattice',
    process_background=True  # 處理背景線條
)

# 當(dāng)表格邊框不完整時(shí)，調(diào)整line_scale提高線條檢測靈敏度
tables = camelot.read_pdf(
    'incomplete_border.pdf',
    flavor='lattice',
    line_scale=60
)

import camelot

# 顯式使用Stream模式
tables = camelot.read_pdf(
    'no_border_table.pdf',
    flavor='stream'
)

# Stream模式默認(rèn)會(huì)把整個(gè)PDF頁面當(dāng)做一個(gè)表格來解析，需要通過參數(shù)指定區(qū)域
tables = camelot.read_pdf(
    'no_border_table.pdf',
    flavor='stream',
    table_areas=['50,700,500,100'],  # 指定表格區(qū)域
    columns=['A', 'B', 'C']  # 手動(dòng)指定列邊界
)

import tabula

# 讀取PDF中的所有表格（默認(rèn)只讀取第一頁）
tables = tabula.read_pdf("example.pdf")

# 讀取所有頁面的所有表格
tables = tabula.read_pdf("example.pdf", pages='all', multiple_tables=True)

# 讀取指定頁面
tables = tabula.read_pdf("example.pdf", pages='1,3,5-7')

# 讀取遠(yuǎn)程PDF
tables = tabula.read_pdf("https://example.com/report.pdf")

# 使用lattice模式（適用于有邊框的表格）
tables = tabula.read_pdf(
    "structured_table.pdf",
    pages='all',
    lattice=True,
    multiple_tables=True
)

# 使用stream模式（適用于無邊框表格）
tables = tabula.read_pdf(
    "text_table.pdf",
    pages='all',
    lattice=False
)

# 直接轉(zhuǎn)換為CSV文件
tabula.convert_into(
    "example.pdf",
    "output.csv",
    output_format="csv",
    pages='all'
)

# 批量轉(zhuǎn)換目錄下所有PDF
tabula.convert_into_by_batch(
    "pdf_directory/",
    output_format='csv',
    pages='all'
)

import camelot

# 提取第1-10頁的所有表格
tables = camelot.read_pdf(
    'large_report.pdf',
    pages='1-10',
    flavor='lattice'
)

# 提取指定頁面：第1、3、5-7頁
tables = camelot.read_pdf(
    'large_report.pdf',
    pages='1,3,5-7'
)

# 提取所有頁面
tables = camelot.read_pdf(
    'large_report.pdf',
    pages='all'
)

import camelot
import pandas as pd

# 提取所有頁面中的表格
tables = camelot.read_pdf('multi_page_table.pdf', pages='all', flavor='lattice')

# 方法1：檢查解析質(zhì)量后選擇性合并
all_data = []
for i, table in enumerate(tables):
    # 獲取解析報(bào)告，過濾低質(zhì)量提取結(jié)果
    report = table.parsing_report
    if report['accuracy'] > 90:  # 只保留準(zhǔn)確率高于90%的表格
        all_data.append(table.df)
        print(f"第{report['page']}頁表格，準(zhǔn)確率：{report['accuracy']}%")

# 縱向拼接所有表格
if all_data:
    merged_df = pd.concat(all_data, ignore_index=True)
    merged_df.to_excel('merged_output.xlsx', index=False)
else:
    print("未找到有效的表格數(shù)據(jù)")

# 高級版本：處理表頭重復(fù)和列對齊
import camelot
import pandas as pd

def merge_multi_page_tables(pdf_path, pages, flavor='lattice'):
    """
    合并跨頁表格，自動(dòng)處理表頭和列對齊問題
    """
    tables = camelot.read_pdf(pdf_path, pages=pages, flavor=flavor)
    
    all_data = []
    header = None
    
    for idx, table in enumerate(tables):
        df = table.df
        
        # 第一頁：記錄表頭
        if header is None:
            header = df.iloc[0].tolist()
            all_data.append(df.iloc[1:])  # 跳過表頭行
        else:
            # 后續(xù)頁面：檢查并去除可能的表頭行
            if df.iloc[0].tolist() == header:
                all_data.append(df.iloc[1:])  # 跳過重復(fù)表頭
            else:
                all_data.append(df)
    
    # 確保所有子表列數(shù)一致
    max_cols = max([df.shape[1] for df in all_data])
    for i, df in enumerate(all_data):
        if df.shape[1] < max_cols:
            for _ in range(max_cols - df.shape[1]):
                df.insert(df.shape[1], f'col_{df.shape[1]}', '')
    
    result = pd.concat(all_data, ignore_index=True)
    result.columns = header
    return result

# 使用示例
merged_df = merge_multi_page_tables(
    'financial_report.pdf',
    pages='5-15',
    flavor='lattice'
)
merged_df.to_excel('merged_financial_data.xlsx', index=False)

import camelot
import pandas as pd
from pathlib import Path
from concurrent.futures import ProcessPoolExecutor

def process_single_pdf(pdf_path):
    """處理單個(gè)PDF文件"""
    try:
        tables = camelot.read_pdf(pdf_path, pages='all', flavor='lattice')
        if len(tables) > 0:
            # 合并所有表格
            df = pd.concat([t.df for t in tables], ignore_index=True)
            return pdf_path.stem, df
    except Exception as e:
        print(f"處理 {pdf_path} 時(shí)出錯(cuò): {e}")
        return pdf_path.stem, None

def batch_process_pdfs(folder_path, output_path):
    """批量處理文件夾中的所有PDF"""
    pdf_files = list(Path(folder_path).glob('*.pdf'))
    
    # 使用多進(jìn)程加速處理
    with ProcessPoolExecutor(max_workers=4) as executor:
        results = list(executor.map(process_single_pdf, pdf_files))
    
    # 將結(jié)果寫入Excel的不同Sheet
    with pd.ExcelWriter(output_path, engine='openpyxl') as writer:
        for filename, df in results:
            if df is not None:
                # Sheet名稱不超過31個(gè)字符
                sheet_name = filename[:31]
                df.to_excel(writer, sheet_name=sheet_name, index=False)
    
    print(f"批量處理完成，結(jié)果保存至 {output_path}")

# 使用示例
batch_process_pdfs('./pdf_reports/', 'batch_output.xlsx')

import tabula
import pandas as pd
from pathlib import Path

# 方式1：批量轉(zhuǎn)換所有PDF為CSV
tabula.convert_into_by_batch(
    "pdf_directory/",
    output_format='csv',
    pages='all'
)

# 方式2：遍歷處理多個(gè)PDF并合并到單個(gè)Excel
def batch_extract_tabula(pdf_folder, output_excel):
    pdf_files = list(Path(pdf_folder).glob('*.pdf'))
    all_data = {}
    
    for pdf_file in pdf_files:
        # 提取所有表格
        tables = tabula.read_pdf(
            str(pdf_file),
            pages='all',
            multiple_tables=True
        )
        
        if tables:
            # 如果單個(gè)PDF有多個(gè)表格，合并它們
            combined = pd.concat(tables, ignore_index=True)
            all_data[pdf_file.stem] = combined
    
    with pd.ExcelWriter(output_excel) as writer:
        for name, df in all_data.items():
            df.to_excel(writer, sheet_name=name[:31], index=False)

batch_extract_tabula('./pdfs/', 'tabula_batch_output.xlsx')

import camelot
import pandas as pd
import numpy as np

def extract_financial_statement(pdf_path, start_page, end_page):
    """
    提取財(cái)務(wù)報(bào)表數(shù)據(jù)
    
    參數(shù):
        pdf_path: PDF文件路徑
        start_page: 起始頁
        end_page: 結(jié)束頁
    """
    # 使用Lattice模式提取所有頁面
    tables = camelot.read_pdf(
        pdf_path,
        pages=f'{start_page}-{end_page}',
        flavor='lattice',
        process_background=True  # 處理可能的背景線條干擾
    )
    
    print(f"共提取到 {len(tables)} 個(gè)表格片段")
    
    # 檢查每個(gè)表格的解析質(zhì)量
    quality_report = []
    for i, table in enumerate(tables):
        report = table.parsing_report
        quality_report.append({
            'table_index': i,
            'page': report['page'],
            'accuracy': report['accuracy'],
            'whitespace': report['whitespace']
        })
        print(f"表格 {i} - 頁碼:{report['page']}, 準(zhǔn)確率:{report['accuracy']}%")
    
    # 篩選高質(zhì)量表格（準(zhǔn)確率>90%）
    high_quality_tables = [
        table for table in tables 
        if table.parsing_report['accuracy'] > 90
    ]
    
    # 拼接所有表格
    all_dfs = [table.df for table in high_quality_tables]
    
    # 處理合并單元格造成的空值
    for df in all_dfs:
        # 前向填充空值（處理合并單元格）
        df.fillna(method='ffill', axis=0, inplace=True)
    
    # 縱向拼接
    final_df = pd.concat(all_dfs, ignore_index=True)
    
    # 數(shù)據(jù)清洗：去除全為空的行和列
    final_df.dropna(how='all', axis=0, inplace=True)
    final_df.dropna(how='all', axis=1, inplace=True)
    
    # 設(shè)置第一行為列名（通常是表頭）
    final_df.columns = final_df.iloc[0]
    final_df = final_df.iloc[1:].reset_index(drop=True)
    
    return final_df, quality_report

# 執(zhí)行提取
pdf_path = 'annual_report_2024.pdf'
financial_data, report = extract_financial_statement(pdf_path, 5, 15)

# 保存結(jié)果
financial_data.to_excel('profit_statement.xlsx', index=False)
print(f"提取完成！共獲得 {len(financial_data)} 行數(shù)據(jù)")

# 數(shù)據(jù)驗(yàn)證與清洗
def validate_extracted_data(df, expected_columns):
    """驗(yàn)證提取數(shù)據(jù)的完整性和準(zhǔn)確性"""
    
    print("=" * 50)
    print("數(shù)據(jù)驗(yàn)證報(bào)告")
    print("=" * 50)
    
    # 1. 檢查列數(shù)是否與預(yù)期一致
    print(f"實(shí)際列數(shù): {len(df.columns)}")
    print(f"預(yù)期列數(shù): {len(expected_columns)}")
    
    # 2. 檢查關(guān)鍵數(shù)據(jù)列是否存在
    for col in expected_columns:
        if col in df.columns:
            print(f"? 列 '{col}' 存在")
        else:
            print(f"? 列 '{col}' 缺失")
    
    # 3. 檢查數(shù)值列的數(shù)據(jù)類型
    numeric_cols = df.select_dtypes(include=[np.number]).columns
    print(f"數(shù)值列: {len(numeric_cols)} 列")
    
    # 4. 檢查空值比例
    null_ratio = df.isnull().sum() / len(df) * 100
    for col, ratio in null_ratio.items():
        if ratio > 10:
            print(f"? 列 '{col}' 空值比例過高: {ratio:.1f}%")
    
    # 5. 數(shù)據(jù)預(yù)覽
    print("\n數(shù)據(jù)預(yù)覽:")
    print(df.head(10))
    
    return df

expected_columns = ['項(xiàng)目', '2024年度', '2023年度', '增減幅度(%)']
validate_extracted_data(financial_data, expected_columns)

from pdf2image import convert_from_path
import pytesseract
import camelot

def extract_with_ocr(pdf_path, pages):
    """
    處理掃描件PDF：先OCR識別，再提取表格
    """
    # 將PDF頁面轉(zhuǎn)換為圖像
    images = convert_from_path(pdf_path, dpi=300, first_page=pages[0], last_page=pages[-1])
    
    # 保存為臨時(shí)PDF文件
    temp_pdf_path = 'temp_ocr_output.pdf'
    images[0].save(temp_pdf_path, save_all=True, append_images=images[1:])
    
    # 對OCR后的PDF進(jìn)行表格提取
    tables = camelot.read_pdf(temp_pdf_path, pages='all', flavor='lattice')
    
    return tables

# 無邊框表格，嘗試Stream模式
tables = camelot.read_pdf(
    'no_border_table.pdf',
    flavor='stream',
    table_areas=['50,700,550,150'],  # 指定表格區(qū)域
    columns=['A', 'B', 'C', 'D']  # 手動(dòng)指定列邊界
)

# 先獲取文本位置信息，確定列邊界
from camelot.utils import get_page_layout

layout = get_page_layout('no_border_table.pdf', pages='1')
print(layout)  # 查看文本塊的x坐標(biāo)，推斷列邊界位置

# 根據(jù)分析結(jié)果設(shè)置列邊界
tables = camelot.read_pdf(
    'no_border_table.pdf',
    flavor='stream',
    columns=['100', '250', '400', '550']  # 列邊界的x坐標(biāo)
)

# 對于傾斜表格，使用Lattice模式并調(diào)整line_scale參數(shù)
tables = camelot.read_pdf(
    'rotated_table.pdf',
    flavor='lattice',
    line_scale=40  # 提高線條檢測靈敏度
)

import cv2
import numpy as np
from pdf2image import convert_from_path

def deskew_table(pdf_path, page_num):
    """檢測并矯正傾斜的表格"""
    images = convert_from_path(pdf_path, first_page=page_num, last_page=page_num)
    img = np.array(images[0])
    
    # 轉(zhuǎn)換為灰度圖
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    
    # 檢測邊緣
    edges = cv2.Canny(gray, 50, 150, apertureSize=3)
    
    # 霍夫變換檢測直線
    lines = cv2.HoughLines(edges, 1, np.pi/180, threshold=100)
    
    if lines is not None:
        # 計(jì)算傾斜角度
        angles = []
        for rho, theta in lines[:,0]:
            angle = theta * 180 / np.pi - 90
            if abs(angle) < 45:
                angles.append(angle)
        
        if angles:
            median_angle = np.median(angles)
            # 旋轉(zhuǎn)矯正
            (h, w) = img.shape[:2]
            center = (w // 2, h // 2)
            M = cv2.getRotationMatrix2D(center, median_angle, 1.0)
            rotated = cv2.warpAffine(img, M, (w, h))
            return rotated
    
    return img

# 使用process_background參數(shù)過濾背景線條
tables = camelot.read_pdf(
    'table_with_background.pdf',
    flavor='lattice',
    process_background=True  # 區(qū)分前景和背景線條
)

def handle_merged_cells(df):
    """
    處理合并單元格造成的空值問題
    """
    # 縱向填充（處理行合并）
    df.fillna(method='ffill', axis=0, inplace=True)
    
    # 橫向填充（處理列合并）
    df.fillna(method='ffill', axis=1, inplace=True)
    
    return df

# 使用示例
df_cleaned = handle_merged_cells(extracted_df)

import camelot

# 提取表格
tables = camelot.read_pdf('problematic_table.pdf', flavor='lattice')

# 可視化檢測到的表格網(wǎng)格
tables[0].plot(kind='grid')
# 可視化檢測到的文本
tables[0].plot(kind='text')
# 可視化表格輪廓
tables[0].plot(kind='contour')

# 保存可視化圖像
import matplotlib.pyplot as plt
plt.savefig('debug_grid.png')