import fitz  # PyMuPDF

# 打開PDF文件
doc = fitz.open('example.pdf')

# 提取所有文本
all_text = ""
for page in doc:
    all_text += page.get_text()

# 提取頁面中的圖像
for page_num in range(len(doc)):
    page = doc.load_page(page_num)
    image_list = page.get_images()
    
    for img_index, img in enumerate(image_list):
        xref = img[0]
        pix = fitz.Pixmap(doc, xref)
        if pix.n - pix.alpha > 3:  # 檢查是否為RGB圖像
            pix = fitz.Pixmap(fitz.csRGB, pix)
        pix.save(f"page_{page_num}_img_{img_index}.png")

# 提取帶格式的文本（保留位置信息）
for page in doc:
    blocks = page.get_text("dict")["blocks"]
    for block in blocks:
        if "lines" in block:  # 文本塊
            for line in block["lines"]:
                for span in line["spans"]:
                    print(f"文本: {span['text']}")
                    print(f"字體: {span['font']}")
                    print(f"大小: {span['size']}")
                    print(f"位置: {span['bbox']}")

doc.close()

import pdfplumber

with pdfplumber.open('example.pdf') as pdf:
    # 提取第一頁的文本
    first_page = pdf.pages[0]
    text = first_page.extract_text()
    print(text)
    
    # 提取表格
    tables = first_page.extract_tables()
    for table in tables:
        for row in table:
            print(row)
    
    # 可視化文本位置（調(diào)試用）
    im = first_page.to_image()
    im.debug_tablefinder().show()

from unstructured.partition.pdf import partition_pdf

# 解析PDF并自動識別元素類型
elements = partition_pdf(
    filename="example.pdf",
    strategy="auto",  # 自動選擇解析策略
    infer_table_structure=True,  # 推斷表格結(jié)構(gòu)
    include_page_breaks=True  # 包含分頁符
)

# 查看識別出的元素類型
for element in elements:
    print(f"類型: {type(element).__name__}")
    print(f"文本: {str(element)[:100]}...")
    print("-" * 50)
    
    # 不同類型的元素有不同的屬性
    if hasattr(element, 'category'):
        print(f"分類: {element.category}")

import fitz

def extract_chinese_pdf(pdf_path):
    doc = fitz.open(pdf_path)
    
    # 方法1：嘗試直接提取
    text = ""
    for page in doc:
        text += page.get_text()
    
    # 如果提取的中文是亂碼，嘗試指定編碼
    if "亂碼" in text or len(text.strip()) < 10:
        print("檢測到可能的編碼問題，嘗試其他方法...")
        
        # 方法2：使用OCR（需要安裝額外的依賴）
        # 這里展示思路，實際需要安裝pytesseract和Pillow
        # import pytesseract
        # from PIL import Image
        # 
        # for page_num in range(len(doc)):
        #     pix = doc[page_num].get_pixmap()
        #     img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
        #     text += pytesseract.image_to_string(img, lang='chi_sim')
    
    return text

import markdown

# 基本使用：將Markdown轉(zhuǎn)換為HTML
md_text = """
# 標(biāo)題一

這是一個段落，包含**加粗**和*斜體*文本。

- 列表項1
- 列表項2

[這是一個鏈接](https://example.com)
"""

html_output = markdown.markdown(md_text)
print(html_output)

# 使用擴展功能
html_with_extensions = markdown.markdown(
    md_text,
    extensions=[
        'toc',           # 目錄生成
        'tables',        # 表格支持
        'fenced_code',   # 代碼塊
        'footnotes'      # 腳注
    ]
)

# 解析為AST（抽象語法樹）
from markdown.extensions import Extension
from markdown.treeprocessors import Treeprocessor

class LinkCollector(Treeprocessor):
    def run(self, root):
        links = []
        for element in root.iter():
            if element.tag == 'a':
                links.append(element.get('href'))
        return links

class MyExtension(Extension):
    def extendMarkdown(self, md):
        md.treeprocessors.register(LinkCollector(md), 'linkcollector', 15)

md = markdown.Markdown(extensions=[MyExtension()])
result = md.convert(md_text)
print(f"找到的鏈接: {md.treeprocessors['linkcollector'].run(md.parser.root)}")

from markdown_it import MarkdownIt
from markdown_it.tree import SyntaxTreeNode

# 創(chuàng)建解析器實例
md = MarkdownIt()

# 解析Markdown
tokens = md.parse(md_text)

# 遍歷語法標(biāo)記
for token in tokens:
    if token.type == 'heading_open' and token.tag == 'h1':
        print("找到一個一級標(biāo)題")
    elif token.type == 'inline':
        print(f"文本內(nèi)容: {token.content}")

# 轉(zhuǎn)換為語法樹
tree = SyntaxTreeNode(tokens)
for node in tree.walk():
    if node.type == 'heading' and node.tag == 'h1':
        print(f"標(biāo)題: {node.children[0].content}")

# 基礎(chǔ)讀取
def read_txt_basic(filepath):
    with open(filepath, 'r', encoding='utf-8') as f:
        content = f.read()
    return content

# 處理大文件（逐行讀?。?
def process_large_txt(filepath):
    with open(filepath, 'r', encoding='utf-8') as f:
        for line_number, line in enumerate(f, 1):
            # 處理每一行
            processed_line = line.strip()
            if processed_line:  # 跳過空行
                print(f"行 {line_number}: {processed_line[:50]}...")

# 自動檢測編碼
import chardet

def read_txt_with_encoding_detection(filepath):
    with open(filepath, 'rb') as f:
        raw_data = f.read()
        result = chardet.detect(raw_data)
        encoding = result['encoding']
        confidence = result['confidence']
        
        print(f"檢測到編碼: {encoding} (置信度: {confidence})")
        
        try:
            return raw_data.decode(encoding)
        except UnicodeDecodeError:
            # 如果檢測失敗，嘗試常見編碼
            for enc in ['utf-8', 'gbk', 'gb2312', 'latin-1']:
                try:
                    return raw_data.decode(enc)
                except UnicodeDecodeError:
                    continue
            raise

# 使用Pandas處理結(jié)構(gòu)化文本數(shù)據(jù)
import pandas as pd

def process_tabular_txt(filepath):
    # 讀取CSV/TSV文件
    try:
        # 嘗試用逗號分隔
        df = pd.read_csv(filepath, encoding='utf-8')
    except:
        try:
            # 嘗試用制表符分隔
            df = pd.read_csv(filepath, sep='\t', encoding='utf-8')
        except:
            # 嘗試自動檢測分隔符
            with open(filepath, 'r') as f:
                first_line = f.readline()
            
            if ',' in first_line:
                df = pd.read_csv(filepath, sep=',', encoding='utf-8')
            elif '\t' in first_line:
                df = pd.read_csv(filepath, sep='\t', encoding='utf-8')
            else:
                df = pd.read_csv(filepath, delim_whitespace=True, encoding='utf-8')
    
    # 數(shù)據(jù)分析示例
    print(f"數(shù)據(jù)形狀: {df.shape}")
    print(f"列名: {df.columns.tolist()}")
    print(f"前5行:\n{df.head()}")
    
    return df

from docx import Document
from docx.document import Document as DocDocument

def read_docx(filepath):
    # 打開文檔
    doc = Document(filepath)
    
    # 提取所有段落
    full_text = []
    for paragraph in doc.paragraphs:
        if paragraph.text.strip():  # 跳過空段落
            full_text.append(paragraph.text)
            print(f"段落: {paragraph.text[:50]}...")
    
    # 提取表格數(shù)據(jù)
    tables_data = []
    for table in doc.tables:
        table_data = []
        for row in table.rows:
            row_data = [cell.text for cell in row.cells]
            table_data.append(row_data)
        tables_data.append(table_data)
        print(f"表格找到，有{len(table.rows)}行{len(table.columns)}列")
    
    # 提取樣式信息
    styled_elements = []
    for paragraph in doc.paragraphs:
        style_info = {
            'text': paragraph.text,
            'style': paragraph.style.name,
            'runs': []
        }
        
        # 獲取運行級別的格式
        for run in paragraph.runs:
            run_info = {
                'text': run.text,
                'bold': run.bold,
                'italic': run.italic,
                'underline': run.underline,
                'font_name': run.font.name,
                'font_size': run.font.size
            }
            style_info['runs'].append(run_info)
        
        if style_info['runs']:
            styled_elements.append(style_info)
    
    # 處理列表
    lists = []
    for paragraph in doc.paragraphs:
        if paragraph.style.name.startswith('List'):
            lists.append({
                'text': paragraph.text,
                'style': paragraph.style.name,
                'level': get_list_level(paragraph.style.name)
            })
    
    return {
        'full_text': '\n'.join(full_text),
        'tables': tables_data,
        'styled_elements': styled_elements,
        'lists': lists
    }

def get_list_level(style_name):
    """獲取列表縮進級別"""
    if '1' in style_name:
        return 1
    elif '2' in style_name:
        return 2
    elif '3' in style_name:
        return 3
    else:
        return 0

# 處理舊版.doc文件（需要額外的庫）
def read_doc_file(filepath):
    # 注意：python-docx只能處理.docx文件
    # 處理.doc文件需要安裝antiword或使用其他方法
    
    # 方法1：使用LibreOffice轉(zhuǎn)換（需要系統(tǒng)安裝LibreOffice）
    # import subprocess
    # subprocess.run(['libreoffice', '--headless', '--convert-to', 'docx', filepath])
    
    # 方法2：使用pywin32（僅Windows）
    # import win32com.client
    # word = win32com.client.Dispatch("Word.Application")
    # doc = word.Documents.Open(filepath)
    # text = doc.Content.Text
    # doc.Close()
    # word.Quit()
    
    print("處理.doc文件需要額外的工具")
    return None

from ebooklib import epub
import html2text

def read_epub(filepath):
    # 打開EPUB文件
    book = epub.read_epub(filepath)
    
    # 獲取書籍元數(shù)據(jù)
    metadata = {
        'title': book.get_metadata('DC', 'title'),
        'creator': book.get_metadata('DC', 'creator'),
        'publisher': book.get_metadata('DC', 'publisher'),
        'date': book.get_metadata('DC', 'date'),
        'language': book.get_metadata('DC', 'language'),
        'identifier': book.get_metadata('DC', 'identifier')
    }
    
    print(f"書名: {metadata['title']}")
    print(f"作者: {metadata['creator']}")
    
    # 提取所有文本內(nèi)容
    h = html2text.HTML2Text()
    h.ignore_links = False
    h.ignore_images = False
    
    full_text = []
    toc_items = []
    
    # 處理目錄
    for item in book.toc:
        if isinstance(item, tuple):
            # 處理嵌套目錄項
            section, subsections = item
            toc_items.append({
                'title': section.title,
                'href': section.href
            })
        else:
            toc_items.append({
                'title': item.title,
                'href': item.href
            })
    
    # 按章節(jié)讀取內(nèi)容
    for item in book.get_items():
        if item.get_type() == ebooklib.ITEM_DOCUMENT:
            # 獲取章節(jié)內(nèi)容（HTML格式）
            content = item.get_content().decode('utf-8')
            
            # 轉(zhuǎn)換為純文本
            text_content = h.handle(content)
            
            # 清理文本
            cleaned_text = clean_epub_text(text_content)
            
            if cleaned_text.strip():
                full_text.append({
                    'title': item.get_name(),
                    'content': cleaned_text,
                    'raw_html': content[:500] + '...'  # 保存部分HTML供參考
                })
    
    # 按目錄順序組織內(nèi)容
    organized_content = organize_by_toc(full_text, toc_items)
    
    return {
        'metadata': metadata,
        'toc': toc_items,
        'content': organized_content,
        'full_text': '\n\n'.join([item['content'] for item in full_text])
    }

def clean_epub_text(text):
    """清理EPUB文本中的多余空白和標(biāo)記"""
    lines = text.split('\n')
    cleaned_lines = []
    
    for line in lines:
        line = line.strip()
        if line and not line.startswith('#' * 4):  # 跳過HTML2Text的標(biāo)題標(biāo)記
            cleaned_lines.append(line)
    
    return '\n'.join(cleaned_lines)

def organize_by_toc(content_items, toc):
    """根據(jù)目錄組織內(nèi)容"""
    organized = []
    
    for toc_item in toc:
        # 查找對應(yīng)章節(jié)
        for content_item in content_items:
            if toc_item['href'] in content_item['title']:
                organized.append({
                    'toc_title': toc_item['title'],
                    'content_title': content_item['title'],
                    'content': content_item['content']
                })
                break
    
    return organized

class UniversalDocumentParser:
    def __init__(self):
        self.supported_formats = {
            '.pdf': self._parse_pdf,
            '.md': self._parse_markdown,
            '.txt': self._parse_text,
            '.docx': self._parse_docx,
            '.epub': self._parse_epub
        }
    
    def parse(self, filepath):
        import os
        
        # 獲取文件擴展名
        _, ext = os.path.splitext(filepath)
        ext = ext.lower()
        
        # 檢查是否支持該格式
        if ext not in self.supported_formats:
            raise ValueError(f"不支持的文件格式: {ext}")
        
        # 調(diào)用對應(yīng)的解析函數(shù)
        return self.supported_formats[ext](filepath)
    
    def _parse_pdf(self, filepath):
        """解析PDF文件"""
        # 根據(jù)需求選擇合適的PDF解析器
        try:
            # 首先嘗試使用PyMuPDF（速度快）
            import fitz
            doc = fitz.open(filepath)
            text = ""
            for page in doc:
                text += page.get_text() + "\n"
            doc.close()
            return {'format': 'pdf', 'content': text, 'parser': 'PyMuPDF'}
        except ImportError:
            # 回退到其他解析器
            try:
                import pdfplumber
                with pdfplumber.open(filepath) as pdf:
                    text = ""
                    for page in pdf.pages:
                        text += page.extract_text() + "\n"
                return {'format': 'pdf', 'content': text, 'parser': 'pdfplumber'}
            except ImportError:
                raise ImportError("請安裝PyMuPDF或pdfplumber以解析PDF文件")
    
    def _parse_markdown(self, filepath):
        """解析Markdown文件"""
        import markdown
        with open(filepath, 'r', encoding='utf-8') as f:
            content = f.read()
        
        # 轉(zhuǎn)換為HTML
        html = markdown.markdown(content)
        
        return {
            'format': 'markdown',
            'raw_content': content,
            'html_content': html
        }
    
    def _parse_text(self, filepath):
        """解析純文本文件"""
        # 自動檢測編碼
        import chardet
        
        with open(filepath, 'rb') as f:
            raw_data = f.read()
            result = chardet.detect(raw_data)
            encoding = result['encoding']
        
        with open(filepath, 'r', encoding=encoding) as f:
            content = f.read()
        
        return {
            'format': 'text',
            'encoding': encoding,
            'content': content
        }
    
    def _parse_docx(self, filepath):
        """解析DOCX文件"""
        from docx import Document
        
        doc = Document(filepath)
        paragraphs = [p.text for p in doc.paragraphs if p.text.strip()]
        
        # 提取表格
        tables = []
        for table in doc.tables:
            table_data = []
            for row in table.rows:
                table_data.append([cell.text for cell in row.cells])
            tables.append(table_data)
        
        return {
            'format': 'docx',
            'paragraphs': paragraphs,
            'tables': tables,
            'full_text': '\n'.join(paragraphs)
        }
    
    def _parse_epub(self, filepath):
        """解析EPUB文件"""
        import ebooklib
        from ebooklib import epub
        import html2text
        
        book = epub.read_epub(filepath)
        h = html2text.HTML2Text()
        h.ignore_links = True
        
        # 提取所有文本內(nèi)容
        text_parts = []
        for item in book.get_items():
            if item.get_type() == ebooklib.ITEM_DOCUMENT:
                content = item.get_content().decode('utf-8')
                text = h.handle(content)
                if text.strip():
                    text_parts.append(text)
        
        full_text = '\n\n'.join(text_parts)
        
        # 提取元數(shù)據(jù)
        metadata = {}
        for key in ['title', 'creator', 'publisher', 'date']:
            meta = book.get_metadata('DC', key)
            if meta:
                metadata[key] = meta[0][0]
        
        return {
            'format': 'epub',
            'metadata': metadata,
            'content': full_text
        }

# 使用示例
parser = UniversalDocumentParser()

# 解析各種格式的文件
formats_to_test = ['document.pdf', 'notes.md', 'data.txt', 'report.docx', 'book.epub']

for file in formats_to_test:
    try:
        result = parser.parse(file)
        print(f"成功解析 {file}: {result['format']} 格式")
        print(f"內(nèi)容預(yù)覽: {result.get('content', result.get('full_text', ''))[:100]}...")
        print("-" * 50)
    except FileNotFoundError:
        print(f"文件不存在: {file}")
    except Exception as e:
        print(f"解析 {file} 時出錯: {e}")

class EfficientDocumentProcessor:
    def __init__(self, chunk_size=1000):
        self.chunk_size = chunk_size  # 每次處理的塊大小
    
    def process_large_pdf(self, filepath, callback=None):
        """流式處理大型PDF文件"""
        import fitz
        
        doc = fitz.open(filepath)
        total_pages = len(doc)
        
        for page_num in range(total_pages):
            page = doc.load_page(page_num)
            text = page.get_text()
            
            # 分塊處理文本
            chunks = self._split_into_chunks(text)
            
            for chunk in chunks:
                if callback:
                    callback(chunk, page_num + 1)
                else:
                    yield chunk, page_num + 1
        
        doc.close()
    
    def _split_into_chunks(self, text, chunk_size=None):
        """將文本分割為指定大小的塊"""
        if chunk_size is None:
            chunk_size = self.chunk_size
        
        chunks = []
        words = text.split()
        
        current_chunk = []
        current_size = 0
        
        for word in words:
            word_size = len(word) + 1  # 加1是空格
            
            if current_size + word_size > chunk_size and current_chunk:
                chunks.append(' '.join(current_chunk))
                current_chunk = [word]
                current_size = word_size
            else:
                current_chunk.append(word)
                current_size += word_size
        
        if current_chunk:
            chunks.append(' '.join(current_chunk))
        
        return chunks

import logging
from functools import wraps

# 配置日志
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

def document_parser_error_handler(func):
    """文檔解析器的錯誤處理裝飾器"""
    @wraps(func)
    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except FileNotFoundError as e:
            logger.error(f"文件未找到: {e.filename}")
            raise
        except PermissionError as e:
            logger.error(f"權(quán)限不足: {e.filename}")
            raise
        except UnicodeDecodeError as e:
            logger.error(f"編碼錯誤: {e.reason}")
            # 嘗試其他編碼
            return handle_encoding_error(*args, **kwargs)
        except Exception as e:
            logger.exception(f"解析文檔時發(fā)生未知錯誤: {e}")
            raise
    return wrapper

@document_parser_error_handler
def safe_document_parse(filepath, parser_func):
    """安全的文檔解析函數(shù)"""
    return parser_func(filepath)