from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
import time

# 配置Chrome無頭模式
chrome_options = Options()
chrome_options.add_argument("--headless")  # 無界面運(yùn)行
chrome_options.add_argument("--disable-gpu")

# 指定ChromeDriver路徑
service = Service(executable_path="/path/to/chromedriver")
driver = webdriver.Chrome(service=service, options=chrome_options)

# 訪問目標(biāo)網(wǎng)頁
url = "https://example.com"
driver.get(url)

# 等待JS執(zhí)行（可替換為顯式等待）
time.sleep(3)  # 簡單等待，實際建議使用WebDriverWait

# 獲取渲染后的HTML
rendered_html = driver.page_source
print(rendered_html)  # 包含JS動態(tài)加載的內(nèi)容

# 提取特定元素
element = driver.find_element(By.CSS_SELECTOR, "div.dynamic-content")
print(element.text)

# 關(guān)閉瀏覽器
driver.quit()

from playwright.sync_api import sync_playwright

# 代理配置
proxyHost = "www.16yun.cn"
proxyPort = "5445"
proxyUser = "16QMSOML"
proxyPass = "280651"

with sync_playwright() as p:
    # 啟動Chromium瀏覽器并配置代理
    browser = p.chromium.launch(
        headless=True,  # 無頭模式
        proxy={
            "server": f"http://{proxyHost}:{proxyPort}",
            "username": proxyUser,
            "password": proxyPass,
        }
    )
    
    # 創(chuàng)建新頁面
    page = browser.new_page()
    
    try:
        # 訪問網(wǎng)頁并等待加載
        page.goto("https://example.com", timeout=10000)  # 增加超時設(shè)置
        page.wait_for_selector("div.dynamic-content")  # 等待目標(biāo)元素出現(xiàn)
        
        # 獲取渲染后的HTML
        rendered_html = page.content()
        print(rendered_html)
        
        # 提取數(shù)據(jù)
        element = page.query_selector("div.dynamic-content")
        if element:
            print(element.inner_text())
        else:
            print("目標(biāo)元素未找到")
            
    except Exception as e:
        print(f"發(fā)生錯誤: {e}")
        
    finally:
        # 確保瀏覽器關(guān)閉
        browser.close()

import asyncio
from pyppeteer import launch

async def fetch_rendered_html():
    # 啟動瀏覽器
    browser = await launch(headless=True)
    page = await browser.newPage()
    
    # 訪問網(wǎng)頁
    await page.goto("https://example.com")
    await page.waitForSelector("div.dynamic-content")  # 等待元素加載
    
    # 獲取HTML
    rendered_html = await page.content()
    print(rendered_html)
    
    # 提取數(shù)據(jù)
    element = await page.querySelector("div.dynamic-content")
    text = await page.evaluate("(element) => element.textContent", element)
    print(text)
    
    # 關(guān)閉瀏覽器
    await browser.close()

# 運(yùn)行異步任務(wù)
asyncio.get_event_loop().run_until_complete(fetch_rendered_html())

from requests_html import HTMLSession

session = HTMLSession()
url = "https://example.com"

# 渲染JS
response = session.get(url)
response.html.render(timeout=20)  # 等待JS執(zhí)行

# 獲取渲染后的HTML
rendered_html = response.html.html
print(rendered_html)

# 提取數(shù)據(jù)
element = response.html.find("div.dynamic-content", first=True)
print(element.text)