import requests
from bs4 import BeautifulSoup
 
# 發(fā)送 HTTP 請(qǐng)求
url = "https://example.com"
response = requests.get(url)
 
# 確保請(qǐng)求成功
if response.status_code == 200:
    # 使用 BeautifulSoup 解析網(wǎng)頁
    soup = BeautifulSoup(response.content, 'html.parser')
    
    # 提取網(wǎng)頁中的標(biāo)題
    title = soup.title.string
    print(f"網(wǎng)頁標(biāo)題：{title}")
    
    # 提取網(wǎng)頁中的所有鏈接
    for link in soup.find_all('a'):
        print(f"鏈接：{link.get('href')}")
else:
    print("網(wǎng)頁請(qǐng)求失敗")

import requests
from lxml import html
 
# 發(fā)送 HTTP 請(qǐng)求
url = "https://example.com"
response = requests.get(url)
 
# 確保請(qǐng)求成功
if response.status_code == 200:
    # 使用 lxml 解析網(wǎng)頁
    tree = html.fromstring(response.content)
    
    # 提取網(wǎng)頁中的標(biāo)題
    title = tree.xpath('//title/text()')
    print(f"網(wǎng)頁標(biāo)題：{title[0] if title else '無標(biāo)題'}")
    
    # 提取所有鏈接
    links = tree.xpath('//a/@href')
    for link in links:
        print(f"鏈接：{link}")
else:
    print("網(wǎng)頁請(qǐng)求失敗")

from selenium import webdriver
from bs4 import BeautifulSoup
import time
 
# 啟動(dòng) WebDriver
driver = webdriver.Chrome(executable_path="path/to/chromedriver")
 
# 訪問網(wǎng)頁
url = "https://example.com"
driver.get(url)
 
# 等待頁面加載
time.sleep(3)
 
# 獲取頁面源代碼
html = driver.page_source
 
# 使用 BeautifulSoup 解析網(wǎng)頁
soup = BeautifulSoup(html, 'html.parser')
 
# 提取網(wǎng)頁中的標(biāo)題
title = soup.title.string
print(f"網(wǎng)頁標(biāo)題：{title}")
 
# 提取網(wǎng)頁中的所有鏈接
for link in soup.find_all('a'):
    print(f"鏈接：{link.get('href')}")
 
# 關(guān)閉瀏覽器
driver.quit()

scrapy startproject myproject

cd myproject
scrapy genspider example_spider example.com

import scrapy
 
class ExampleSpider(scrapy.Spider):
    name = 'example_spider'
    start_urls = ['https://example.com']
 
    def parse(self, response):
        # 提取網(wǎng)頁標(biāo)題
        title = response.css('title::text').get()
        print(f"網(wǎng)頁標(biāo)題：{title}")
 
        # 提取所有鏈接
        links = response.css('a::attr(href)').getall()
        for link in links:
            print(f"鏈接：{link}")

scrapy crawl example_spider

from pyquery import PyQuery as pq
import requests
 
# 發(fā)送 HTTP 請(qǐng)求
url = "https://example.com"
response = requests.get(url)
 
# 使用 PyQuery 解析網(wǎng)頁
doc = pq(response.content)
 
# 提取網(wǎng)頁標(biāo)題
title = doc('title').text()
print(f"網(wǎng)頁標(biāo)題：{title}")
 
# 提取網(wǎng)頁中的所有鏈接
for link in doc('a').items():
    print(f"鏈接：{link.attr('href')}")