pip install requests beautifulsoup4

import requests
from bs4 import BeautifulSoup

# 發(fā)送 HTTP GET 請求
url = "https://www.example.com"
response = requests.get(url)

# 解析 HTML 內容
soup = BeautifulSoup(response.text, 'html.parser')

# 提取標題
title = soup.title.text
print("網(wǎng)頁標題:", title)

# 提取所有鏈接
links = soup.find_all('a')  # 查找所有 <a> 標簽
for link in links:
    href = link.get('href')
    print("鏈接:", href)

import requests
from bs4 import BeautifulSoup

url = "https://quotes.toscrape.com/"  # 一個簡單的網(wǎng)頁，包含名言和作者
response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser')

# 提取每個名言和作者
quotes = soup.find_all('div', class_='quote')
for quote in quotes:
    text = quote.find('span', class_='text').text
    author = quote.find('small', class_='author').text
    print(f"名言: {text}, 作者: {author}")

pip install requests lxml

import requests
from lxml import html

# 發(fā)送請求
url = "https://quotes.toscrape.com/"
response = requests.get(url)

# 解析 HTML
tree = html.fromstring(response.text)

# 提取名言和作者
quotes = tree.xpath('//div[@class="quote"]')
for quote in quotes:
    text = quote.xpath('.//span[@class="text"]/text()')[0]
    author = quote.xpath('.//small[@class="author"]/text()')[0]
    print(f"名言: {text}, 作者: {author}")

pip install selenium

from selenium import webdriver
from selenium.webdriver.common.by import By
import time

# 設置 ChromeDriver 路徑
driver = webdriver.Chrome(executable_path='/path/to/chromedriver')

# 打開網(wǎng)頁
driver.get("https://quotes.toscrape.com/js/")  # 一個動態(tài)加載的網(wǎng)頁

# 等待頁面加載
time.sleep(2)

# 獲取頁面內容
quotes = driver.find_elements(By.CLASS_NAME, 'quote')
for quote in quotes:
    text = quote.find_element(By.CLASS_NAME, 'text').text
    author = quote.find_element(By.CLASS_NAME, 'author').text
    print(f"名言: {text}, 作者: {author}")

# 關閉瀏覽器
driver.quit()

pip install scrapy

scrapy startproject myspider

import scrapy

class QuotesSpider(scrapy.Spider):
    name = "quotes"
    start_urls = ['https://quotes.toscrape.com/']

    def parse(self, response):
        for quote in response.css('div.quote'):
            yield {
                'text': quote.css('span.text::text').get(),
                'author': quote.css('small.author::text').get(),
            }
        
        # 下一頁
        next_page = response.css('li.next a::attr(href)').get()
        if next_page:
            yield response.follow(next_page, self.parse)

scrapy crawl quotes

pip install pyquery

from pyquery import PyQuery as pq

# 獲取網(wǎng)頁內容
url = "https://quotes.toscrape.com/"
doc = pq(url)

# 提取名言和作者
for quote in doc('.quote').items():
    text = quote('.text').text()
    author = quote('.author').text()
    print(f"名言: {text}, 作者: {author}")

pip install requests-html

from requests_html import HTMLSession

session = HTML

Session()
url = "https://quotes.toscrape.com/js/"
response = session.get(url)

# 渲染 JavaScript
response.html.render()

# 提取名言和作者
quotes = response.html.find('.quote')
for quote in quotes:
    text = quote.find('.text', first=True).text
    author = quote.find('.author', first=True).text
    print(f"名言: {text}, 作者: {author}")