使用 Python 请求和代理进行网页抓取

import random
import time
import requests
from bs4 import BeautifulSoup
from requests.exceptions import RequestException

USER, PASS, HOST = "USER", "PASS", "proxy.s4m.online"
POOL = [
    f"socks5h://{USER}:{PASS}@{HOST}:1080",  # needs requests[socks]
    f"http://{USER}:{PASS}@{HOST}:3128",
]

HEADERS = {
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
    "Accept-Language": "en-US,en;q=0.9",
}

def make_session(proxy):
    s = requests.Session()
    s.headers.update(HEADERS)
    s.proxies = {"http": proxy, "https": proxy}
    return s

def fetch(url, retries=4):
    for attempt in range(1, retries + 1):
        proxy = random.choice(POOL)
        try:
            s = make_session(proxy)
            r = s.get(url, timeout=(5, 20))
            if r.status_code == 429:
                raise RequestException("rate limited")
            r.raise_for_status()
            return r
        except RequestException as e:
            wait = 2 ** attempt + random.random()  # exponential backoff
            print(f"[{attempt}/{retries}] {proxy} -> {e}; sleeping {wait:.1f}s")
            time.sleep(wait)
    raise SystemExit(f"giving up on {url}")

html = fetch("https://quotes.toscrape.com/").text
soup = BeautifulSoup(html, "html.parser")
for q in soup.select(".quote"):
    text = q.select_one(".text").get_text(strip=True)
    who = q.select_one(".author").get_text(strip=True)
    print(f"{who}: {text}")