Scrapling proxies for web scrapingstealth scrapingblocked sitesprice trackingbig crawls

import os
from scrapling.fetchers import Fetcher

# http://USER:[email protected]:10000
proxy = os.environ["PROXY_URL"]

page = Fetcher.get("https://news.ycombinator.com/", proxy=proxy)
print(page.css(".titleline > a::text").getall()[:5])

import os
from scrapling.fetchers import DynamicSession

proxy = os.environ["PROXY_URL"]

# Chromium renders the page, every request exits through the proxy
with DynamicSession(headless=True, proxy=proxy) as session:
    page = session.fetch("https://news.ycombinator.com/")
    print(page.css(".titleline > a::text").getall()[:5])

import os
from scrapling.fetchers import StealthySession

proxy = os.environ["PROXY_URL"]

# Keep one sticky residential IP for the whole browser session
with StealthySession(headless=True, proxy=proxy, block_webrtc=True) as session:
    page = session.fetch("https://example.com/")
    print(page.status)

import os
from scrapling.fetchers import FetcherSession, ProxyRotator

# Sticky endpoints from the dashboard, one per session
rotator = ProxyRotator([
    os.environ["PROXY_URL_1"],
    os.environ["PROXY_URL_2"],
    os.environ["PROXY_URL_3"],
])

with FetcherSession(proxy_rotator=rotator, impersonate="chrome") as session:
    for n in range(1, 4):
        page = session.get(f"https://example.com/catalog?page={n}")
        print(page.status, page.meta["proxy"])