Enterprise-grade scraping proxies for Google SearchGoogle MapsAmazonsocial media

import asyncio
import os

import httpx

PROXY_URL = os.environ["PROXY_URL"]  # rotating endpoint from the dashboard
urls = [f"https://example.com/products?page={n}" for n in range(1, 101)]


async def fetch(client, url):
    response = await client.get(url)
    return response.status_code


async def main():
    limits = httpx.Limits(max_connections=50)
    async with httpx.AsyncClient(proxy=PROXY_URL, limits=limits, timeout=30) as client:
        statuses = await asyncio.gather(*(fetch(client, url) for url in urls))
    print(statuses.count(200), "of", len(urls), "pages OK")


asyncio.run(main())

import { chromium } from "playwright";

// Sticky German endpoint from the dashboard: one IP for this profile
const browser = await chromium.launch({
  proxy: {
    server: process.env.PROXY_SERVER,
    username: process.env.PROXY_USER,
    password: process.env.PROXY_PASS,
  },
});

// Match the browser to the exit IP
const context = await browser.newContext({
  locale: "de-DE",
  timezoneId: "Europe/Berlin",
});

const page = await context.newPage();
await page.goto("https://ipinfo.io/json");
console.log(await page.textContent("body"));
await browser.close();

import puppeteer from "puppeteer";

const browser = await puppeteer.launch({
  args: [`--proxy-server=${process.env.PROXY_SERVER}`],
});

const page = await browser.newPage();
await page.authenticate({
  username: process.env.PROXY_USER,
  password: process.env.PROXY_PASS,
});
await page.emulateTimezone("Europe/Berlin");

await page.goto("https://ipinfo.io/json");
console.log(await page.evaluate(() => document.body.innerText));
await browser.close();

import os

import scrapy


class ProductsSpider(scrapy.Spider):
    name = "products"
    custom_settings = {"CONCURRENT_REQUESTS": 64}

    def start_requests(self):
        proxy = os.environ["PROXY_URL"]
        for page in range(1, 101):
            url = f"https://example.com/products?page={page}"
            yield scrapy.Request(url, meta={"proxy": proxy})

    def parse(self, response):
        for product in response.css(".product"):
            yield {
                "name": product.css("h2::text").get(),
                "price": product.css(".price::text").get(),
            }

import { PlaywrightCrawler, ProxyConfiguration } from "crawlee";

const proxyConfiguration = new ProxyConfiguration({
  proxyUrls: [process.env.PROXY_URL],
});

const crawler = new PlaywrightCrawler({
  proxyConfiguration,
  maxConcurrency: 50,
  async requestHandler({ request, page, enqueueLinks }) {
    console.log(request.url, await page.title());
    await enqueueLinks({ globs: ["https://example.com/products/**"] });
  },
});

await crawler.run(["https://example.com/products"]);

# Rotating endpoint: a new residential IP on every request
for i in 1 2 3; do
  curl -s -x "$PROXY_URL" https://ipinfo.io/ip
  echo
done