How to use proxies with Crawlee

import { CheerioCrawler, ProxyConfiguration } from "crawlee";

const proxyConfiguration = new ProxyConfiguration({
  proxyUrls: ["http://USERNAME:PASSWORD@HOST:PORT"],
});

const crawler = new CheerioCrawler({
  proxyConfiguration,
  maxConcurrency: 20,
  async requestHandler({ request, json, $, log }) {
    if (json) log.info("Exit IP: " + json.ip);
    else log.info(request.url + ": " + $("title").text());
  },
});

await crawler.run(["https://api.ipify.org?format=json"]);

import { PlaywrightCrawler, ProxyConfiguration } from "crawlee";
import { readFileSync } from "node:fs";

// one sticky line per row: http://USERNAME:PASSWORD@HOST:PORT
const proxyUrls = readFileSync("sticky.txt", "utf8").split("\n").filter(Boolean);

const crawler = new PlaywrightCrawler({
  proxyConfiguration: new ProxyConfiguration({ proxyUrls }),
  useSessionPool: true,
  persistCookiesPerSession: true,
  sessionPoolOptions: { maxPoolSize: proxyUrls.length },
  async requestHandler({ page, session, log }) {
    log.info("session " + session?.id + ": " + (await page.title()));
  },
});

await crawler.run(["https://example.com"]);

import { CheerioCrawler, ProxyConfiguration } from "crawlee";
import { readFileSync } from "node:fs";

const ispUrls = readFileSync("isp.txt", "utf8").split("\n").filter(Boolean);

const proxyConfiguration = new ProxyConfiguration({
  tieredProxyUrls: [
    ispUrls, // tier 0: static ISP lines
    ["http://USERNAME:PASSWORD@HOST:PORT"], // tier 1: rotating residential
  ],
});

const crawler = new CheerioCrawler({
  proxyConfiguration,
  async requestHandler({ request, $ }) {
    console.log(request.url, $("title").text());
  },
});