Crawl Service

Access the crawl and scrape services for full-page extraction and website crawling.

Access

// Via the ScrapingCrawl container service
client.scrapingCrawl.scrape    // ScrapeService
client.scrapingCrawl.crawl     // CrawlService

// Convenience methods
client.scrapingCrawl.scrapeUrl(...)
client.scrapingCrawl.crawlUrl(...)

ScrapeService Methods

scrapeUrl(url, params?, pollInterval?)

Scrape a URL and wait for the result.

const result = await client.scrapingCrawl.scrapeUrl(
  'https://example.com',
  { formats: ['markdown', 'links'] },
  5000 // Optional poll interval
);

asyncScrapeUrl(url, params?)

Start a scrape without waiting.

const { id } = await client.scrapingCrawl.scrape.asyncScrapeUrl(
  'https://example.com',
  { formats: ['markdown'] }
);

checkScrapeStatus(id)

const status = await client.scrapingCrawl.scrape.checkScrapeStatus(id);

batchScrapeUrls(urls, params?, pollInterval?, ignoreInvalid?)

const result = await client.scrapingCrawl.scrape.batchScrapeUrls(
  ['https://a.com', 'https://b.com'],
  { formats: ['markdown'] }
);

asyncBatchScrapeUrls(urls, params?, ignoreInvalid?)

const { id } = await client.scrapingCrawl.scrape.asyncBatchScrapeUrls(
  urls, { formats: ['markdown'] }
);

checkBatchScrapeStatus(id)

const status = await client.scrapingCrawl.scrape.checkBatchScrapeStatus(id);

CrawlService Methods

crawlUrl(url, params?, pollInterval?)

Crawl a website and wait for completion.

const result = await client.scrapingCrawl.crawlUrl(
  'https://example.com',
  { limit: 50, maxDepth: 3 }
);

asyncCrawlUrl(url, params?)

const { id } = await client.scrapingCrawl.crawl.asyncCrawlUrl(
  'https://example.com',
  { limit: 100, maxDepth: 4 }
);

checkCrawlStatus(id?)

const status = await client.scrapingCrawl.crawl.checkCrawlStatus(id);

checkCrawlErrors(id)

const errors = await client.scrapingCrawl.crawl.checkCrawlErrors(id);

cancelCrawl(id)

await client.scrapingCrawl.crawl.cancelCrawl(id);

Types

interface ScrapeParams {
  formats?: ('markdown' | 'html' | 'rawHtml' | 'links' | 'screenshot' | 'json')[];
  headers?: Record<string, string>;
  includeTags?: string[];
  excludeTags?: string[];
  onlyMainContent?: boolean;
  timeout?: number;
  waitFor?: number;
  extract?: {
    schema?: Record<string, unknown>;
    systemPrompt?: string;
    prompt?: string;
  };
}

interface CrawlParams {
  excludePaths?: string[];
  includePaths?: string[];
  maxDepth?: number;
  limit?: number;
  allowBackwardLinks?: boolean;
  allowExternalLinks?: boolean;
  ignoreSitemap?: boolean;
  scrapeOptions?: CrawlScrapeOptions;
  webhook?: string;
}