Batch Scraping

Scrape multiple URLs in a single request. All URLs are processed in parallel and results are returned together.

Basic Batch Scrape

import { ScrayleClient } from '@scrayle/sdk';

const client = new ScrayleClient();

const result = await client.scrapingCrawl.scrape.batchScrapeUrls(
  [
    'https://example.com/page-1',
    'https://example.com/page-2',
    'https://example.com/page-3',
  ],
  { formats: ['markdown'] }
);

for (const doc of result.data || []) {
  console.log(doc.metadata?.sourceURL, doc.markdown?.substring(0, 100));
}

With Options

const result = await client.scrapingCrawl.scrape.batchScrapeUrls(
  urls,
  {
    formats: ['markdown', 'links'],
    onlyMainContent: true,
    timeout: 30000,
  },
  5000,  // Poll interval in ms (default)
  true   // Ignore invalid URLs instead of failing
);

Async Batch Scrape

// Start batch without waiting
const { id } = await client.scrapingCrawl.scrape.asyncBatchScrapeUrls(
  ['https://example.com/a', 'https://example.com/b'],
  { formats: ['markdown'] }
);

// Check status later
const status = await client.scrapingCrawl.scrape.checkBatchScrapeStatus(id);
console.log(status.status);     // 'scraping' | 'completed' | 'failed'
console.log(status.completed);  // Number of URLs done
console.log(status.total);      // Total URLs

Response Format

interface BatchScrapeStatusResponse {
  status: 'scraping' | 'completed' | 'failed';
  total: number;
  completed: number;
  data?: ScrapingCrawlDocument[];
}

interface ScrapingCrawlDocument {
  markdown?: string;
  html?: string;
  rawHtml?: string;
  links?: string[];
  screenshot?: string;
  json?: Record<string, unknown>;
  metadata?: {
    title?: string;
    description?: string;
    sourceURL?: string;
    statusCode?: number;
    // ...
  };
}