Crawl Websites

Crawl an entire website by following links from a starting URL. Control depth, path filters, and output formats.

Basic Crawl

import { ScrayleClient } from '@scrayle/sdk';

const client = new ScrayleClient();

const result = await client.scrapingCrawl.crawlUrl(
  'https://example.com',
  { limit: 50, maxDepth: 3 }
);

console.log(result.status);     // 'completed'
console.log(result.total);      // Total pages found
console.log(result.completed);  // Pages scraped

for (const doc of result.data || []) {
  console.log(doc.metadata?.sourceURL, doc.metadata?.title);
}

Crawl Options

Path Filtering

const result = await client.scrapingCrawl.crawlUrl(
  'https://docs.example.com',
  {
    limit: 100,
    maxDepth: 5,
    includePaths: ['/docs/*', '/guides/*'],
    excludePaths: ['/blog/*', '/changelog/*'],
    scrapeOptions: {
      formats: ['markdown'],
      onlyMainContent: true,
    },
  }
);

Async Crawl

// Start crawl in the background
const { id } = await client.scrapingCrawl.crawl.asyncCrawlUrl(
  'https://example.com',
  { limit: 200, maxDepth: 4 }
);

// Poll for status
const status = await client.scrapingCrawl.crawl.checkCrawlStatus(id);
console.log(status.status, status.completed, '/', status.total);

// Check for errors
const errors = await client.scrapingCrawl.crawl.checkCrawlErrors(id);
for (const err of errors.errors) {
  console.log(err.url, err.error);
}

// Cancel if needed
await client.scrapingCrawl.crawl.cancelCrawl(id);

Webhook Delivery

await client.scrapingCrawl.crawl.asyncCrawlUrl(
  'https://example.com',
  {
    limit: 50,
    webhook: 'https://your-server.com/webhook/crawl-done',
  }
);
// Results will be POSTed to your webhook URL when complete