Function bodies 44 total

export class Webclaw {
  private readonly apiKey: string;
  private readonly baseUrl: string;
  private readonly timeout: number;

  constructor(config: WebclawConfig) {
    if (!config.apiKey) throw new Error("apiKey is required");
    this.apiKey = config.apiKey;
    this.baseUrl = (config.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
    this.timeout = config.timeout ?? DEFAULT_TIMEOUT;
  }

  // -- Public API methods --

  /**
   * Scrape a single URL and extract its content.
   * @param params - URL and extraction options (formats, selectors, caching).
   * @returns Extracted content in the requested formats.
   * @throws {WebclawError} On network or API errors.
   */
  async scrape(params: ScrapeRequest): Promise<ScrapeResponse> {
    if (!params.url) throw new Error("url is required");
    return this.post<ScrapeResponse>("/v1/scrape", params);
  }

  /**
   * Start an async crawl job that discovers and scrapes pages from a root URL.
   * @param params - Root URL and crawl l

  constructor(config: WebclawConfig) {
    if (!config.apiKey) throw new Error("apiKey is required");
    this.apiKey = config.apiKey;
    this.baseUrl = (config.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
    this.timeout = config.timeout ?? DEFAULT_TIMEOUT;
  }

  async scrape(params: ScrapeRequest): Promise<ScrapeResponse> {
    if (!params.url) throw new Error("url is required");
    return this.post<ScrapeResponse>("/v1/scrape", params);
  }

  async crawl(params: CrawlRequest): Promise<CrawlJob> {
    const res = await this.post<CrawlStartResponse>("/v1/crawl", params);
    return new CrawlJob(res.id, this);
  }

  async getCrawlStatus(id: string): Promise<CrawlStatusResponse> {
    return this.get<CrawlStatusResponse>(`/v1/crawl/${encodeURIComponent(id)}`);
  }

  async map(params: MapRequest): Promise<MapResponse> {
    return this.post<MapResponse>("/v1/map", params);
  }

  async batch(params: BatchRequest): Promise<BatchResponse> {
    if (!params.urls?.length) throw new Error("urls must be a non-empty array");
    return this.post<BatchResponse>("/v1/batch", params);
  }

  async extract(params: ExtractRequest): Promise<ExtractResponse> {
    if (!params.url) throw new Error("url is required");
    return this.post<ExtractResponse>("/v1/extract", params);
  }

  async summarize(params: SummarizeRequest): Promise<SummarizeResponse> {
    return this.post<SummarizeResponse>("/v1/summarize", params);
  }

  async brand(params: BrandRequest): Promise<BrandResponse> {
    return this.post<BrandResponse>("/v1/brand", params);
  }

  async search(params: SearchRequest): Promise<SearchResponse> {
    if (!params.query) throw new Error("query is required");
    return this.post<SearchResponse>("/v1/search", params);
  }

  async diff(params: DiffRequest): Promise<DiffResponse> {
    return this.post<DiffResponse>("/v1/diff", params);
  }

  async research(
    params: ResearchRequest,
    opts: ResearchPollOptions = {},
  ): Promise<ResearchResponse> {
    if (!params.query) throw new Error("query is required");
    const start = await this.post<ResearchStartResponse>(
      "/v1/research",
      params,
    );

    const interval = opts.interval ?? 2_000;
    const defaultMax = params.deep ? 1_200_000 : 600_000;
    const maxWait = opts.maxWait ?? defaultMax;

    return pollUntilDone(
      () => this.getResearchStatus(start.id),
      (r) => r.status === "completed" || r.status === "failed",
      { interval, timeout: maxWait },
    );
  }

  async waitForResearch(
    id: string,
    opts: ResearchPollOptions = {},
  ): Promise<ResearchResponse> {
    const interval = opts.interval ?? 2_000;
    // No `deep` hint here since we don't have the original request;
    // pick the longer window so shallow jobs finish comfortably.
    const maxWait = opts.maxWait ?? 1_200_000;
    return pollUntilDone(
      () => this.getResearchStatus(id),
      (r) => r.status === "completed" || r.status === "failed",
      { interval, timeout: maxWait },
    );
  }

  async waitForCrawl(
    id: string,
    opts: CrawlPollOptions = {},
  ): Promise<CrawlStatusResponse> {
    const interval = opts.interval ?? 2_000;
    const maxWait = opts.maxWait ?? 300_000;
    return pollUntilDone(
      () => this.getCrawlStatus(id),
      (s) => s.status === "completed" || s.status === "failed",
      { interval, timeout: maxWait },
    );
  }

  async getResearchStatus(id: string): Promise<ResearchResponse> {
    return this.get<ResearchResponse>(`/v1/research/${encodeURIComponent(id)}`);
  }

  async watchCreate(params: WatchCreateRequest): Promise<WatchResponse> {
    if (!params.url) throw new Error("url is required");
    return this.post<WatchResponse>("/v1/watch", params);
  }

  async watchList(limit?: number, offset?: number): Promise<WatchResponse[]> {
    const query = new URLSearchParams();
    if (limit !== undefined) query.set("limit", String(limit));
    if (offset !== undefined) query.set("offset", String(offset));
    const qs = query.toString();
    return this.get<WatchResponse[]>(`/v1/watch${qs ? `?${qs}` : ""}`);
  }

  async watchGet(id: string): Promise<WatchResponse> {
    return this.get<WatchResponse>(`/v1/watch/${encodeURIComponent(id)}`);
  }

  async watchDelete(id: string): Promise<void> {
    await this.del(`/v1/watch/${encodeURIComponent(id)}`);
  }

  async watchCheck(id: string): Promise<WatchResponse> {
    return this.post<WatchResponse>(
      `/v1/watch/${encodeURIComponent(id)}/check`,
      {},
    );
  }

  private async request<T>(path: string, init: RequestInit): Promise<T> {
    const url = `${this.baseUrl}${path}`;
    const controller = new AbortController();
    const timer = setTimeout(() => controller.abort(), this.timeout);

    let res: Response;
    try {
      res = await fetch(url, { ...init, signal: controller.signal });
    } catch (err: unknown) {
      if (isAbortError(err)) {
        throw new TimeoutError(this.timeout);
      }
      throw new WebclawError(
        err instanceof Error ? err.message : "Network request failed",
      );
    } finally {
      clearTimeout(timer);
    }

    // DELETE with 204 has no body
    if (res.ok && res.status === 204) {
      return undefined as T;
    }

    if (res.ok) {
      try {
        return (await res.json()) as T;
      } catch {
        throw new WebclawError("Invalid JSON in response body", res.status);
      }
    }

    const body = await res.text().catch(() => null);
    const parsed = tryParseJson(body);
    const

  private post<T>(path: string, body: unknown): Promise<T> {
    return this.request<T>(path, {
      method: "POST",
      headers: {
        "Content-Type": "application/json",
        Authorization: `Bearer ${this.apiKey}`,
      },
      body: JSON.stringify(body),
    });
  }

  private get<T>(path: string): Promise<T> {
    return this.request<T>(path, {
      method: "GET",
      headers: { Authorization: `Bearer ${this.apiKey}` },
    });
  }

  private del(path: string): Promise<void> {
    return this.request<void>(path, {
      method: "DELETE",
      headers: { Authorization: `Bearer ${this.apiKey}` },
    });
  }

export class CrawlJob {
  constructor(
    public readonly id: string,
    private readonly client: Webclaw,
  ) {}

  async getStatus(): Promise<CrawlStatusResponse> {
    return this.client.getCrawlStatus(this.id);
  }

  async waitForCompletion(
    opts: CrawlPollOptions = {},
  ): Promise<CrawlStatusResponse> {
    const interval = opts.interval ?? 2_000;
    const maxWait = opts.maxWait ?? 300_000;

    return pollUntilDone(
      () => this.getStatus(),
      (s) => s.status === "completed" || s.status === "failed",
      { interval, timeout: maxWait },
    );
  }
}

  constructor(
    public readonly id: string,
    private readonly client: Webclaw,
  ) {}

  async getStatus(): Promise<CrawlStatusResponse> {
    return this.client.getCrawlStatus(this.id);
  }

  async waitForCompletion(
    opts: CrawlPollOptions = {},
  ): Promise<CrawlStatusResponse> {
    const interval = opts.interval ?? 2_000;
    const maxWait = opts.maxWait ?? 300_000;

    return pollUntilDone(
      () => this.getStatus(),
      (s) => s.status === "completed" || s.status === "failed",
      { interval, timeout: maxWait },
    );
  }

async function pollUntilDone<T>(
  checkFn: () => Promise<T>,
  isDone: (result: T) => boolean,
  options: { interval: number; timeout: number },

function isAbortError(err: unknown): boolean {
  if (err instanceof DOMException && err.name === "AbortError") return true;
  if (err instanceof Error && err.name === "AbortError") return true;
  return false;
}

function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function tryParseJson(text: string | null): unknown {
  if (!text) return null;
  try {
    return JSON.parse(text);
  } catch {
    return null;
  }
}

function parseRetryAfter(header: string | null): number | null {
  if (!header) return null;
  const seconds = Number(header);
  return Number.isFinite(seconds) ? seconds : null;
}

export class WebclawError extends Error {
  constructor(
    message: string,
    public readonly status?: number,
    public readonly body?: unknown,
  ) {
    super(message);
    this.name = "WebclawError";
  }
}

  constructor(
    message: string,
    public readonly status?: number,
    public readonly body?: unknown,
  ) {
    super(message);
    this.name = "WebclawError";
  }

export class AuthenticationError extends WebclawError {
  constructor(message = "Invalid or missing API key") {
    super(message, 401);
    this.name = "AuthenticationError";
  }
}

  constructor(message = "Invalid or missing API key") {
    super(message, 401);
    this.name = "AuthenticationError";
  }

export class RateLimitError extends WebclawError {
  public readonly retryAfter: number | null;

  constructor(retryAfterSeconds: number | null = null) {
    super("Rate limit exceeded", 429);
    this.name = "RateLimitError";
    this.retryAfter = retryAfterSeconds;
  }
}

  constructor(retryAfterSeconds: number | null = null) {
    super("Rate limit exceeded", 429);
    this.name = "RateLimitError";
    this.retryAfter = retryAfterSeconds;
  }

export class NotFoundError extends WebclawError {
  constructor(message = "Resource not found") {
    super(message, 404);
    this.name = "NotFoundError";
  }
}

  constructor(message = "Resource not found") {
    super(message, 404);
    this.name = "NotFoundError";
  }

export class TimeoutError extends WebclawError {
  constructor(timeoutMs: number) {
    super(`Request timed out after ${timeoutMs}ms`);
    this.name = "TimeoutError";
  }
}

  constructor(timeoutMs: number) {
    super(`Request timed out after ${timeoutMs}ms`);
    this.name = "TimeoutError";
  }