// Intercrawl for Cloudflare Pages: save this file as functions/_middleware.ts in your
// Pages project, and add the environment variable under Settings, Variables and
// Secrets (as a secret):
//
//   INTERCRAWL_TOKEN   your API token (Account, API tokens)
//
// Humans are never touched. A crawler gets a prerendered page from Intercrawl, and if
// Intercrawl cannot answer within 10 seconds your own page is served as before.
// Cloudflare gives the function each visitor's real IP, so every crawler is checked
// against its operator's published addresses.

// Single file, no dependencies of its own. See intercrawl.com for updates.
var SERVING_BOTS = [
  { id: "googlebot", name: "Googlebot", operator: "Google", uaTokens: ["googlebot", "google-inspectiontool", "googleother", "storebot-google"], rangeLists: ["googlebot"] },
  { id: "google-special", name: "Google special crawler", operator: "Google", uaTokens: ["adsbot-google", "mediapartners-google", "apis-google", "google-safety"], rangeLists: ["googleSpecial"] },
  { id: "google-user", name: "Google user fetcher", operator: "Google", uaTokens: ["google-site-verification", "feedfetcher-google", "google-read-aloud"], rangeLists: ["googleUser"] },
  { id: "bingbot", name: "Bingbot", operator: "Microsoft", uaTokens: ["bingbot", "adidxbot", "bingpreview"], rangeLists: ["bingbot"] },
  { id: "gptbot", name: "GPTBot", operator: "OpenAI", uaTokens: ["gptbot"], rangeLists: ["gptbot"] },
  { id: "oai-searchbot", name: "OAI-SearchBot", operator: "OpenAI", uaTokens: ["oai-searchbot"], rangeLists: ["oaiSearchbot"] },
  { id: "chatgpt-user", name: "ChatGPT-User", operator: "OpenAI", uaTokens: ["chatgpt-user"], rangeLists: ["chatgptUser"] },
  { id: "claudebot", name: "ClaudeBot", operator: "Anthropic", uaTokens: ["claudebot"], rangeLists: ["anthropic"] },
  { id: "claude-searchbot", name: "Claude-SearchBot", operator: "Anthropic", uaTokens: ["claude-searchbot"], rangeLists: ["anthropic"] },
  { id: "claude-user", name: "Claude-User", operator: "Anthropic", uaTokens: ["claude-user"], rangeLists: ["anthropic"] },
  { id: "perplexitybot", name: "PerplexityBot", operator: "Perplexity", uaTokens: ["perplexitybot"], rangeLists: ["perplexitybot"] },
  { id: "perplexity-user", name: "Perplexity-User", operator: "Perplexity", uaTokens: ["perplexity-user"], rangeLists: ["perplexityUser"] },
  { id: "applebot", name: "Applebot", operator: "Apple", uaTokens: ["applebot"], rangeLists: ["applebot"] },
  // No published ranges: link previews and smaller engines. Rendered, always
  // "unverified", so their counts stay apart from the verified crawlers.
  { id: "duckduckbot", name: "DuckDuckBot", operator: "DuckDuckGo", uaTokens: ["duckduckbot"], rangeLists: [] },
  { id: "yandexbot", name: "YandexBot", operator: "Yandex", uaTokens: ["yandexbot"], rangeLists: [] },
  { id: "facebookexternalhit", name: "facebookexternalhit", operator: "Meta", uaTokens: ["facebookexternalhit", "facebookcatalog"], rangeLists: [] },
  { id: "twitterbot", name: "Twitterbot", operator: "X", uaTokens: ["twitterbot"], rangeLists: [] },
  { id: "linkedinbot", name: "LinkedInBot", operator: "LinkedIn", uaTokens: ["linkedinbot"], rangeLists: [] },
  { id: "slackbot", name: "Slackbot", operator: "Slack", uaTokens: ["slackbot"], rangeLists: [] },
  { id: "discordbot", name: "Discordbot", operator: "Discord", uaTokens: ["discordbot"], rangeLists: [] },
  { id: "whatsapp", name: "WhatsApp", operator: "Meta", uaTokens: ["whatsapp/"], rangeLists: [] },
  { id: "telegrambot", name: "TelegramBot", operator: "Telegram", uaTokens: ["telegrambot"], rangeLists: [] }
];
var MAX_UA_CHARS = 512;
function botForUserAgent(userAgent) {
  if (!userAgent) return null;
  const ua = userAgent.slice(0, MAX_UA_CHARS).toLowerCase();
  return SERVING_BOTS.find((b) => b.uaTokens.some((t) => ua.includes(t))) ?? null;
}

// src/core.ts
var DEFAULT_RENDER_URL = "https://render.intercrawl.com";
var DEFAULT_TIMEOUT_MS = 9e3;
var ASSET = /\.(?:avif|bmp|css|csv|eot|gif|gz|ico|jpe?g|js|json|map|mjs|mp3|mp4|ogg|otf|pdf|png|svg|ttf|txt|wasm|webm|webp|woff2?|xml|zip)$/i;
var AUDIT_HEADER = "x-intercrawl-audit";
var AUDIT_HEADER_MAX_CHARS = 200;
var PASSED = ["content-type", "location", "cache-control", "vary", "x-robots-tag", "link"];
function cloudflareClientIp(headers) {
  const ip = headers.get("cf-connecting-ip")?.trim() ?? null;
  const first = ip ? Number(ip.split(".")[0]) : NaN;
  if (ip && /^\d+\.\d+\.\d+\.\d+$/.test(ip) && first >= 240 && first <= 255) return headers.get("cf-connecting-ipv6")?.trim() || ip;
  return ip;
}
function shouldRender(request) {
  if (request.method !== "GET" && request.method !== "HEAD") return false;
  if (request.headers.has("x-prerender") || request.headers.has("x-intercrawl-render")) return false;
  let pathname;
  try {
    pathname = new URL(request.url).pathname;
  } catch {
    return false;
  }
  if (ASSET.test(pathname)) return false;
  return botForUserAgent(request.headers.get("user-agent")) !== null;
}
async function renderForCrawler(request, options) {
  try {
    if (!options.token || !shouldRender(request)) return null;
    const base = (options.renderUrl || DEFAULT_RENDER_URL).replace(/\/+$/, "");
    const headers = new Headers({
      "x-prerender-token": options.token,
      "user-agent": request.headers.get("user-agent") ?? "",
      "accept-language": request.headers.get("accept-language") ?? "en-US"
    });
    if (options.clientIp) headers.set("x-intercrawl-client-ip", options.clientIp);
    const audit = request.headers.get(AUDIT_HEADER)?.trim();
    if (audit && audit.length <= AUDIT_HEADER_MAX_CHARS) headers.set(AUDIT_HEADER, audit);
    const controller = new AbortController();
    const timer = setTimeout(() => controller.abort(), options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
    let res;
    try {
      res = await (options.fetchFn ?? fetch)(`${base}/${request.url}`, {
        method: request.method,
        headers,
        redirect: "manual",
        signal: controller.signal
      });
    } finally {
      clearTimeout(timer);
    }
    const source = res.headers.get("x-intercrawl-source");
    if (source !== "cache" && source !== "fresh") {
      await res.body?.cancel().catch(() => {
      });
      return null;
    }
    const out = new Headers();
    for (const name of PASSED) {
      const value = res.headers.get(name);
      if (value !== null) out.set(name, value);
    }
    res.headers.forEach((value, name) => {
      if (name.startsWith("x-intercrawl-")) out.set(name, value);
    });
    return new Response(request.method === "HEAD" ? null : res.body, { status: res.status, headers: out });
  } catch {
    return null;
  }
}

// src/pages.ts
async function onRequest(context) {
  const page = await renderForCrawler(context.request, {
    token: context.env.INTERCRAWL_TOKEN,
    renderUrl: context.env.INTERCRAWL_RENDER_URL,
    clientIp: cloudflareClientIp(context.request.headers)
  });
  return page ?? context.next();
}
export {
  onRequest
};