middleware.ts

import { next } from "@vercel/edge"

export default async function middleware(req: Request) {
  const userAgent = req.headers.get("user-agent")
  console.log("userAgent", userAgent)
  console.log("req", req)
  
  if (req.url.includes("/robots.txt")) {
    return next()
  }

  if (userAgent && shouldPrerender(userAgent)) {
    console.log("prerendering", req.url)
    const prerenderUrl = `http://service.prerender.io/${req.url}`
    const prerenderServiceRes = await fetch(prerenderUrl, {
      headers: {
        "X-Prerender-Token": process.env.PRERENDER_TOKEN ?? "",
      },
    })

    const body = await prerenderServiceRes.text()
    return new Response(body, {
      status: prerenderServiceRes.status,
      headers: {
        "Content-Type": "text/html",
      },
    })
  }

  return next()
}

function shouldPrerender(userAgent: string) {
  if (!userAgent) return false

  const crawlerUserAgents = [
    "googlebot",
    "google",
    "bingbot",
    "yandex",
    "baiduspider",
    "facebookexternalhit",
    "twitterbot",
    "rogerbot",
    "linkedinbot",
    "embedly",
    "quora link preview",
    "showyoubot",
    "outbrain",
    "pinterest",
    "developers.google.com/+/web/snippet",
    "slackbot",
    "vkShare",
    "W3C_Validator",
    "redditbot",
    "Applebot",
    "WhatsApp",
    "flipboard",
    "tumblr",
    "bitlybot",
    "skypeuripreview",
    "nuzzel",
    "discordbot",
    "Google PageSpeed Insights",
    "xing-contenttabreceiver",
    "Chrome-Lighthouse",
    "facebot",
    "whatsapp",
    "linkedin",
    "TwitterShare",
    "coccocbot",
    "exabot",
    "seznambot",
    "genieo",
    "ia_archiver",
    "id-bot",
    "yeti",
    "yodaobot",
    "admantx",
    "ahrefsbot",
    "alexabot",
    "archive.org_bot",
    "careerbot",
    "ccbot",
    "duckduckgo",
    "MJ12bot",
    "seokicks",
    "serpstatbot",
    "sogou",
    "spbot",
    "surdotlybot",
    "teoma",
    "uptimebot",
    "urlresolver",
    "voilabot",
    "woxbot",
    "zumBot",
    "semrushbot",
    "crawler",
    "spider",
    "fetch",
    "slurp",
  ]

  return crawlerUserAgents.some((crawler) =>
    userAgent.toLowerCase().includes(crawler)
  )
}