lamps.com Cloudflare Worker — adds bot classification headers (CF-Verified-Bot, CF-ASN, CF-Bot-Category) to Shopify-bound HTML requests. Works on Business tier; forward-compatible with Enterprise Bot Management (JA3/JA4/score auto-populate).

lampscom-bot-signal-worker.js

// Cloudflare Worker for lamps.com (Shopify O2O)
// Adds bot classification headers to Shopify-bound HTML requests so the
// origin can route verified crawlers to a higher wall-clock budget.
//
// Works on Cloudflare Business tier (cf.verifiedBot + cf.asn).
// Forward-compatible with Enterprise Bot Management (JA3/JA4/score lines
// guard with `if` — they populate the day BM is enabled, no code change).

// UA patterns — used ONLY to label category, never to flip verified-bot
// status. cf.verifiedBot is the authoritative signal because Cloudflare
// validates the claimed bot's IP/ASN at the edge; raw UA is spoofable.
const SEARCH_BOTS = /googlebot|bingbot|slurp|duckduckbot|yandexbot|baiduspider|applebot|petalbot/i;
const SOCIAL_BOTS = /facebookexternalhit|twitterbot|linkedinbot|pinterestbot|whatsapp|telegrambot/i;

// Pure helper — easy to unit-test. Returns the Headers we want on the
// outgoing request; caller wraps in `new Request(request, { headers })`.
export function withBotSignals(request) {
  const cf = request.cf || {};
  const bm = cf.botManagement || {}; // populated on Enterprise BM; empty on Business
  const ua = request.headers.get('user-agent') || '';

  // Authoritative bot signal — Cloudflare-validated, not client-spoofable.
  const isVerifiedBot = cf.verifiedBot === true;

  // Strip ALL incoming CF-* headers before setting our own — prevents
  // clients from spoofing CF-Bot-Category, CF-Bot-Score, CF-JA3, etc.
  // Cloudflare's own edge-set headers (CF-Connecting-IP, CF-Ray) are
  // re-added by the edge after the Worker runs, so we don't break those.
  const headers = new Headers(request.headers);
  for (const k of [...headers.keys()]) {
    if (k.toLowerCase().startsWith('cf-')) headers.delete(k);
  }

  // Always-on signals — Shopify uses ASN/country for geo + classification
  // on both human and bot traffic.
  headers.set('CF-Verified-Bot', String(isVerifiedBot));
  if (cf.asn != null) headers.set('CF-ASN', String(cf.asn));
  if (cf.country) headers.set('CF-Country', cf.country);

  // Bot-only signals — only meaningful when CF has verified the bot identity.
  if (isVerifiedBot) {
    const isSearchBot = SEARCH_BOTS.test(ua);
    const isSocialBot = SOCIAL_BOTS.test(ua);
    headers.set('CF-Bot-Category',
      isSearchBot ? 'search-engine' :
      isSocialBot ? 'social-media' :
      bm.verifiedBotCategory || 'verified-other'
    );

    // Forward-compat: populated on Enterprise BM, no-op on Business.
    if (bm.ja3Hash)        headers.set('CF-JA3', bm.ja3Hash);
    if (bm.ja4)            headers.set('CF-JA4', bm.ja4);
    if (bm.score != null)  headers.set('CF-Bot-Score', String(bm.score));
  }

  return { headers };
}

export default {
  async fetch(request) {
    if (request.method !== 'GET') return fetch(request);

    const url = new URL(request.url);
    const path = url.pathname.toLowerCase();

    // Static assets — bypass bot signaling (Shopify CDN is already fast)
    if (path.match(/\.(?:css|js|mjs|map|png|jpe?g|gif|webp|svg|ico|woff2?|ttf|eot|otf|pdf|zip|gz|br|json|xml|txt)$/)) {
      return fetch(request);
    }

    // Shopify CDN + infra paths — same, no point
    if (
      path.startsWith("/cdn/") ||
      path.startsWith("/assets/") ||
      path.startsWith("/files/") ||
      path.includes("/s/files/")
    ) return fetch(request);

    // Shopify critical flows — NEVER redirect, and bots don't typically hit these
    if (
      path.startsWith("/checkout") ||
      path.startsWith("/checkouts/") ||
      path.startsWith("/orders/") ||
      path.startsWith("/tools/") ||
      path.startsWith("/wallets/") ||
      path.startsWith("/cart") ||
      path.startsWith("/apps/") ||
      path.startsWith("/a/") ||
      path.startsWith("/services/") ||
      path.startsWith("/account") ||
      path.startsWith("/wpm@") ||
      path.startsWith("/.well-known/") ||
      path.startsWith("/policies/") ||
      path.startsWith("/password") ||
      path.startsWith("/recommendations/")
    ) return fetch(request);

    // Strip Algolia attribution params
    const STRIP = new Set(["queryID", "objectID", "indexName", "collection_link"]);
    let changed = false;
    for (const k of STRIP) {
      if (url.searchParams.has(k)) { url.searchParams.delete(k); changed = true; }
    }
    if (changed) {
      if ([...url.searchParams.keys()].length === 0) url.search = "";
      return Response.redirect(url.toString(), 301);
      // Bot follows redirect → re-hits this worker with clean URL → falls through to signaled fetch below
    }

    // Renderable HTML path — inject bot signals so Shopify can classify the request budget
    const { headers } = withBotSignals(request);
    return fetch(new Request(request, { headers }));
  }
};

lampscom-bot-signal-worker.test.mjs

import { describe, it } from 'node:test';
import { strict as assert } from 'node:assert';
import { withBotSignals } from './worker.js';

// Build a request-like object with a Headers instance + cf property.
// We don't use `new Request()` because cf is non-standard and CF-specific —
// attaching it as a JS property mirrors how the CF runtime exposes it.
function mockRequest(cf, headerInit = {}) {
  return {
    headers: new Headers(headerInit),
    cf,
  };
}

describe('withBotSignals', () => {
  describe('Layer A — bot-vs-human classification', () => {
    it('marks Googlebot from a verified Google IP as a verified bot', () => {
      const { headers } = withBotSignals(mockRequest(
        { verifiedBot: true, asn: 15169, country: 'US' },
        { 'user-agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' }
      ));
      assert.equal(headers.get('CF-Verified-Bot'), 'true');
      assert.equal(headers.get('CF-Bot-Category'), 'search-engine');
      assert.equal(headers.get('CF-ASN'), '15169');
      assert.equal(headers.get('CF-Country'), 'US');
    });

    it('marks Bingbot from verified Bing IP as a verified bot', () => {
      const { headers } = withBotSignals(mockRequest(
        { verifiedBot: true, asn: 8075, country: 'US' },
        { 'user-agent': 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)' }
      ));
      assert.equal(headers.get('CF-Verified-Bot'), 'true');
      assert.equal(headers.get('CF-Bot-Category'), 'search-engine');
    });

    it('marks Facebook crawler as social-media category', () => {
      const { headers } = withBotSignals(mockRequest(
        { verifiedBot: true, asn: 32934 },
        { 'user-agent': 'facebookexternalhit/1.1' }
      ));
      assert.equal(headers.get('CF-Verified-Bot'), 'true');
      assert.equal(headers.get('CF-Bot-Category'), 'social-media');
    });

    it('marks human traffic as NOT verified bot', () => {
      const { headers } = withBotSignals(mockRequest(
        { verifiedBot: false, asn: 7922, country: 'US' },
        { 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36' }
      ));
      assert.equal(headers.get('CF-Verified-Bot'), 'false');
      assert.equal(headers.get('CF-Bot-Category'), null, 'humans should not get a bot category');
      assert.equal(headers.get('CF-ASN'), '7922', 'ASN still passes through for humans');
      assert.equal(headers.get('CF-Country'), 'US', 'country still passes through for humans');
    });
  });

  describe('Layer B — security: UA-spoof rejection', () => {
    it('rejects spoofed Googlebot UA from non-verified IP', () => {
      // Attacker on Comcast residential ASN claiming to be Googlebot
      const { headers } = withBotSignals(mockRequest(
        { verifiedBot: false, asn: 7922, country: 'US' },
        { 'user-agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' }
      ));
      assert.equal(headers.get('CF-Verified-Bot'), 'false', 'spoofed UA must NOT flip verified-bot to true');
      assert.equal(headers.get('CF-Bot-Category'), null, 'no category for unverified UA-spoof');
    });

    it('rejects spoofed Bingbot UA from non-verified IP', () => {
      const { headers } = withBotSignals(mockRequest(
        { verifiedBot: false, asn: 12345 },
        { 'user-agent': 'bingbot/2.0' }
      ));
      assert.equal(headers.get('CF-Verified-Bot'), 'false');
      assert.equal(headers.get('CF-Bot-Category'), null);
    });

    it('rejects spoofed Facebook crawler UA', () => {
      const { headers } = withBotSignals(mockRequest(
        { verifiedBot: false },
        { 'user-agent': 'facebookexternalhit/1.1' }
      ));
      assert.equal(headers.get('CF-Verified-Bot'), 'false');
    });
  });

  describe('Layer C — security: header injection rejection', () => {
    it('strips client-injected CF-Bot-Category', () => {
      const { headers } = withBotSignals(mockRequest(
        { verifiedBot: false, asn: 7922 },
        {
          'user-agent': 'Mozilla/5.0',
          'CF-Bot-Category': 'search-engine', // <-- attacker tries to set this
        }
      ));
      assert.equal(headers.get('CF-Bot-Category'), null, 'client-injected category must be stripped');
    });

    it('strips client-injected CF-Bot-Score', () => {
      const { headers } = withBotSignals(mockRequest(
        { verifiedBot: false },
        { 'user-agent': 'Mozilla/5.0', 'CF-Bot-Score': '99' }
      ));
      assert.equal(headers.get('CF-Bot-Score'), null);
    });

    it('strips client-injected CF-JA3 and CF-JA4', () => {
      const { headers } = withBotSignals(mockRequest(
        { verifiedBot: false },
        { 'user-agent': 'Mozilla/5.0', 'CF-JA3': 'fake-hash', 'CF-JA4': 'fake-ja4' }
      ));
      assert.equal(headers.get('CF-JA3'), null);
      assert.equal(headers.get('CF-JA4'), null);
    });

    it('strips client-injected CF-Verified-Bot (overwrites with our value)', () => {
      const { headers } = withBotSignals(mockRequest(
        { verifiedBot: false },
        { 'user-agent': 'Mozilla/5.0', 'CF-Verified-Bot': 'true' } // attacker tries to claim bot status
      ));
      assert.equal(headers.get('CF-Verified-Bot'), 'false', 'must reflect our trusted value, not the client claim');
    });

    it('strips arbitrary CF-* prefixed headers from clients', () => {
      const { headers } = withBotSignals(mockRequest(
        { verifiedBot: false },
        {
          'user-agent': 'Mozilla/5.0',
          'CF-Custom-Junk': 'whatever',
          'cf-lowercase-junk': 'sneaky',
        }
      ));
      assert.equal(headers.get('CF-Custom-Junk'), null);
      assert.equal(headers.get('cf-lowercase-junk'), null);
    });

    it('preserves non-CF-prefixed headers like Accept-Language', () => {
      const { headers } = withBotSignals(mockRequest(
        { verifiedBot: false },
        { 'user-agent': 'Mozilla/5.0', 'Accept-Language': 'en-US' }
      ));
      assert.equal(headers.get('Accept-Language'), 'en-US');
    });
  });

  describe('Layer D — robustness', () => {
    it('handles missing cf property gracefully', () => {
      const { headers } = withBotSignals({ headers: new Headers({ 'user-agent': 'Mozilla/5.0' }) });
      assert.equal(headers.get('CF-Verified-Bot'), 'false');
      assert.equal(headers.get('CF-ASN'), null);
      assert.equal(headers.get('CF-Country'), null);
    });

    it('handles empty cf object gracefully', () => {
      const { headers } = withBotSignals(mockRequest({}, { 'user-agent': 'Mozilla/5.0' }));
      assert.equal(headers.get('CF-Verified-Bot'), 'false');
    });

    it('handles missing user-agent', () => {
      const { headers } = withBotSignals(mockRequest({ verifiedBot: true, asn: 15169 }, {}));
      assert.equal(headers.get('CF-Verified-Bot'), 'true');
      assert.equal(headers.get('CF-Bot-Category'), 'verified-other', 'unknown UA falls back to verified-other');
    });

    it('handles asn=0 correctly (does set the header)', () => {
      const { headers } = withBotSignals(mockRequest({ asn: 0 }, { 'user-agent': 'Mozilla/5.0' }));
      assert.equal(headers.get('CF-ASN'), '0', 'asn=0 is valid and should pass through');
    });
  });

  describe('Layer E — Enterprise BM forward-compat', () => {
    it('emits JA3/JA4/Score when botManagement is populated', () => {
      const { headers } = withBotSignals(mockRequest(
        {
          verifiedBot: true,
          asn: 15169,
          botManagement: {
            ja3Hash: 'aabbccdd',
            ja4: 't13d1517h2_8daaf6152771_b186095e22b6',
            score: 5,
            verifiedBotCategory: 'SearchEngineCrawler',
          },
        },
        { 'user-agent': 'Googlebot/2.1' }
      ));
      assert.equal(headers.get('CF-JA3'), 'aabbccdd');
      assert.equal(headers.get('CF-JA4'), 't13d1517h2_8daaf6152771_b186095e22b6');
      assert.equal(headers.get('CF-Bot-Score'), '5');
    });

    it('omits BM fields when botManagement is empty (Business tier)', () => {
      const { headers } = withBotSignals(mockRequest(
        { verifiedBot: true, asn: 15169 },
        { 'user-agent': 'Googlebot/2.1' }
      ));
      assert.equal(headers.get('CF-JA3'), null);
      assert.equal(headers.get('CF-JA4'), null);
      assert.equal(headers.get('CF-Bot-Score'), null);
    });

    it('omits BM fields entirely for non-bots even if BM is populated', () => {
      // Sanity: BM fields are bot-only, not human-only
      const { headers } = withBotSignals(mockRequest(
        {
          verifiedBot: false,
          asn: 7922,
          botManagement: { ja3Hash: 'aabbccdd', score: 5 },
        },
        { 'user-agent': 'Mozilla/5.0' }
      ));
      assert.equal(headers.get('CF-JA3'), null);
      assert.equal(headers.get('CF-Bot-Score'), null);
    });
  });

  describe('Layer F — unknown verified bot (CF-validated, UA unknown to us)', () => {
    it('still classifies as verified-other when UA is novel', () => {
      const { headers } = withBotSignals(mockRequest(
        { verifiedBot: true, asn: 99999, botManagement: { verifiedBotCategory: 'AcademicResearch' } },
        { 'user-agent': 'NovelCrawler/1.0' }
      ));
      assert.equal(headers.get('CF-Verified-Bot'), 'true');
      assert.equal(headers.get('CF-Bot-Category'), 'AcademicResearch', 'falls through to BM category');
    });

    it('falls all the way back to verified-other when nothing matches', () => {
      const { headers } = withBotSignals(mockRequest(
        { verifiedBot: true, asn: 99999 },
        { 'user-agent': 'NovelCrawler/1.0' }
      ));
      assert.equal(headers.get('CF-Bot-Category'), 'verified-other');
    });
  });
});