Browser Network Observer

gistfile1.txt

// ==UserScript==
// @name         BrowserNetworkObserver
// @description  Captures all fetch/XHR traffic via callback and Observable interfaces.
// @match        *://*/*
// @run-at       document-start
// @grant        none
// @source       https://gist.githubusercontent.com/jonashw/d314c2b1dd0e08948bb588245344a4ef
// @version      2026-06-08
// ==/UserScript==
// Specification: https://gist.github.com/jonashw/df8c914c706a50ce1a73e86740bd210c

//
// HOW TO USE
// ----------
// 1. Install Tampermonkey and paste this file. Set @match for your target site(s).
//
// 2. Callback style:
//      const unsub = NetworkObserver.on.response(e => {
//        console.log(e.method, e.url, e.status, Math.round(e.duration) + 'ms');
//      });
//      unsub(); // stop listening
//
// 3. Observable style (fluent):
//      NetworkObserver.stream.response
//        .filter(e => e.host.includes('forgerock.io'))
//        .map(e => ({ path: e.path, status: e.status }))
//        .take(5)
//        .subscribe(e => console.table([e]));
//
// 4. Teardown: NetworkObserver.destroy();
//

(function () {
  'use strict';

  // ===========================================================================
  // CORE: interception engine
  // ===========================================================================

  var replayWindow = 5 * 60 * 1000;
  var listeners = { request: [], response: [], error: [] };
  var buffer = { request: [], response: [], error: [] };
  var idCounter = 0;
  var destroyed = false;
  var seenByPatch = new Set();

  var originalFetch = window.fetch;
  var originalXHROpen = XMLHttpRequest.prototype.open;
  var originalXHRSend = XMLHttpRequest.prototype.send;
  var originalXHRSetRequestHeader = XMLHttpRequest.prototype.setRequestHeader;

  function nextId() {
    return 'nr-' + (++idCounter) + '-' + Math.random().toString(36).slice(2, 8);
  }

  function now() { return performance.now(); }

  function emit(type, event) {
    if (destroyed) return;
    if (type === 'request') seenByPatch.add(event.url + '|' + event.startTime.toFixed(0));
    event._time = Date.now();
    buffer[type].push(event);
    pruneBuffer(type);
    var cbs = listeners[type];
    for (var i = 0; i < cbs.length; i++) {
      try { cbs[i](event); } catch (_) {}
    }
  }

  function pruneBuffer(type) {
    var cutoff = Date.now() - replayWindow;
    var buf = buffer[type];
    while (buf.length && buf[0]._time < cutoff) buf.shift();
  }

  function parseUrl(raw) {
    try {
      var u = new URL(raw, location.origin);
      return { host: u.host, path: u.pathname };
    } catch (_) {
      return { host: '', path: raw };
    }
  }

  function headersToRecord(headers) {
    var rec = {};
    if (!headers) return rec;
    if (typeof headers.forEach === 'function') {
      headers.forEach(function (v, k) { rec[k] = v; });
    } else if (typeof headers === 'object') {
      var entries = headers instanceof Headers
        ? Array.from(headers.entries())
        : Object.entries(headers);
      for (var i = 0; i < entries.length; i++) rec[entries[i][0]] = entries[i][1];
    }
    return rec;
  }

  function subscribe(type, cb) {
    if (destroyed) return function () {};
    listeners[type].push(cb);
    pruneBuffer(type);
    var buf = buffer[type];
    for (var i = 0; i < buf.length; i++) {
      try { cb(buf[i]); } catch (_) {}
    }
    return function unsubscribe() {
      var idx = listeners[type].indexOf(cb);
      if (idx !== -1) listeners[type].splice(idx, 1);
    };
  }

  // --- Patch fetch ---
  function patchedFetch(input, init) {
    var requestId = nextId();
    var method = ((init && init.method) || 'GET').toUpperCase();
    var url = typeof input === 'string' ? input : (input && input.url) || String(input);
    var parsed = parseUrl(url);
    var reqHeaders = headersToRecord((init && init.headers) || (input && input.headers));
    var body = (init && init.body) || null;
    var startTime = now();

    emit('request', { requestId: requestId, url: url, host: parsed.host, path: parsed.path, method: method, headers: reqHeaders, body: body, startTime: startTime });

    return originalFetch.apply(this, arguments).then(function (response) {
      var endTime = now();
      var respHeaders = headersToRecord(response.headers);
      var cl = response.headers.get('content-length');
      emit('response', {
        requestId: requestId, url: url, host: parsed.host, path: parsed.path, method: method,
        status: response.status, requestHeaders: reqHeaders, responseHeaders: respHeaders,
        body: null, startTime: startTime, endTime: endTime,
        duration: endTime - startTime,
        responseSize: cl !== null ? parseInt(cl, 10) : null
      });
      return response;
    }, function (err) {
      var endTime = now();
      emit('error', {
        requestId: requestId, url: url, host: parsed.host, path: parsed.path, method: method,
        error: err, startTime: startTime, endTime: endTime, duration: endTime - startTime
      });
      throw err;
    });
  }
  window.fetch = patchedFetch;

  // --- PerformanceObserver fallback ---
  var perfObserver = null;
  if (typeof PerformanceObserver !== 'undefined') {
    perfObserver = new PerformanceObserver(function (list) {
      var entries = list.getEntries();
      for (var i = 0; i < entries.length; i++) {
        var entry = entries[i];
        if (entry.initiatorType !== 'fetch' && entry.initiatorType !== 'xmlhttprequest') continue;
        var key = entry.name + '|' + Math.round(entry.startTime);
        if (seenByPatch.has(key)) continue;
        var found = false;
        for (var k of seenByPatch) {
          if (k.startsWith(entry.name + '|')) {
            var t = parseInt(k.split('|')[1], 10);
            if (Math.abs(t - entry.startTime) < 50) { found = true; break; }
          }
        }
        if (found) continue;
        var parsed = parseUrl(entry.name);
        emit('response', {
          requestId: nextId(), url: entry.name, host: parsed.host, path: parsed.path,
          method: '', status: entry.responseStatus || 0,
          requestHeaders: {}, responseHeaders: {}, body: null,
          startTime: entry.startTime, endTime: entry.startTime + entry.duration,
          duration: entry.duration, responseSize: entry.transferSize || null
        });
      }
    });
    perfObserver.observe({ type: 'resource', buffered: true });
  }

  // --- Patch XMLHttpRequest ---
  XMLHttpRequest.prototype.open = function (method, url) {
    var parsed = parseUrl(String(url));
    this.__no = { method: (method || 'GET').toUpperCase(), url: String(url), host: parsed.host, path: parsed.path, headers: {} };
    return originalXHROpen.apply(this, arguments);
  };

  XMLHttpRequest.prototype.setRequestHeader = function (name, value) {
    if (this.__no) this.__no.headers[name] = value;
    return originalXHRSetRequestHeader.apply(this, arguments);
  };

  XMLHttpRequest.prototype.send = function (body) {
    if (!this.__no) return originalXHRSend.apply(this, arguments);
    var meta = this.__no;
    var requestId = nextId();
    var startTime = now();

    emit('request', {
      requestId: requestId, url: meta.url, host: meta.host, path: meta.path, method: meta.method,
      headers: meta.headers, body: body || null, startTime: startTime
    });

    this.addEventListener('load', function () {
      var endTime = now();
      var respHeaders = {};
      (this.getAllResponseHeaders() || '').trim().split(/\r?\n/).forEach(function (line) {
        var idx = line.indexOf(':');
        if (idx > 0) respHeaders[line.slice(0, idx).trim().toLowerCase()] = line.slice(idx + 1).trim();
      });
      var cl = respHeaders['content-length'];
      emit('response', {
        requestId: requestId, url: meta.url, host: meta.host, path: meta.path, method: meta.method,
        status: this.status, requestHeaders: meta.headers, responseHeaders: respHeaders,
        body: null, startTime: startTime, endTime: endTime, duration: endTime - startTime,
        responseSize: cl != null ? parseInt(cl, 10) : null
      });
    });

    this.addEventListener('error', function () {
      var endTime = now();
      emit('error', { requestId: requestId, url: meta.url, host: meta.host, path: meta.path, method: meta.method, error: new Error('XHR network error'), startTime: startTime, endTime: endTime, duration: endTime - startTime });
    });

    this.addEventListener('abort', function () {
      var endTime = now();
      emit('error', { requestId: requestId, url: meta.url, host: meta.host, path: meta.path, method: meta.method, error: new Error('XHR aborted'), startTime: startTime, endTime: endTime, duration: endTime - startTime });
    });

    this.addEventListener('timeout', function () {
      var endTime = now();
      emit('error', { requestId: requestId, url: meta.url, host: meta.host, path: meta.path, method: meta.method, error: new Error('XHR timeout'), startTime: startTime, endTime: endTime, duration: endTime - startTime });
    });

    return originalXHRSend.apply(this, arguments);
  };

  // ===========================================================================
  // STREAM: fluent Observable with chainable operators
  // ===========================================================================

  function createStream(subscribeFn) {
    var stream = {
      subscribe: function (next) {
        return subscribeFn(next || function () {});
      },
      filter: function (predicate) {
        var source = this;
        return createStream(function (next) {
          return source.subscribe(function (v) { if (predicate(v)) next(v); });
        });
      },
      map: function (project) {
        var source = this;
        return createStream(function (next) {
          return source.subscribe(function (v) { next(project(v)); });
        });
      },
      tap: function (fn) {
        var source = this;
        return createStream(function (next) {
          return source.subscribe(function (v) { try { fn(v); } catch (_) {} next(v); });
        });
      },
      scan: function (accumulator, seed) {
        var source = this;
        return createStream(function (next) {
          var acc = seed;
          return source.subscribe(function (v) { acc = accumulator(acc, v); next(acc); });
        });
      },
      debounceTime: function (ms) {
        var source = this;
        return createStream(function (next) {
          var timer = null, latest;
          return source.subscribe(function (v) {
            latest = v;
            if (timer) clearTimeout(timer);
            timer = setTimeout(function () { next(latest); }, ms);
          });
        });
      },
      throttleTime: function (ms) {
        var source = this;
        return createStream(function (next) {
          var last = 0;
          return source.subscribe(function (v) {
            var t = Date.now();
            if (t - last >= ms) { last = t; next(v); }
          });
        });
      },
      take: function (count) {
        var source = this;
        return createStream(function (next) {
          var taken = 0;
          var sub = source.subscribe(function (v) {
            if (taken < count) {
              taken++;
              next(v);
              if (taken >= count && sub) sub.unsubscribe();
            }
          });
          if (taken >= count) sub.unsubscribe();
          return sub;
        });
      },
      takeUntil: function (signal) {
        var source = this;
        return createStream(function (next) {
          var sub = source.subscribe(next);
          var sigSub = signal.subscribe(function () {
            sub.unsubscribe();
            sigSub.unsubscribe();
          });
          return sub;
        });
      }
    };
    return stream;
  }

  function streamForType(type) {
    return createStream(function (next) {
      var unsub = subscribe(type, next);
      return { unsubscribe: unsub };
    });
  }

  function streamAll() {
    return createStream(function (next) {
      var u1 = subscribe('request', function (e) { next(Object.assign({ type: 'request' }, e)); });
      var u2 = subscribe('response', function (e) { next(Object.assign({ type: 'response' }, e)); });
      var u3 = subscribe('error', function (e) { next(Object.assign({ type: 'error' }, e)); });
      return { unsubscribe: function () { u1(); u2(); u3(); } };
    });
  }

  // ===========================================================================
  // PUBLIC API: window.NetworkObserver
  // ===========================================================================

  window.NetworkObserver = {
    on: {
      request: function (cb) { return subscribe('request', cb); },
      response: function (cb) { return subscribe('response', cb); },
      error: function (cb) { return subscribe('error', cb); }
    },
    stream: {
      all: streamAll(),
      request: streamForType('request'),
      response: streamForType('response'),
      error: streamForType('error')
    },
    destroy: function () {
      destroyed = true;
      window.fetch = originalFetch;
      XMLHttpRequest.prototype.open = originalXHROpen;
      XMLHttpRequest.prototype.send = originalXHRSend;
      XMLHttpRequest.prototype.setRequestHeader = originalXHRSetRequestHeader;
      if (perfObserver) perfObserver.disconnect();
      listeners.request.length = 0;
      listeners.response.length = 0;
      listeners.error.length = 0;
    }
  };

  // ===========================================================================
  // IFRAME PATCHING: same-origin iframes (ForgeRock appAuthHelper pattern)
  // ===========================================================================

  var iframeIdCounter = 0;

  function patchIframeWindow(iWin) {
    try {
      if (!iWin || !iWin.fetch || iWin.__netObserverPatched) return;
      var iOrigFetch = iWin.fetch.bind(iWin);
      iWin.fetch = function (input, init) {
        var method = ((init && init.method) || 'GET').toUpperCase();
        var url = typeof input === 'string' ? input : (input && input.url) || String(input);
        var host = '', path = url;
        try { var u = new URL(url, location.origin); host = u.host; path = u.pathname; } catch (_) {}
        var reqHeaders = {};
        var rawH = (init && init.headers) || (input && input.headers);
        if (rawH) {
          if (typeof rawH.forEach === 'function') rawH.forEach(function (v, k) { reqHeaders[k] = v; });
          else if (typeof rawH === 'object') Object.entries(rawH).forEach(function (e) { reqHeaders[e[0]] = e[1]; });
        }
        var body = (init && init.body) || null;
        var startTime = performance.now();
        var requestId = 'nr-if-' + (++iframeIdCounter) + '-' + Math.random().toString(36).slice(2, 8);

        emit('request', { requestId: requestId, url: url, host: host, path: path, method: method, headers: reqHeaders, body: body, startTime: startTime });

        return iOrigFetch.apply(iWin, arguments).then(function (response) {
          var endTime = performance.now();
          var respHeaders = {};
          response.headers.forEach(function (v, k) { respHeaders[k] = v; });
          var cl = response.headers.get('content-length');
          emit('response', {
            requestId: requestId, url: url, host: host, path: path, method: method,
            status: response.status, requestHeaders: reqHeaders, responseHeaders: respHeaders,
            body: null, startTime: startTime, endTime: endTime, duration: endTime - startTime,
            responseSize: cl !== null ? parseInt(cl, 10) : null
          });
          return response;
        }, function (err) {
          var endTime = performance.now();
          emit('error', {
            requestId: requestId, url: url, host: host, path: path, method: method,
            error: err, startTime: startTime, endTime: endTime, duration: endTime - startTime
          });
          throw err;
        });
      };
      iWin.__netObserverPatched = true;
    } catch (e) { /* cross-origin -- skip */ }
  }

  function scanIframes() {
    var iframes = document.querySelectorAll('iframe');
    for (var i = 0; i < iframes.length; i++) patchIframeWindow(iframes[i].contentWindow);
  }

  if (document.readyState === 'loading') {
    document.addEventListener('DOMContentLoaded', scanIframes);
  } else {
    scanIframes();
  }
  document.addEventListener('load', function (e) {
    if (e.target && e.target.tagName === 'IFRAME') patchIframeWindow(e.target.contentWindow);
  }, true);

  new MutationObserver(function (mutations) {
    for (var i = 0; i < mutations.length; i++) {
      var nodes = mutations[i].addedNodes;
      for (var j = 0; j < nodes.length; j++) {
        if (nodes[j].tagName === 'IFRAME') {
          nodes[j].addEventListener('load', function () { patchIframeWindow(this.contentWindow); });
          patchIframeWindow(nodes[j].contentWindow);
        }
      }
    }
  }).observe(document.documentElement, { childList: true, subtree: true });
})();

//
// IMPLEMENTATION NOTES
// --------------------
// Interception layers (ordered by fidelity):
//   1. Monkey-patched window.fetch -- catches all fetch() from the main page.
//   2. Monkey-patched XMLHttpRequest.prototype -- catches XHR.
//   3. Same-origin iframe patching -- patches fetch inside hidden iframes (e.g., ForgeRock
//      appAuthHelper, which delegates OAuth-authenticated calls via postMessage).
//      Discovered via MutationObserver + iframe load events.
//   4. PerformanceObserver (type: 'resource') -- fallback for fetch/XHR that bypasses
//      layers 1-3 (stale closed-over refs). Limited: no headers/body/method.
//
// Deduplication: URL+startTime fingerprint in a Set. PerformanceObserver skips +-50ms matches.
//
// Observable: fluent chainable streams. Every operator (.filter, .map, .take, etc.)
// returns a new stream with the same method set. subscribe() returns { unsubscribe() }.
// Streams are multicast (share underlying listener). Replay inherited from callback layer.
//