PathsExtractorWithNeatGUI.js

(function URLScout(){
  const PANEL_ID = "__url_scout_panel__";
  const STYLE_ID = "__url_scout_style__";
  const SEEN_KEY = "__url_scout_seen__";

  // --- helpers -------------------------------------------------------------
  const likelyStart = (s)=> /^(https?:|wss?:|data:|blob:|mailto:|tel:|\/|\.\/|\.\.\/|\/\/)/i.test(s||"");
  const urlRe = /(https?:\/\/[^\s"'`<>]+|wss?:\/\/[^\s"'`<>]+|\/\/[^\s"'`<>]+|\/(?:[A-Za-z0-9\-._~!$&'()*+,;=:@\/?%#])+)/g;
  const srcsetSplit = (v)=> (v||"").split(',').map(x=>x.trim().split(/\s+/)[0]).filter(Boolean);

  const stripWrap = (s)=> (s||"")
    .replace(/^[\s\u00A0"'`\(\[\{<]+/, '')
    .replace(/[\s\u00A0"'`,;:>\]\)\}]+$/, '');

  const stripUnbalanced = (s)=>{
    let out=s;
    while(out.endsWith(')') && (out.split(')').length-1) > (out.split('(').length-1)) out=out.slice(0,-1);
    while(out.endsWith(']') && (out.split(']').length-1) > (out.split('[').length-1)) out=out.slice(0,-1);
    return out;
  };
  const stripRegexFlags = (s)=> s.replace(/\/(?:[gimuy]{1,5})(?=$|[\s,;:>\)\]\}])/i,'');
  const preClean = (raw)=>{
    if(!raw||typeof raw!=="string") return raw; let s=stripWrap(raw); s=stripRegexFlags(s); s=stripUnbalanced(s);
    s=s.replace(/\)\s*;?$/, '').replace(/,\s*$/, '').replace(/:;?$/, '');
    return s;
  };

  const normalize = (raw)=>{
    const cleaned = preClean(raw); if(!cleaned) return null; if(/^javascript:/i.test(cleaned)) return null;
    if(/^(data:|blob:|mailto:|tel:)/i.test(cleaned)) return cleaned;
    try{ let u=cleaned; if(u.startsWith('//')) u=location.protocol+u; const abs=new URL(u, location.href); abs.hash=''; return abs.href; }catch(_){ return null; }
  };

  const ext = (u)=>{ try{ const {pathname}=new URL(u,location.href); const m=pathname.match(/\.([a-z0-9]+)$/i); return m?m[1].toLowerCase():''; }catch(_){ return ''; } };
  const typeOf = (u)=>{ if(!u) return 'other'; if(/^data:/i.test(u)) return 'data'; if(/^blob:/i.test(u)) return 'blob'; if(/^wss?:/i.test(u)) return 'websocket';
    let t='other'; const e=ext(u); const p=(()=>{ try{ return new URL(u,location.href).pathname; }catch(_){ return u; } })();
    const isApi = /(^|\/)api(\/|$)/i.test(p) || /(^|[?&])_?type=|[?&](format|action|tag|op|cmd|do)=|\.do(\?|$)/i.test(u);
    if(isApi) t='api';
    const img=new Set(['png','jpg','jpeg','gif','webp','bmp','ico']); const font=new Set(['woff','woff2','ttf','otf','eot']);
    if(e==='js') t='script'; else if(e==='css') t='style'; else if(e==='svg') t=t==='api'?t:'svg'; else if(e==='json') t=t==='api'?t:'json';
    else if(img.has(e)) t='image'; else if(font.has(e)) t='font'; else if(['html','htm','php','aspx','jsp'].includes(e) || (!e && !isApi)) t=t==='api'?t:'page';
    return t; };

  const isNamespaceUri = (u)=>{ try{ const url=new URL(u,location.href); return /(^|\.)w3\.org$/i.test(url.hostname) && /^\/(1999|2000)\//.test(url.pathname); }catch(_){ return false; } };
  const isKnownNoise = (u)=>{ try{ const {hostname,pathname}=new URL(u,location.href); if(hostname==='svgjs.com' && /^\/svgjs\/?$/i.test(pathname)) return true; return false; }catch(_){ return false; } };

  const STOP_LAST = new Set(['a','i','p','div','span','ul','li','label','option','select','script','style','function','html','css','javascript','windows','msie','iphone','ipad','android','step']);
  const STOP_FIRST = new Set(['midp','ucweb','rv','msie','some']);
  const tokenish = (seg)=> /^[A-Za-z0-9+/_-]{16,}$/.test(seg) && !/\./.test(seg);

  const isProbableGarbage = (u)=>{
    try{
      if(isNamespaceUri(u) || isKnownNoise(u)) return true;
      const url = new URL(u, location.href);
      const qs = url.search || '';
      // Root with real query (e.g., /?_type=...) should be kept
      if(url.pathname==='/' && /[?&][^=&]+=/.test(qs)) return false;
      // If there are key=value pairs in query at all, keep it (likely API)
      if(/[?&][^=&]+=/.test(qs)) return false;

      const segs = url.pathname.split('/').filter(Boolean);
      const first = segs[0]||''; const last = segs[segs.length-1]||'';
      const hasExt = /\.[a-z0-9]{2,}$/i.test(last);
      const hasQuery = !!qs;

      if(segs.length===0) return true; // bare '/'
      if(STOP_FIRST.has(first.toLowerCase()) && !hasExt && !hasQuery) return true;
      if(segs.length===1 && !hasExt && !hasQuery){
        const low=last.toLowerCase(); if(STOP_LAST.has(low)) return true; if(/^[0-9]{1,3}$/.test(last)) return true; if(last.length<=2) return true;
      }
      if(segs.some(tokenish) && !hasExt && !hasQuery) return true;
      if(/[(){}:;]/.test(url.pathname)) return true;
      if(url.pathname.includes('+') && !hasExt && !hasQuery) return true;
      return false;
    }catch(_){ return true; }
  };

  // --- collection ----------------------------------------------------------
  const results = new Map();
  const add = (raw, meta={})=>{
    if(!raw) return; if(Array.isArray(raw)){ raw.forEach(r=>add(r,meta)); return; }
    const candidates = typeof raw==='string' ? (raw.match(urlRe)||[]) : [];
    (candidates.length?candidates:[raw]).forEach(val=>{
      if(!likelyStart(val)) return; const n=normalize(val); if(!n) return; if(isProbableGarbage(n)) return;
      if(!results.has(n)) results.set(n, {type:typeOf(n), firstParty:(()=>{ try{return new URL(n,location.href).host===location.host;}catch(_){return false;}})(), sources:new Set([meta.source||'unknown'])});
      else results.get(n).sources.add(meta.source||'unknown');
    });
  };

  // Attributes & srcset
  document.querySelectorAll('*').forEach(el=>{
    for(const a of el.attributes||[]){
      const name=a.name.toLowerCase(); const val=(a.value||'').trim(); if(!val) continue;
      if(name==='srcset') add(srcsetSplit(val), {source:`attr:srcset <${el.tagName.toLowerCase()}>`});
      else if(['href','src','action','formaction','poster','cite','data-src','data-href','content','data-url'].includes(name)) add(val,{source:`attr:${name} <${el.tagName.toLowerCase()}>`});
      else if(/url\(/i.test(val)) (val.match(/url\(([^)]+)\)/gi)||[]).forEach(seg=> add(seg.replace(/url\(|\)|['"]/g,'').trim(), {source:`style-url <${el.tagName.toLowerCase()}>`}));
      else if(urlRe.test(val)) add(val, {source:`attr:${name} <${el.tagName.toLowerCase()}>`});
    }
  });

  // Scripts / JSON / Styles
  document.querySelectorAll('script').forEach(s=>{ const t=(s.innerText||''); if(t) add(t,{source:'script-text'}); });
  document.querySelectorAll('script[type="application/ld+json"],script[type="application/json"]').forEach(s=>{ try{ const obj=JSON.parse(s.textContent||'{}'); add(JSON.stringify(obj), {source:'script:json'});}catch(_){}});
  document.querySelectorAll('style').forEach(st=>{ const t=st.textContent||''; (t.match(/url\(([^)]+)\)/gi)||[]).forEach(seg=> add(seg.replace(/url\(|\)|['"]/g,'').trim(), {source:'style-block'})); });

  // Comments & visible text (lightweight)
  try{ const walker=document.createTreeWalker(document.body||document, NodeFilter.SHOW_COMMENT); let n; while((n=walker.nextNode())) add(n.textContent||'', {source:'comment'});}catch(_){ }
  document.querySelectorAll('meta,link,script,style,noscript').forEach(el=>el.setAttribute(SEEN_KEY,'1'));
  const textNodes=document.createTreeWalker(document.body||document, NodeFilter.SHOW_TEXT);
  let tn,count=0; while((tn=textNodes.nextNode())){ if(count>5000) break; const p=tn.parentElement; if(!p||p.hasAttribute(SEEN_KEY)) continue; const t=(tn.textContent||'').trim(); if(!t) continue; count+=t.length; add(t,{source:`text <${p.tagName.toLowerCase()}>`}); }

  const arr = Array.from(results.entries()).map(([url,meta])=>({url, type:meta.type, firstParty:meta.firstParty, sources:Array.from(meta.sources)}))
    .sort((a,b)=> (a.firstParty===b.firstParty?0:(a.firstParty?-1:1)) || a.type.localeCompare(b.type) || a.url.localeCompare(b.url));

  // --- rendering -----------------------------------------------------------
  const orderedTypes=['api','page','script','style','image','font','json','svg','websocket','data','blob','other'];
  const renderHtml=(data)=>{
    const groups=data.reduce((acc,it)=>{(acc[it.type]||(acc[it.type]=[])).push(it);return acc;},{});
    const total=data.length; const fp=data.filter(x=>x.firstParty).length; const tp=total-fp;
    const esc=(s)=>String(s).replace(/[&<>]/g,c=>({"&":"&amp;","<":"&lt;",">":"&gt;"}[c]));
    const list=orderedTypes.map(t=>{ const list=groups[t]||[]; if(!list.length) return ''; return `<section class="blk"><details ${['api','script','page'].includes(t)?'open':''}><summary>${t.toUpperCase()} <span class="pill">${list.length}</span></summary><ul>`+list.map(it=>`<li data-type="${t}" data-firstparty="${it.firstParty?1:0}"><code title="${esc((it.sources||[]).join(', '))}"><a href="${esc(it.url)}" target="_blank" rel="noreferrer noopener">${esc(it.url)}</a></code></li>`).join('')+`</ul></details></section>`; }).join('');
    return `<!doctype html><html><head><meta charset="utf-8"/><title>URL Scout — ${esc(location.hostname)} (${total})</title>
      <style>:root{color-scheme:light dark}body{margin:0;font:13px/1.4 ui-sans-serif,system-ui,-apple-system,Segoe UI,Roboto,Helvetica,Arial}.hdr{position:sticky;top:0;display:flex;gap:8px;align-items:center;justify-content:space-between;padding:10px 12px;border-bottom:1px solid #e5e7eb;background:Canvas;color:CanvasText}.ttl{font-weight:700;font-size:14px}.btn{cursor:pointer;border:1px solid #e5e7eb;padding:6px 10px;border-radius:10px;background:ButtonFace;color:ButtonText}.row{display:flex;gap:8px;align-items:center;padding:8px 12px;flex-wrap:wrap}.pill{font-size:12px;padding:2px 8px;border-radius:999px;border:1px solid #e5e7eb}.inp{flex:1;min-width:200px;padding:6px 10px;border-radius:10px;border:1px solid #e5e7eb}details{margin:6px 12px;border:1px solid #e5e7eb;border-radius:10px}summary{cursor:pointer;list-style:none;padding:8px 10px;font-weight:600}ul{margin:0;padding:8px 0;max-height:260px;overflow:auto}li{padding:2px 10px}a{color:inherit;text-decoration:none}code{white-space:nowrap;overflow:hidden;text-overflow:ellipsis;display:block}</style>
    </head><body>
      <div class="hdr"><div class="ttl">URL Scout <span class="pill">${total} total</span> <span class="pill">First‑party: ${fp}</span> <span class="pill">Third‑party: ${tp}</span></div>
        <div>
          <button class="btn" id="copy-list">Copy list</button>
          <button class="btn" id="copy-json">Copy JSON</button>
          <button class="btn" id="download-json">Download JSON</button>
        </div>
      </div>
      <div class="row">
        <input class="inp" id="filter" placeholder="Filter…"/>
        <label class="pill" style="display:flex;gap:6px;align-items:center"><input type="checkbox" id="fp-only"/> First‑party only</label>
        <label class="pill" style="display:flex;gap:6px;align-items:center"><input type="checkbox" id="strict"/> Strict mode</label>
      </div>
      <main id="list">${list}</main>
      <script>
        const data=${JSON.stringify(arr)}; const list=document.getElementById('list');
        const q=document.getElementById('filter'); const fpOnly=document.getElementById('fp-only'); const strict=document.getElementById('strict');
        function apply(){ const str=(q.value||'').toLowerCase(); const only=fpOnly.checked; const sm=strict.checked;
          list.querySelectorAll('li').forEach(li=>{ const text=li.textContent.toLowerCase(); const okQ=!str||text.includes(str); const okFp=!only||li.dataset.firstparty==='1'; let ok=okQ&&okFp; if(sm){ try{ const u=new URL(li.textContent.trim()); const segs=u.pathname.split('/').filter(Boolean); const last=segs[segs.length-1]||''; const hasExt=/\.[a-z0-9]{2,}$/i.test(last); const hasQuery=!!u.search; if(segs.length===1 && !hasExt && !hasQuery){ if(last.length<=2 || /^[0-9]{1,3}$/.test(last)) ok=false; } }catch{} }
            li.style.display = ok ? '' : 'none';
          });
        }
        q.addEventListener('input',apply); fpOnly.addEventListener('change',apply); strict.addEventListener('change',apply);
        document.getElementById('copy-list').addEventListener('click',()=>{ const txt=data.map(x=>x.url).join('\n'); navigator.clipboard?.writeText(txt).then(()=>alert('Copied list')).catch(()=>prompt('Copy the list:',txt)); });
        document.getElementById('copy-json').addEventListener('click',()=>{ const txt=JSON.stringify(data,null,2); navigator.clipboard?.writeText(txt).then(()=>alert('Copied JSON')).catch(()=>prompt('Copy the JSON:',txt)); });
        document.getElementById('download-json').addEventListener('click',()=>{ const blob=new Blob([JSON.stringify(data,null,2)],{type:'application/json'}); const a=document.createElement('a'); a.href=URL.createObjectURL(blob); a.download='url-scout-'+location.hostname+'.json'; a.click(); setTimeout(()=>URL.revokeObjectURL(a.href),1500); });
      </script>
    </body></html>`;
  };

  // New tab via Blob URL (with data: fallback)
  const openInNewTab=(data)=>{
    const html=renderHtml(data);
    try{ const blob=new Blob([html],{type:'text/html'}); const url=URL.createObjectURL(blob); const w=window.open(url,'_blank','noopener'); if(w){ setTimeout(()=>URL.revokeObjectURL(url),10000); return true; } }catch(_){ }
    try{ const dataUrl='data:text/html;charset=utf-8,'+encodeURIComponent(html); const w2=window.open(dataUrl,'_blank','noopener'); if(w2) return true; }catch(_){ }
    return false;
  };

  // Panel fallback (compact)
  function renderPanel(data){
    try{ if(window[PANEL_ID]){ window[PANEL_ID].remove(); delete window[PANEL_ID]; } document.getElementById(STYLE_ID)?.remove(); }catch(_){ }
    const style=document.createElement('style'); style.id=STYLE_ID; style.textContent=`#${PANEL_ID}{all:initial;position:fixed;top:16px;right:16px;width:420px;max-height:80vh;overflow:auto;background:#fff;color:#111;font:13px/1.4 ui-sans-serif,system-ui,-apple-system,Segoe UI,Roboto,Helvetica,Arial;border:1px solid #e5e7eb;border-radius:14px;box-shadow:0 10px 30px rgba(0,0,0,.18);z-index:2147483647}#${PANEL_ID} .hdr{display:flex;align-items:center;justify-content:space-between;padding:10px 12px;border-bottom:1px solid #e5e7eb}.btn{cursor:pointer;border:1px solid #e5e7eb;padding:6px 10px;border-radius:10px;background:#f9fafb}`;
    document.head.appendChild(style);
    const panel=document.createElement('div'); panel.id=PANEL_ID;
    const mk=(h)=>{const d=document.createElement('div'); d.innerHTML=h.trim(); return d.firstElementChild;};
    panel.appendChild(mk(`<div class="hdr"><div>URL Scout</div><div><button class="btn" data-act="open">Open in new tab</button> <button class="btn" data-act="close">Close</button></div></div>`));
    const ul=document.createElement('ul'); data.forEach(it=>{ const li=document.createElement('li'); const a=document.createElement('a'); a.href=it.url; a.textContent=it.url; a.target='_blank'; a.rel='noreferrer noopener'; li.appendChild(a); ul.appendChild(li); }); panel.appendChild(ul);
    panel.addEventListener('click',(e)=>{ const b=e.target.closest('[data-act]'); if(!b) return; const act=b.getAttribute('data-act'); if(act==='open'){ openInNewTab(data) && panel.remove(); } if(act==='close'){ panel.remove(); document.getElementById(STYLE_ID)?.remove(); }});
    document.body.appendChild(panel); window[PANEL_ID]=panel;
  }

  const data = arr; // collected above
  if(!openInNewTab(data)) renderPanel(data);
})();