| <!doctype html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8"> |
| <meta name="viewport" content="width=device-width,initial-scale=1"> |
| <title>Web graph embeddings — 2D map (streaming)</title> |
| |
| <link rel="preconnect" href="https://fonts.googleapis.com"> |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> |
| <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Libre+Franklin:wght@400;500;600;700&family=IBM+Plex+Mono:wght@400;500;600&display=swap"> |
| |
| |
| |
| <style> |
| |
| |
| |
| :root { |
| --font: 'Libre Franklin', system-ui, -apple-system, 'Segoe UI', sans-serif; |
| --mono: 'IBM Plex Mono', ui-monospace, SFMono-Regular, Menlo, monospace; |
| --bg: #0d1929; --panel: rgba(21,42,71,.80); --solid: #152a47; --ink: #ffffff; |
| --muted: #8d99b0; --line: #2a456d; --accent: #4a90e8; --hot: #ff9220; |
| --shadow: 0 8px 30px rgba(0,0,0,.45); |
| } |
| @media (prefers-color-scheme: light) { |
| :root { |
| --bg: #f6f7fa; --panel: rgba(255,255,255,.86); --solid: #ffffff; --ink: #152a47; |
| --muted: #6e7191; --line: #dde2e8; --accent: #0050bd; --hot: #ff9220; |
| --shadow: 0 8px 26px rgba(21,42,71,.16); |
| } |
| } |
| :root[data-theme="dark"] { |
| --bg: #0d1929; --panel: rgba(21,42,71,.80); --solid: #152a47; --ink: #ffffff; |
| --muted: #8d99b0; --line: #2a456d; --accent: #4a90e8; --hot: #ff9220; |
| --shadow: 0 8px 30px rgba(0,0,0,.45); |
| } |
| :root[data-theme="light"] { |
| --bg: #f6f7fa; --panel: rgba(255,255,255,.86); --solid: #ffffff; --ink: #152a47; |
| --muted: #6e7191; --line: #dde2e8; --accent: #0050bd; --hot: #ff9220; |
| --shadow: 0 8px 26px rgba(21,42,71,.16); |
| } |
| * { box-sizing: border-box; } |
| html, body { margin: 0; height: 100%; } |
| body { |
| background: var(--bg); color: var(--ink); overflow: hidden; |
| font-family: var(--font); |
| -webkit-font-smoothing: antialiased; |
| } |
| #stage { position: fixed; inset: 0; } |
| canvas { display: block; width: 100%; height: 100%; cursor: crosshair; touch-action: none; } |
| |
| .panel { |
| position: fixed; background: var(--panel); border: 1px solid var(--line); |
| border-radius: 12px; box-shadow: var(--shadow); |
| backdrop-filter: blur(14px) saturate(1.2); -webkit-backdrop-filter: blur(14px) saturate(1.2); |
| } |
| #rail { top: 16px; left: 16px; width: 268px; padding: 14px 15px 15px; display: flex; |
| flex-direction: column; gap: 13px; } |
| .brand { display: flex; align-items: baseline; gap: 8px; } |
| .brand h1 { margin: 0; font-size: 14px; font-weight: 650; letter-spacing: -.01em; } |
| .brand .n { margin-left: auto; font: 600 11px/1 var(--mono); |
| color: var(--accent); font-variant-numeric: tabular-nums; } |
| .lede { margin: 0; font-size: 11.5px; line-height: 1.45; color: var(--muted); } |
| .lede a { color: var(--accent); overflow-wrap: anywhere; word-break: break-word; } |
| .grp { display: flex; flex-direction: column; gap: 6px; } |
| .lbl { font-size: 10px; text-transform: uppercase; letter-spacing: .09em; color: var(--muted); |
| font-weight: 600; } |
| .seg { display: grid; grid-template-columns: 1fr 1fr; gap: 5px; } |
| .seg button { |
| appearance: none; border: 1px solid var(--line); background: transparent; color: var(--ink); |
| padding: 7px 8px; border-radius: 8px; font: 500 12px/1 var(--font); cursor: pointer; |
| transition: background .12s, border-color .12s; text-align: left; |
| } |
| .seg button:hover { background: color-mix(in srgb, var(--accent) 12%, transparent); } |
| .seg button[aria-pressed="true"] { |
| border-color: var(--accent); background: color-mix(in srgb, var(--accent) 20%, transparent); |
| color: var(--ink); font-weight: 600; |
| } |
| .seg button:focus-visible { outline: 2px solid var(--accent); outline-offset: 1px; } |
| .search { position: relative; } |
| .search input { |
| width: 100%; padding: 8px 10px; border-radius: 8px; border: 1px solid var(--line); |
| background: var(--solid); color: var(--ink); font: 500 12.5px var(--mono); |
| } |
| .search input::placeholder { color: var(--muted); } |
| .search input:focus-visible { outline: 2px solid var(--accent); outline-offset: 0; border-color: var(--accent); } |
| #hits { list-style: none; margin: 5px 0 0; padding: 0; max-height: 176px; overflow-y: auto; |
| border-radius: 8px; } |
| #hits li { padding: 6px 9px; border-radius: 6px; cursor: pointer; |
| font: 500 12px var(--mono); color: var(--ink); display: flex; gap: 8px; } |
| #hits li:hover, #hits li[aria-selected="true"] { background: color-mix(in srgb, var(--hot) 22%, transparent); } |
| #hits li .deg { margin-left: auto; color: var(--muted); font-variant-numeric: tabular-nums; } |
| |
| #legend { left: 16px; bottom: 16px; max-width: 268px; padding: 11px 13px; display: none; |
| flex-direction: column; gap: 8px; } |
| #legend.on { display: flex; } |
| #legend .lbl { margin-bottom: 1px; } |
| .swatches { display: flex; flex-direction: column; gap: 2px; max-height: 240px; overflow-y: auto; } |
| .sw { display: flex; align-items: center; gap: 7px; font-size: 11px; color: var(--ink); |
| appearance: none; border: 1px solid transparent; background: transparent; text-align: left; |
| width: 100%; padding: 3px 6px; border-radius: 6px; cursor: pointer; font-family: inherit; |
| transition: background .12s, border-color .12s; } |
| .sw:hover { background: color-mix(in srgb, var(--accent) 12%, transparent); } |
| .sw[aria-pressed="true"] { border-color: var(--accent); |
| background: color-mix(in srgb, var(--accent) 18%, transparent); } |
| .sw:focus-visible { outline: 2px solid var(--accent); outline-offset: 1px; } |
| .sw i { width: 10px; height: 10px; border-radius: 3px; display: inline-block; flex: none; } |
| .sw .nm { flex: 1; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } |
| .sw .ct { color: var(--muted); font-variant-numeric: tabular-nums; |
| font: 500 10px var(--mono); } |
| |
| #tip { |
| position: fixed; pointer-events: none; z-index: 40; padding: 8px 10px; border-radius: 9px; |
| background: var(--solid); border: 1px solid var(--line); box-shadow: var(--shadow); |
| font: 500 12px var(--mono); max-width: 300px; display: none; |
| transform: translate(12px, 12px); |
| } |
| #tip .host { color: var(--accent); word-break: break-all; font-weight: 600; } |
| #tip .row { color: var(--muted); margin-top: 3px; display: flex; gap: 7px; } |
| #tip .row b { color: var(--ink); font-weight: 600; } |
| |
| #hud { right: 16px; bottom: 16px; padding: 8px 11px; display: flex; gap: 12px; align-items: center; |
| font: 500 11px var(--mono); color: var(--muted); font-variant-numeric: tabular-nums; } |
| #hud b { color: var(--ink); font-weight: 600; } |
| #hud button { appearance: none; border: 1px solid var(--line); background: transparent; color: var(--muted); |
| border-radius: 7px; padding: 4px 8px; cursor: pointer; font: inherit; } |
| #hud button:hover { color: var(--ink); border-color: var(--accent); } |
| #hud button[aria-pressed="true"] { color: var(--ink); border-color: var(--accent); |
| background: color-mix(in srgb, var(--accent) 18%, transparent); } |
| |
| #spin { position: fixed; top: 16px; right: 16px; z-index: 45; width: 17px; height: 17px; |
| border-radius: 50%; border: 2.5px solid var(--line); border-top-color: var(--accent); |
| box-shadow: var(--shadow); opacity: 0; pointer-events: none; transition: opacity .2s; |
| animation: spin .7s linear infinite; } |
| #spin.on { opacity: .95; } |
| @keyframes spin { to { transform: rotate(360deg); } } |
| |
| #err { position: fixed; inset: 0; display: none; place-items: center; padding: 24px; text-align: center; } |
| #err div { max-width: 400px; color: var(--muted); font-size: 13px; line-height: 1.5; } |
| |
| #toprail { position: fixed; top: 16px; right: 16px; z-index: 45; display: flex; |
| align-items: center; gap: 10px; pointer-events: none; } |
| #spin { position: relative; top: auto; right: auto; flex: none; } |
| #selpill { position: relative; display: none; pointer-events: auto; align-items: stretch; |
| border-radius: 999px; overflow: hidden; } |
| #selpill.on { display: flex; } |
| #selpill.open { border-color: var(--accent); } |
| #seln, #selclr { appearance: none; border: 0; background: transparent; color: var(--ink); |
| cursor: pointer; font: 600 11.5px/1.35 var(--mono); |
| font-variant-numeric: tabular-nums; } |
| #seln { position: relative; padding: 6px 11px; overflow: hidden; } |
| #seln:hover { background: color-mix(in srgb, var(--accent) 12%, transparent); } |
| #selpill.open #seln { background: color-mix(in srgb, var(--accent) 18%, transparent); } |
| #selclr { padding: 5px 11px 6px; color: var(--muted); font-size: 15px; line-height: 1.1; |
| border-left: 1px solid var(--line); } |
| #selclr:hover { color: var(--ink); background: color-mix(in srgb, var(--hot) 22%, transparent); } |
| |
| #seln:focus-visible, #selclr:focus-visible { outline: 2px solid var(--accent); |
| outline-offset: -2px; } |
| #seln::after { content: ""; position: absolute; left: 0; bottom: 0; height: 2px; width: 0; |
| background: var(--accent); transition: width .15s; } |
| #seln.scanning::after { width: calc(var(--p, 0) * 100%); } |
| |
| #selbox { position: fixed; z-index: 30; display: none; pointer-events: none; border-radius: 3px; |
| border: 1.5px dashed var(--accent); background: color-mix(in srgb, var(--accent) 13%, transparent); } |
| #selbox.on { display: block; } |
| #selbox.set { border-style: solid; } |
| |
| #sel { display: none; right: 16px; top: 58px; bottom: 16px; width: 452px; padding: 0; |
| flex-direction: column; overflow: hidden; z-index: 44; } |
| #sel.on { display: flex; } |
| #sel header { display: flex; align-items: baseline; gap: 8px; padding: 12px 14px 9px; |
| border-bottom: 1px solid var(--line); } |
| #sel h2 { margin: 0; font-size: 13px; font-weight: 650; letter-spacing: -.01em; } |
| #sel .sub { color: var(--muted); font-size: 10.5px; padding: 8px 14px 0; line-height: 1.45; } |
| #selx, #helpx { margin-left: auto; appearance: none; border: 1px solid var(--line); background: transparent; |
| color: var(--muted); border-radius: 7px; padding: 1px 8px 3px; cursor: pointer; font: inherit; } |
| #selx:hover, #helpx:hover { color: var(--ink); border-color: var(--accent); } |
| #selx:focus-visible, #helpx:focus-visible { outline: 2px solid var(--accent); outline-offset: 1px; } |
| .selrow { display: grid; grid-template-columns: 1fr 58px 132px; gap: 9px; padding: 5px 14px; |
| align-items: baseline; font: 500 11.5px var(--mono); } |
| .selrow > span { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } |
| .selrow .d { color: var(--muted); text-align: right; font-variant-numeric: tabular-nums; } |
| .selrow .c { color: var(--muted); font-size: 10.5px; } |
| .selhd { color: var(--muted); text-transform: uppercase; letter-spacing: .08em; font-size: 9.5px; |
| font-weight: 600; padding: 9px 14px 7px; border-bottom: 1px solid var(--line); } |
| #selrows { flex: 1; overflow-y: auto; overscroll-behavior: contain; } |
| #selrows .selrow { cursor: pointer; } |
| #selrows .selrow:nth-child(2n) { background: color-mix(in srgb, var(--ink) 5%, transparent); } |
| #selrows .selrow:hover { background: color-mix(in srgb, var(--accent) 16%, transparent); } |
| #selrows .selrow:focus-visible { outline: 2px solid var(--accent); outline-offset: -2px; } |
| #selrows .empty { color: var(--muted); padding: 16px 14px; font-size: 12px; } |
| #selfoot { border-top: 1px solid var(--line); padding: 9px 14px; display: flex; gap: 10px; |
| align-items: center; flex-wrap: wrap; |
| font: 500 10.5px var(--mono); color: var(--muted); } |
| #selfoot .warn { color: var(--hot); } |
| #selfoot button { appearance: none; border: 1px solid var(--line); background: transparent; |
| color: var(--muted); border-radius: 7px; padding: 3px 8px; cursor: pointer; font: inherit; |
| margin-left: auto; } |
| #selfoot button:hover { color: var(--ink); border-color: var(--accent); } |
| |
| body.sheet #hud { right: 484px; } |
| |
| |
| #helpbtn { appearance: none; width: 14px; height: 14px; padding: 0; margin-left: 4px; |
| border: 1px solid var(--line); border-radius: 50%; background: transparent; color: var(--muted); |
| font: 600 9px/1 var(--font); cursor: pointer; vertical-align: 1px; } |
| #helpbtn:hover { color: var(--ink); border-color: var(--accent); } |
| #helpbtn:focus-visible { outline: 2px solid var(--accent); outline-offset: 1px; } |
| #scrim { position: fixed; inset: 0; z-index: 40; background: rgba(3, 10, 20, .55); } |
| |
| |
| #help { position: fixed; z-index: 41; top: 50%; left: 50%; transform: translate(-50%, -50%); |
| width: min(560px, calc(100vw - 32px)); max-height: min(78vh, 720px); |
| display: flex; flex-direction: column; |
| background: var(--solid); backdrop-filter: none; -webkit-backdrop-filter: none; } |
| #help header { display: flex; align-items: baseline; gap: 10px; |
| padding: 12px 16px 10px; border-bottom: 1px solid var(--line); } |
| #help h2 { margin: 0; font-size: 13px; } |
| #helpbody { overflow-y: auto; overscroll-behavior: contain; padding: 2px 16px 14px; } |
| #helpbody h3 { margin: 14px 0 3px; font-size: 11.5px; color: var(--ink); text-transform: capitalize; } |
| #helpbody p { margin: 0; font-size: 11.5px; line-height: 1.5; color: var(--muted); } |
| #helpbody a, #help footer a { color: var(--accent); } |
| #help footer { border-top: 1px solid var(--line); padding: 10px 16px; |
| font-size: 11px; line-height: 1.5; color: var(--muted); } |
| [hidden] { display: none !important; } |
| |
| @media (max-width: 560px) { |
| #rail { width: calc(100vw - 32px); } |
| #legend { max-width: calc(100vw - 32px); } |
| } |
| @media (max-width: 560px) { |
| #sel { left: 16px; width: auto; } |
| body.sheet #hud { right: 16px; } |
| } |
| @media (prefers-reduced-motion: reduce) { * { transition: none !important; } #spin { animation: none; } } |
| </style> |
| </head> |
| <body> |
| <div id="stage"><canvas id="cv" aria-label="2D map of host embeddings"></canvas></div> |
|
|
| <section id="rail" class="panel" aria-label="Controls"> |
| <div class="brand"><h1>Web graph embeddings</h1><span class="n" id="nnodes">—</span></div> |
| <p class="lede" id="lede">The link-prediction embeddings, projected to 2D and streamed by tile. |
| Structure the model learned is the shape you see. Recolor to ask which properties it separates.</p> |
| <div class="grp"> |
| <span class="lbl">Color by <button id="helpbtn" type="button" |
| aria-label="What the colours mean" title="What the colours mean">?</button></span> |
| <div class="seg" id="colorby" role="group" aria-label="Color by"></div> |
| </div> |
| <div class="grp search"> |
| <span class="lbl" id="findlbl">Find a top-20k host</span> |
| <input id="q" type="search" placeholder="search hostname…" autocomplete="off" spellcheck="false" /> |
| <ul id="hits" role="listbox" aria-label="Search results"></ul> |
| </div> |
| </section> |
|
|
| <aside id="legend" class="panel" aria-label="Legend"></aside> |
| <div id="toprail"> |
| <div id="selpill" class="panel"> |
| <button id="seln" type="button" aria-expanded="false" aria-controls="sel"></button> |
| <button id="selclr" type="button" title="Clear selection (Esc)" |
| aria-label="Clear selection">×</button> |
| </div> |
| <div id="spin" role="status" aria-label="Loading tiles" aria-hidden="true"></div> |
| </div> |
| <div id="selbox" aria-hidden="true"></div> |
| <section id="sel" class="panel" role="dialog" aria-labelledby="seltitle"> |
| <header><h2 id="seltitle">Selected hosts</h2><button id="selx" type="button" |
| aria-label="Close list">×</button></header> |
| <p class="sub" id="selsub"></p> |
| <div class="selrow selhd" id="selhd"></div> |
| <div id="selrows"></div> |
| <footer id="selfoot"></footer> |
| </section> |
| <div id="scrim" hidden></div> |
| <section id="help" class="panel" role="dialog" aria-modal="true" aria-labelledby="helptitle" hidden> |
| <header><h2 id="helptitle">What the colours mean</h2><button id="helpx" type="button" |
| aria-label="Close">×</button></header> |
| <div id="helpbody"></div> |
| <footer>The model never saw any of these properties — only anonymous host ids and the links |
| between them. Any structure here was learned from links alone. |
| <a href="https://huggingface.co/datasets/commoncrawl/web-graph-embeddings" |
| target="_blank" rel="noopener">Dataset</a> · |
| <a href="https://github.com/commoncrawl/web-graph-embeddings" |
| target="_blank" rel="noopener">Code</a></footer> |
| </section> |
| <div id="tip" role="tooltip"></div> |
| <div id="hud" class="panel"> |
| <span><b id="fps">—</b> fps</span><span><b id="tiles">0</b> tiles</span> |
| <span>zoom <b id="zoom">1.0×</b></span> |
| <button id="selarm" aria-pressed="false" |
| title="Drag a rectangle to select hosts (or hold Shift and drag)">select</button> |
| <button id="reset" title="Reset the view, the selection and the pinned host">reset</button> |
| </div> |
| <div id="err"><div id="errmsg"></div></div> |
|
|
| <script> |
| (function () { |
| "use strict"; |
| const $ = (id) => document.getElementById(id); |
| function fail(msg) { $("errmsg").textContent = msg; $("err").style.display = "grid"; } |
| |
| const params = new URLSearchParams(location.search); |
| const BASE = (params.get("base") || ".").replace(/\/$/, ""); |
| |
| |
| |
| |
| const CAT = [ |
| [85, 158, 231], [237, 70, 29], [29, 237, 29], [240, 76, 208], |
| [190, 211, 105], [29, 237, 153], [142, 76, 240], [133, 237, 29], |
| [200, 65, 119], [237, 174, 29], [223, 144, 152], [105, 211, 115], |
| [29, 237, 237], [184, 144, 223], [65, 92, 200], [144, 207, 223], |
| [200, 119, 65], [90, 226, 185], [65, 200, 65], [29, 133, 237], |
| [244, 123, 244], [29, 237, 91], [184, 218, 47], [186, 65, 200], |
| ]; |
| const UNLAB = [110, 120, 135]; |
| function rampColor(t) { |
| t = Math.max(0, Math.min(1, t)); |
| const stops = [[36, 52, 96], [50, 130, 190], [86, 199, 245], [255, 180, 84]]; |
| const s = t * (stops.length - 1); const i = Math.floor(s), f = s - i; |
| const a = stops[i], b = stops[Math.min(i + 1, stops.length - 1)]; |
| return [a[0] + (b[0] - a[0]) * f, a[1] + (b[1] - a[1]) * f, a[2] + (b[2] - a[2]) * f]; |
| } |
| const DIM = 26; |
| let DEG_BUCKETS = 6; |
| function degBucket(b) { return Math.min(DEG_BUCKETS - 1, Math.floor(b / (256 / DEG_BUCKETS))); } |
| function fmtN(n) { |
| n = Math.round(n); |
| if (n >= 1e6) return (n / 1e6).toFixed(1) + "M"; |
| if (n >= 1e3) return (n / 1e3).toFixed(n >= 1e4 ? 0 : 1) + "k"; |
| return "" + n; |
| } |
| |
| |
| const cv = $("cv"); |
| const gl = cv.getContext("webgl", { antialias: true, alpha: false, premultipliedAlpha: false }); |
| if (!gl) { fail("This viewer needs WebGL, which your browser has disabled."); return; } |
| function sh(type, src) { |
| const s = gl.createShader(type); gl.shaderSource(s, src); gl.compileShader(s); |
| if (!gl.getShaderParameter(s, gl.COMPILE_STATUS)) throw new Error(gl.getShaderInfoLog(s)); |
| return s; |
| } |
| const prog = gl.createProgram(); |
| gl.attachShader(prog, sh(gl.VERTEX_SHADER, ` |
| attribute vec2 a_xy; attribute vec3 a_col; attribute float a_deg; attribute float a_alpha; |
| uniform vec2 u_scale; uniform vec2 u_off; uniform float u_pt; uniform float u_ptMin; uniform float u_matchOnly; |
| varying vec3 v_col; varying float v_alpha; |
| void main() { |
| vec2 p = a_xy * u_scale + u_off; |
| gl_Position = vec4(p, 0.0, 1.0); |
| gl_PointSize = max(u_ptMin, u_pt * (0.65 + 1.9 * a_deg)); |
| v_col = a_col; v_alpha = a_alpha; |
| // foreground pass: draw only the filter-matched points (alpha==1), cull the dimmed rest |
| if (u_matchOnly > 0.5 && a_alpha < 0.99) { gl_PointSize = 0.0; gl_Position = vec4(2.0, 2.0, 2.0, 1.0); } |
| }`)); |
| gl.attachShader(prog, sh(gl.FRAGMENT_SHADER, ` |
| precision mediump float; varying vec3 v_col; varying float v_alpha; uniform float u_alpha; |
| void main() { |
| vec2 d = gl_PointCoord - 0.5; float r = dot(d, d); |
| if (r > 0.25) discard; |
| float a = smoothstep(0.25, 0.14, r); |
| gl_FragColor = vec4(v_col, a * u_alpha * v_alpha); |
| }`)); |
| gl.linkProgram(prog); |
| if (!gl.getProgramParameter(prog, gl.LINK_STATUS)) { fail("Shader link failed."); return; } |
| gl.useProgram(prog); |
| const A = { |
| xy: gl.getAttribLocation(prog, "a_xy"), col: gl.getAttribLocation(prog, "a_col"), |
| deg: gl.getAttribLocation(prog, "a_deg"), alpha: gl.getAttribLocation(prog, "a_alpha"), |
| }; |
| const U = { |
| scale: gl.getUniformLocation(prog, "u_scale"), off: gl.getUniformLocation(prog, "u_off"), |
| pt: gl.getUniformLocation(prog, "u_pt"), alpha: gl.getUniformLocation(prog, "u_alpha"), |
| ptMin: gl.getUniformLocation(prog, "u_ptMin"), matchOnly: gl.getUniformLocation(prog, "u_matchOnly"), |
| }; |
| gl.enableVertexAttribArray(A.xy); gl.enableVertexAttribArray(A.col); |
| gl.enableVertexAttribArray(A.deg); gl.enableVertexAttribArray(A.alpha); |
| |
| |
| let META = null, N = 0, nCat = 0, active = 0, filterVal = -1, defaultSet = 0; |
| let view = { s: 0.9, cx: 0, cy: 0 }; |
| let asp = 1, dpr = Math.min(window.devicePixelRatio || 1, 2), dirty = true; |
| const tileMeta = new Map(); |
| const colorRank = {}; |
| const ready = new Map(); |
| let activeIds = new Set(); |
| const MAXKEEP = 600; |
| let inflight = 0; const MAXFETCH = 8; const queue = []; |
| let pin = null; |
| |
| |
| |
| |
| const Q = 32767; |
| const EPS = 1e-4; |
| const sel = { rect: null, cls: null, count: 0, done: 0, todo: 0, errs: 0, |
| exact: false, epoch: 0, partN: new Map() }; |
| let scanSig = null; |
| let band = null; |
| let selArmed = false, sheetOpen = false; |
| const list = { band: 0, buf: null, tiles: [], pos: 0, rows: 0, |
| eof: false, busy: false, approx: false }; |
| |
| |
| const scanCache = new Map(), nameCache = new Map(); |
| const scanWait = new Map(), nameWait = new Map(); |
| let scanBytes = 0, nameBytes = 0; |
| const SCAN_BYTES_MAX = 96 << 20, NAME_BYTES_MAX = 48 << 20; |
| const SCAN_FETCH = 6; |
| const PAGE = 200, LIST_MAX = 10000, BAND_MAX = 3e6; |
| |
| |
| fetch(BASE + "/manifest.json").then((r) => { |
| if (!r.ok) throw new Error("manifest.json " + r.status); |
| return r.json(); |
| }).then((m) => { META = m; boot(); }).catch((e) => |
| fail("Could not load manifest.json from " + BASE + " — serve this folder over HTTP " |
| + "(python -m http.server) and open the page from there. (" + e.message + ")")); |
| |
| function boot() { |
| N = META.n; DEG_BUCKETS = META.degBuckets || 6; |
| nCat = META.labelSets.filter((l) => l.kind === "cat").length; |
| META.tiles.forEach((t) => tileMeta.set(t.id, t)); |
| |
| META.labelSets.forEach((ls, i) => { |
| if (ls.kind !== "cat") return; |
| const cnt = ls.counts || ls.classes.map(() => 0); |
| const rank = new Array(ls.classes.length); |
| ls.classes.map((_, k) => k).sort((a, b) => cnt[b] - cnt[a]).forEach((k, pos) => { rank[k] = pos; }); |
| colorRank[i] = rank; |
| }); |
| $("nnodes").textContent = N >= 1e6 ? (N / 1e6).toFixed(N % 1e6 ? 1 : 0) + "M" |
| : (N / 1e3).toFixed(0) + "k"; |
| if (META.lede) $("lede").innerHTML = META.lede; |
| defaultSet = Math.max(0, META.labelSets.findIndex((l) => l.name === "language")); |
| const nSearch = (META.search || []).length; |
| if (nSearch) $("findlbl").textContent = "Find a top-" + fmtN(nSearch) + " host"; |
| buildColorBy(); buildSearch(); buildSelHead(); |
| resize(); recolor(defaultSet); requestAnimationFrame(loop); |
| } |
| |
| |
| function pump() { |
| while (inflight < MAXFETCH && queue.length) { const id = queue.shift(); doLoad(id); } |
| } |
| function startLoad(id) { |
| if (ready.has(id) || queue.includes(id)) return; |
| ready.set(id, { id, status: "queued" }); queue.push(id); pump(); |
| } |
| function doLoad(id) { |
| const t = ready.get(id); const meta = tileMeta.get(id); if (!t || !meta) return; |
| |
| const tdir = "/t/" + (meta.d != null ? meta.d + "/" : "") + id; |
| t.status = "loading"; inflight++; |
| fetch(BASE + tdir + ".bin").then((r) => r.arrayBuffer()).then((buf) => { |
| const K = meta.n; let o = 0; |
| t.n = K; |
| t.xy = new Int16Array(buf, o, K * 2); o += K * 4; |
| t.labs = []; |
| for (let c = 0; c < nCat; c++) { t.labs.push(new Uint8Array(buf, o, K)); o += K; } |
| t.deg = new Uint8Array(buf, o, K); o += K; |
| t.bXY = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, t.bXY); |
| gl.bufferData(gl.ARRAY_BUFFER, t.xy, gl.STATIC_DRAW); |
| t.bDeg = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, t.bDeg); |
| gl.bufferData(gl.ARRAY_BUFFER, t.deg, gl.STATIC_DRAW); |
| t.bCol = gl.createBuffer(); t.bAlpha = gl.createBuffer(); |
| t.names = null; |
| colorTile(t); alphaTile(t); |
| t.status = "ready"; dirty = true; |
| if (META.hasNames) fetch(BASE + tdir + ".nm").then((r) => r.text()) |
| .then((txt) => { t.names = txt.length ? txt.split("\n") : []; }); |
| }).catch(() => { ready.delete(id); }).finally(() => { inflight--; pump(); }); |
| } |
| function evictTile(t) { |
| if (t.bXY) gl.deleteBuffer(t.bXY); if (t.bDeg) gl.deleteBuffer(t.bDeg); |
| if (t.bCol) gl.deleteBuffer(t.bCol); if (t.bAlpha) gl.deleteBuffer(t.bAlpha); |
| } |
| |
| function colorTile(t) { |
| if (t.status === "queued" || !t.deg) return; |
| const set = META.labelSets[active], K = t.n, buf = new Uint8Array(K * 3); |
| if (set.kind === "degree") { |
| for (let i = 0; i < K; i++) { const c = rampColor(t.deg[i] / 255), j = i * 3; |
| buf[j] = c[0]; buf[j + 1] = c[1]; buf[j + 2] = c[2]; } |
| } else { |
| const lab = t.labs[active], rank = colorRank[active]; |
| for (let i = 0; i < K; i++) { const v = lab[i], c = v === 255 ? UNLAB : CAT[rank[v] % CAT.length]; |
| const j = i * 3; buf[j] = c[0]; buf[j + 1] = c[1]; buf[j + 2] = c[2]; } |
| } |
| gl.bindBuffer(gl.ARRAY_BUFFER, t.bCol); gl.bufferData(gl.ARRAY_BUFFER, buf, gl.STATIC_DRAW); |
| } |
| function alphaTile(t) { |
| if (t.status === "queued" || !t.deg) return; |
| const set = META.labelSets[active], K = t.n, buf = new Uint8Array(K); |
| if (filterVal < 0) { |
| buf.fill(255); |
| } else if (set.kind === "degree") { |
| for (let i = 0; i < K; i++) buf[i] = degBucket(t.deg[i]) === filterVal ? 255 : DIM; |
| } else { |
| const lab = t.labs[active]; |
| for (let i = 0; i < K; i++) buf[i] = lab[i] === filterVal ? 255 : DIM; |
| } |
| gl.bindBuffer(gl.ARRAY_BUFFER, t.bAlpha); gl.bufferData(gl.ARRAY_BUFFER, buf, gl.DYNAMIC_DRAW); |
| } |
| |
| |
| |
| |
| |
| const LOD_RATE = 0.7, LOD_BIAS = 0.4; |
| function lodLevel() { |
| return Math.max(0, Math.min(META.zMax, Math.round(Math.log2(view.s) * LOD_RATE + LOD_BIAS))); |
| } |
| |
| |
| function toWorld(px, py) { |
| const w = cv.clientWidth, h = cv.clientHeight; |
| return { x: ((px / w) * 2 - 1) / (view.s * asp) + view.cx, |
| y: (1 - (py / h) * 2) / view.s + view.cy }; |
| } |
| function toScreenX(wx) { return ((wx - view.cx) * view.s * asp + 1) * cv.clientWidth / 2; } |
| function toScreenY(wy) { return (1 - (wy - view.cy) * view.s) * cv.clientHeight / 2; } |
| function viewport() { |
| const hw = 1 / (view.s * asp), hh = 1 / view.s, m = 0.25; |
| return [view.cx - hw * (1 + m), view.cy - hh * (1 + m), |
| view.cx + hw * (1 + m), view.cy + hh * (1 + m)]; |
| } |
| function updateActive() { |
| |
| |
| |
| const maxZ = lodLevel(), vp = viewport(); activeIds = new Set(); |
| for (const t of META.tiles) { |
| if (t.z > maxZ) continue; |
| const b = t.bbox; |
| if (b[0] < vp[2] && b[2] > vp[0] && b[1] < vp[3] && b[3] > vp[1]) { |
| activeIds.add(t.id); if (!ready.has(t.id)) startLoad(t.id); |
| } |
| } |
| if (ready.size > MAXKEEP) { |
| for (const [id, t] of ready) { |
| if (ready.size <= MAXKEEP) break; |
| if (!activeIds.has(id) && t.status === "ready") { evictTile(t); ready.delete(id); } |
| } |
| } |
| } |
| |
| |
| function drawTiles() { |
| let drawn = 0; |
| for (const id of activeIds) { |
| const t = ready.get(id); if (!t || t.status !== "ready") continue; |
| gl.bindBuffer(gl.ARRAY_BUFFER, t.bXY); gl.vertexAttribPointer(A.xy, 2, gl.SHORT, true, 0, 0); |
| gl.bindBuffer(gl.ARRAY_BUFFER, t.bCol); gl.vertexAttribPointer(A.col, 3, gl.UNSIGNED_BYTE, true, 0, 0); |
| gl.bindBuffer(gl.ARRAY_BUFFER, t.bDeg); gl.vertexAttribPointer(A.deg, 1, gl.UNSIGNED_BYTE, true, 0, 0); |
| gl.bindBuffer(gl.ARRAY_BUFFER, t.bAlpha); gl.vertexAttribPointer(A.alpha, 1, gl.UNSIGNED_BYTE, true, 0, 0); |
| gl.drawArrays(gl.POINTS, 0, t.n); drawn++; |
| } |
| return drawn; |
| } |
| function draw() { |
| const bg = getComputedStyle(document.body).backgroundColor.match(/\d+/g) || [11, 14, 21]; |
| gl.clearColor(bg[0] / 255, bg[1] / 255, bg[2] / 255, 1); gl.clear(gl.COLOR_BUFFER_BIT); |
| gl.enable(gl.BLEND); gl.blendFunc(gl.SRC_ALPHA, gl.ONE_MINUS_SRC_ALPHA); |
| gl.uniform2f(U.scale, view.s * asp, view.s); |
| gl.uniform2f(U.off, -view.cx * view.s * asp, -view.cy * view.s); |
| |
| |
| |
| const L = Math.log2(view.s); |
| gl.uniform1f(U.pt, Math.max(1.2, dpr * (2.0 + 5.0 * Math.exp(-(L - 2.0) * (L - 2.0) / 8.0)))); |
| |
| |
| |
| gl.uniform1f(U.ptMin, dpr * 10 * Math.min(1, Math.max(0, (L - 4.6) / 2.4))); |
| |
| gl.uniform1f(U.matchOnly, 0); |
| gl.uniform1f(U.alpha, N > 4e6 ? 0.5 : 0.75); |
| const drawn = drawTiles(); |
| |
| if (filterVal >= 0) { |
| gl.uniform1f(U.matchOnly, 1); |
| gl.uniform1f(U.alpha, N > 4e6 ? 0.9 : 1.0); |
| drawTiles(); |
| gl.uniform1f(U.matchOnly, 0); |
| } |
| $("tiles").textContent = drawn; |
| if (pin) drawPin(); |
| syncSelBox(); |
| } |
| const bPin = gl.createBuffer(); |
| function drawPin() { |
| const t = ready.get(pin.id); if (!t || t.status !== "ready") return; |
| gl.bindBuffer(gl.ARRAY_BUFFER, bPin); |
| gl.bufferData(gl.ARRAY_BUFFER, new Int16Array([t.xy[pin.li * 2], t.xy[pin.li * 2 + 1]]), gl.STATIC_DRAW); |
| gl.vertexAttribPointer(A.xy, 2, gl.SHORT, true, 0, 0); |
| const hot = getComputedStyle(document.body).getPropertyValue("--hot").trim(); |
| const m = hot.match(/\w\w/g).map((h) => parseInt(h, 16)); |
| gl.disableVertexAttribArray(A.col); gl.disableVertexAttribArray(A.deg); |
| gl.disableVertexAttribArray(A.alpha); |
| gl.vertexAttrib3f(A.col, m[0] / 255, m[1] / 255, m[2] / 255); |
| gl.vertexAttrib1f(A.deg, 1); gl.vertexAttrib1f(A.alpha, 1); |
| gl.uniform1f(U.pt, 16 * dpr); gl.uniform1f(U.alpha, 1); |
| gl.drawArrays(gl.POINTS, 0, 1); |
| gl.enableVertexAttribArray(A.col); gl.enableVertexAttribArray(A.deg); |
| gl.enableVertexAttribArray(A.alpha); |
| } |
| |
| |
| function recolor(idx) { |
| active = idx; filterVal = -1; |
| for (const t of ready.values()) { colorTile(t); alphaTile(t); } |
| drawLegend(idx); dirty = true; |
| } |
| function buildColorBy() { |
| const seg = $("colorby"); |
| META.labelSets.forEach((ls, i) => { |
| const b = document.createElement("button"); b.textContent = ls.name; |
| b.setAttribute("aria-pressed", i === defaultSet ? "true" : "false"); |
| b.onclick = () => { [...seg.children].forEach((c) => c.setAttribute("aria-pressed", "false")); |
| b.setAttribute("aria-pressed", "true"); recolor(i); }; |
| seg.appendChild(b); |
| }); |
| } |
| function degLabel(b) { return Math.round(Math.pow(10, (b / 255) * (META.degLogMax || 4))); } |
| function drawLegend(idx) { |
| const ls = META.labelSets[idx], el = $("legend"); |
| el.innerHTML = ""; |
| const head = document.createElement("span"); head.className = "lbl"; |
| head.textContent = ls.name + (ls.note ? " · " + ls.note : ""); |
| el.appendChild(head); |
| let entries; |
| if (ls.kind === "degree") { |
| const w = 256 / DEG_BUCKETS, c = META.degCounts || []; entries = []; |
| for (let k = 0; k < DEG_BUCKETS; k++) { |
| const lo = degLabel(Math.round(k * w)), hi = degLabel(Math.round(Math.min(255, (k + 1) * w))); |
| entries.push({ val: k, color: rampColor((k + 0.5) * w / 255), |
| text: fmtN(lo) + "–" + fmtN(hi), count: c[k] || 0 }); |
| } |
| } else { |
| const c = ls.counts || ls.classes.map(() => 0), rank = colorRank[idx]; |
| entries = ls.classes.map((cl, k) => ({ val: k, color: CAT[rank[k] % CAT.length], text: cl, |
| count: c[k] || 0 })).filter((e) => e.count > 0).sort((a, b) => b.count - a.count); |
| const na = N - c.reduce((s, v) => s + (v || 0), 0); |
| if (na > 0) entries.push({ val: 255, color: UNLAB, text: "N/A", count: na }); |
| } |
| const wrap = document.createElement("div"); wrap.className = "swatches"; |
| entries.forEach((e) => { |
| const b = document.createElement("button"); b.type = "button"; b.className = "sw"; |
| b.setAttribute("aria-pressed", filterVal === e.val ? "true" : "false"); |
| b.title = "Show only these — click again to clear"; |
| const c = e.color; |
| const sw = document.createElement("i"); |
| sw.style.background = `rgb(${c[0] | 0},${c[1] | 0},${c[2] | 0})`; |
| const nm = document.createElement("span"); nm.className = "nm"; nm.textContent = e.text; |
| const ct = document.createElement("span"); ct.className = "ct"; ct.textContent = fmtN(e.count); |
| b.append(sw, nm, ct); |
| b.onclick = () => { |
| filterVal = (filterVal === e.val) ? -1 : e.val; |
| for (const t of ready.values()) alphaTile(t); |
| drawLegend(idx); dirty = true; |
| }; |
| wrap.appendChild(b); |
| }); |
| el.appendChild(wrap); el.classList.add("on"); |
| } |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| const clamp1 = (v) => Math.max(-1, Math.min(1, v)); |
| |
| function classify(r) { |
| const full = [], part = []; let freeN = 0; |
| for (const t of META.tiles) { |
| const b = t.bbox; |
| const x0 = clamp1(b[0] - EPS), y0 = clamp1(b[1] - EPS); |
| const x1 = clamp1(b[2] + EPS), y1 = clamp1(b[3] + EPS); |
| if (x0 > r.x1 || x1 < r.x0 || y0 > r.y1 || y1 < r.y0) continue; |
| if (x0 >= r.x0 && x1 <= r.x1 && y0 >= r.y0 && y1 <= r.y1) { full.push(t); freeN += t.n; } |
| else part.push(t); |
| } |
| return { full, part, freeN }; |
| } |
| |
| |
| function quantRect(r) { return [r.x0 * Q, r.y0 * Q, r.x1 * Q, r.y1 * Q]; } |
| function hits(xy, K, q, out) { |
| const ax = q[0], ay = q[1], bx = q[2], by = q[3]; let c = 0; |
| for (let i = 0; i < K; i++) { |
| const x = xy[i * 2], y = xy[i * 2 + 1]; |
| if (x >= ax && x <= bx && y >= ay && y <= by) { if (out) out[c] = i; c++; } |
| } |
| return c; |
| } |
| |
| |
| function tdirOf(t) { return BASE + "/t/" + (t.d != null ? t.d + "/" : "") + t.id; } |
| function lru(cache, max, cur) { |
| for (const [k, v] of cache) { if (cur <= max) break; cache.delete(k); cur -= v.bytes; } |
| return cur; |
| } |
| function scanTile(t, sig) { |
| const hit = scanCache.get(t.id); |
| if (hit) { scanCache.delete(t.id); scanCache.set(t.id, hit); return Promise.resolve(hit); } |
| if (scanWait.has(t.id)) return scanWait.get(t.id); |
| const p = fetch(tdirOf(t) + ".bin", { signal: sig }).then((r) => { |
| if (!r.ok) throw new Error(t.id + " " + r.status); |
| return r.arrayBuffer(); |
| }).then((buf) => { |
| const K = t.n; let o = 0; |
| const v = { n: K, xy: new Int16Array(buf, o, K * 2), labs: [], deg: null, |
| bytes: buf.byteLength }; |
| o += K * 4; |
| for (let c = 0; c < nCat; c++) { v.labs.push(new Uint8Array(buf, o, K)); o += K; } |
| v.deg = new Uint8Array(buf, o, K); |
| scanCache.set(t.id, v); scanBytes = lru(scanCache, SCAN_BYTES_MAX, scanBytes + v.bytes); |
| return v; |
| }).finally(() => scanWait.delete(t.id)); |
| scanWait.set(t.id, p); return p; |
| } |
| function scanNames(t, sig) { |
| const hit = nameCache.get(t.id); |
| if (hit) { nameCache.delete(t.id); nameCache.set(t.id, hit); return Promise.resolve(hit); } |
| if (nameWait.has(t.id)) return nameWait.get(t.id); |
| const p = fetch(tdirOf(t) + ".nm", { signal: sig }).then((r) => { |
| if (!r.ok) throw new Error(t.id + " " + r.status); |
| return r.text(); |
| }).then((txt) => { |
| const v = { names: txt.length ? txt.split("\n") : [], bytes: txt.length }; |
| nameCache.set(t.id, v); nameBytes = lru(nameCache, NAME_BYTES_MAX, nameBytes + v.bytes); |
| return v; |
| }).finally(() => nameWait.delete(t.id)); |
| nameWait.set(t.id, p); return p; |
| } |
| function pool(items, k, fn) { |
| let i = 0; |
| const run = async () => { while (i < items.length) { const j = i++; await fn(items[j]); } }; |
| return Promise.all(Array.from({ length: Math.min(k, items.length) || 1 }, run)); |
| } |
| |
| |
| function cancelScan() { |
| sel.epoch++; |
| if (scanSig) { scanSig.abort(); scanSig = null; } |
| } |
| function setSelection(r) { |
| cancelScan(); listReset(); |
| Object.assign(sel, { rect: r, cls: null, count: 0, done: 0, todo: 0, errs: 0, |
| exact: false, partN: new Map() }); |
| if (!r) { |
| $("selpill").classList.remove("on"); closeSheet(); syncSelBox(); dirty = true; return; |
| } |
| const ep = sel.epoch, ac = new AbortController(); scanSig = ac; |
| const cls = sel.cls = classify(r); |
| sel.count = cls.freeN; sel.todo = cls.part.length; sel.exact = !cls.part.length; |
| $("selpill").classList.add("on"); paintBadge(); syncSelBox(); dirty = true; |
| if (!cls.part.length) return; |
| const q = quantRect(r); |
| cls.part.sort((a, b) => a.z - b.z); |
| pool(cls.part, SCAN_FETCH, async (t) => { |
| if (ep !== sel.epoch) return; |
| let v; |
| try { v = await scanTile(t, ac.signal); } catch { if (ep === sel.epoch) sel.errs++; return; } |
| if (ep !== sel.epoch) return; |
| const c = hits(v.xy, t.n, q, null); |
| sel.partN.set(t.id, c); sel.count += c; sel.done++; paintBadge(); |
| }).then(() => { |
| if (ep !== sel.epoch) return; |
| sel.exact = !sel.errs; paintBadge(); |
| }); |
| } |
| |
| let badgeRaf = 0; |
| function paintBadge() { |
| if (badgeRaf) return; |
| badgeRaf = requestAnimationFrame(() => { |
| badgeRaf = 0; const b = $("seln"), scanning = !sel.exact && sel.done < sel.todo; |
| b.textContent = fmtFull(sel.count) + (sel.exact ? " selected" : "+ scanning…"); |
| b.title = sel.exact |
| ? "Hosts inside the rectangle, across all degree bands — not just the points drawn at this " |
| + "zoom. Click to list them, highest degree first." |
| : "reading " + sel.done + " / " + sel.todo + " edge tiles" |
| + (sel.errs ? " · " + sel.errs + " failed" : ""); |
| b.style.setProperty("--p", sel.todo ? sel.done / sel.todo : 1); |
| b.classList.toggle("scanning", scanning); |
| b.setAttribute("aria-live", sel.exact ? "polite" : "off"); |
| if (sheetOpen) { listSub(); listFoot(); } |
| }); |
| } |
| function fmtFull(v) { return v.toLocaleString("en-US"); } |
| |
| |
| |
| function syncSelBox() { |
| const el = $("selbox"); |
| let x0, y0, x1, y1, set = false; |
| if (band) { |
| x0 = Math.min(band.x0, band.x1); x1 = Math.max(band.x0, band.x1); |
| y0 = Math.min(band.y0, band.y1); y1 = Math.max(band.y0, band.y1); |
| } else if (sel.rect) { |
| const r = sel.rect, C = 4000; |
| x0 = Math.max(-C, toScreenX(r.x0)); x1 = Math.min(C, toScreenX(r.x1)); |
| y0 = Math.max(-C, toScreenY(r.y1)); y1 = Math.min(C, toScreenY(r.y0)); |
| set = true; |
| if (x1 <= x0 || y1 <= y0) { el.classList.remove("on"); return; } |
| } else { el.classList.remove("on"); return; } |
| el.style.left = x0 + "px"; el.style.top = y0 + "px"; |
| el.style.width = (x1 - x0) + "px"; el.style.height = (y1 - y0) + "px"; |
| el.classList.toggle("set", set); el.classList.add("on"); |
| } |
| |
| |
| |
| |
| |
| |
| |
| |
| function listReset() { |
| Object.assign(list, { band: 0, buf: null, tiles: [], pos: 0, rows: 0, |
| eof: false, busy: false, approx: false }); |
| const rows = $("selrows"); |
| rows.textContent = ""; rows.scrollTop = 0; |
| listSub(); listFoot(); |
| } |
| |
| let scratch = new Uint32Array(0); |
| async function nextBand(ep) { |
| while (list.band <= META.zMax) { |
| const z = list.band++; |
| const ts = []; |
| for (const t of sel.cls.full) if (t.z === z) ts.push(t); |
| for (const t of sel.cls.part) if (t.z === z) ts.push(t); |
| if (!ts.length) continue; |
| let want = 0; |
| for (const t of ts) want += sel.partN.has(t.id) ? sel.partN.get(t.id) : t.n; |
| if (!want) continue; |
| const cap = Math.min(want, BAND_MAX); |
| const li = new Uint16Array(cap), ti = new Uint16Array(cap), dg = new Uint8Array(cap); |
| const lb = new Uint8Array(cap * nCat); |
| const slot = []; let m = 0; |
| await pool(ts, SCAN_FETCH, async (t) => { |
| if (ep !== sel.epoch || m >= cap) return; |
| let v; |
| try { v = await scanTile(t, scanSig && scanSig.signal); } catch { return; } |
| if (ep !== sel.epoch) return; |
| |
| if (scratch.length < t.n) scratch = new Uint32Array(t.n); |
| const c = hits(v.xy, t.n, quantRect(sel.rect), scratch); |
| if (!c) return; |
| const s = slot.push(t) - 1; |
| for (let i = 0; i < c && m < cap; i++, m++) { |
| const j = scratch[i]; |
| ti[m] = s; li[m] = j; dg[m] = v.deg[j]; |
| for (let k = 0; k < nCat; k++) lb[m * nCat + k] = v.labs[k][j]; |
| } |
| }); |
| if (ep !== sel.epoch) return false; |
| if (m >= cap && want > cap) list.approx = true; |
| |
| const cnt = new Uint32Array(256); |
| for (let i = 0; i < m; i++) cnt[dg[i]]++; |
| const at = new Uint32Array(256); let acc = 0; |
| for (let v = 255; v >= 0; v--) { at[v] = acc; acc += cnt[v]; } |
| const ord = new Uint32Array(m); |
| for (let i = 0; i < m; i++) ord[at[dg[i]]++] = i; |
| list.buf = { li, ti, dg, lb, ord, m }; list.tiles = slot; list.pos = 0; |
| if (m) return true; |
| } |
| list.eof = true; return false; |
| } |
| |
| |
| |
| function rowEl(t, j, deg8, lb, i) { |
| const el = document.createElement("div"); |
| el.className = "selrow"; el.tabIndex = 0; el.title = "Show on the map"; |
| const host = document.createElement("span"); host.className = "h"; host.textContent = "…"; |
| const deg = document.createElement("span"); deg.className = "d"; |
| deg.textContent = "~" + fmtN(degLabel(deg8)); |
| el.append(host, deg); |
| |
| |
| |
| const known = [], full = []; |
| let k2 = 0; |
| META.labelSets.forEach((ls) => { |
| if (ls.kind === "degree") return; |
| const v = lb[i * nCat + k2++]; |
| if (v === 255) return; |
| known.push(ls.classes[v]); |
| full.push(ls.name + ": " + ls.classes[v]); |
| }); |
| const sp = document.createElement("span"); sp.className = "c"; |
| sp.textContent = known.length ? known.join(" · ") : "—"; |
| |
| sp.title = full.join(" · ") || "no labels"; |
| el.appendChild(sp); |
| const go = () => flyTo([host.textContent, t.id, j, deg8]); |
| el.onclick = go; |
| el.onkeydown = (ev) => { |
| if (ev.key === "Enter" || ev.key === " ") { ev.preventDefault(); go(); } |
| }; |
| return { el, host }; |
| } |
| |
| async function listFill() { |
| if (list.busy || list.eof || list.rows >= LIST_MAX || !sel.rect) return; |
| list.busy = true; const ep = sel.epoch; |
| try { |
| while (!list.buf || list.pos >= list.buf.m) { |
| list.buf = null; |
| const more = await nextBand(ep); |
| if (ep !== sel.epoch) return; |
| if (!more) { listEmpty(); listFoot(); return; } |
| } |
| const b = list.buf, take = Math.min(PAGE, b.m - list.pos, LIST_MAX - list.rows); |
| const frag = document.createDocumentFragment(), need = new Map(); |
| for (let k = 0; k < take; k++) { |
| const i = b.ord[list.pos + k], t = list.tiles[b.ti[i]], j = b.li[i]; |
| const r = rowEl(t, j, b.dg[i], b.lb, i); |
| if (!need.has(t)) need.set(t, []); |
| need.get(t).push([r.host, j]); |
| frag.appendChild(r.el); |
| } |
| list.pos += take; list.rows += take; |
| $("selrows").appendChild(frag); listFoot(); |
| |
| |
| pool([...need.keys()], 4, async (t) => { |
| let v; |
| try { v = await scanNames(t, scanSig && scanSig.signal); } catch { v = null; } |
| if (ep !== sel.epoch) return; |
| for (const pair of need.get(t)) { |
| pair[0].textContent = (v && v.names[pair[1]]) || "(unnamed)"; |
| } |
| }); |
| |
| const e = $("selrows"); |
| if (e.scrollHeight <= e.clientHeight && !list.eof && list.rows < LIST_MAX) { |
| requestAnimationFrame(listFill); |
| } |
| } finally { list.busy = false; } |
| } |
| |
| function listEmpty() { |
| if (list.rows) return; |
| const d = document.createElement("div"); d.className = "empty"; |
| d.textContent = sel.count |
| ? "Nothing to list — the tiles for this region could not be read." |
| : "No hosts in this region."; |
| $("selrows").appendChild(d); |
| } |
| |
| function buildSelHead() { |
| const hd = $("selhd"); hd.textContent = ""; |
| const cats = META.labelSets.filter((ls) => ls.kind !== "degree").map((ls) => ls.name); |
| ["host", "degree", "labels"].forEach((c, k) => { |
| const s = document.createElement("span"); s.textContent = c; |
| if (k === 2) s.title = cats.join(", ") + " — whichever are known for that host"; |
| hd.appendChild(s); |
| }); |
| } |
| function listSub() { |
| if (!sel.rect) { $("selsub").textContent = ""; return; } |
| $("selsub").textContent = |
| "Every host in the rectangle, across all degree bands — not just the points drawn at this " |
| + "zoom. Highest degree first. Click a row to fly to it."; |
| } |
| function note(cls, text) { |
| const s = document.createElement("span"); s.className = cls; s.textContent = text; return s; |
| } |
| function listFoot() { |
| const f = $("selfoot"); f.textContent = ""; |
| f.appendChild(note("", fmtFull(list.rows) + " of " + fmtFull(sel.count) |
| + (sel.exact ? "" : "+") + " shown")); |
| if (list.rows >= LIST_MAX) { |
| f.appendChild(note("warn", "capped at " + fmtFull(LIST_MAX) |
| + " rows — shrink the rectangle to reach further down")); |
| } |
| if (list.approx) { |
| f.appendChild(note("warn", "order approximate past " + fmtFull(BAND_MAX) |
| + " rows in one band")); |
| } |
| if (sel.errs) f.appendChild(note("warn", sel.errs + " tile(s) failed to load")); |
| const c = document.createElement("button"); |
| c.type = "button"; c.textContent = "clear"; |
| c.onclick = () => setSelection(null); |
| f.appendChild(c); |
| } |
| |
| function openSheet() { |
| if (!sel.rect) return; |
| sheetOpen = true; |
| $("sel").classList.add("on"); $("seln").setAttribute("aria-expanded", "true"); |
| $("selpill").classList.add("open"); |
| document.body.classList.add("sheet"); |
| listSub(); $("selx").focus(); listFill(); |
| } |
| function closeSheet() { |
| if (!sheetOpen) return; |
| sheetOpen = false; |
| $("sel").classList.remove("on"); $("seln").setAttribute("aria-expanded", "false"); |
| $("selpill").classList.remove("open"); |
| document.body.classList.remove("sheet"); |
| $("seln").focus(); |
| } |
| |
| |
| const PICK_PX = 15; |
| function pick(px, py) { |
| const w = cv.clientWidth, h = cv.clientHeight; |
| const p = toWorld(px, py), sx = p.x * Q, sy = p.y * Q; |
| const supx = (2 / w) / (view.s * asp) * Q, supy = (2 / h) / view.s * Q; |
| let best = null, bd = PICK_PX * PICK_PX; |
| for (const id of activeIds) { |
| const t = ready.get(id); if (!t || t.status !== "ready") continue; |
| const xy = t.xy; |
| for (let i = 0; i < t.n; i++) { |
| const dxp = (xy[i * 2] - sx) / supx, dyp = (xy[i * 2 + 1] - sy) / supy; |
| const d = dxp * dxp + dyp * dyp; |
| if (d < bd) { bd = d; best = { id, li: i, t }; } |
| } |
| } |
| return best; |
| } |
| function hover(px, py) { |
| const hit = pick(px, py); const tip = $("tip"); |
| if (!hit) { tip.style.display = "none"; return; } |
| const t = hit.t, i = hit.li; |
| const name = t.names ? (t.names[i] || null) : null; |
| |
| tip.textContent = ""; |
| const h = document.createElement("div"); h.className = "host"; |
| h.textContent = name || "(loading name…)"; |
| tip.appendChild(h); |
| tip.appendChild(tipRow("degree", "~" + fmtN(degLabel(t.deg[i])))); |
| META.labelSets.forEach((ls, k) => { |
| if (ls.kind === "degree") return; |
| const v = t.labs[k][i]; |
| tip.appendChild(tipRow(ls.name, v === 255 ? "—" : ls.classes[v])); |
| }); |
| tip.style.left = px + "px"; tip.style.top = py + "px"; tip.style.display = "block"; |
| } |
| |
| function tipRow(label, value) { |
| const row = document.createElement("span"); row.className = "row"; |
| const b = document.createElement("b"); b.textContent = label; |
| const v = document.createElement("span"); v.textContent = value; |
| row.append(b, v); return row; |
| } |
| |
| |
| function buildSearch() { |
| const idxList = META.search || []; |
| const lo = idxList.map((e) => e[0].toLowerCase()); |
| const q = $("q"), hits = $("hits"); |
| q.addEventListener("input", () => { |
| const term = q.value.trim().toLowerCase(); hits.innerHTML = ""; |
| if (!term) return; |
| const out = []; |
| for (let k = 0; k < idxList.length && out.length < 30; k++) |
| if (lo[k].includes(term)) out.push(idxList[k]); |
| out.forEach((e) => { |
| const li = document.createElement("li"); li.setAttribute("role", "option"); |
| const nm = document.createElement("span"); nm.textContent = e[0]; |
| const dg = document.createElement("span"); dg.className = "deg"; |
| dg.textContent = "~" + fmtN(degLabel(e[3])); |
| li.append(nm, dg); |
| li.onclick = () => flyTo(e); hits.appendChild(li); |
| }); |
| }); |
| } |
| function flyTo(entry) { |
| const [, tid, liLocal] = entry; const meta = tileMeta.get(tid); if (!meta) return; |
| startLoad(tid); |
| const b = meta.bbox, tx = (b[0] + b[2]) / 2, ty = (b[1] + b[3]) / 2; |
| const finish = () => { |
| const t = ready.get(tid); |
| const px = (t && t.xy) ? t.xy[liLocal * 2] / Q : tx; |
| const py = (t && t.xy) ? t.xy[liLocal * 2 + 1] / Q : ty; |
| pin = { id: tid, li: liLocal }; |
| const s0 = view.s, cx0 = view.cx, cy0 = view.cy, ts = Math.max(40, view.s), t0 = performance.now(), D = 620; |
| const reduce = matchMedia("(prefers-reduced-motion: reduce)").matches; |
| (function step(now) { |
| const p = reduce ? 1 : Math.min(1, (now - t0) / D), e = 1 - Math.pow(1 - p, 3); |
| view.s = s0 + (ts - s0) * e; view.cx = cx0 + (px - cx0) * e; view.cy = cy0 + (py - cy0) * e; |
| dirty = true; if (p < 1) requestAnimationFrame(step); |
| })(t0); |
| }; |
| const t = ready.get(tid); |
| if (t && t.status === "ready") finish(); |
| else { const iv = setInterval(() => { const tt = ready.get(tid); |
| if (tt && tt.status === "ready") { clearInterval(iv); finish(); } }, 60); } |
| } |
| |
| |
| let drag = null; |
| |
| |
| cv.addEventListener("pointerdown", (e) => { |
| cv.setPointerCapture(e.pointerId); |
| if (e.shiftKey || selArmed) { |
| band = { x0: e.clientX, y0: e.clientY, x1: e.clientX, y1: e.clientY }; |
| $("tip").style.display = "none"; syncSelBox(); return; |
| } |
| drag = { x: e.clientX, y: e.clientY }; |
| }); |
| function endBand(commit) { |
| if (!band) return; |
| const b = band; band = null; |
| if (!commit || Math.abs(b.x1 - b.x0) < 4 || Math.abs(b.y1 - b.y0) < 4) { |
| setSelection(null); return; |
| } |
| |
| |
| |
| const p = toWorld(b.x0, b.y0), q = toWorld(b.x1, b.y1); |
| setSelection({ x0: Math.min(p.x, q.x), y0: Math.min(p.y, q.y), |
| x1: Math.max(p.x, q.x), y1: Math.max(p.y, q.y) }); |
| } |
| cv.addEventListener("pointerup", () => { endBand(true); drag = null; }); |
| cv.addEventListener("pointercancel", () => { endBand(false); drag = null; }); |
| cv.addEventListener("pointermove", (e) => { |
| if (band) { band.x1 = e.clientX; band.y1 = e.clientY; syncSelBox(); return; } |
| if (drag) { |
| const w = cv.clientWidth, h = cv.clientHeight; |
| view.cx -= ((e.clientX - drag.x) / w) * 2 / (view.s * asp); |
| view.cy += ((e.clientY - drag.y) / h) * 2 / view.s; |
| drag.x = e.clientX; drag.y = e.clientY; dirty = true; $("tip").style.display = "none"; |
| } else { hover(e.clientX, e.clientY); } |
| }); |
| cv.addEventListener("pointerleave", () => { $("tip").style.display = "none"; }); |
| |
| |
| |
| |
| const HELP = { |
| topic: ["What the sites are about, in 24 categories from the WebOrganizer corpus. Labels are " + |
| "assigned per page and rolled up to the host, so only a small share of hosts carry one \u2014 " + |
| "but topic is the property these embeddings separate best.", |
| [["https://huggingface.co/datasets/WebOrganizer/Corpus-200B", "WebOrganizer/Corpus-200B"]]], |
| language: ["The main content language of a host's pages, across 120 languages. Detected per " + |
| "page with CLD2 and read from Common Crawl's columnar URL index. This has the widest " + |
| "coverage of the four. Sites tend to link to sites in the same language, so it separates " + |
| "cleanly once a host has enough links.", |
| [["https://github.com/commoncrawl/language-detection-cld2", "Language detection (CLD2)"], |
| ["https://commoncrawl.org/blog/index-to-warc-files-and-urls-in-columnar-format", |
| "Columnar URL index"]]], |
| quality: ["A four-way content-quality annotation from GneissWeb. Its source crawls predate this " + |
| "graph, so coverage is thin and tilted toward long-lived sites.", |
| [["https://huggingface.co/datasets/commoncrawl/gneissweb-annotation-host-testing-v1", |
| "gneissweb-annotation-host-testing-v1"]]], |
| degree: ["How many links a host has in the web graph, counting both directions. The only one of " + |
| "the four that is not a label \u2014 it is read straight off the graph, so every host has a " + |
| "value. High-degree hubs pull toward the dense centre.", |
| [["https://commoncrawl.org/web-graphs", "Common Crawl web graphs"]]], |
| }; |
| let helpBuilt = false; |
| |
| function buildHelp() { |
| if (helpBuilt) return; |
| helpBuilt = true; |
| const seen = new Set(META.labelSets.map((ls) => ls.name)); |
| $("helpbody").innerHTML = Object.keys(HELP).filter((k) => seen.has(k)).map((k) => { |
| const [text, links] = HELP[k]; |
| const hrefs = links |
| .map(([href, label]) => `<a href="${href}" target="_blank" rel="noopener">${label}</a>`) |
| .join(" · "); |
| return `<h3>${k}</h3><p>${text} ${hrefs}</p>`; |
| }).join(""); |
| } |
| |
| function openHelp() { |
| buildHelp(); |
| $("scrim").hidden = false; $("help").hidden = false; |
| $("helpx").focus(); |
| } |
| |
| function closeHelp() { |
| $("scrim").hidden = true; $("help").hidden = true; |
| $("helpbtn").focus(); |
| } |
| |
| $("helpbtn").onclick = () => ($("help").hidden ? openHelp() : closeHelp()); |
| $("helpx").onclick = closeHelp; |
| $("scrim").onclick = closeHelp; |
| |
| $("seln").onclick = () => (sheetOpen ? closeSheet() : openSheet()); |
| $("selclr").onclick = () => setSelection(null); |
| $("selx").onclick = () => closeSheet(); |
| $("selarm").onclick = () => { |
| selArmed = !selArmed; |
| $("selarm").setAttribute("aria-pressed", selArmed ? "true" : "false"); |
| cv.style.cursor = selArmed ? "cell" : "crosshair"; |
| }; |
| window.addEventListener("keydown", (e) => { |
| if (e.key !== "Escape" || e.target === $("q")) return; |
| if (!$("help").hidden) closeHelp(); |
| else if (sheetOpen) closeSheet(); else if (sel.rect) setSelection(null); |
| }); |
| $("selrows").addEventListener("scroll", () => { |
| const e = $("selrows"); |
| if (e.scrollTop + e.clientHeight > e.scrollHeight - 400) listFill(); |
| }, { passive: true }); |
| cv.addEventListener("wheel", (e) => { |
| e.preventDefault(); |
| const w = cv.clientWidth, h = cv.clientHeight; |
| const ndcx = (e.clientX / w) * 2 - 1, ndcy = 1 - (e.clientY / h) * 2; |
| const wx = (ndcx / (view.s * asp)) + view.cx, wy = (ndcy / view.s) + view.cy; |
| const f = Math.exp(-e.deltaY * 0.0016); view.s = Math.max(0.15, Math.min(1200, view.s * f)); |
| view.cx = wx - (ndcx / (view.s * asp)); view.cy = wy - (ndcy / view.s); dirty = true; |
| }, { passive: false }); |
| |
| |
| $("reset").onclick = () => { |
| view = { s: 0.9, cx: 0, cy: 0 }; pin = null; setSelection(null); dirty = true; |
| }; |
| |
| function resize() { |
| const w = cv.clientWidth, h = cv.clientHeight; |
| cv.width = Math.round(w * dpr); cv.height = Math.round(h * dpr); |
| gl.viewport(0, 0, cv.width, cv.height); asp = h / w; dirty = true; |
| } |
| window.addEventListener("resize", resize); |
| |
| |
| let fFrames = 0, fT = performance.now(), spinOn = false, spinUntil = 0; |
| function syncSpinner(now) { |
| const busy = inflight > 0 || queue.length > 0; |
| if (busy) spinUntil = now + 350; |
| const show = busy || now < spinUntil; |
| if (show !== spinOn) { spinOn = show; $("spin").classList.toggle("on", show); } |
| } |
| function loop(now) { |
| syncSpinner(now); |
| if (dirty) { |
| updateActive(); draw(); dirty = false; |
| $("zoom").textContent = view.s.toFixed(view.s < 10 ? 1 : 0) + "×"; |
| } |
| fFrames++; |
| if (now - fT > 500) { $("fps").textContent = Math.round((fFrames * 1000) / (now - fT)); fFrames = 0; fT = now; } |
| requestAnimationFrame(loop); |
| } |
| })(); |
| </script> |
| </body> |
| </html> |
|
|
