dashboard: messages on one timeline, when they went

Each node's message stream comes its own way to the page, a subscribed
node's over the network to it: messages logged 16 ms apart on three
nodes ran over a second apart, as their streams arrived.

Messages now run when they went, on this node's clock: each at the
time its node logged it, taken back by how far that node's clock is
ahead of this node's (the map's clock chain, `aheadOf`), a while
behind real time (`replayDelay`): the most any stream lagged in the
last 30 s and a tenth of a second more, shown after the sliders
(`behind: …`), each stream's lag behind the fastest in its tooltip.
The browser's clock is against all streams alike and drops out. A
message known only from its receiver (its sender not subscribed) is
placed at its arrival, starting a crossing before. Messages lagging
more than 10 s (a gap) don't run, nor hold the map behind.

1 files changed, 120 insertions, 28 deletions

diff --git a/src/api/dashboard/dashboard.html b/src/api/dashboard/dashboard.html
index b7b45aa..b53c7b1 100644
--- a/src/api/dashboard/dashboard.html
+++ b/src/api/dashboard/dashboard.html
@@ -214,6 +214,8 @@
     drop-shadow(-2px 0 0 var(--panel)) drop-shadow(0 2px 0 var(--panel))
     drop-shadow(0 -2px 0 var(--panel)); }
   .map-overlay button, .map-overlay input, .map-overlay label { pointer-events: auto; }
+  /* How far behind the messages run: pointed at for its tooltip, which says why. */
+  .map-overlay #replay-delay { pointer-events: auto; cursor: help; }
   /* How hard links pull, beside the layout's toggle while dynamic, as tall. */
   .map-controls .slider { display: inline-flex; align-items: center; gap: 4px;
     font-size: 12px; color: var(--muted); height: 1.4em; }
@@ -2906,7 +2908,14 @@ graph.addEventListener("mouseleave", () => hover(null));
 // link's round trip) and leaves an afterglow that draws back into where it went
 // (`FLASH_GLOW_MS`). Dots are kept above everything else on the map (`flows`, after
 // what `show` places), flashes over the lines but below their labels and the nodes. ---
-const FLOW_SPREAD_MS = 2000, FLOWS_AT_MOST = 6;
+const FLOWS_AT_MOST = 6;
+// The timeline the messages run on (`runMessages`, below).
+const REPLAY_WINDOW_MS = 30000, REPLAY_MARGIN_MS = 100, REPLAY_LEAST_MS = 150,
+  REPLAY_MOST_MS = 10000;
+// By node whose stream it is: [when, lag] of its recent batches, the oldest first.
+const lags = new Map();
+let replayDelay = 500;
+let clocksKnown = null, clocksAt = -Infinity;
 // The line from `from` to `to` a message went over, as drawn: over `transport`'s
 // line, else (none known, or none drawn for it) the one in use. `at(t)`: the point `t`
 // along (0 at `from`, 1 at `to`) and the direction there (a unit vector); `length`,
@@ -3030,7 +3039,10 @@ function showMessagesLegend() {
       el("span", { class: "message", style: "--colour: " + colour },
         KIND_NAMES[kind] || kind)),
       el("label", { class: "slider" }, "size", slider),
-      el("label", { class: "slider" }, "speed", speed));
+      el("label", { class: "slider" }, "speed", speed),
+      el("span", { id: "replay-delay", class: "hint" }));
+    document.getElementById("replay-delay").addEventListener("pointerenter", explainReplay);
+    showReplay();
   }
   dotSlider.disabled = !flowsShown;
   speedSlider.disabled = flowsMode !== "dots";
@@ -3076,33 +3088,110 @@ function flowsNow() {
   return flowsShown && !calm.matches && !document.hidden &&
     !document.getElementById("map").classList.contains("collapsed");
 }
-// Messages as a stream brought them, this node's or a subscribed node's (each named by
-// both ends, `from` and `to`), run along their links as they come: started as far
-// apart as they were sent (`at`, by the clock of the node that logged them, so only
-// against each other), `FLOWS_AT_MOST` at most per link and way of a batch, shared out
-// by kind (`dotsFor`) where more came. Of a batch that spans more than
-// `FLOW_SPREAD_MS` (one after a gap: the computer slept, the stream stalled), only
-// those of its last `FLOW_SPREAD_MS`.
-function runMessages(list) {
-  if (!flowsNow() || list.length === 0) return;
+// --- One timeline for the map's messages: each stream (this node's, each subscribed
+// node's) comes over its own way, the slower ones later, so messages run not as they
+// arrive but when they were logged, on this node's clock (each node's `at` taken back by
+// how far its clock is ahead of this node's, as the map chains them: `aheadOf`), a
+// while behind (`replayDelay`): as long as the slowest stream lagged within the last
+// `REPLAY_WINDOW_MS`, and a margin. So messages of different nodes run as far apart
+// as they went, give or take how well their clocks are known. The browser's own clock
+// is against all of them alike, so it drops out. ---
+// How far `node`'s clock is ahead of this node's, as the map chains them (`swarmClocks`,
+// worked out again at most every second); 0 where not known.
+function aheadOf(node) {
+  if (!state || node === state.node.id) return 0;
   const now = performance.now();
-  const latest = Math.max(...list.map(message => message.at));
-  const recent = list.filter(message => latest - message.at <= FLOW_SPREAD_MS);
-  const first = Math.min(...recent.map(message => message.at));
+  if (now - clocksAt > 1000) {
+    clocksKnown = swarmClocks();
+    clocksAt = now;
+  }
+  const self = clocksKnown[state.node.id], it = clocksKnown[node];
+  return self && it ? it.offsetMs - self.offsetMs : 0;
+}
+// `node`'s stream brought a batch whose earliest message lagged `lag` ms (by the
+// browser's clock, against this node's): the most of it, which the delay must cover.
+function noteLag(node, lag) {
+  const now = performance.now();
+  const kept = (lags.get(node) || []).filter(([at]) => now - at <= REPLAY_WINDOW_MS);
+  kept.push([now, lag]);
+  lags.set(node, kept);
+  for (const [other, samples] of lags) {
+    if (!samples.some(([at]) => now - at <= REPLAY_WINDOW_MS)) lags.delete(other);
+  }
+  const slowest = Math.max(...[...lags.values()].flat().map(([, lag]) => lag));
+  replayDelay = Math.min(REPLAY_MOST_MS, Math.max(REPLAY_LEAST_MS, slowest + REPLAY_MARGIN_MS));
+  showReplay();
+}
+// `behind: …` in the messages' legend: how far behind they run, and in its tooltip how
+// that came about: the stream that lagged the most within the window, plus the margin
+// (or the bounds, where they held it), and the three slowest streams, each with the
+// most and least it lagged. Lags are as the page measures them, by the browser's
+// clock against this node's: one that is off adds to all of them alike. The tooltip is
+// written as the pointer comes onto it, and stays as it is while it's there (a tooltip
+// changing under the pointer goes away).
+function showReplay() {
+  const shown = document.getElementById("replay-delay");
+  if (!shown) return;
+  // Its text node kept, its text changed: the element under the pointer stays itself.
+  const words = "behind: " + (replayDelay / 1000).toFixed(2) + " s";
+  if (shown.firstChild) shown.firstChild.data = words;
+  else shown.textContent = words;
+}
+function explainReplay() {
+  const shown = document.getElementById("replay-delay");
+  if (!shown) return;
+  const s = ms => (ms / 1000).toFixed(2) + " s";
+  const each = [...lags].map(([node, samples]) => ({ node,
+    least: Math.min(...samples.map(([, lag]) => lag)),
+    most: Math.max(...samples.map(([, lag]) => lag)) }))
+    .sort((x, y) => y.most - x.most);
+  if (each.length === 0) {
+    shown.title = "Messages run when they went, this far behind real time; no stream yet.";
+    return;
+  }
+  const slowest = each[0];
+  const bound = slowest.most + REPLAY_MARGIN_MS < REPLAY_LEAST_MS ? ", raised to the least, " + s(REPLAY_LEAST_MS)
+    : slowest.most + REPLAY_MARGIN_MS > REPLAY_MOST_MS ? ", cut to the most, " + s(REPLAY_MOST_MS) : "";
+  shown.title = "Messages run when they went, this far behind real time, so the slowest" +
+    " stream's are in too:\n" + s(replayDelay) + " = " + name(slowest.node) + "'s lag, the most" +
+    " of any stream in the last 30 s (" + s(slowest.most) + "), + " + s(REPLAY_MARGIN_MS) +
+    bound + "\n\nThe slowest streams (the most and the least each lagged, last 30 s):\n" +
+    each.slice(0, 3).map(({ node, least, most }, i) => (i + 1) + ". " + name(node) + ": " +
+      s(most) + " (least " + s(least) + ")").join("\n") +
+    (each.length > 3 ? "\n(" + (each.length - 3) + " more)" : "");
+}
+// Messages as `node`'s stream brought them (each named by both ends, `from` and `to`;
+// `arrived`: logged by its receiver, the sender's not read, so it went its link's
+// crossing before), run along their links at their places on the timeline.
+// `FLOWS_AT_MOST` at most per link and way of a batch, shared out by kind (`dotsFor`)
+// over the moment where more came. Those lagging more than `REPLAY_MOST_MS` (after a
+// gap: the computer slept, the stream stalled) don't run.
+function runMessages(list, node) {
+  if (list.length === 0 || !state) return;
+  const ahead = aheadOf(node), wall = Date.now(), now = performance.now();
+  // On the timeline: when the message was logged, as `performance.now()` has it, now
+  // and behind.
+  const placed = list.map(message => ({ ...message, lag: wall - (message.at - ahead) }));
+  // Not those of a gap, which would hold the map that far behind for a while.
+  const fresh = placed.filter(message => message.lag <= REPLAY_MOST_MS);
+  if (fresh.length > 0) noteLag(node, Math.max(...fresh.map(message => message.lag)));
+  if (!flowsNow()) return;
   const links = new Map();
-  for (const message of recent) {
+  for (const message of placed) {
+    if (message.lag > REPLAY_MOST_MS) continue;
     const transport = schemeOf(message.address);
-    const key = message.from + " " + message.to + " " + (transport || "");
+    const key = message.from + " " + message.to + " " + (transport || "") +
+      (message.arrived ? " arrived" : "");
     const link = links.get(key) ||
-      { from: message.from, to: message.to, transport, messages: [] };
+      { from: message.from, to: message.to, transport, arrived: !!message.arrived, messages: [] };
     link.messages.push(message);
     links.set(key, link);
   }
-  for (const { from, to, transport, messages: sent } of links.values()) {
-    const start = message => now + (message.at - first);
+  const at = message => now + replayDelay - message.lag;
+  for (const { from, to, transport, arrived, messages: sent } of links.values()) {
     if (sent.length <= FLOWS_AT_MOST) {
       for (const message of sent) {
-        flows.push({ from, to, transport, start: start(message), kind: message.counted,
+        flows.push({ from, to, transport, arrived, at: at(message), kind: message.counted,
           r: dotRadius(message.bytes) });
       }
       continue;
@@ -3113,11 +3202,10 @@ function runMessages(list) {
       kind.messages += 1;
       kind.bytes += message.bytes;
     }
-    const span = Math.max(...sent.map(message => message.at)) - Math.min(...sent.map(message => message.at));
-    const from0 = start(sent.reduce((a, b) => (a.at <= b.at ? a : b)));
+    const times = sent.map(at), first = Math.min(...times), span = Math.max(...times) - first;
     const shown = dotsFor(kinds, sent.length);
     shown.forEach((dot, i) => {
-      flows.push({ from, to, transport, start: from0 + i * span / shown.length,
+      flows.push({ from, to, transport, arrived, at: first + i * span / shown.length,
         kind: dot.kind, r: dotRadius(dot.bytes) });
     });
   }
@@ -3132,9 +3220,9 @@ function ownMessagesArrived(list) {
   const self = state.node.id;
   runMessages(list.filter(message => message.way === "sent" ||
     (message.way === "received" && !logging(message.peer)))
-    .map(message => ({ ...message,
+    .map(message => ({ ...message, arrived: message.way === "received",
       from: message.way === "sent" ? self : message.peer,
-      to: message.way === "sent" ? message.peer : self })));
+      to: message.way === "sent" ? message.peer : self })), self);
 }
 // A subscribed node's messages (`follow`), as they come: listed in the console and
 // run along its links, as this node's own. Its messages with this node are this node's
@@ -3165,11 +3253,11 @@ function remoteMessagesArrived(id, entry, list) {
     if (message.peer !== self && !(message.way === "received" && message.peer !== id &&
         logging(message.peer))) kept.push(made);
     if (message.way === "sent" || !(message.peer === self || logging(message.peer))) {
-      running.push(made);
+      running.push({ ...made, arrived: message.way === "received" });
     }
   }
   checkDeliveries();
-  runMessages(running);
+  runMessages(running, id);
   if (kept.length > 0) messagesArrived(kept);
 }
 // Whether the page reads `id`'s messages one by one: a subscribed node's, while they
@@ -3516,15 +3604,19 @@ function drawFlows() {
   // to 1) as long as the line is now; as a flash, its head and its afterglow's tail
   // (each 0 to 1) and how bright. Arrived (a flash: its afterglow gone), or its link
   // gone, it goes.
+  // Each starts when it was logged (`at`), or, logged as it arrived, its crossing
+  // before: as a flash, half the link's round trip; as a dot, as long as it runs.
   const where = flow => {
     const line = route(flow.from, flow.to, flow.transport);
     if (!line) return null;
     if (flowsMode === "flash") {
       const travel = flow.travel || (flow.travel = Math.max(1, oneWayMs(flow.from, flow.to)));
+      flow.start = flow.arrived ? flow.at - travel : flow.at;
       const age = now - flow.start, glow = Math.max(0, (age - travel) / FLASH_GLOW_MS);
       return { line, done: glow >= 1, head: Math.min(1, age / travel),
         tail: 1 - (1 - Math.min(1, glow)) ** 2, opacity: 1 - Math.min(1, glow) };
     }
+    flow.start = flow.arrived ? flow.at - line.length / flowSpeed : flow.at;
     const t = (now - flow.start) * flowSpeed / line.length;
     return { line, done: t >= 1, t };
   };
@@ -3535,8 +3627,8 @@ function drawFlows() {
   const dots = [], streaks = [];
   const f = n => n.toFixed(1);
   for (const flow of flows) {
-    if (now < flow.start) continue;
     const at = where(flow);
+    if (now < flow.start) continue;
     const r = flow.r * dotScale, colour = kindColour(flow.kind);
     if (flowsMode === "flash") {
       // From the tail, a point, widening to the head, rounded, along the line: brightest