From 5480825403ceae8bafb809e9eb0432000d41a91a Mon Sep 17 00:00:00 2001 From: Khaliq Date: Fri, 7 Aug 2026 13:34:03 +0200 Subject: [PATCH 1/3] docs(evidence): replace the inferred sub-200ms latency claim with measurement The public "sub-200ms end-to-end including measurement overhead that exceeds the signal" was never a one-way measurement. It came from the 2026-07-26 evidence, which measured a round trip (median 315.5 ms, n=12, 5 ms polling echo) and halved it. This replaces the inference with a direct one-way measurement, sf-mini as receiver. Both halves of the old claim are wrong, in opposite directions. "sub-200ms" is false for realistic change sets. A single small file propagates in a median of 20.2 ms (p95 161.7 ms, n=20), but a repo-sized change set of 11 files / ~14 KB takes a median of 216.7 ms (p95 303.9 ms, n=20). The receive path fetches each file with its own server round trip (syncer.go:3310,3320,3339), so cost scales with file count, not bytes. Agents commit change sets; small single files were never the case that mattered. "measurement overhead that exceeds the signal" is false by more than an order of magnitude and is retired. The watcher's own detection delay was measured against a local control, not assumed: median 1.225 ms, max 2.455 ms over 25 pairs. Every figure holds only under the stated topology precondition, which is repeated beside each median in RESULTS.md: the server ran on the sender's own machine, so sender->server was loopback and the only network hop was server->receiver over a Tailscale LAN. These are LAN best cases and do not license any claim about the hosted product, faster or slower. RESULTS.md states the three prerequisites a product claim would need. No sub-100ms claim is made. Two things the method caught that a simpler one would have missed: - The hosts' clock offset drifted 8.323 ms across a ~21 minute run (-6.441 ms to -14.765 ms), comparable to the small-file signal itself. The offset is therefore interpolated to each trial's own send time rather than pinned. A single offset measurement would have looked reasonable and been quietly wrong. - sf-mini's status/live flipped online<->offline four times per gate window while its heartbeat advanced monotonically, and an MCP query_nodes call reported it flatly offline with a 38 s old, advancing heartbeat. Liveness is gated on heartbeat advance across >=90 s, before and after the trials. Trusting status would have abandoned the run against a healthy host. 52 trials, all HTTP 202, zero lost files, zero partial change sets, no sample near the ~30 s websocket-off fallback. The headline batches are the clean 20+20; an earlier 12-trial batch cut short by an interrupt is preserved as correctness evidence only and contributes to no percentile. 26/26 named assertions pass via harness/assertions.py. Isolation: fresh server on a separate port with a separate state dir, fresh workspace, distinct mount path. The pre-existing .dev-collab-stack/ and .salvaged-from-minis/ trees, their processes and ports, were not touched or reused. Teardown in CLEANUP.md. Test credentials were ephemeral, kept outside the repo, and are absent from every artifact. Co-Authored-By: Claude Opus 5 --- .../mount-latency-20260807/CLEANUP.md | 74 +++++ .../mount-latency-20260807/METHODOLOGY.md | 187 ++++++++++++ .../mount-latency-20260807/RESULTS.md | 243 +++++++++++++++ .../mount-latency-20260807/harness/analyse.py | 192 ++++++++++++ .../harness/assertions.py | 197 +++++++++++++ .../harness/clock-offset.py | 140 +++++++++ .../harness/control-local.py | 66 +++++ .../harness/dev-authd.py | 146 +++++++++ .../harness/receiver-watch.py | 134 +++++++++ .../harness/sender-trials.py | 146 +++++++++ .../raw/clock-offset-post.jsonl | 201 +++++++++++++ .../raw/clock-offset-pre.jsonl | 145 +++++++++ .../raw/control-create.jsonl | 25 ++ .../raw/control-watch.jsonl | 29 ++ .../raw/heartbeat-gate-post.jsonl | 14 + .../raw/heartbeat-gate-pre.jsonl | 14 + .../raw/heartbeat-gate-pre.sh | 15 + .../raw/mount-watch.jsonl | 278 ++++++++++++++++++ .../raw/trials-repo.jsonl | 20 ++ .../raw/trials-small.jsonl | 32 ++ docs/multi-agent-collaboration-assessment.md | 21 +- 21 files changed, 2315 insertions(+), 4 deletions(-) create mode 100644 docs/evidence/mount-latency-20260807/CLEANUP.md create mode 100644 docs/evidence/mount-latency-20260807/METHODOLOGY.md create mode 100644 docs/evidence/mount-latency-20260807/RESULTS.md create mode 100644 docs/evidence/mount-latency-20260807/harness/analyse.py create mode 100644 docs/evidence/mount-latency-20260807/harness/assertions.py create mode 100644 docs/evidence/mount-latency-20260807/harness/clock-offset.py create mode 100644 docs/evidence/mount-latency-20260807/harness/control-local.py create mode 100644 docs/evidence/mount-latency-20260807/harness/dev-authd.py create mode 100644 docs/evidence/mount-latency-20260807/harness/receiver-watch.py create mode 100644 docs/evidence/mount-latency-20260807/harness/sender-trials.py create mode 100644 docs/evidence/mount-latency-20260807/raw/clock-offset-post.jsonl create mode 100644 docs/evidence/mount-latency-20260807/raw/clock-offset-pre.jsonl create mode 100644 docs/evidence/mount-latency-20260807/raw/control-create.jsonl create mode 100644 docs/evidence/mount-latency-20260807/raw/control-watch.jsonl create mode 100644 docs/evidence/mount-latency-20260807/raw/heartbeat-gate-post.jsonl create mode 100644 docs/evidence/mount-latency-20260807/raw/heartbeat-gate-pre.jsonl create mode 100755 docs/evidence/mount-latency-20260807/raw/heartbeat-gate-pre.sh create mode 100644 docs/evidence/mount-latency-20260807/raw/mount-watch.jsonl create mode 100644 docs/evidence/mount-latency-20260807/raw/trials-repo.jsonl create mode 100644 docs/evidence/mount-latency-20260807/raw/trials-small.jsonl diff --git a/docs/evidence/mount-latency-20260807/CLEANUP.md b/docs/evidence/mount-latency-20260807/CLEANUP.md new file mode 100644 index 00000000..fac401e3 --- /dev/null +++ b/docs/evidence/mount-latency-20260807/CLEANUP.md @@ -0,0 +1,74 @@ +# Cleanup for the 2026-08-07 mount latency run + +Everything this run created is disposable. Nothing here touches the +pre-existing `.dev-collab-stack/` or `.salvaged-from-minis/` directories, +their processes, their ports, or the existing sf-mini mounts — those were +deliberately left alone and must stay that way. + +## What this run started + +**On the sender host (`khaliqs-macbook-pro`, Tailscale `100.89.219.17`)** + +| Thing | Where | +|---|---| +| `relayfile-server` | bound to `100.89.219.17:18299` (Tailscale address only, not `0.0.0.0`) | +| `dev-authd.py serve` (JWKS) | `127.0.0.1:19091`, loopback only | +| Server state file | `/latency-run/state/state.json` — outside the repo | +| Throwaway RSA private key + minted tokens | `/latency-run/keys/` — outside the repo, mode 0600 | + +**On the receiver host (`sf-mac-mini`, Tailscale `100.102.30.76`)** + +| Thing | Where | +|---|---| +| `relayfile-cli mount ws_latency_20260807` | mirror at `~/relayfile-latency-mount-20260807` | +| `receiver-watch.py` | writing `~/.relayfile-latency-harness/raw/` | +| `clock-offset.py server` | port `19299` | +| Deployed harness + receiver token | `~/.relayfile-latency-harness/` | + +## Teardown + +Receiver: + +```sh +ssh sf-mini ' + pkill -f "relayfile-cli mount ws_latency_20260807" + pkill -f receiver-watch.py + pkill -f "clock-offset.py server" + rm -rf ~/relayfile-latency-mount-20260807 + rm -rf ~/.relayfile-latency-harness +' +``` + +Sender: + +```sh +pkill -f "bin/relayfile-server" +pkill -f "dev-authd.py serve" +rm -rf /latency-run +``` + +`pkill -f "relayfile-cli mount ws_latency_20260807"` is deliberately matched on +the full workspace name. sf-mini also runs unrelated pre-existing mounts +(`relayfile-dev-collab`, `relay-dev-collab`); a looser pattern would kill them. + +## Verifying nothing else was disturbed + +```sh +lsof -nP -iTCP:8299 -sTCP:LISTEN # dev-collab server port: expected untouched +ssh sf-mini 'pgrep -fl "relayfile-cli.*dev-collab"' # pre-existing mounts still up +git -C status --short # .dev-collab-stack/ and .salvaged-from-minis/ still untracked, unmodified +``` + +## Credentials + +The RSA key and the bearer tokens minted for this run are throwaway, scoped to +workspace `ws_latency_20260807`, short-lived, and were never written into any +committed artifact or sent over Relay. Deleting the scratch directory and +`~/.relayfile-latency-harness` on the receiver destroys them. + +Separately, and unrelated to this run: a routine `ps` on sf-mini exposes live +`RELAY_API_KEY` and agent-token values in broker process argv, because they are +passed as command-line arguments. Those are pre-existing production credentials, +readable by any local process, and were reported for rotation. This run +deliberately passed its own receiver token via the `RELAYFILE_TOKEN` environment +variable rather than `--token` so as not to add to that exposure. diff --git a/docs/evidence/mount-latency-20260807/METHODOLOGY.md b/docs/evidence/mount-latency-20260807/METHODOLOGY.md new file mode 100644 index 00000000..f286b678 --- /dev/null +++ b/docs/evidence/mount-latency-20260807/METHODOLOGY.md @@ -0,0 +1,187 @@ +# One-way mount propagation latency — methodology + +Date: 2026-08-07 +Author: `herdr-t2-relayfile-latency-b1` +Status: written **before** any trial was run, per the measurement brief. + +## Why this run exists + +The public claim is currently: + +> sub-200ms end-to-end including measurement overhead that exceeds the signal + +That wording is deliberately hedged because the number behind it is not a +one-way measurement. It comes from +`docs/evidence/real-time-collaboration-2026-07-26/`, which measured a **round +trip** — `sf-initiated` median 315.526 ms / p95 372.479 ms over n=12, and +`finn-initiated` median 373.230 ms over n=12 — using +`scripts/measure-mount-latency.rb`. That script's own header says the +initiator "measures ping-to-ack time with its own monotonic clock, so +separate-machine wall-clock skew is absent". Avoiding skew that way is sound, +but it means: + +1. The one-way figure was inferred by halving a round trip. A round trip is + not symmetric: the ack leg includes a *second* full write-and-propagate, + plus the responder's scheduling delay. +2. The responder polled the directory every 5 ms + (`measure-mount-latency.rb:39,93`), so every sample carries up to 5 ms of + quantisation on each leg. +3. n=12 per direction is too small for a credible p95. + +This run replaces the inference with a directly measured one-way number. + +## What is being measured + +**End-to-end one-way propagation:** from the instant a writer on the sender +host issues a file write, to the instant that file's content is readable on +the receiver host's mounted workspace. + +The receiver is `sf-mac-mini` (Tailscale `100.102.30.76`), as required by the +brief. The sender and the relayfile server are `khaliqs-macbook-pro` +(Tailscale `100.89.219.17`). + +### Which latency the watcher actually observes + +This matters, and it was settled by reading the delivery path before any trial +was run. + +`relayfile-cli mount` runs in `poll` mode by default +(`cmd/relayfile-cli/main.go:58`). Despite the name, `poll` does **not** mean +"poll the server for changes" — it means "materialise a mirror of real files +on local disk", as opposed to `fuse` (which this build hard-refuses, +`cmd/relayfile-cli/main.go:6687-6689`). Remote changes arrive over a +websocket subscription to `/v1/workspaces/{id}/fs/ws` +(`internal/mountsync/syncer.go:3251-3260`). On each event the daemon calls +`applyWebSocketEvent` (`syncer.go:3310`), which does `ReadFile` +(`syncer.go:3320`) then `applyRemoteFile` (`syncer.go:3339`), which +`writeFileAtomic`s the bytes to the local path (`syncer.go:6033-6051`). + +Two consequences: + +- This is a **push**, not a pull. The receiver's `stat`/`read` on the mirror + is a purely local syscall with no network hop. So an in-box watcher on the + receiver measures propagation, not its own fetch. Had the mount been FUSE, + the websocket would only *invalidate* cache + (`internal/mountfuse/wsinvalidate.go:158-165`) and the next lookup would + trigger a synchronous server fetch — a watcher would then have been + measuring its own pull, and the number would have been meaningless. +- The measured interval therefore covers: sender write → server ingest and + `publishEvent` → websocket fan-out → receiver daemon `ReadFile` round trip → + `writeFileAtomic` to local disk → watcher observation. That is the full + chain an agent on the receiver actually waits for, which is what "end-to-end" + should mean. + +### Known windows on the path, and how they are handled + +| Window | Default | Handling | +|---|---|---| +| Server envelope coalesce | 3 s (`internal/relayfile/store.go:883-885`) | Applies to duplicate inbound *provider envelopes* (`store.go:3605`), not to direct fs writes. Neutralised anyway: every trial writes a unique path, and trials are spaced beyond the window. | +| Mount reconcile tick | 30 s, and only every 10th cycle when websocket is on ≈ 5 min (`cmd/relayfile-cli/main.go:57,13209-13211`) | This is the safety net, not the delivery path. Any sample anywhere near these values means the websocket did not deliver; such samples are reported, never silently dropped. | +| SDK `subscribe()` coalesce | 200 ms (`packages/sdk/typescript/src/client.ts:189`) | Not on this path — the SDK is not used. Noted because it would otherwise silently add 200 ms to an SDK-based measurement. | +| Receive-path debounce | none in poll mode | `applyWebSocketEvent` applies inline with no timer. | + +## Clock handling + +The two hosts' clocks are **not** assumed equal, and were not equal: sf-mini's +realtime clock measured **6.441 ms behind** this laptop. On a ~150 ms signal +that is a ~4% systematic error, and on any faster path it would matter much +more. + +Offset is measured with NTP's four-timestamp formula over a raw TCP exchange +on the Tailscale LAN (`harness/clock-offset.py`): + + delay = (t3 - t0) - (t2 - t1) + offset = ((t1 - t0) + (t2 - t3)) / 2 # receiver_clock - sender_clock + +Both formulas assume path symmetry, which is weakest under queueing, so many +samples are taken and the one with the **smallest delay** is selected — the +least-queued sample is the least asymmetric. Residual uncertainty is bounded +at ±delay/2 and is reported alongside the result, so the final latency carries +an honest error bar rather than a false precision. + +An ssh-based clock comparison was rejected: its round trip is of the same order +as the signal being measured, so it could not bound the offset usefully. + +Offset is measured **before and after** the trial block. The difference bounds +relative clock drift over the run; if drift is material it is reported as part +of the uncertainty rather than ignored. + +## Receiver watcher + +An in-box resident watcher runs on sf-mini and records arrival timestamps +**locally**, using sf-mini's own `CLOCK_REALTIME` via `time.time_ns()`. No +timestamp is taken over ssh, because ssh round-trip would be added to every +sample. + +Detection uses a tight `stat` poll loop over the local mirror. Since the mirror +is real local disk (established above), each poll is a local syscall costing +microseconds, so the loop can run at a ~1 ms period without meaningful cost. +That 1 ms is the quantisation floor, versus 5 ms in the prior run. + +**Measurement overhead is itself measured, not assumed.** A control experiment +creates files locally on sf-mini — same directory, same watcher, no network +involved — and records the watcher's own detection delay distribution. The +reported latency is quoted both raw and with this control subtracted, so the +"measurement overhead exceeds the signal" hedge can be either retired or +confirmed with a number instead of a guess. + +## Trial design + +Two populations, per the brief: + +- **Small-file trials (n ≥ 20).** A few hundred bytes. Isolates propagation + latency with transfer time near zero. +- **Realistic repo-sized change-set trials (n ≥ 20).** A change set with the + size and file-count profile of an actual commit in this repository, so the + number quoted is one a reader can expect from real agent work rather than + from a synthetic best case. + +Each trial writes a unique path, so no trial can be coalesced into, or +satisfied by, a previous one. Raw per-trial records are appended to +`raw/*.jsonl` **immediately** as each trial completes, so a mid-run tool or +host failure leaves usable evidence rather than nothing. + +## Liveness gate + +sf-mini's participation is gated on its **own `lastHeartbeatAt` advancing** +across ≥90 s, sampled before the trials and again at result time. + +Two weaker signals are explicitly rejected: + +- **Absence from a fleet listing is not evidence of offline.** The listing + returns nondeterministic subsets. +- **`status` and `live` are registration fields, not liveness fields.** During + the pre-trial gate window sf-mini's `status` flipped `online`↔`offline` four + times while its heartbeat advanced monotonically, and an MCP `query_nodes` + call at 11:08Z reported it `offline`/`live:false` while its heartbeat was + 38 s old and advancing. Only monotonic advance is trusted. + +## Failure policy + +If the propagation path or either liveness gate fails, partial raw data is +preserved and an explicit blocker artifact is written. **No median is salvaged +from a failed run.** Until valid results exist the public claim stays exactly: + +> sub-200ms end-to-end including measurement overhead that exceeds the signal + +and is never stated as sub-100ms. + +**Outcome (added after the run):** both liveness gates passed, the propagation +path held for all 52 trials, and 26/26 named assertions pass. Results are in +[`RESULTS.md`](RESULTS.md). Both halves of the claim above turned out to be +wrong — "sub-200ms" is false for realistic repo-sized change sets (median +216.7 ms), and the measurement overhead is ~1.2 ms against signals of 20.2 ms +and 216.7 ms, so it does not exceed the signal and that wording is retired. No +sub-100ms claim is made: the figures come from a loopback-plus-LAN topology +that does not represent the hosted product path. + +## Isolation and cleanup + +The pre-existing `.dev-collab-stack/` and `.salvaged-from-minis/` directories, +their processes, ports, state, and the existing sf-mini mounts are **not +touched or reused**. This run stands up a fresh server on a separate port with +a separate state directory, a fresh workspace, and a distinct receiver mount +path. Cleanup instructions are recorded in `CLEANUP.md`. + +Test credentials are minted fresh for this run, are short-lived, and are never +written to any artifact or transmitted over Relay. diff --git a/docs/evidence/mount-latency-20260807/RESULTS.md b/docs/evidence/mount-latency-20260807/RESULTS.md new file mode 100644 index 00000000..6f4f2fe3 --- /dev/null +++ b/docs/evidence/mount-latency-20260807/RESULTS.md @@ -0,0 +1,243 @@ +# One-way mount propagation latency — measured result + +Date: 2026-08-07 +Base commit: `ea67a73` (`chore(release): v0.10.39`) +Sender: `khaliqs-macbook-pro`, Tailscale `100.89.219.17` +Receiver: `sf-mac-mini` ("sf-mini"), Tailscale `100.102.30.76` +Method: [`METHODOLOGY.md`](METHODOLOGY.md) — written before any trial was run +Assertions: 26/26 pass (`harness/assertions.py`) + +--- + +## The headline + +**A realistic repo-sized change set does not propagate in under 200 ms.** + +Two populations were measured, and they land on opposite sides of the claim +that was being checked: + +| Change set | n | median | p95 | min | max | +|---|---|---|---|---|---| +| Single small file (~300 B) | 20 | **20.2 ms** | 161.7 ms | 12.3 ms | 183.3 ms | +| Repo-sized change set (11 files, ~14 KB) | 20 | **216.7 ms** | 303.9 ms | 165.9 ms | 328.9 ms | + +**Every number in this table holds only under this precondition: the relayfile +server ran on the sender's own machine, so the sender→server leg was loopback, +and the only network hop was server→receiver across a Tailscale LAN between two +Macs.** This is a best case, not the product's path. See +[Topology precondition](#topology-precondition) — it is repeated beside every +figure in this document deliberately, so no figure can be lifted out without it. + +### Quotable statement + +> Measured 2026-08-07 on relayfile `ea67a73`: with the server on the sender's +> own machine and a Tailscale LAN between two Macs, a single small file +> propagates to a second machine's mount in a median of 20.2 ms (p95 161.7 ms, +> n=20), while a repo-sized change set of 11 files / ~14 KB takes a median of +> 216.7 ms (p95 303.9 ms, n=20). These are LAN best-case figures with one +> network hop; they are not measurements of the hosted product path. + +## What this does to the existing claim + +The claim under test was: + +> sub-200ms end-to-end including measurement overhead that exceeds the signal + +Both halves are wrong, in opposite directions. + +**"sub-200ms end-to-end" is false for realistic change sets.** It holds only +for a single small file. A repo-sized change set — the shape actual agent work +produces — has a median of 216.7 ms and a p95 of 303.9 ms, and that is *with* +the sender→server leg on loopback. Add a real network hop on that leg and the +figure can only get worse. Small single files were never the interesting case; +agents commit change sets. + +**"measurement overhead that exceeds the signal" is false by more than an +order of magnitude, and is retired by this run.** The watcher's own detection +delay was measured, not assumed: 25 local create/detect pairs on the receiver, +same clock, same watcher code, same filesystem, no network, published by atomic +rename to match how the mount daemon materialises remote content. + +| Watcher detection delay | min | median | p95 | max | +|---|---|---|---|---| +| Control, n=25 | 0.360 ms | **1.225 ms** | 2.349 ms | 2.455 ms | + +1.2 ms of overhead against a 20.2 ms small-file signal and a 216.7 ms +change-set signal. The overhead is ~6% of the smaller signal and ~0.6% of the +larger one. No result here is overhead-limited, and the hedge should not be +repeated. + +### Where the old number came from + +`docs/evidence/real-time-collaboration-2026-07-26/` measured a **round trip**: +sf-initiated median 315.526 ms / p95 372.479 ms (n=12), finn-initiated median +373.230 ms (n=12). The public one-way figure was that round trip halved. Three +problems: a round trip is not symmetric (the ack leg is a second full +write-and-propagate plus the responder's scheduling delay); the responder +polled at 5 ms granularity (`scripts/measure-mount-latency.rb:39,93`); and +n=12 per direction cannot support a p95. That run was honest about avoiding +clock skew — it used a single monotonic clock deliberately — but the cost of +that choice was that it could not produce a one-way number at all. + +## Leg decomposition + +Because the sender records both its send time and the server's acknowledgement, +the two legs separate cleanly. + +| Leg | Small file (n=20) | Repo-sized (n=20) | +|---|---|---| +| A — sender → server (**loopback, not network**) | median 3.1 ms, p95 7.0 ms | median 3.7 ms, p95 8.1 ms | +| B — server → receiver mount (**Tailscale LAN**) | median 15.8 ms, p95 146.8 ms | median 212.4 ms, p95 300.3 ms | +| End-to-end | median 20.2 ms, p95 161.7 ms | median 216.7 ms, p95 303.9 ms | + +Leg A is ~3 ms because it never touches a network. In any real deployment leg A +is a WAN request and this decomposition is the reason the end-to-end figures +above cannot be carried over to the product. + +Leg B carries essentially all of the change-set cost: 212.4 ms of the 216.7 ms +median. The receive path is `applyWebSocketEvent` → `ReadFile` → +`writeFileAtomic` per file (`internal/mountsync/syncer.go:3310,3320,3339`), so +an 11-file change set costs 11 sequential server round trips on the receiver's +side after the single websocket notification. That is the dominant term, and it +scales with file count rather than with bytes — which is why the change-set +median is ~10× the single-file median for only ~47× the bytes. + +## Topology precondition + +Repeated here in full because every figure in this document depends on it: + +- The relayfile server ran on the **sender's own machine** + (`100.89.219.17:18299`). The sender→server leg was loopback. +- The only network hop was **server→receiver**, across a Tailscale LAN between + two Macs on the same tailnet, min RTT ~4.5 ms. +- The receiver was a Mac mini on that same tailnet, not a typical end-user + machine on a typical network. +- Fresh isolated server, fresh workspace `ws_latency_20260807`, fresh mount. + +### What a product claim would require + +None of the figures above license a claim about the hosted product. To make +one, the measurement needs all three of: + +1. **The server off the sender.** Leg A must be a real request over the + network, not loopback. As measured, leg A contributes ~3 ms; in production + it is a WAN round trip and will likely dominate the small-file case. +2. **A real WAN path**, not a same-tailnet LAN with ~4.5 ms RTT. Tailscale + here negotiated a direct connection over the local network. +3. **A receiver that is not a Mac mini on the sender's own tailnet** — an + ordinary client on an ordinary network, including the tail of poor + connectivity that a p95 is supposed to capture. + +Until then the honest public position is that the sub-200 ms claim is +unsupported for realistic change sets *and* that no faster claim can be made +either. **This run does not support a sub-100 ms claim and none should be +made**, notwithstanding that the small-file median is 20.2 ms — that figure +describes a loopback-plus-LAN path, not a product. + +## Clock handling, and a finding + +The two hosts' clocks were **not** equal and **did not stay** at a constant +offset. Measured with NTP's four-timestamp formula over raw TCP on the LAN, +selecting the minimum-delay sample: + +| | offset (receiver − sender) | min delay | uncertainty | samples | +|---|---|---|---|---| +| Before trials | −6.441 ms | 4.537 ms | ±2.268 ms | 145 | +| After trials | −14.765 ms | 4.577 ms | ±2.288 ms | 200 | + +**The offset moved 8.323 ms across a ~21 minute run** — roughly 8 ppm of +relative drift, and comparable to the entire small-file signal. Pinning a +single offset would have biased every trial by up to the full drift; an +early-run trial and a late-run trial would have been corrected by amounts +differing by more than a third of the small-file median. + +The analyser therefore linearly interpolates the offset to **each trial's own +send time** between the two anchors. Zero trials fell outside the anchor span. +This correction only exists because the offset was measured twice; a single +measurement would have looked perfectly reasonable and been quietly wrong. + +Residual uncertainty from the symmetry assumption is ±2.3 ms, which is ~11% of +the small-file median and ~1% of the change-set median. + +## Liveness gates + +sf-mini's participation was gated on its **own `lastHeartbeatAt` advancing** +across ≥90 s, before the trials and again at result time. Both passed. + +| Gate | Window | Samples | Distinct heartbeats observed | +|---|---|---|---| +| Pre-trial | 11:10:48Z → 11:13:16Z (148 s) | 14 | 11:09:54Z → 11:11:00Z → 11:12:00Z → 11:13:00Z | +| Post-trial | 11:29:21Z → 11:31:36Z (135 s) | 13 | 11:29:08Z → 11:30:08Z → 11:31:09Z | + +The node was present in every sample of both windows. + +**Two weaker signals were explicitly rejected, and the run demonstrates why.** +Within the pre-trial window sf-mini's `status`/`live` flipped +`online`↔`offline` four times while its heartbeat advanced monotonically, and +the post-trial window showed the same flapping. Separately, an MCP +`query_nodes` call at 11:08Z reported sf-mini `status: "offline"`, +`live: false`, `handlersLive: false` while its heartbeat was 38 s old and +advancing. Had either signal been trusted, this run would have been abandoned +against a perfectly healthy host. `status` and `live` are registration fields, +not liveness fields; and absence from a fleet listing is not evidence of +offline, because the listing returns nondeterministic subsets. + +## Trials and data integrity + +| Batch | n | complete | incomplete | non-202 | status | +|---|---|---|---|---|---| +| `r2` small file | 20 | 20 | 0 | 0 | **headline** | +| `r2` repo-sized | 20 | 20 | 0 | 0 | **headline** | +| `run20260807` small file | 12 | 12 | 0 | 0 | **correctness evidence only** | + +The `run20260807` batch was cut short at 12 of 20 by an operator interrupt. It +is preserved in the raw record and is **not** a source of any percentile in +this document; it is retained solely as evidence that the path delivered +correctly (12/12 accepted, 12/12 arrived, 0 lost). All headline statistics come +from the clean `r2` batches. + +Across all 52 trials: every write returned HTTP 202, every expected file +arrived, and no change set was partially delivered. No sample came near the +~30 s websocket-off reconcile fallback, confirming every measurement is of the +websocket delivery path rather than the polling safety net. + +Each trial wrote a unique path, so no trial could be satisfied by a previous +one or collapsed into another by coalescing. A change set counts as complete +when its **last** file arrives; ordering across files is not assumed. + +## Reproducing + +```sh +cd docs/evidence/mount-latency-20260807 +python3 harness/assertions.py # 26 named assertions over the raw data +python3 harness/analyse.py raw/clock-offset-pre.jsonl raw/clock-offset-post.jsonl \ + raw/trials-repo.jsonl raw/mount-watch.jsonl repo r2 +``` + +Raw evidence, appended live as each trial completed: + +| File | Contents | +|---|---| +| `raw/trials-small.jsonl` | sender records, both small-file batches | +| `raw/trials-repo.jsonl` | sender records, repo-sized batch | +| `raw/mount-watch.jsonl` | receiver arrival timestamps, sf-mini's own clock | +| `raw/control-create.jsonl`, `raw/control-watch.jsonl` | watcher-overhead control | +| `raw/clock-offset-pre.jsonl`, `raw/clock-offset-post.jsonl` | clock offset anchors | +| `raw/heartbeat-gate-pre.jsonl`, `raw/heartbeat-gate-post.jsonl` | liveness gates | + +Teardown and isolation verification: [`CLEANUP.md`](CLEANUP.md). + +## Follow-ups this run surfaced + +1. **Per-file sequential fetch on the receive path.** Leg B scales with file + count, not bytes — 11 files cost ~212 ms while one file costs ~16 ms. + Batching or parallelising the receiver's `ReadFile` calls after a + multi-file event is the obvious lever, and would move the change-set median + more than any transport change. +2. **`docs/guides/collaboration.md` and the mount help text.** The 2026-07-26 + assessment flagged the help text as underselling propagation speed. With + these numbers the correction is not simply "it's faster" — it is + size-dependent, and any replacement wording should say so. +3. **The hosted path is still unmeasured.** Both this run and the 2026-07-26 + run put the server on one of the two participating machines. No evidence + currently exists for the product topology. diff --git a/docs/evidence/mount-latency-20260807/harness/analyse.py b/docs/evidence/mount-latency-20260807/harness/analyse.py new file mode 100644 index 00000000..32414d7f --- /dev/null +++ b/docs/evidence/mount-latency-20260807/harness/analyse.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Pair sender records with receiver arrivals and emit the latency summary. + +Latency for one trial is: + + (arrival_on_receiver - clock_offset) - send_on_sender + +where `clock_offset` is receiver_clock - sender_clock, measured separately +(`clock-offset.py`). Without that correction the two hosts' clocks differ by +several milliseconds and the subtraction is meaningless at this scale. + +For a multi-file change set the trial is complete only when its LAST file has +arrived, so completion uses max(arrival) across the trial's paths. Ordering +across files is not assumed. + +Rules this script enforces rather than papers over: + - A trial with any missing arrival is reported as INCOMPLETE and excluded + from percentiles, but is always counted and named in the output. Losses + are never silently dropped. + - The daemon's atomic-write temp files (`.name.tmp-`) are not arrivals. + - Percentiles are computed with linear interpolation, and p95 on n=20 is + reported with an explicit note that it rests on the top few samples. + +The offset is NOT a constant. Measured before and after the trial block, it +moved by ~8 ms -- comparable to the signal itself, because the two hosts' clocks +drift relative to each other at roughly 8 ppm. Pinning a single offset would +therefore bias every trial by up to the full drift. Instead the offset is +linearly interpolated to each trial's own send time between the two anchor +measurements. Trials outside the anchor span are extrapolated and flagged. + +Usage: + analyse.py OFFSET_PRE_JSONL OFFSET_POST_JSONL SENDS_JSONL ARRIVALS_JSONL [LABEL] +""" + +import json +import os +import sys + + +def percentile(values, fraction): + """Linear-interpolated percentile over a sorted list.""" + if not values: + return None + if len(values) == 1: + return values[0] + position = fraction * (len(values) - 1) + lower = int(position) + upper = min(lower + 1, len(values) - 1) + weight = position - lower + return values[lower] * (1 - weight) + values[upper] * weight + + +def load_arrivals(path): + """Map workspace path -> earliest observation, ignoring daemon temp files.""" + arrivals = {} + for line in open(path): + line = line.strip() + if not line: + continue + record = json.loads(line) + if record.get("kind"): + continue + relative = record["path"] + base = os.path.basename(relative) + # writeFileAtomic publishes through `..tmp-`; the temp + # becoming visible is not the file arriving. + if ".tmp-" in base or base.endswith(".tmp"): + continue + key = "/" + relative + if key not in arrivals or record["observed_ns"] < arrivals[key]: + arrivals[key] = record["observed_ns"] + return arrivals + + +def anchor(path): + """Return (reference_time_ns, offset_ns) from the least-queued exchange.""" + # The trailing summary record also mentions delay (`min_delay_ns`), so + # select on the parsed per-sample shape rather than on the raw text. + samples = [ + record + for record in (json.loads(line) for line in open(path) if line.strip()) + if "t0_client_ns" in record and not record.get("kind") + ] + best = min(samples, key=lambda record: record["delay_ns"]) + # Midpoint of the exchange is the instant the offset describes. + return (best["t0_client_ns"] + best["t3_client_ns"]) // 2, best["offset_ns"] + + +def offset_at(send_ns, pre, post): + """Linearly interpolate the clock offset to a trial's own send time.""" + (t_pre, offset_pre), (t_post, offset_post) = pre, post + if t_post == t_pre: + return offset_pre, False + fraction = (send_ns - t_pre) / (t_post - t_pre) + extrapolated = fraction < 0 or fraction > 1 + return offset_pre + (offset_post - offset_pre) * fraction, extrapolated + + +def main(): + pre = anchor(sys.argv[1]) + post = anchor(sys.argv[2]) + sends_path, arrivals_path = sys.argv[3], sys.argv[4] + label = sys.argv[5] if len(sys.argv) > 5 else os.path.basename(sends_path) + # Optional run-id filter. The raw files are append-only and hold every + # batch that was ever run, including a batch cut short by an interrupt; + # headline statistics must come from one clean batch, so select rather + # than edit the raw evidence. + run_filter = sys.argv[6] if len(sys.argv) > 6 else None + + arrivals = load_arrivals(arrivals_path) + latencies = [] + extrapolated_trials = [] + incomplete = [] + non_202 = [] + + for line in open(sends_path): + line = line.strip() + if not line: + continue + send = json.loads(line) + if run_filter and send.get("run_id") != run_filter: + continue + if send.get("http_status") != 202: + non_202.append((send["correlation_id"], send.get("http_status"), send.get("error"))) + continue + + observed = [arrivals.get(path) for path in send["paths"]] + if any(value is None for value in observed): + missing = [p for p, v in zip(send["paths"], observed) if v is None] + incomplete.append((send["correlation_id"], len(missing), len(send["paths"]))) + continue + + # Change set is complete when its last file lands. + completion_ns = max(observed) + # Leg A: sender -> server. In this topology the server runs on the + # sender's own machine, so this leg is LOOPBACK, not network. + # Leg B: server -> receiver mount. This is the only leg that crosses + # the Tailscale LAN, and it is the one a cloud deployment would + # replace with a WAN path. + trial_offset_ns, extrapolated = offset_at(send["t_send_ns"], pre, post) + if extrapolated: + extrapolated_trials.append(send["correlation_id"]) + corrected_arrival_ns = completion_ns - trial_offset_ns + latencies.append( + { + "correlation_id": send["correlation_id"], + "files": len(send["paths"]), + "bytes": send["expected_bytes"], + "offset_applied_ms": trial_offset_ns / 1e6, + "leg_a_loopback_ms": send["ack_ms"], + "leg_b_lan_ms": (corrected_arrival_ns - send["t_ack_ns"]) / 1e6, + "latency_ms": (corrected_arrival_ns - send["t_send_ns"]) / 1e6, + } + ) + + values = sorted(item["latency_ms"] for item in latencies) + acks = sorted(item["leg_a_loopback_ms"] for item in latencies) + legs_b = sorted(item["leg_b_lan_ms"] for item in latencies) + summary = { + "label": label, + "clock_offset_pre_ms": pre[1] / 1e6, + "clock_offset_post_ms": post[1] / 1e6, + "clock_drift_across_run_ms": (post[1] - pre[1]) / 1e6, + "clock_correction": "linearly interpolated to each trial's send time", + "trials_extrapolated_outside_anchors": extrapolated_trials, + "trials_sent": len(latencies) + len(incomplete) + len(non_202), + "trials_complete": len(values), + "trials_incomplete": len(incomplete), + "trials_non_202": len(non_202), + "incomplete_detail": incomplete, + "non_202_detail": non_202, + "latency_ms": { + "min": values[0] if values else None, + "median": percentile(values, 0.50), + "p95": percentile(values, 0.95), + "max": values[-1] if values else None, + }, + "leg_a_sender_to_server_loopback_ms": { + "median": percentile(acks, 0.50), + "p95": percentile(acks, 0.95), + }, + "leg_b_server_to_receiver_lan_ms": { + "median": percentile(legs_b, 0.50), + "p95": percentile(legs_b, 0.95), + }, + "per_trial": latencies, + } + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/docs/evidence/mount-latency-20260807/harness/assertions.py b/docs/evidence/mount-latency-20260807/harness/assertions.py new file mode 100644 index 00000000..9ca08e06 --- /dev/null +++ b/docs/evidence/mount-latency-20260807/harness/assertions.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Named assertions gating the 2026-08-07 mount latency result. + +Every claim in RESULTS.md is checked here against the raw evidence, so the +result cannot drift from the data it came from. Run from the evidence +directory. Exit status is non-zero if any assertion fails. + +Deliberately, this fails the run rather than downgrading it: if a gate does not +pass there is no median to publish, only a blocker. +""" + +import json +import os +import subprocess +import sys + +RESULTS = [] + + +def check(name, condition, detail): + RESULTS.append((name, bool(condition), detail)) + + +def load(path): + return [json.loads(line) for line in open(path) if line.strip()] + + +def heartbeat_gate(path, label): + """sf-mini is live iff its OWN lastHeartbeatAt advances across >=90s.""" + if not os.path.exists(path): + check(f"{label}_gate_present", False, f"{path} missing") + return + samples = load(path) + stamps = [s["node"].get("lastHeartbeatAt") for s in samples] + distinct = [s for i, s in enumerate(stamps) if s and (i == 0 or s != stamps[i - 1])] + span = None + if len(samples) >= 2: + first, last = samples[0]["sampledAtLocalUtc"], samples[-1]["sampledAtLocalUtc"] + span = (first, last) + + check( + f"{label}_window_at_least_90s", + len(samples) >= 9, + f"{len(samples)} samples at ~10s spacing spanning {span}", + ) + check( + f"{label}_heartbeat_advanced", + len(distinct) >= 2, + f"{len(distinct)} distinct heartbeat values: {distinct}", + ) + check( + f"{label}_node_present_every_sample", + all(s["node"].get("present") for s in samples), + "absence from a listing would not have been treated as offline anyway", + ) + + +def main(): + # --- liveness gates, before and after ------------------------------- + heartbeat_gate("raw/heartbeat-gate-pre.jsonl", "heartbeat_pre") + heartbeat_gate("raw/heartbeat-gate-post.jsonl", "heartbeat_post") + + # --- clock offset measured, not assumed ----------------------------- + offsets = {} + for label in ("pre", "post"): + path = f"raw/clock-offset-{label}.jsonl" + if not os.path.exists(path): + check(f"clock_offset_{label}_present", False, f"{path} missing") + continue + samples = [r for r in load(path) if "delay_ns" in r] + check( + f"clock_offset_{label}_sampled", + len(samples) >= 50, + f"{len(samples)} NTP-style exchanges", + ) + best = min(samples, key=lambda r: r["delay_ns"]) + offsets[label] = best["offset_ns"] + check( + f"clock_offset_{label}_uncertainty_under_5ms", + best["delay_ns"] / 2e6 < 5.0, + f"min delay {best['delay_ns']/1e6:.3f} ms -> +/-{best['delay_ns']/2e6:.3f} ms", + ) + + if "pre" in offsets and "post" in offsets: + drift_ms = abs(offsets["post"] - offsets["pre"]) / 1e6 + check( + "clock_drift_small_vs_signal", + drift_ms < 10.0, + f"offset moved {drift_ms:.3f} ms across the run", + ) + check( + "clocks_were_not_equal", + abs(offsets["pre"]) > 1e6, + f"pre-run offset {offsets['pre']/1e6:.3f} ms — assuming equal clocks " + f"would have been wrong by this much", + ) + + # --- trial counts and completeness ---------------------------------- + # Headline statistics come from the clean batch "r2" only. The earlier + # batch was cut short by an interrupt and is retained as correctness + # evidence, not as a source of percentiles. + CLEAN_BATCH = "r2" + summaries = {} + for shape, sends in (("small", "raw/trials-small.jsonl"), ("repo", "raw/trials-repo.jsonl")): + if not os.path.exists(sends): + check(f"{shape}_trials_present", False, f"{sends} missing") + continue + output = subprocess.run( + [sys.executable, "harness/analyse.py", + "raw/clock-offset-pre.jsonl", "raw/clock-offset-post.jsonl", sends, + "raw/mount-watch.jsonl", shape, CLEAN_BATCH], + capture_output=True, text=True, + ) + summary = json.loads(output.stdout) + summaries[shape] = summary + check( + f"{shape}_at_least_20_complete_trials", + summary["trials_complete"] >= 20, + f"{summary['trials_complete']} complete of {summary['trials_sent']} sent", + ) + check( + f"{shape}_no_lost_changes", + summary["trials_incomplete"] == 0, + f"incomplete: {summary['incomplete_detail']}", + ) + check( + f"{shape}_all_writes_accepted", + summary["trials_non_202"] == 0, + f"non-202: {summary['non_202_detail']}", + ) + check( + f"{shape}_clock_correction_interpolated_not_pinned", + summary["trials_extrapolated_outside_anchors"] == [], + f"offset moved {summary['clock_drift_across_run_ms']:.3f} ms across " + f"the run; every trial interpolated within the anchor span", + ) + check( + f"{shape}_no_sample_hit_the_polling_fallback", + (summary["latency_ms"]["max"] or 0) < 30_000, + f"max {summary['latency_ms']['max']:.1f} ms; the websocket-off " + f"reconcile safety net is ~30 s and would be obvious here", + ) + + # --- the measurement-overhead claim --------------------------------- + if os.path.exists("raw/control-watch.jsonl") and os.path.exists("raw/control-create.jsonl"): + observed = {} + for record in load("raw/control-watch.jsonl"): + if record.get("kind"): + continue + observed.setdefault(record["path"], record["observed_ns"]) + delays = sorted( + (observed[c["path"]] - c["t_create_ns"]) / 1e6 + for c in load("raw/control-create.jsonl") + if c["path"] in observed + ) + check("control_paired_at_least_20", len(delays) >= 20, f"{len(delays)} pairs") + overhead_median = delays[len(delays) // 2] + check( + "watcher_overhead_measured_not_assumed", + len(delays) > 0, + f"watcher detection delay median {overhead_median:.3f} ms, " + f"max {delays[-1]:.3f} ms", + ) + if "small" in summaries and summaries["small"]["latency_ms"]["median"]: + signal = summaries["small"]["latency_ms"]["median"] + check( + "overhead_does_not_exceed_signal", + overhead_median < signal, + f"overhead {overhead_median:.3f} ms vs signal {signal:.1f} ms — the " + f"existing public wording asserts the opposite", + ) + + # --- isolation ------------------------------------------------------ + status = subprocess.run( + ["git", "status", "--porcelain", ".dev-collab-stack", ".salvaged-from-minis"], + capture_output=True, text=True, cwd="../../..", + ).stdout.strip().splitlines() + disturbed = [line for line in status if not line.startswith("??")] + check( + "preexisting_untracked_dirs_untouched", + not disturbed, + f"{len(status)} entries, all still untracked: {[s[:3] for s in status]}", + ) + + # --- report --------------------------------------------------------- + width = max(len(name) for name, _, _ in RESULTS) + failed = 0 + for name, passed, detail in RESULTS: + marker = "PASS" if passed else "FAIL" + failed += 0 if passed else 1 + print(f"{marker} {name.ljust(width)} {detail}") + print(f"\n{len(RESULTS) - failed}/{len(RESULTS)} assertions passed") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/evidence/mount-latency-20260807/harness/clock-offset.py b/docs/evidence/mount-latency-20260807/harness/clock-offset.py new file mode 100644 index 00000000..bcd45f3f --- /dev/null +++ b/docs/evidence/mount-latency-20260807/harness/clock-offset.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Measure the realtime-clock offset between two hosts over the Tailscale LAN. + +We must NOT assume the sender and receiver clocks agree. A one-way latency of +order 100ms is smaller than the drift two unsynchronised macOS hosts can +accumulate, so an uncorrected cross-host subtraction can produce a negative or +absurd "latency". + +Method: NTP's four-timestamp formula over a raw TCP exchange on the Tailscale +LAN, which keeps the round trip in the low-single-digit-millisecond range (an +ssh-based comparison would have an RTT far larger than the signal). + + t0 = client realtime before send + t1 = server realtime on receipt (server sends t1 and t2) + t2 = server realtime before reply + t3 = client realtime after receive + + delay = (t3 - t0) - (t2 - t1) # round-trip minus server think time + offset = ((t1 - t0) + (t2 - t3)) / 2 # server_clock - client_clock + +Both formulas assume a symmetric path. That assumption is weakest when the +network is congested, so we take many samples and select the one with the +SMALLEST delay -- the least-queued sample is the least asymmetric. The residual +uncertainty on that offset is bounded by +/- delay/2, which we report so the +final latency number can be quoted with an honest error bar. + +Usage: + clock-offset.py server [BIND_HOST] [PORT] + clock-offset.py client HOST PORT SAMPLES OUTPUT_JSONL +""" + +import json +import socket +import sys +import time + + +def serve(bind_host, port): + """Reply to each probe with receipt and reply realtime timestamps.""" + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind((bind_host, port)) + listener.listen(8) + sys.stderr.write(f"clock-offset server listening on {bind_host}:{port}\n") + sys.stderr.flush() + + while True: + connection, _ = listener.accept() + connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + with connection: + stream = connection.makefile("rwb") + for line in stream: + if not line.strip(): + continue + receipt_ns = time.time_ns() + if line.strip() == b"QUIT": + return + reply_ns = time.time_ns() + stream.write(f"{receipt_ns} {reply_ns}\n".encode()) + stream.flush() + + +def measure(host, port, samples, output_path): + """Take `samples` NTP-style exchanges and keep the minimum-delay estimate.""" + connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + connection.settimeout(10) + connection.connect((host, port)) + stream = connection.makefile("rwb") + + observations = [] + with open(output_path, "a") as raw: + for index in range(samples): + t0 = time.time_ns() + stream.write(b"PROBE\n") + stream.flush() + reply = stream.readline() + t3 = time.time_ns() + if not reply: + raise RuntimeError("clock-offset server closed the connection early") + + t1_text, t2_text = reply.split() + t1 = int(t1_text) + t2 = int(t2_text) + + delay_ns = (t3 - t0) - (t2 - t1) + offset_ns = ((t1 - t0) + (t2 - t3)) // 2 + observation = { + "sample": index + 1, + "t0_client_ns": t0, + "t1_server_recv_ns": t1, + "t2_server_send_ns": t2, + "t3_client_ns": t3, + "delay_ns": delay_ns, + "offset_ns": offset_ns, + } + observations.append(observation) + raw.write(json.dumps(observation) + "\n") + raw.flush() + time.sleep(0.005) + + stream.write(b"QUIT\n") + stream.flush() + connection.close() + + best = min(observations, key=lambda obs: obs["delay_ns"]) + offsets = sorted(obs["offset_ns"] for obs in observations) + summary = { + "kind": "clock_offset_summary", + "host": host, + "samples": len(observations), + # server_clock - client_clock, from the least-queued exchange + "offset_ns": best["offset_ns"], + "offset_ms": best["offset_ns"] / 1e6, + "min_delay_ns": best["delay_ns"], + "min_delay_ms": best["delay_ns"] / 1e6, + # symmetry assumption residual: the true offset lies within +/- delay/2 + "uncertainty_ms": best["delay_ns"] / 2e6, + "offset_median_ms": offsets[len(offsets) // 2] / 1e6, + "offset_spread_ms": (offsets[-1] - offsets[0]) / 1e6, + "measured_at_client_utc": time.strftime( + "%Y-%m-%dT%H:%M:%SZ", time.gmtime() + ), + } + with open(output_path, "a") as raw: + raw.write(json.dumps(summary) + "\n") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + if len(sys.argv) >= 2 and sys.argv[1] == "server": + serve( + sys.argv[2] if len(sys.argv) > 2 else "0.0.0.0", + int(sys.argv[3]) if len(sys.argv) > 3 else 19299, + ) + elif len(sys.argv) == 6 and sys.argv[1] == "client": + measure(sys.argv[2], int(sys.argv[3]), int(sys.argv[4]), sys.argv[5]) + else: + sys.stderr.write(__doc__) + sys.exit(2) diff --git a/docs/evidence/mount-latency-20260807/harness/control-local.py b/docs/evidence/mount-latency-20260807/harness/control-local.py new file mode 100644 index 00000000..e0c6f9cb --- /dev/null +++ b/docs/evidence/mount-latency-20260807/harness/control-local.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Control experiment: how much of the measured latency is the watcher itself? + +Runs entirely on the RECEIVER host. It creates files locally -- same clock, +same watcher code, same filesystem, no network anywhere -- and records the +creation timestamp. Pairing those against the watcher's observation timestamps +gives the watcher's own detection delay distribution. + +This is what turns "measurement overhead that exceeds the signal" from an +assumption into a measured quantity that can be subtracted or dismissed. + +Files are published with an atomic rename, matching how the mount daemon +materialises remote content (`writeFileAtomic`, syncer.go:8285), so the +control exercises the same visibility transition the real trials do. + +Usage: + control-local.py CONTROL_DIR RUN_ID COUNT SPACING_SECONDS OUT_JSONL +""" + +import json +import os +import sys +import time + + +def main(): + control_dir, run_id, count, spacing, out_path = sys.argv[1:6] + count = int(count) + spacing = float(spacing) + os.makedirs(control_dir, exist_ok=True) + + with open(out_path, "a") as raw: + for trial in range(1, count + 1): + directory = os.path.join(control_dir, f"control-{trial:03d}") + os.makedirs(directory, exist_ok=True) + final_path = os.path.join(directory, "probe.txt") + temporary_path = final_path + ".tmp" + content = (f"control={run_id}-{trial:03d} ").ljust(300, "x") + + with open(temporary_path, "w") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + + # The instant the path becomes visible, by the same mechanism the + # mount daemon uses. + t_create_ns = time.time_ns() + os.rename(temporary_path, final_path) + + record = { + "kind": "control_create", + "run_id": run_id, + "trial": trial, + "path": os.path.relpath(final_path, control_dir), + "t_create_ns": t_create_ns, + "host": "sf-mini", + "clock": "CLOCK_REALTIME", + } + raw.write(json.dumps(record) + "\n") + raw.flush() + if trial < count: + time.sleep(spacing) + + +if __name__ == "__main__": + main() diff --git a/docs/evidence/mount-latency-20260807/harness/dev-authd.py b/docs/evidence/mount-latency-20260807/harness/dev-authd.py new file mode 100644 index 00000000..544f0174 --- /dev/null +++ b/docs/evidence/mount-latency-20260807/harness/dev-authd.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Minimal RS256 token issuer + JWKS endpoint for the latency measurement run. + +The relayfile server verifies bearer tokens as RS256 against a JWKS document +(`internal/httpapi/auth.go:203-215`, `:262-276`). Rather than boot the full +relayauth monorepo for a measurement run, this issues an ephemeral keypair and +serves the matching JWKS, so the whole auth dependency is one self-contained, +reproducible file. + +The private key is written OUTSIDE the repository (a scratch directory passed +in by the caller) and is a throwaway credential scoped to this run only. It is +never committed and never leaves the two hosts. + +Usage: + dev-authd.py serve KEY_DIR BIND_HOST PORT + dev-authd.py mint KEY_DIR WORKSPACE_ID AGENT_NAME TTL_SECONDS +""" + +import base64 +import hashlib +import http.server +import json +import os +import sys +import time + +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import padding, rsa + +# Scopes the mount daemon needs, mirroring scripts/generate-dev-token.sh. +SCOPES = [ + "fs:read", + "fs:write", + "sync:read", + "sync:trigger", + "ops:read", + "ops:replay", + "admin:read", + "admin:replay", +] + + +def b64url(raw: bytes) -> str: + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def load_or_create_key(key_dir): + """Return (private_key, kid), generating a fresh keypair on first use.""" + os.makedirs(key_dir, mode=0o700, exist_ok=True) + key_path = os.path.join(key_dir, "latency-run-key.pem") + if os.path.exists(key_path): + with open(key_path, "rb") as handle: + private_key = serialization.load_pem_private_key(handle.read(), password=None) + else: + private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + # 0600 before any bytes land on disk. + descriptor = os.open(key_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(descriptor, "wb") as handle: + handle.write(pem) + + numbers = private_key.public_key().public_numbers() + modulus = numbers.n.to_bytes((numbers.n.bit_length() + 7) // 8, "big") + kid = hashlib.sha256(modulus).hexdigest()[:16] + return private_key, kid + + +def jwks_document(private_key, kid): + numbers = private_key.public_key().public_numbers() + modulus = numbers.n.to_bytes((numbers.n.bit_length() + 7) // 8, "big") + exponent = numbers.e.to_bytes((numbers.e.bit_length() + 7) // 8, "big") + return { + "keys": [ + { + "kid": kid, + "kty": "RSA", + "alg": "RS256", + "use": "sig", + "n": b64url(modulus), + "e": b64url(exponent), + } + ] + } + + +def mint(key_dir, workspace_id, agent_name, ttl_seconds): + private_key, kid = load_or_create_key(key_dir) + header = {"alg": "RS256", "typ": "JWT", "kid": kid} + now = int(time.time()) + payload = { + "wks": workspace_id, + "workspace_id": workspace_id, + "sub": agent_name, + "agent_name": agent_name, + "aud": ["relayfile"], + "iat": now, + "exp": now + int(ttl_seconds), + "scopes": SCOPES, + } + signing_input = ( + b64url(json.dumps(header, separators=(",", ":")).encode()) + + "." + + b64url(json.dumps(payload, separators=(",", ":")).encode()) + ) + signature = private_key.sign( + signing_input.encode(), padding.PKCS1v15(), hashes.SHA256() + ) + print(signing_input + "." + b64url(signature)) + + +def serve(key_dir, bind_host, port): + private_key, kid = load_or_create_key(key_dir) + document = json.dumps(jwks_document(private_key, kid)).encode() + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + if self.path.startswith("/.well-known/jwks.json"): + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(document))) + self.end_headers() + self.wfile.write(document) + else: + self.send_response(404) + self.end_headers() + + def log_message(self, *args): + pass # keep the trial logs clean + + sys.stderr.write(f"dev-authd JWKS on {bind_host}:{port} kid={kid}\n") + sys.stderr.flush() + http.server.HTTPServer((bind_host, port), Handler).serve_forever() + + +if __name__ == "__main__": + if len(sys.argv) == 5 and sys.argv[1] == "serve": + serve(sys.argv[2], sys.argv[3], int(sys.argv[4])) + elif len(sys.argv) == 6 and sys.argv[1] == "mint": + mint(sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5]) + else: + sys.stderr.write(__doc__) + sys.exit(2) diff --git a/docs/evidence/mount-latency-20260807/harness/receiver-watch.py b/docs/evidence/mount-latency-20260807/harness/receiver-watch.py new file mode 100644 index 00000000..67797d09 --- /dev/null +++ b/docs/evidence/mount-latency-20260807/harness/receiver-watch.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""In-box resident watcher: records file arrival times on the RECEIVER host. + +Runs on sf-mini, inside the box, and timestamps with sf-mini's own +`CLOCK_REALTIME` (`time.time_ns()`). Nothing is timestamped over ssh, because +an ssh round trip would be added to every sample and would dwarf the signal. + +The mount is a synced mirror of real local files (poll mode writes bytes to +disk via `writeFileAtomic`, `internal/mountsync/syncer.go:6033-6051`), so each +probe below is a purely local syscall with no network hop. That is what makes +an in-box watcher a measure of propagation rather than of its own fetch. + +Because the daemon publishes each file with an atomic rename, a path becoming +visible already has its complete content -- there is no torn-read window. The +recorded size is emitted anyway so the analyser can assert it. + +Output is one JSON line per newly observed file, appended and flushed +IMMEDIATELY so that a host or tool failure mid-run leaves usable raw evidence +rather than nothing. + +Usage: + receiver-watch.py WATCH_DIR OUTPUT_JSONL DURATION_SECONDS [POLL_SECONDS] +""" + +import json +import os +import sys +import time + + +def scan(root, seen, output, loop_periods): + """Record every path under `root` not already in `seen`.""" + stack = [root] + while stack: + current = stack.pop() + try: + with os.scandir(current) as entries: + for entry in entries: + try: + if entry.is_dir(follow_symlinks=False): + # .relay holds mount bookkeeping, not workspace + # content; watching it would time our own daemon's + # state writes instead of the payload. + if entry.name != ".relay": + stack.append(entry.path) + continue + if entry.path in seen: + continue + observed_ns = time.time_ns() + seen.add(entry.path) + try: + size = entry.stat(follow_symlinks=False).st_size + except OSError: + size = None + record = { + "path": os.path.relpath(entry.path, root), + "observed_ns": observed_ns, + "size": size, + "host": "sf-mini", + "clock": "CLOCK_REALTIME", + } + output.write(json.dumps(record) + "\n") + output.flush() + except OSError: + # A file can vanish or be mid-rename between the + # directory listing and the stat; skip and re-see it + # on the next pass. + continue + except OSError: + continue + + +def main(): + watch_dir = sys.argv[1] + output_path = sys.argv[2] + duration = float(sys.argv[3]) + poll_seconds = float(sys.argv[4]) if len(sys.argv) > 4 else 0.001 + + os.makedirs(watch_dir, exist_ok=True) + seen = set() + loop_periods = [] + + # Prime `seen` with whatever already exists, so pre-existing content is not + # reported as an arrival. + with open(os.devnull, "w") as sink: + scan(watch_dir, seen, sink, loop_periods) + + started = time.monotonic() + with open(output_path, "a") as output: + output.write( + json.dumps( + { + "kind": "watcher_started", + "watch_dir": watch_dir, + "poll_seconds": poll_seconds, + "primed_paths": len(seen), + "started_ns": time.time_ns(), + } + ) + + "\n" + ) + output.flush() + + while time.monotonic() - started < duration: + loop_start = time.monotonic() + scan(watch_dir, seen, output, loop_periods) + loop_periods.append(time.monotonic() - loop_start) + time.sleep(poll_seconds) + + # The scan cost bounds detection granularity; report it rather than + # assume the poll interval alone is the quantisation. + loop_periods.sort() + output.write( + json.dumps( + { + "kind": "watcher_finished", + "scans": len(loop_periods), + "scan_ms_median": loop_periods[len(loop_periods) // 2] * 1e3 + if loop_periods + else None, + "scan_ms_p95": loop_periods[int(len(loop_periods) * 0.95)] * 1e3 + if loop_periods + else None, + "scan_ms_max": loop_periods[-1] * 1e3 if loop_periods else None, + "finished_ns": time.time_ns(), + } + ) + + "\n" + ) + output.flush() + + +if __name__ == "__main__": + main() diff --git a/docs/evidence/mount-latency-20260807/harness/sender-trials.py b/docs/evidence/mount-latency-20260807/harness/sender-trials.py new file mode 100644 index 00000000..10da755c --- /dev/null +++ b/docs/evidence/mount-latency-20260807/harness/sender-trials.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Sender side of the one-way latency run. Runs on the laptop (sender host). + +Each trial writes a UNIQUE path, so no trial can be satisfied by a previous +trial's content and none can be collapsed into another by any coalescing. + +`t_send_ns` is taken on this host's CLOCK_REALTIME immediately before the HTTP +request is issued -- that is the instant a writing agent "has written". The +receiver's watcher timestamps arrival on its own clock; the analyser subtracts +the separately measured clock offset. `t_ack_ns` is recorded too, so the +server-side write leg can be separated from the propagation leg. + +Raw records are appended and flushed IMMEDIATELY after each trial so that a +crash mid-run leaves usable evidence. + +Trial shapes: + small -- one ~300 byte file. Isolates propagation with transfer time ~0. + repo -- 11 files / ~14 KB, the p75 change set in this repository's own + history (median 6 files/105 lines, p75 11 files/357 lines over the + last 300 non-merge commits). A number quoted from this shape is one + a reader can expect from real agent work. + +Usage: + sender-trials.py SHAPE SERVER WORKSPACE TOKEN_FILE RUN_ID COUNT SPACING_S OUT_JSONL +""" + +import json +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +SMALL_BYTES = 300 +REPO_FILE_COUNT = 11 +REPO_TOTAL_BYTES = 14_000 + + +def body_for(shape, run_id, trial): + """Return (paths, request_path, payload) for one trial.""" + if shape == "small": + path = f"/trials/{run_id}/small-{trial:03d}/probe.txt" + # Deterministic filler; unique per trial so content hashes differ. + content = f"trial={run_id}-{trial:03d} " .ljust(SMALL_BYTES, "x") + return ( + [path], + "fs/file", + {"path": path, "content": content}, + ) + + per_file = REPO_TOTAL_BYTES // REPO_FILE_COUNT + files = [] + paths = [] + for index in range(REPO_FILE_COUNT): + path = f"/trials/{run_id}/repo-{trial:03d}/src/module_{index:02d}.go" + header = f"// trial={run_id}-{trial:03d} file={index:02d}\npackage main\n" + content = header + ("// filler line to reach a realistic size\n" * 1000) + content = content[:per_file] + paths.append(path) + files.append( + {"path": path, "contentType": "text/plain", "content": content} + ) + return paths, "fs/bulk", {"files": files} + + +def request(server, workspace, endpoint, payload, token, correlation_id): + url = f"{server}/v1/workspaces/{workspace}/{endpoint}" + method = "POST" + if endpoint == "fs/file": + url += "?path=" + urllib.parse.quote(payload.pop("path"), safe="") + method = "PUT" + encoded = json.dumps(payload).encode() + req = urllib.request.Request(url, data=encoded, method=method) + req.add_header("Authorization", f"Bearer {token}") + req.add_header("Content-Type", "application/json") + req.add_header("X-Correlation-Id", correlation_id) + if method == "PUT": + req.add_header("If-Match", "*") + with urllib.request.urlopen(req, timeout=30) as response: + return response.status, response.read() + + +def main(): + ( + shape, + server, + workspace, + token_file, + run_id, + count, + spacing, + out_path, + ) = sys.argv[1:9] + count = int(count) + spacing = float(spacing) + with open(token_file) as handle: + token = handle.read().strip() + + with open(out_path, "a") as raw: + for trial in range(1, count + 1): + paths, endpoint, payload = body_for(shape, run_id, trial) + correlation_id = f"{run_id}-{shape}-{trial:03d}" + expected_bytes = sum( + len(f["content"]) for f in payload.get("files", []) + ) or len(payload.get("content", "")) + + t_send_ns = time.time_ns() + try: + status, _ = request( + server, workspace, endpoint, dict(payload), token, correlation_id + ) + error = None + except urllib.error.HTTPError as exc: + status, error = exc.code, exc.read().decode()[:400] + except Exception as exc: # noqa: BLE001 - record, never lose a trial + status, error = None, repr(exc) + t_ack_ns = time.time_ns() + + record = { + "kind": "send", + "shape": shape, + "run_id": run_id, + "trial": trial, + "correlation_id": correlation_id, + "paths": paths, + "expected_bytes": expected_bytes, + "t_send_ns": t_send_ns, + "t_ack_ns": t_ack_ns, + "ack_ms": (t_ack_ns - t_send_ns) / 1e6, + "http_status": status, + "error": error, + "host": "khaliqs-macbook-pro", + "clock": "CLOCK_REALTIME", + } + raw.write(json.dumps(record) + "\n") + raw.flush() + print( + f"{shape} trial {trial}/{count} status={status} " + f"ack={record['ack_ms']:.1f}ms" + ) + if trial < count: + time.sleep(spacing) + + +if __name__ == "__main__": + main() diff --git a/docs/evidence/mount-latency-20260807/raw/clock-offset-post.jsonl b/docs/evidence/mount-latency-20260807/raw/clock-offset-post.jsonl new file mode 100644 index 00000000..3e6866b4 --- /dev/null +++ b/docs/evidence/mount-latency-20260807/raw/clock-offset-post.jsonl @@ -0,0 +1,201 @@ +{"sample": 1, "t0_client_ns": 1786102163116809000, "t1_server_recv_ns": 1786102163106072000, "t2_server_send_ns": 1786102163106079000, "t3_client_ns": 1786102163124000000, "delay_ns": 7184000, "offset_ns": -14329000} +{"sample": 2, "t0_client_ns": 1786102163131068000, "t1_server_recv_ns": 1786102163118757000, "t2_server_send_ns": 1786102163118759000, "t3_client_ns": 1786102163138773000, "delay_ns": 7703000, "offset_ns": -16162500} +{"sample": 3, "t0_client_ns": 1786102163161592000, "t1_server_recv_ns": 1786102163150541000, "t2_server_send_ns": 1786102163150543000, "t3_client_ns": 1786102163171108000, "delay_ns": 9514000, "offset_ns": -15808000} +{"sample": 4, "t0_client_ns": 1786102163179815000, "t1_server_recv_ns": 1786102163167704000, "t2_server_send_ns": 1786102163167706000, "t3_client_ns": 1786102163185724000, "delay_ns": 5907000, "offset_ns": -15064500} +{"sample": 5, "t0_client_ns": 1786102163209046000, "t1_server_recv_ns": 1786102163196964000, "t2_server_send_ns": 1786102163196966000, "t3_client_ns": 1786102163214182000, "delay_ns": 5134000, "offset_ns": -14649000} +{"sample": 6, "t0_client_ns": 1786102163237742000, "t1_server_recv_ns": 1786102163225610000, "t2_server_send_ns": 1786102163225611000, "t3_client_ns": 1786102163242721000, "delay_ns": 4978000, "offset_ns": -14621000} +{"sample": 7, "t0_client_ns": 1786102163265631000, "t1_server_recv_ns": 1786102163253155000, "t2_server_send_ns": 1786102163253157000, "t3_client_ns": 1786102163270210000, "delay_ns": 4577000, "offset_ns": -14764500} +{"sample": 8, "t0_client_ns": 1786102163293986000, "t1_server_recv_ns": 1786102163281863000, "t2_server_send_ns": 1786102163281864000, "t3_client_ns": 1786102163299101000, "delay_ns": 5114000, "offset_ns": -14680000} +{"sample": 9, "t0_client_ns": 1786102163324180000, "t1_server_recv_ns": 1786102163312252000, "t2_server_send_ns": 1786102163312253000, "t3_client_ns": 1786102163329959000, "delay_ns": 5778000, "offset_ns": -14817000} +{"sample": 10, "t0_client_ns": 1786102163352748000, "t1_server_recv_ns": 1786102163340540000, "t2_server_send_ns": 1786102163340541000, "t3_client_ns": 1786102163357799000, "delay_ns": 5050000, "offset_ns": -14733000} +{"sample": 11, "t0_client_ns": 1786102163382257000, "t1_server_recv_ns": 1786102163370148000, "t2_server_send_ns": 1786102163370149000, "t3_client_ns": 1786102163387517000, "delay_ns": 5259000, "offset_ns": -14738500} +{"sample": 12, "t0_client_ns": 1786102163411719000, "t1_server_recv_ns": 1786102163400469000, "t2_server_send_ns": 1786102163400470000, "t3_client_ns": 1786102163417627000, "delay_ns": 5907000, "offset_ns": -14203500} +{"sample": 13, "t0_client_ns": 1786102163432261000, "t1_server_recv_ns": 1786102163420439000, "t2_server_send_ns": 1786102163420440000, "t3_client_ns": 1786102163437997000, "delay_ns": 5735000, "offset_ns": -14689500} +{"sample": 14, "t0_client_ns": 1786102163463167000, "t1_server_recv_ns": 1786102163451014000, "t2_server_send_ns": 1786102163451016000, "t3_client_ns": 1786102163468147000, "delay_ns": 4978000, "offset_ns": -14642000} +{"sample": 15, "t0_client_ns": 1786102163489159000, "t1_server_recv_ns": 1786102163477047000, "t2_server_send_ns": 1786102163477048000, "t3_client_ns": 1786102163495090000, "delay_ns": 5930000, "offset_ns": -15077000} +{"sample": 16, "t0_client_ns": 1786102163518599000, "t1_server_recv_ns": 1786102163506189000, "t2_server_send_ns": 1786102163506190000, "t3_client_ns": 1786102163523397000, "delay_ns": 4797000, "offset_ns": -14808500} +{"sample": 17, "t0_client_ns": 1786102163541912000, "t1_server_recv_ns": 1786102163529727000, "t2_server_send_ns": 1786102163529728000, "t3_client_ns": 1786102163546849000, "delay_ns": 4936000, "offset_ns": -14653000} +{"sample": 18, "t0_client_ns": 1786102163561723000, "t1_server_recv_ns": 1786102163549774000, "t2_server_send_ns": 1786102163549776000, "t3_client_ns": 1786102163566933000, "delay_ns": 5208000, "offset_ns": -14553000} +{"sample": 19, "t0_client_ns": 1786102163590346000, "t1_server_recv_ns": 1786102163578316000, "t2_server_send_ns": 1786102163578317000, "t3_client_ns": 1786102163595758000, "delay_ns": 5411000, "offset_ns": -14735500} +{"sample": 20, "t0_client_ns": 1786102163607659000, "t1_server_recv_ns": 1786102163595460000, "t2_server_send_ns": 1786102163595461000, "t3_client_ns": 1786102163613008000, "delay_ns": 5348000, "offset_ns": -14873000} +{"sample": 21, "t0_client_ns": 1786102163626245000, "t1_server_recv_ns": 1786102163614103000, "t2_server_send_ns": 1786102163614105000, "t3_client_ns": 1786102163631290000, "delay_ns": 5043000, "offset_ns": -14663500} +{"sample": 22, "t0_client_ns": 1786102163643248000, "t1_server_recv_ns": 1786102163631317000, "t2_server_send_ns": 1786102163631319000, "t3_client_ns": 1786102163648637000, "delay_ns": 5387000, "offset_ns": -14624500} +{"sample": 23, "t0_client_ns": 1786102163670878000, "t1_server_recv_ns": 1786102163659807000, "t2_server_send_ns": 1786102163659808000, "t3_client_ns": 1786102163677027000, "delay_ns": 6148000, "offset_ns": -14145000} +{"sample": 24, "t0_client_ns": 1786102163699550000, "t1_server_recv_ns": 1786102163687532000, "t2_server_send_ns": 1786102163687534000, "t3_client_ns": 1786102163704697000, "delay_ns": 5145000, "offset_ns": -14590500} +{"sample": 25, "t0_client_ns": 1786102163710256000, "t1_server_recv_ns": 1786102163698111000, "t2_server_send_ns": 1786102163698113000, "t3_client_ns": 1786102163715290000, "delay_ns": 5032000, "offset_ns": -14661000} +{"sample": 26, "t0_client_ns": 1786102163732276000, "t1_server_recv_ns": 1786102163720253000, "t2_server_send_ns": 1786102163720256000, "t3_client_ns": 1786102163737397000, "delay_ns": 5118000, "offset_ns": -14582000} +{"sample": 27, "t0_client_ns": 1786102163760930000, "t1_server_recv_ns": 1786102163748712000, "t2_server_send_ns": 1786102163748715000, "t3_client_ns": 1786102163765956000, "delay_ns": 5023000, "offset_ns": -14729500} +{"sample": 28, "t0_client_ns": 1786102163780082000, "t1_server_recv_ns": 1786102163768228000, "t2_server_send_ns": 1786102163768230000, "t3_client_ns": 1786102163785462000, "delay_ns": 5378000, "offset_ns": -14543000} +{"sample": 29, "t0_client_ns": 1786102163799785000, "t1_server_recv_ns": 1786102163787676000, "t2_server_send_ns": 1786102163787677000, "t3_client_ns": 1786102163804897000, "delay_ns": 5111000, "offset_ns": -14664500} +{"sample": 30, "t0_client_ns": 1786102163827337000, "t1_server_recv_ns": 1786102163815167000, "t2_server_send_ns": 1786102163815169000, "t3_client_ns": 1786102163835827000, "delay_ns": 8488000, "offset_ns": -16414000} +{"sample": 31, "t0_client_ns": 1786102163851093000, "t1_server_recv_ns": 1786102163838936000, "t2_server_send_ns": 1786102163838937000, "t3_client_ns": 1786102163856134000, "delay_ns": 5040000, "offset_ns": -14677000} +{"sample": 32, "t0_client_ns": 1786102163879524000, "t1_server_recv_ns": 1786102163867515000, "t2_server_send_ns": 1786102163867516000, "t3_client_ns": 1786102163884741000, "delay_ns": 5216000, "offset_ns": -14617000} +{"sample": 33, "t0_client_ns": 1786102163907948000, "t1_server_recv_ns": 1786102163895867000, "t2_server_send_ns": 1786102163895868000, "t3_client_ns": 1786102163913278000, "delay_ns": 5329000, "offset_ns": -14745500} +{"sample": 34, "t0_client_ns": 1786102163936603000, "t1_server_recv_ns": 1786102163925058000, "t2_server_send_ns": 1786102163925059000, "t3_client_ns": 1786102163942296000, "delay_ns": 5692000, "offset_ns": -14391000} +{"sample": 35, "t0_client_ns": 1786102163966626000, "t1_server_recv_ns": 1786102163954403000, "t2_server_send_ns": 1786102163954405000, "t3_client_ns": 1786102163971525000, "delay_ns": 4897000, "offset_ns": -14671500} +{"sample": 36, "t0_client_ns": 1786102163983477000, "t1_server_recv_ns": 1786102163971341000, "t2_server_send_ns": 1786102163971343000, "t3_client_ns": 1786102163989317000, "delay_ns": 5838000, "offset_ns": -15055000} +{"sample": 37, "t0_client_ns": 1786102164010040000, "t1_server_recv_ns": 1786102163997964000, "t2_server_send_ns": 1786102163997965000, "t3_client_ns": 1786102164016298000, "delay_ns": 6257000, "offset_ns": -15204500} +{"sample": 38, "t0_client_ns": 1786102164035864000, "t1_server_recv_ns": 1786102164023791000, "t2_server_send_ns": 1786102164023792000, "t3_client_ns": 1786102164044196000, "delay_ns": 8331000, "offset_ns": -16238500} +{"sample": 39, "t0_client_ns": 1786102164067522000, "t1_server_recv_ns": 1786102164060865000, "t2_server_send_ns": 1786102164060866000, "t3_client_ns": 1786102164082975000, "delay_ns": 15452000, "offset_ns": -14383000} +{"sample": 40, "t0_client_ns": 1786102164095626000, "t1_server_recv_ns": 1786102164083669000, "t2_server_send_ns": 1786102164083669000, "t3_client_ns": 1786102164100888000, "delay_ns": 5262000, "offset_ns": -14588000} +{"sample": 41, "t0_client_ns": 1786102164112286000, "t1_server_recv_ns": 1786102164100089000, "t2_server_send_ns": 1786102164100090000, "t3_client_ns": 1786102164117978000, "delay_ns": 5691000, "offset_ns": -15042500} +{"sample": 42, "t0_client_ns": 1786102164137853000, "t1_server_recv_ns": 1786102164126222000, "t2_server_send_ns": 1786102164126225000, "t3_client_ns": 1786102164143482000, "delay_ns": 5626000, "offset_ns": -14444000} +{"sample": 43, "t0_client_ns": 1786102164158887000, "t1_server_recv_ns": 1786102164146716000, "t2_server_send_ns": 1786102164146717000, "t3_client_ns": 1786102164165561000, "delay_ns": 6673000, "offset_ns": -15507500} +{"sample": 44, "t0_client_ns": 1786102164180911000, "t1_server_recv_ns": 1786102164169222000, "t2_server_send_ns": 1786102164169223000, "t3_client_ns": 1786102164186278000, "delay_ns": 5366000, "offset_ns": -14372000} +{"sample": 45, "t0_client_ns": 1786102164207585000, "t1_server_recv_ns": 1786102164195562000, "t2_server_send_ns": 1786102164195564000, "t3_client_ns": 1786102164213247000, "delay_ns": 5660000, "offset_ns": -14853000} +{"sample": 46, "t0_client_ns": 1786102164228635000, "t1_server_recv_ns": 1786102164216564000, "t2_server_send_ns": 1786102164216566000, "t3_client_ns": 1786102164233685000, "delay_ns": 5048000, "offset_ns": -14595000} +{"sample": 47, "t0_client_ns": 1786102164255781000, "t1_server_recv_ns": 1786102164243846000, "t2_server_send_ns": 1786102164243848000, "t3_client_ns": 1786102164261183000, "delay_ns": 5400000, "offset_ns": -14635000} +{"sample": 48, "t0_client_ns": 1786102164277998000, "t1_server_recv_ns": 1786102164265745000, "t2_server_send_ns": 1786102164265747000, "t3_client_ns": 1786102164282954000, "delay_ns": 4954000, "offset_ns": -14730000} +{"sample": 49, "t0_client_ns": 1786102164305825000, "t1_server_recv_ns": 1786102164293660000, "t2_server_send_ns": 1786102164293661000, "t3_client_ns": 1786102164310932000, "delay_ns": 5106000, "offset_ns": -14718000} +{"sample": 50, "t0_client_ns": 1786102164332791000, "t1_server_recv_ns": 1786102164320984000, "t2_server_send_ns": 1786102164320986000, "t3_client_ns": 1786102164338174000, "delay_ns": 5381000, "offset_ns": -14497500} +{"sample": 51, "t0_client_ns": 1786102164362147000, "t1_server_recv_ns": 1786102164350374000, "t2_server_send_ns": 1786102164350375000, "t3_client_ns": 1786102164367493000, "delay_ns": 5345000, "offset_ns": -14445500} +{"sample": 52, "t0_client_ns": 1786102164391083000, "t1_server_recv_ns": 1786102164379160000, "t2_server_send_ns": 1786102164379161000, "t3_client_ns": 1786102164396888000, "delay_ns": 5804000, "offset_ns": -14825000} +{"sample": 53, "t0_client_ns": 1786102164414363000, "t1_server_recv_ns": 1786102164402665000, "t2_server_send_ns": 1786102164402666000, "t3_client_ns": 1786102164419845000, "delay_ns": 5481000, "offset_ns": -14438500} +{"sample": 54, "t0_client_ns": 1786102164439004000, "t1_server_recv_ns": 1786102164426951000, "t2_server_send_ns": 1786102164426953000, "t3_client_ns": 1786102164444103000, "delay_ns": 5097000, "offset_ns": -14601500} +{"sample": 55, "t0_client_ns": 1786102164456471000, "t1_server_recv_ns": 1786102164444390000, "t2_server_send_ns": 1786102164444391000, "t3_client_ns": 1786102164461732000, "delay_ns": 5260000, "offset_ns": -14711000} +{"sample": 56, "t0_client_ns": 1786102164485068000, "t1_server_recv_ns": 1786102164472892000, "t2_server_send_ns": 1786102164472894000, "t3_client_ns": 1786102164490085000, "delay_ns": 5015000, "offset_ns": -14683500} +{"sample": 57, "t0_client_ns": 1786102164512319000, "t1_server_recv_ns": 1786102164500235000, "t2_server_send_ns": 1786102164500237000, "t3_client_ns": 1786102164517562000, "delay_ns": 5241000, "offset_ns": -14704500} +{"sample": 58, "t0_client_ns": 1786102164542382000, "t1_server_recv_ns": 1786102164530258000, "t2_server_send_ns": 1786102164530259000, "t3_client_ns": 1786102164547627000, "delay_ns": 5244000, "offset_ns": -14746000} +{"sample": 59, "t0_client_ns": 1786102164554805000, "t1_server_recv_ns": 1786102164542638000, "t2_server_send_ns": 1786102164542639000, "t3_client_ns": 1786102164559839000, "delay_ns": 5033000, "offset_ns": -14683500} +{"sample": 60, "t0_client_ns": 1786102164583601000, "t1_server_recv_ns": 1786102164571983000, "t2_server_send_ns": 1786102164571985000, "t3_client_ns": 1786102164589426000, "delay_ns": 5823000, "offset_ns": -14529500} +{"sample": 61, "t0_client_ns": 1786102164605882000, "t1_server_recv_ns": 1786102164594528000, "t2_server_send_ns": 1786102164594529000, "t3_client_ns": 1786102164612424000, "delay_ns": 6541000, "offset_ns": -14624500} +{"sample": 62, "t0_client_ns": 1786102164632266000, "t1_server_recv_ns": 1786102164621338000, "t2_server_send_ns": 1786102164621339000, "t3_client_ns": 1786102164641399000, "delay_ns": 9132000, "offset_ns": -15494000} +{"sample": 63, "t0_client_ns": 1786102164661903000, "t1_server_recv_ns": 1786102164669748000, "t2_server_send_ns": 1786102164669751000, "t3_client_ns": 1786102164688548000, "delay_ns": 26642000, "offset_ns": -5476000} +{"sample": 64, "t0_client_ns": 1786102164711168000, "t1_server_recv_ns": 1786102164699739000, "t2_server_send_ns": 1786102164699741000, "t3_client_ns": 1786102164717490000, "delay_ns": 6320000, "offset_ns": -14589000} +{"sample": 65, "t0_client_ns": 1786102164738694000, "t1_server_recv_ns": 1786102164726653000, "t2_server_send_ns": 1786102164726655000, "t3_client_ns": 1786102164743996000, "delay_ns": 5300000, "offset_ns": -14691000} +{"sample": 66, "t0_client_ns": 1786102164765570000, "t1_server_recv_ns": 1786102164753537000, "t2_server_send_ns": 1786102164753539000, "t3_client_ns": 1786102164770855000, "delay_ns": 5283000, "offset_ns": -14674500} +{"sample": 67, "t0_client_ns": 1786102164795525000, "t1_server_recv_ns": 1786102164783622000, "t2_server_send_ns": 1786102164783624000, "t3_client_ns": 1786102164801157000, "delay_ns": 5630000, "offset_ns": -14718000} +{"sample": 68, "t0_client_ns": 1786102164824541000, "t1_server_recv_ns": 1786102164812578000, "t2_server_send_ns": 1786102164812581000, "t3_client_ns": 1786102164830910000, "delay_ns": 6366000, "offset_ns": -15146000} +{"sample": 69, "t0_client_ns": 1786102164854248000, "t1_server_recv_ns": 1786102164842081000, "t2_server_send_ns": 1786102164842082000, "t3_client_ns": 1786102164859434000, "delay_ns": 5185000, "offset_ns": -14759500} +{"sample": 70, "t0_client_ns": 1786102164874771000, "t1_server_recv_ns": 1786102164862574000, "t2_server_send_ns": 1786102164862576000, "t3_client_ns": 1786102164880342000, "delay_ns": 5569000, "offset_ns": -14981500} +{"sample": 71, "t0_client_ns": 1786102164890069000, "t1_server_recv_ns": 1786102164878192000, "t2_server_send_ns": 1786102164878194000, "t3_client_ns": 1786102164895469000, "delay_ns": 5398000, "offset_ns": -14576000} +{"sample": 72, "t0_client_ns": 1786102164912226000, "t1_server_recv_ns": 1786102164899985000, "t2_server_send_ns": 1786102164899987000, "t3_client_ns": 1786102164917539000, "delay_ns": 5311000, "offset_ns": -14896500} +{"sample": 73, "t0_client_ns": 1786102164932914000, "t1_server_recv_ns": 1786102164922387000, "t2_server_send_ns": 1786102164922389000, "t3_client_ns": 1786102164939660000, "delay_ns": 6744000, "offset_ns": -13899000} +{"sample": 74, "t0_client_ns": 1786102164957584000, "t1_server_recv_ns": 1786102164945401000, "t2_server_send_ns": 1786102164945402000, "t3_client_ns": 1786102164962822000, "delay_ns": 5237000, "offset_ns": -14801500} +{"sample": 75, "t0_client_ns": 1786102164986560000, "t1_server_recv_ns": 1786102164974142000, "t2_server_send_ns": 1786102164974143000, "t3_client_ns": 1786102164992230000, "delay_ns": 5669000, "offset_ns": -15252500} +{"sample": 76, "t0_client_ns": 1786102165012848000, "t1_server_recv_ns": 1786102165001332000, "t2_server_send_ns": 1786102165001332000, "t3_client_ns": 1786102165019472000, "delay_ns": 6624000, "offset_ns": -14828000} +{"sample": 77, "t0_client_ns": 1786102165043458000, "t1_server_recv_ns": 1786102165031629000, "t2_server_send_ns": 1786102165031630000, "t3_client_ns": 1786102165049955000, "delay_ns": 6496000, "offset_ns": -15077000} +{"sample": 78, "t0_client_ns": 1786102165063521000, "t1_server_recv_ns": 1786102165052320000, "t2_server_send_ns": 1786102165052321000, "t3_client_ns": 1786102165069634000, "delay_ns": 6112000, "offset_ns": -14257000} +{"sample": 79, "t0_client_ns": 1786102165088797000, "t1_server_recv_ns": 1786102165076655000, "t2_server_send_ns": 1786102165076656000, "t3_client_ns": 1786102165093933000, "delay_ns": 5135000, "offset_ns": -14709500} +{"sample": 80, "t0_client_ns": 1786102165112031000, "t1_server_recv_ns": 1786102165100268000, "t2_server_send_ns": 1786102165100270000, "t3_client_ns": 1786102165117451000, "delay_ns": 5418000, "offset_ns": -14472000} +{"sample": 81, "t0_client_ns": 1786102165130730000, "t1_server_recv_ns": 1786102165119472000, "t2_server_send_ns": 1786102165119473000, "t3_client_ns": 1786102165136652000, "delay_ns": 5921000, "offset_ns": -14218500} +{"sample": 82, "t0_client_ns": 1786102165161618000, "t1_server_recv_ns": 1786102165149988000, "t2_server_send_ns": 1786102165149990000, "t3_client_ns": 1786102165168161000, "delay_ns": 6541000, "offset_ns": -14900500} +{"sample": 83, "t0_client_ns": 1786102165192261000, "t1_server_recv_ns": 1786102165180077000, "t2_server_send_ns": 1786102165180078000, "t3_client_ns": 1786102165197645000, "delay_ns": 5383000, "offset_ns": -14875500} +{"sample": 84, "t0_client_ns": 1786102165217567000, "t1_server_recv_ns": 1786102165205571000, "t2_server_send_ns": 1786102165205572000, "t3_client_ns": 1786102165222698000, "delay_ns": 5130000, "offset_ns": -14561000} +{"sample": 85, "t0_client_ns": 1786102165242785000, "t1_server_recv_ns": 1786102165230664000, "t2_server_send_ns": 1786102165230665000, "t3_client_ns": 1786102165250128000, "delay_ns": 7342000, "offset_ns": -15792000} +{"sample": 86, "t0_client_ns": 1786102165273064000, "t1_server_recv_ns": 1786102165261093000, "t2_server_send_ns": 1786102165261094000, "t3_client_ns": 1786102165278168000, "delay_ns": 5103000, "offset_ns": -14522500} +{"sample": 87, "t0_client_ns": 1786102165299768000, "t1_server_recv_ns": 1786102165287593000, "t2_server_send_ns": 1786102165287594000, "t3_client_ns": 1786102165304792000, "delay_ns": 5023000, "offset_ns": -14686500} +{"sample": 88, "t0_client_ns": 1786102165329882000, "t1_server_recv_ns": 1786102165317825000, "t2_server_send_ns": 1786102165317826000, "t3_client_ns": 1786102165335028000, "delay_ns": 5145000, "offset_ns": -14629500} +{"sample": 89, "t0_client_ns": 1786102165350651000, "t1_server_recv_ns": 1786102165338500000, "t2_server_send_ns": 1786102165338501000, "t3_client_ns": 1786102165355725000, "delay_ns": 5073000, "offset_ns": -14687500} +{"sample": 90, "t0_client_ns": 1786102165373792000, "t1_server_recv_ns": 1786102165361600000, "t2_server_send_ns": 1786102165361601000, "t3_client_ns": 1786102165379336000, "delay_ns": 5543000, "offset_ns": -14963500} +{"sample": 91, "t0_client_ns": 1786102165403986000, "t1_server_recv_ns": 1786102165391880000, "t2_server_send_ns": 1786102165391881000, "t3_client_ns": 1786102165409980000, "delay_ns": 5993000, "offset_ns": -15102500} +{"sample": 92, "t0_client_ns": 1786102165418693000, "t1_server_recv_ns": 1786102165406485000, "t2_server_send_ns": 1786102165406486000, "t3_client_ns": 1786102165423769000, "delay_ns": 5075000, "offset_ns": -14745500} +{"sample": 93, "t0_client_ns": 1786102165435086000, "t1_server_recv_ns": 1786102165422799000, "t2_server_send_ns": 1786102165422800000, "t3_client_ns": 1786102165440019000, "delay_ns": 4932000, "offset_ns": -14753000} +{"sample": 94, "t0_client_ns": 1786102165464778000, "t1_server_recv_ns": 1786102165452639000, "t2_server_send_ns": 1786102165452639000, "t3_client_ns": 1786102165470040000, "delay_ns": 5262000, "offset_ns": -14770000} +{"sample": 95, "t0_client_ns": 1786102165486977000, "t1_server_recv_ns": 1786102165475300000, "t2_server_send_ns": 1786102165475301000, "t3_client_ns": 1786102165492574000, "delay_ns": 5596000, "offset_ns": -14475000} +{"sample": 96, "t0_client_ns": 1786102165505772000, "t1_server_recv_ns": 1786102165494445000, "t2_server_send_ns": 1786102165494446000, "t3_client_ns": 1786102165511901000, "delay_ns": 6128000, "offset_ns": -14391000} +{"sample": 97, "t0_client_ns": 1786102165519802000, "t1_server_recv_ns": 1786102165507507000, "t2_server_send_ns": 1786102165507508000, "t3_client_ns": 1786102165524759000, "delay_ns": 4956000, "offset_ns": -14773000} +{"sample": 98, "t0_client_ns": 1786102165543317000, "t1_server_recv_ns": 1786102165530890000, "t2_server_send_ns": 1786102165530891000, "t3_client_ns": 1786102165548128000, "delay_ns": 4810000, "offset_ns": -14832000} +{"sample": 99, "t0_client_ns": 1786102165570797000, "t1_server_recv_ns": 1786102165559251000, "t2_server_send_ns": 1786102165559252000, "t3_client_ns": 1786102165576381000, "delay_ns": 5583000, "offset_ns": -14337500} +{"sample": 100, "t0_client_ns": 1786102165595039000, "t1_server_recv_ns": 1786102165582923000, "t2_server_send_ns": 1786102165582924000, "t3_client_ns": 1786102165600993000, "delay_ns": 5953000, "offset_ns": -15092500} +{"sample": 101, "t0_client_ns": 1786102165621105000, "t1_server_recv_ns": 1786102165608862000, "t2_server_send_ns": 1786102165608863000, "t3_client_ns": 1786102165626641000, "delay_ns": 5535000, "offset_ns": -15010500} +{"sample": 102, "t0_client_ns": 1786102165650594000, "t1_server_recv_ns": 1786102165639386000, "t2_server_send_ns": 1786102165639387000, "t3_client_ns": 1786102165660043000, "delay_ns": 9448000, "offset_ns": -15932000} +{"sample": 103, "t0_client_ns": 1786102165685109000, "t1_server_recv_ns": 1786102165672915000, "t2_server_send_ns": 1786102165672916000, "t3_client_ns": 1786102165690396000, "delay_ns": 5286000, "offset_ns": -14837000} +{"sample": 104, "t0_client_ns": 1786102165713630000, "t1_server_recv_ns": 1786102165701836000, "t2_server_send_ns": 1786102165701837000, "t3_client_ns": 1786102165719160000, "delay_ns": 5529000, "offset_ns": -14558500} +{"sample": 105, "t0_client_ns": 1786102165737804000, "t1_server_recv_ns": 1786102165725602000, "t2_server_send_ns": 1786102165725603000, "t3_client_ns": 1786102165742993000, "delay_ns": 5188000, "offset_ns": -14796000} +{"sample": 106, "t0_client_ns": 1786102165761386000, "t1_server_recv_ns": 1786102165749207000, "t2_server_send_ns": 1786102165749208000, "t3_client_ns": 1786102165768817000, "delay_ns": 7430000, "offset_ns": -15894000} +{"sample": 107, "t0_client_ns": 1786102165791683000, "t1_server_recv_ns": 1786102165779566000, "t2_server_send_ns": 1786102165779567000, "t3_client_ns": 1786102165799684000, "delay_ns": 8000000, "offset_ns": -16117000} +{"sample": 108, "t0_client_ns": 1786102165818009000, "t1_server_recv_ns": 1786102165805814000, "t2_server_send_ns": 1786102165805816000, "t3_client_ns": 1786102165822971000, "delay_ns": 4960000, "offset_ns": -14675000} +{"sample": 109, "t0_client_ns": 1786102165832931000, "t1_server_recv_ns": 1786102165820742000, "t2_server_send_ns": 1786102165820744000, "t3_client_ns": 1786102165838705000, "delay_ns": 5772000, "offset_ns": -15075000} +{"sample": 110, "t0_client_ns": 1786102165863030000, "t1_server_recv_ns": 1786102165850886000, "t2_server_send_ns": 1786102165850888000, "t3_client_ns": 1786102165868185000, "delay_ns": 5153000, "offset_ns": -14720500} +{"sample": 111, "t0_client_ns": 1786102165889163000, "t1_server_recv_ns": 1786102165877071000, "t2_server_send_ns": 1786102165877072000, "t3_client_ns": 1786102165894731000, "delay_ns": 5567000, "offset_ns": -14875500} +{"sample": 112, "t0_client_ns": 1786102165913138000, "t1_server_recv_ns": 1786102165901212000, "t2_server_send_ns": 1786102165901214000, "t3_client_ns": 1786102165918488000, "delay_ns": 5348000, "offset_ns": -14600000} +{"sample": 113, "t0_client_ns": 1786102165930125000, "t1_server_recv_ns": 1786102165918366000, "t2_server_send_ns": 1786102165918368000, "t3_client_ns": 1786102165936440000, "delay_ns": 6313000, "offset_ns": -14915500} +{"sample": 114, "t0_client_ns": 1786102165958670000, "t1_server_recv_ns": 1786102165946828000, "t2_server_send_ns": 1786102165946830000, "t3_client_ns": 1786102165965997000, "delay_ns": 7325000, "offset_ns": -15504500} +{"sample": 115, "t0_client_ns": 1786102165980215000, "t1_server_recv_ns": 1786102165968142000, "t2_server_send_ns": 1786102165968144000, "t3_client_ns": 1786102165985362000, "delay_ns": 5145000, "offset_ns": -14645500} +{"sample": 116, "t0_client_ns": 1786102166004266000, "t1_server_recv_ns": 1786102165992128000, "t2_server_send_ns": 1786102165992130000, "t3_client_ns": 1786102166009304000, "delay_ns": 5036000, "offset_ns": -14656000} +{"sample": 117, "t0_client_ns": 1786102166032233000, "t1_server_recv_ns": 1786102166020763000, "t2_server_send_ns": 1786102166020765000, "t3_client_ns": 1786102166037904000, "delay_ns": 5669000, "offset_ns": -14304500} +{"sample": 118, "t0_client_ns": 1786102166061184000, "t1_server_recv_ns": 1786102166048896000, "t2_server_send_ns": 1786102166048897000, "t3_client_ns": 1786102166066345000, "delay_ns": 5160000, "offset_ns": -14868000} +{"sample": 119, "t0_client_ns": 1786102166087837000, "t1_server_recv_ns": 1786102166075393000, "t2_server_send_ns": 1786102166075394000, "t3_client_ns": 1786102166093409000, "delay_ns": 5571000, "offset_ns": -15229500} +{"sample": 120, "t0_client_ns": 1786102166109562000, "t1_server_recv_ns": 1786102166097429000, "t2_server_send_ns": 1786102166097431000, "t3_client_ns": 1786102166114769000, "delay_ns": 5205000, "offset_ns": -14735500} +{"sample": 121, "t0_client_ns": 1786102166139839000, "t1_server_recv_ns": 1786102166127405000, "t2_server_send_ns": 1786102166127406000, "t3_client_ns": 1786102166145320000, "delay_ns": 5480000, "offset_ns": -15174000} +{"sample": 122, "t0_client_ns": 1786102166162548000, "t1_server_recv_ns": 1786102166151199000, "t2_server_send_ns": 1786102166151200000, "t3_client_ns": 1786102166168371000, "delay_ns": 5822000, "offset_ns": -14260000} +{"sample": 123, "t0_client_ns": 1786102166179106000, "t1_server_recv_ns": 1786102166166850000, "t2_server_send_ns": 1786102166166852000, "t3_client_ns": 1786102166184066000, "delay_ns": 4958000, "offset_ns": -14735000} +{"sample": 124, "t0_client_ns": 1786102166203237000, "t1_server_recv_ns": 1786102166190976000, "t2_server_send_ns": 1786102166190977000, "t3_client_ns": 1786102166208238000, "delay_ns": 5000000, "offset_ns": -14761000} +{"sample": 125, "t0_client_ns": 1786102166232493000, "t1_server_recv_ns": 1786102166221125000, "t2_server_send_ns": 1786102166221126000, "t3_client_ns": 1786102166238984000, "delay_ns": 6490000, "offset_ns": -14613000} +{"sample": 126, "t0_client_ns": 1786102166246348000, "t1_server_recv_ns": 1786102166236218000, "t2_server_send_ns": 1786102166236220000, "t3_client_ns": 1786102166253564000, "delay_ns": 7214000, "offset_ns": -13737000} +{"sample": 127, "t0_client_ns": 1786102166269081000, "t1_server_recv_ns": 1786102166256894000, "t2_server_send_ns": 1786102166256895000, "t3_client_ns": 1786102166274060000, "delay_ns": 4978000, "offset_ns": -14676000} +{"sample": 128, "t0_client_ns": 1786102166298138000, "t1_server_recv_ns": 1786102166285965000, "t2_server_send_ns": 1786102166285966000, "t3_client_ns": 1786102166303136000, "delay_ns": 4997000, "offset_ns": -14671500} +{"sample": 129, "t0_client_ns": 1786102166328111000, "t1_server_recv_ns": 1786102166315844000, "t2_server_send_ns": 1786102166315845000, "t3_client_ns": 1786102166333478000, "delay_ns": 5366000, "offset_ns": -14950000} +{"sample": 130, "t0_client_ns": 1786102166356122000, "t1_server_recv_ns": 1786102166343986000, "t2_server_send_ns": 1786102166343988000, "t3_client_ns": 1786102166365892000, "delay_ns": 9768000, "offset_ns": -17020000} +{"sample": 131, "t0_client_ns": 1786102166389977000, "t1_server_recv_ns": 1786102166377763000, "t2_server_send_ns": 1786102166377763000, "t3_client_ns": 1786102166394859000, "delay_ns": 4882000, "offset_ns": -14655000} +{"sample": 132, "t0_client_ns": 1786102166413100000, "t1_server_recv_ns": 1786102166400911000, "t2_server_send_ns": 1786102166400912000, "t3_client_ns": 1786102166418125000, "delay_ns": 5024000, "offset_ns": -14701000} +{"sample": 133, "t0_client_ns": 1786102166440510000, "t1_server_recv_ns": 1786102166428322000, "t2_server_send_ns": 1786102166428324000, "t3_client_ns": 1786102166445587000, "delay_ns": 5075000, "offset_ns": -14725500} +{"sample": 134, "t0_client_ns": 1786102166468224000, "t1_server_recv_ns": 1786102166456842000, "t2_server_send_ns": 1786102166456844000, "t3_client_ns": 1786102166477156000, "delay_ns": 8930000, "offset_ns": -15847000} +{"sample": 135, "t0_client_ns": 1786102166501095000, "t1_server_recv_ns": 1786102166488966000, "t2_server_send_ns": 1786102166488967000, "t3_client_ns": 1786102166506064000, "delay_ns": 4968000, "offset_ns": -14613000} +{"sample": 136, "t0_client_ns": 1786102166527816000, "t1_server_recv_ns": 1786102166516759000, "t2_server_send_ns": 1786102166516761000, "t3_client_ns": 1786102166534220000, "delay_ns": 6402000, "offset_ns": -14258000} +{"sample": 137, "t0_client_ns": 1786102166549846000, "t1_server_recv_ns": 1786102166537863000, "t2_server_send_ns": 1786102166537864000, "t3_client_ns": 1786102166555027000, "delay_ns": 5180000, "offset_ns": -14573000} +{"sample": 138, "t0_client_ns": 1786102166576616000, "t1_server_recv_ns": 1786102166564496000, "t2_server_send_ns": 1786102166564497000, "t3_client_ns": 1786102166583019000, "delay_ns": 6402000, "offset_ns": -15321000} +{"sample": 139, "t0_client_ns": 1786102166601656000, "t1_server_recv_ns": 1786102166589481000, "t2_server_send_ns": 1786102166589482000, "t3_client_ns": 1786102166606743000, "delay_ns": 5086000, "offset_ns": -14718000} +{"sample": 140, "t0_client_ns": 1786102166629372000, "t1_server_recv_ns": 1786102166617138000, "t2_server_send_ns": 1786102166617139000, "t3_client_ns": 1786102166634321000, "delay_ns": 4948000, "offset_ns": -14708000} +{"sample": 141, "t0_client_ns": 1786102166656837000, "t1_server_recv_ns": 1786102166646952000, "t2_server_send_ns": 1786102166646953000, "t3_client_ns": 1786102166664400000, "delay_ns": 7562000, "offset_ns": -13666000} +{"sample": 142, "t0_client_ns": 1786102166682595000, "t1_server_recv_ns": 1786102166670331000, "t2_server_send_ns": 1786102166670333000, "t3_client_ns": 1786102166687669000, "delay_ns": 5072000, "offset_ns": -14800000} +{"sample": 143, "t0_client_ns": 1786102166711521000, "t1_server_recv_ns": 1786102166699344000, "t2_server_send_ns": 1786102166699346000, "t3_client_ns": 1786102166716585000, "delay_ns": 5062000, "offset_ns": -14708000} +{"sample": 144, "t0_client_ns": 1786102166737853000, "t1_server_recv_ns": 1786102166725617000, "t2_server_send_ns": 1786102166725619000, "t3_client_ns": 1786102166742866000, "delay_ns": 5011000, "offset_ns": -14741500} +{"sample": 145, "t0_client_ns": 1786102166764507000, "t1_server_recv_ns": 1786102166753460000, "t2_server_send_ns": 1786102166753461000, "t3_client_ns": 1786102166774222000, "delay_ns": 9714000, "offset_ns": -15904000} +{"sample": 146, "t0_client_ns": 1786102166794823000, "t1_server_recv_ns": 1786102166785276000, "t2_server_send_ns": 1786102166785278000, "t3_client_ns": 1786102166802470000, "delay_ns": 7645000, "offset_ns": -13369500} +{"sample": 147, "t0_client_ns": 1786102166822902000, "t1_server_recv_ns": 1786102166810737000, "t2_server_send_ns": 1786102166810738000, "t3_client_ns": 1786102166828128000, "delay_ns": 5225000, "offset_ns": -14777500} +{"sample": 148, "t0_client_ns": 1786102166848909000, "t1_server_recv_ns": 1786102166836872000, "t2_server_send_ns": 1786102166836874000, "t3_client_ns": 1786102166853745000, "delay_ns": 4834000, "offset_ns": -14454000} +{"sample": 149, "t0_client_ns": 1786102166873935000, "t1_server_recv_ns": 1786102166861965000, "t2_server_send_ns": 1786102166861967000, "t3_client_ns": 1786102166883191000, "delay_ns": 9254000, "offset_ns": -16597000} +{"sample": 150, "t0_client_ns": 1786102166908770000, "t1_server_recv_ns": 1786102166896750000, "t2_server_send_ns": 1786102166896752000, "t3_client_ns": 1786102166920136000, "delay_ns": 11364000, "offset_ns": -17702000} +{"sample": 151, "t0_client_ns": 1786102166931913000, "t1_server_recv_ns": 1786102166919838000, "t2_server_send_ns": 1786102166919839000, "t3_client_ns": 1786102166937395000, "delay_ns": 5481000, "offset_ns": -14815500} +{"sample": 152, "t0_client_ns": 1786102166962482000, "t1_server_recv_ns": 1786102166950515000, "t2_server_send_ns": 1786102166950517000, "t3_client_ns": 1786102166968244000, "delay_ns": 5760000, "offset_ns": -14847000} +{"sample": 153, "t0_client_ns": 1786102166988755000, "t1_server_recv_ns": 1786102166976492000, "t2_server_send_ns": 1786102166976494000, "t3_client_ns": 1786102166994301000, "delay_ns": 5544000, "offset_ns": -15035000} +{"sample": 154, "t0_client_ns": 1786102167016496000, "t1_server_recv_ns": 1786102167004722000, "t2_server_send_ns": 1786102167004723000, "t3_client_ns": 1786102167022026000, "delay_ns": 5529000, "offset_ns": -14538500} +{"sample": 155, "t0_client_ns": 1786102167036584000, "t1_server_recv_ns": 1786102167024773000, "t2_server_send_ns": 1786102167024775000, "t3_client_ns": 1786102167042209000, "delay_ns": 5623000, "offset_ns": -14622500} +{"sample": 156, "t0_client_ns": 1786102167061291000, "t1_server_recv_ns": 1786102167049155000, "t2_server_send_ns": 1786102167049156000, "t3_client_ns": 1786102167067173000, "delay_ns": 5881000, "offset_ns": -15076500} +{"sample": 157, "t0_client_ns": 1786102167090831000, "t1_server_recv_ns": 1786102167078290000, "t2_server_send_ns": 1786102167078291000, "t3_client_ns": 1786102167095494000, "delay_ns": 4662000, "offset_ns": -14872000} +{"sample": 158, "t0_client_ns": 1786102167112680000, "t1_server_recv_ns": 1786102167100754000, "t2_server_send_ns": 1786102167100756000, "t3_client_ns": 1786102167117988000, "delay_ns": 5306000, "offset_ns": -14579000} +{"sample": 159, "t0_client_ns": 1786102167136649000, "t1_server_recv_ns": 1786102167124671000, "t2_server_send_ns": 1786102167124672000, "t3_client_ns": 1786102167141850000, "delay_ns": 5200000, "offset_ns": -14578000} +{"sample": 160, "t0_client_ns": 1786102167151733000, "t1_server_recv_ns": 1786102167139429000, "t2_server_send_ns": 1786102167139430000, "t3_client_ns": 1786102167156562000, "delay_ns": 4828000, "offset_ns": -14718000} +{"sample": 161, "t0_client_ns": 1786102167169575000, "t1_server_recv_ns": 1786102167157385000, "t2_server_send_ns": 1786102167157386000, "t3_client_ns": 1786102167174535000, "delay_ns": 4959000, "offset_ns": -14669500} +{"sample": 162, "t0_client_ns": 1786102167196113000, "t1_server_recv_ns": 1786102167183727000, "t2_server_send_ns": 1786102167183728000, "t3_client_ns": 1786102167201126000, "delay_ns": 5012000, "offset_ns": -14892000} +{"sample": 163, "t0_client_ns": 1786102167213125000, "t1_server_recv_ns": 1786102167200731000, "t2_server_send_ns": 1786102167200732000, "t3_client_ns": 1786102167217930000, "delay_ns": 4804000, "offset_ns": -14796000} +{"sample": 164, "t0_client_ns": 1786102167234364000, "t1_server_recv_ns": 1786102167222094000, "t2_server_send_ns": 1786102167222095000, "t3_client_ns": 1786102167239213000, "delay_ns": 4848000, "offset_ns": -14694000} +{"sample": 165, "t0_client_ns": 1786102167261412000, "t1_server_recv_ns": 1786102167249754000, "t2_server_send_ns": 1786102167249755000, "t3_client_ns": 1786102167267321000, "delay_ns": 5908000, "offset_ns": -14612000} +{"sample": 166, "t0_client_ns": 1786102167291117000, "t1_server_recv_ns": 1786102167278894000, "t2_server_send_ns": 1786102167278895000, "t3_client_ns": 1786102167296176000, "delay_ns": 5058000, "offset_ns": -14752000} +{"sample": 167, "t0_client_ns": 1786102167314976000, "t1_server_recv_ns": 1786102167302752000, "t2_server_send_ns": 1786102167302753000, "t3_client_ns": 1786102167320002000, "delay_ns": 5025000, "offset_ns": -14736500} +{"sample": 168, "t0_client_ns": 1786102167341629000, "t1_server_recv_ns": 1786102167329548000, "t2_server_send_ns": 1786102167329549000, "t3_client_ns": 1786102167346783000, "delay_ns": 5153000, "offset_ns": -14657500} +{"sample": 169, "t0_client_ns": 1786102167365019000, "t1_server_recv_ns": 1786102167353342000, "t2_server_send_ns": 1786102167353344000, "t3_client_ns": 1786102167371585000, "delay_ns": 6564000, "offset_ns": -14959000} +{"sample": 170, "t0_client_ns": 1786102167387975000, "t1_server_recv_ns": 1786102167376196000, "t2_server_send_ns": 1786102167376197000, "t3_client_ns": 1786102167394709000, "delay_ns": 6733000, "offset_ns": -15145500} +{"sample": 171, "t0_client_ns": 1786102167418175000, "t1_server_recv_ns": 1786102167406070000, "t2_server_send_ns": 1786102167406072000, "t3_client_ns": 1786102167423196000, "delay_ns": 5019000, "offset_ns": -14614500} +{"sample": 172, "t0_client_ns": 1786102167439049000, "t1_server_recv_ns": 1786102167426782000, "t2_server_send_ns": 1786102167426783000, "t3_client_ns": 1786102167444072000, "delay_ns": 5022000, "offset_ns": -14778000} +{"sample": 173, "t0_client_ns": 1786102167463415000, "t1_server_recv_ns": 1786102167452095000, "t2_server_send_ns": 1786102167452098000, "t3_client_ns": 1786102167470166000, "delay_ns": 6748000, "offset_ns": -14694000} +{"sample": 174, "t0_client_ns": 1786102167491947000, "t1_server_recv_ns": 1786102167479893000, "t2_server_send_ns": 1786102167479895000, "t3_client_ns": 1786102167497353000, "delay_ns": 5404000, "offset_ns": -14756000} +{"sample": 175, "t0_client_ns": 1786102167506480000, "t1_server_recv_ns": 1786102167494665000, "t2_server_send_ns": 1786102167494667000, "t3_client_ns": 1786102167512794000, "delay_ns": 6312000, "offset_ns": -14971000} +{"sample": 176, "t0_client_ns": 1786102167523522000, "t1_server_recv_ns": 1786102167511336000, "t2_server_send_ns": 1786102167511337000, "t3_client_ns": 1786102167528606000, "delay_ns": 5083000, "offset_ns": -14727500} +{"sample": 177, "t0_client_ns": 1786102167548507000, "t1_server_recv_ns": 1786102167537188000, "t2_server_send_ns": 1786102167537190000, "t3_client_ns": 1786102167555388000, "delay_ns": 6879000, "offset_ns": -14758500} +{"sample": 178, "t0_client_ns": 1786102167578491000, "t1_server_recv_ns": 1786102167566395000, "t2_server_send_ns": 1786102167566396000, "t3_client_ns": 1786102167584288000, "delay_ns": 5796000, "offset_ns": -14994000} +{"sample": 179, "t0_client_ns": 1786102167599237000, "t1_server_recv_ns": 1786102167587084000, "t2_server_send_ns": 1786102167587085000, "t3_client_ns": 1786102167605441000, "delay_ns": 6203000, "offset_ns": -15254500} +{"sample": 180, "t0_client_ns": 1786102167629524000, "t1_server_recv_ns": 1786102167617428000, "t2_server_send_ns": 1786102167617430000, "t3_client_ns": 1786102167634696000, "delay_ns": 5170000, "offset_ns": -14681000} +{"sample": 181, "t0_client_ns": 1786102167654680000, "t1_server_recv_ns": 1786102167642855000, "t2_server_send_ns": 1786102167642857000, "t3_client_ns": 1786102167660039000, "delay_ns": 5357000, "offset_ns": -14503500} +{"sample": 182, "t0_client_ns": 1786102167682377000, "t1_server_recv_ns": 1786102167670477000, "t2_server_send_ns": 1786102167670478000, "t3_client_ns": 1786102167687802000, "delay_ns": 5424000, "offset_ns": -14612000} +{"sample": 183, "t0_client_ns": 1786102167704694000, "t1_server_recv_ns": 1786102167692075000, "t2_server_send_ns": 1786102167692075000, "t3_client_ns": 1786102167709309000, "delay_ns": 4615000, "offset_ns": -14926500} +{"sample": 184, "t0_client_ns": 1786102167731795000, "t1_server_recv_ns": 1786102167721016000, "t2_server_send_ns": 1786102167721017000, "t3_client_ns": 1786102167738229000, "delay_ns": 6433000, "offset_ns": -13995500} +{"sample": 185, "t0_client_ns": 1786102167762427000, "t1_server_recv_ns": 1786102167750182000, "t2_server_send_ns": 1786102167750183000, "t3_client_ns": 1786102167767960000, "delay_ns": 5532000, "offset_ns": -15011000} +{"sample": 186, "t0_client_ns": 1786102167787449000, "t1_server_recv_ns": 1786102167775268000, "t2_server_send_ns": 1786102167775270000, "t3_client_ns": 1786102167792553000, "delay_ns": 5102000, "offset_ns": -14732000} +{"sample": 187, "t0_client_ns": 1786102167815753000, "t1_server_recv_ns": 1786102167803640000, "t2_server_send_ns": 1786102167803641000, "t3_client_ns": 1786102167820866000, "delay_ns": 5112000, "offset_ns": -14669000} +{"sample": 188, "t0_client_ns": 1786102167832880000, "t1_server_recv_ns": 1786102167821480000, "t2_server_send_ns": 1786102167821481000, "t3_client_ns": 1786102167838659000, "delay_ns": 5778000, "offset_ns": -14289000} +{"sample": 189, "t0_client_ns": 1786102167863625000, "t1_server_recv_ns": 1786102167851790000, "t2_server_send_ns": 1786102167851791000, "t3_client_ns": 1786102167869003000, "delay_ns": 5377000, "offset_ns": -14523500} +{"sample": 190, "t0_client_ns": 1786102167887754000, "t1_server_recv_ns": 1786102167875727000, "t2_server_send_ns": 1786102167875729000, "t3_client_ns": 1786102167892697000, "delay_ns": 4941000, "offset_ns": -14497500} +{"sample": 191, "t0_client_ns": 1786102167916645000, "t1_server_recv_ns": 1786102167906496000, "t2_server_send_ns": 1786102167906499000, "t3_client_ns": 1786102167926925000, "delay_ns": 10277000, "offset_ns": -15287500} +{"sample": 192, "t0_client_ns": 1786102167951055000, "t1_server_recv_ns": 1786102167940777000, "t2_server_send_ns": 1786102167940779000, "t3_client_ns": 1786102167957980000, "delay_ns": 6923000, "offset_ns": -13739500} +{"sample": 193, "t0_client_ns": 1786102167982209000, "t1_server_recv_ns": 1786102167970369000, "t2_server_send_ns": 1786102167970371000, "t3_client_ns": 1786102167988652000, "delay_ns": 6441000, "offset_ns": -15060500} +{"sample": 194, "t0_client_ns": 1786102168001311000, "t1_server_recv_ns": 1786102167989940000, "t2_server_send_ns": 1786102167989942000, "t3_client_ns": 1786102168007213000, "delay_ns": 5900000, "offset_ns": -14321000} +{"sample": 195, "t0_client_ns": 1786102168016231000, "t1_server_recv_ns": 1786102168004400000, "t2_server_send_ns": 1786102168004401000, "t3_client_ns": 1786102168022121000, "delay_ns": 5889000, "offset_ns": -14775500} +{"sample": 196, "t0_client_ns": 1786102168039433000, "t1_server_recv_ns": 1786102168027220000, "t2_server_send_ns": 1786102168027222000, "t3_client_ns": 1786102168047685000, "delay_ns": 8250000, "offset_ns": -16338000} +{"sample": 197, "t0_client_ns": 1786102168061444000, "t1_server_recv_ns": 1786102168049299000, "t2_server_send_ns": 1786102168049300000, "t3_client_ns": 1786102168069034000, "delay_ns": 7589000, "offset_ns": -15939500} +{"sample": 198, "t0_client_ns": 1786102168076556000, "t1_server_recv_ns": 1786102168064265000, "t2_server_send_ns": 1786102168064266000, "t3_client_ns": 1786102168082373000, "delay_ns": 5816000, "offset_ns": -15199000} +{"sample": 199, "t0_client_ns": 1786102168106186000, "t1_server_recv_ns": 1786102168094047000, "t2_server_send_ns": 1786102168094048000, "t3_client_ns": 1786102168111577000, "delay_ns": 5390000, "offset_ns": -14834000} +{"sample": 200, "t0_client_ns": 1786102168136597000, "t1_server_recv_ns": 1786102168124890000, "t2_server_send_ns": 1786102168124891000, "t3_client_ns": 1786102168142058000, "delay_ns": 5460000, "offset_ns": -14437000} +{"kind": "clock_offset_summary", "host": "100.102.30.76", "samples": 200, "offset_ns": -14764500, "offset_ms": -14.7645, "min_delay_ns": 4577000, "min_delay_ms": 4.577, "uncertainty_ms": 2.2885, "offset_median_ms": -14.718, "offset_spread_ms": 12.226, "measured_at_client_utc": "2026-08-07T11:29:28Z"} diff --git a/docs/evidence/mount-latency-20260807/raw/clock-offset-pre.jsonl b/docs/evidence/mount-latency-20260807/raw/clock-offset-pre.jsonl new file mode 100644 index 00000000..bfb49c87 --- /dev/null +++ b/docs/evidence/mount-latency-20260807/raw/clock-offset-pre.jsonl @@ -0,0 +1,145 @@ +{"sample": 1, "t0_client_ns": 1786101151055427000, "t1_server_recv_ns": 1786101151052227000, "t2_server_send_ns": 1786101151052230000, "t3_client_ns": 1786101151071421000, "delay_ns": 15991000, "offset_ns": -11195500} +{"sample": 2, "t0_client_ns": 1786101151095227000, "t1_server_recv_ns": 1786101151091543000, "t2_server_send_ns": 1786101151091546000, "t3_client_ns": 1786101151100577000, "delay_ns": 5347000, "offset_ns": -6357500} +{"sample": 3, "t0_client_ns": 1786101151113599000, "t1_server_recv_ns": 1786101151109826000, "t2_server_send_ns": 1786101151109828000, "t3_client_ns": 1786101151119355000, "delay_ns": 5754000, "offset_ns": -6650000} +{"sample": 4, "t0_client_ns": 1786101151143919000, "t1_server_recv_ns": 1786101151139725000, "t2_server_send_ns": 1786101151139727000, "t3_client_ns": 1786101151151824000, "delay_ns": 7903000, "offset_ns": -8145500} +{"sample": 5, "t0_client_ns": 1786101151182816000, "t1_server_recv_ns": 1786101151178964000, "t2_server_send_ns": 1786101151178965000, "t3_client_ns": 1786101151189724000, "delay_ns": 6907000, "offset_ns": -7305500} +{"sample": 6, "t0_client_ns": 1786101151214989000, "t1_server_recv_ns": 1786101151211286000, "t2_server_send_ns": 1786101151211287000, "t3_client_ns": 1786101151221473000, "delay_ns": 6483000, "offset_ns": -6944500} +{"sample": 7, "t0_client_ns": 1786101151241839000, "t1_server_recv_ns": 1786101151237964000, "t2_server_send_ns": 1786101151237966000, "t3_client_ns": 1786101151249899000, "delay_ns": 8058000, "offset_ns": -7904000} +{"sample": 8, "t0_client_ns": 1786101151267263000, "t1_server_recv_ns": 1786101151263709000, "t2_server_send_ns": 1786101151263710000, "t3_client_ns": 1786101151273716000, "delay_ns": 6452000, "offset_ns": -6780000} +{"sample": 9, "t0_client_ns": 1786101151299082000, "t1_server_recv_ns": 1786101151295304000, "t2_server_send_ns": 1786101151295305000, "t3_client_ns": 1786101151305072000, "delay_ns": 5989000, "offset_ns": -6772500} +{"sample": 10, "t0_client_ns": 1786101151327450000, "t1_server_recv_ns": 1786101151323648000, "t2_server_send_ns": 1786101151323649000, "t3_client_ns": 1786101151332446000, "delay_ns": 4995000, "offset_ns": -6299500} +{"sample": 11, "t0_client_ns": 1786101151355392000, "t1_server_recv_ns": 1786101151352476000, "t2_server_send_ns": 1786101151352477000, "t3_client_ns": 1786101151363839000, "delay_ns": 8446000, "offset_ns": -7139000} +{"sample": 12, "t0_client_ns": 1786101151382716000, "t1_server_recv_ns": 1786101151378812000, "t2_server_send_ns": 1786101151378814000, "t3_client_ns": 1786101151389297000, "delay_ns": 6579000, "offset_ns": -7193500} +{"sample": 13, "t0_client_ns": 1786101151410480000, "t1_server_recv_ns": 1786101151406816000, "t2_server_send_ns": 1786101151406817000, "t3_client_ns": 1786101151418901000, "delay_ns": 8420000, "offset_ns": -7874000} +{"sample": 14, "t0_client_ns": 1786101151445862000, "t1_server_recv_ns": 1786101151442011000, "t2_server_send_ns": 1786101151442013000, "t3_client_ns": 1786101151450942000, "delay_ns": 5078000, "offset_ns": -6390000} +{"sample": 15, "t0_client_ns": 1786101151464682000, "t1_server_recv_ns": 1786101151480106000, "t2_server_send_ns": 1786101151480107000, "t3_client_ns": 1786101151489952000, "delay_ns": 25269000, "offset_ns": 2789500} +{"sample": 16, "t0_client_ns": 1786101151510072000, "t1_server_recv_ns": 1786101151506204000, "t2_server_send_ns": 1786101151506205000, "t3_client_ns": 1786101151514978000, "delay_ns": 4905000, "offset_ns": -6320500} +{"sample": 17, "t0_client_ns": 1786101151540015000, "t1_server_recv_ns": 1786101151536408000, "t2_server_send_ns": 1786101151536409000, "t3_client_ns": 1786101151547788000, "delay_ns": 7772000, "offset_ns": -7493000} +{"sample": 18, "t0_client_ns": 1786101151566220000, "t1_server_recv_ns": 1786101151562268000, "t2_server_send_ns": 1786101151562270000, "t3_client_ns": 1786101151571081000, "delay_ns": 4859000, "offset_ns": -6381500} +{"sample": 19, "t0_client_ns": 1786101151594015000, "t1_server_recv_ns": 1786101151590101000, "t2_server_send_ns": 1786101151590103000, "t3_client_ns": 1786101151598848000, "delay_ns": 4831000, "offset_ns": -6329500} +{"sample": 20, "t0_client_ns": 1786101151612380000, "t1_server_recv_ns": 1786101151608797000, "t2_server_send_ns": 1786101151608798000, "t3_client_ns": 1786101151619595000, "delay_ns": 7214000, "offset_ns": -7190000} +{"sample": 21, "t0_client_ns": 1786101151640391000, "t1_server_recv_ns": 1786101151636493000, "t2_server_send_ns": 1786101151636495000, "t3_client_ns": 1786101151646310000, "delay_ns": 5917000, "offset_ns": -6856500} +{"sample": 22, "t0_client_ns": 1786101151665216000, "t1_server_recv_ns": 1786101151661346000, "t2_server_send_ns": 1786101151661347000, "t3_client_ns": 1786101151670296000, "delay_ns": 5079000, "offset_ns": -6409500} +{"sample": 23, "t0_client_ns": 1786101151690710000, "t1_server_recv_ns": 1786101151686784000, "t2_server_send_ns": 1786101151686786000, "t3_client_ns": 1786101151697371000, "delay_ns": 6659000, "offset_ns": -7255500} +{"sample": 24, "t0_client_ns": 1786101151722053000, "t1_server_recv_ns": 1786101151719085000, "t2_server_send_ns": 1786101151719086000, "t3_client_ns": 1786101151731113000, "delay_ns": 9059000, "offset_ns": -7497500} +{"sample": 25, "t0_client_ns": 1786101151749449000, "t1_server_recv_ns": 1786101151745784000, "t2_server_send_ns": 1786101151745785000, "t3_client_ns": 1786101151754558000, "delay_ns": 5108000, "offset_ns": -6219000} +{"sample": 26, "t0_client_ns": 1786101151777316000, "t1_server_recv_ns": 1786101151773892000, "t2_server_send_ns": 1786101151773894000, "t3_client_ns": 1786101151782494000, "delay_ns": 5176000, "offset_ns": -6012000} +{"sample": 27, "t0_client_ns": 1786101151803111000, "t1_server_recv_ns": 1786101151799150000, "t2_server_send_ns": 1786101151799152000, "t3_client_ns": 1786101151808079000, "delay_ns": 4966000, "offset_ns": -6444000} +{"sample": 28, "t0_client_ns": 1786101151831825000, "t1_server_recv_ns": 1786101151827949000, "t2_server_send_ns": 1786101151827950000, "t3_client_ns": 1786101151836773000, "delay_ns": 4947000, "offset_ns": -6349500} +{"sample": 29, "t0_client_ns": 1786101151860224000, "t1_server_recv_ns": 1786101151857054000, "t2_server_send_ns": 1786101151857055000, "t3_client_ns": 1786101151869086000, "delay_ns": 8861000, "offset_ns": -7600500} +{"sample": 30, "t0_client_ns": 1786101151878095000, "t1_server_recv_ns": 1786101151874207000, "t2_server_send_ns": 1786101151874209000, "t3_client_ns": 1786101151883028000, "delay_ns": 4931000, "offset_ns": -6353500} +{"sample": 31, "t0_client_ns": 1786101151906770000, "t1_server_recv_ns": 1786101151905808000, "t2_server_send_ns": 1786101151905810000, "t3_client_ns": 1786101151915384000, "delay_ns": 8612000, "offset_ns": -5268000} +{"sample": 32, "t0_client_ns": 1786101151936840000, "t1_server_recv_ns": 1786101151933894000, "t2_server_send_ns": 1786101151933896000, "t3_client_ns": 1786101151946349000, "delay_ns": 9507000, "offset_ns": -7699500} +{"sample": 33, "t0_client_ns": 1786101151958064000, "t1_server_recv_ns": 1786101151955455000, "t2_server_send_ns": 1786101151955456000, "t3_client_ns": 1786101151965868000, "delay_ns": 7803000, "offset_ns": -6510500} +{"sample": 34, "t0_client_ns": 1786101151990873000, "t1_server_recv_ns": 1786101151987350000, "t2_server_send_ns": 1786101151987352000, "t3_client_ns": 1786101151996262000, "delay_ns": 5387000, "offset_ns": -6216500} +{"sample": 35, "t0_client_ns": 1786101152006348000, "t1_server_recv_ns": 1786101152002576000, "t2_server_send_ns": 1786101152002578000, "t3_client_ns": 1786101152012780000, "delay_ns": 6430000, "offset_ns": -6987000} +{"sample": 36, "t0_client_ns": 1786101152033532000, "t1_server_recv_ns": 1786101152029688000, "t2_server_send_ns": 1786101152029690000, "t3_client_ns": 1786101152038459000, "delay_ns": 4925000, "offset_ns": -6306500} +{"sample": 37, "t0_client_ns": 1786101152058398000, "t1_server_recv_ns": 1786101152054613000, "t2_server_send_ns": 1786101152054614000, "t3_client_ns": 1786101152063490000, "delay_ns": 5091000, "offset_ns": -6330500} +{"sample": 38, "t0_client_ns": 1786101152088028000, "t1_server_recv_ns": 1786101152084499000, "t2_server_send_ns": 1786101152084501000, "t3_client_ns": 1786101152094522000, "delay_ns": 6492000, "offset_ns": -6775000} +{"sample": 39, "t0_client_ns": 1786101152106424000, "t1_server_recv_ns": 1786101152102595000, "t2_server_send_ns": 1786101152102596000, "t3_client_ns": 1786101152111738000, "delay_ns": 5313000, "offset_ns": -6485500} +{"sample": 40, "t0_client_ns": 1786101152132500000, "t1_server_recv_ns": 1786101152128801000, "t2_server_send_ns": 1786101152128803000, "t3_client_ns": 1786101152137712000, "delay_ns": 5210000, "offset_ns": -6304000} +{"sample": 41, "t0_client_ns": 1786101152155781000, "t1_server_recv_ns": 1786101152151968000, "t2_server_send_ns": 1786101152151969000, "t3_client_ns": 1786101152161296000, "delay_ns": 5514000, "offset_ns": -6570000} +{"sample": 42, "t0_client_ns": 1786101152171016000, "t1_server_recv_ns": 1786101152167334000, "t2_server_send_ns": 1786101152167335000, "t3_client_ns": 1786101152176298000, "delay_ns": 5281000, "offset_ns": -6322500} +{"sample": 43, "t0_client_ns": 1786101152194413000, "t1_server_recv_ns": 1786101152191261000, "t2_server_send_ns": 1786101152191262000, "t3_client_ns": 1786101152200069000, "delay_ns": 5655000, "offset_ns": -5979500} +{"sample": 44, "t0_client_ns": 1786101152209486000, "t1_server_recv_ns": 1786101152205934000, "t2_server_send_ns": 1786101152205936000, "t3_client_ns": 1786101152216642000, "delay_ns": 7154000, "offset_ns": -7129000} +{"sample": 45, "t0_client_ns": 1786101152237726000, "t1_server_recv_ns": 1786101152233992000, "t2_server_send_ns": 1786101152233993000, "t3_client_ns": 1786101152242887000, "delay_ns": 5160000, "offset_ns": -6314000} +{"sample": 46, "t0_client_ns": 1786101152260332000, "t1_server_recv_ns": 1786101152257187000, "t2_server_send_ns": 1786101152257188000, "t3_client_ns": 1786101152266547000, "delay_ns": 6214000, "offset_ns": -6252000} +{"sample": 47, "t0_client_ns": 1786101152287933000, "t1_server_recv_ns": 1786101152285048000, "t2_server_send_ns": 1786101152285049000, "t3_client_ns": 1786101152298690000, "delay_ns": 10756000, "offset_ns": -8263000} +{"sample": 48, "t0_client_ns": 1786101152322678000, "t1_server_recv_ns": 1786101152319833000, "t2_server_send_ns": 1786101152319834000, "t3_client_ns": 1786101152330261000, "delay_ns": 7582000, "offset_ns": -6636000} +{"sample": 49, "t0_client_ns": 1786101152343595000, "t1_server_recv_ns": 1786101152340698000, "t2_server_send_ns": 1786101152340698000, "t3_client_ns": 1786101152349678000, "delay_ns": 6083000, "offset_ns": -5938500} +{"sample": 50, "t0_client_ns": 1786101152360480000, "t1_server_recv_ns": 1786101152356743000, "t2_server_send_ns": 1786101152356744000, "t3_client_ns": 1786101152366365000, "delay_ns": 5884000, "offset_ns": -6679000} +{"sample": 51, "t0_client_ns": 1786101152383772000, "t1_server_recv_ns": 1786101152380375000, "t2_server_send_ns": 1786101152380376000, "t3_client_ns": 1786101152389093000, "delay_ns": 5320000, "offset_ns": -6057000} +{"sample": 52, "t0_client_ns": 1786101152412051000, "t1_server_recv_ns": 1786101152408184000, "t2_server_send_ns": 1786101152408185000, "t3_client_ns": 1786101152416952000, "delay_ns": 4900000, "offset_ns": -6317000} +{"sample": 53, "t0_client_ns": 1786101152434726000, "t1_server_recv_ns": 1786101152430938000, "t2_server_send_ns": 1786101152430939000, "t3_client_ns": 1786101152439926000, "delay_ns": 5199000, "offset_ns": -6387500} +{"sample": 54, "t0_client_ns": 1786101152450353000, "t1_server_recv_ns": 1786101152446423000, "t2_server_send_ns": 1786101152446424000, "t3_client_ns": 1786101152455217000, "delay_ns": 4863000, "offset_ns": -6361500} +{"sample": 55, "t0_client_ns": 1786101152472241000, "t1_server_recv_ns": 1786101152468502000, "t2_server_send_ns": 1786101152468503000, "t3_client_ns": 1786101152477368000, "delay_ns": 5126000, "offset_ns": -6302000} +{"sample": 56, "t0_client_ns": 1786101152501040000, "t1_server_recv_ns": 1786101152497803000, "t2_server_send_ns": 1786101152497804000, "t3_client_ns": 1786101152506711000, "delay_ns": 5670000, "offset_ns": -6072000} +{"sample": 57, "t0_client_ns": 1786101152520295000, "t1_server_recv_ns": 1786101152516531000, "t2_server_send_ns": 1786101152516532000, "t3_client_ns": 1786101152525262000, "delay_ns": 4966000, "offset_ns": -6247000} +{"sample": 58, "t0_client_ns": 1786101152545783000, "t1_server_recv_ns": 1786101152542274000, "t2_server_send_ns": 1786101152542276000, "t3_client_ns": 1786101152551167000, "delay_ns": 5382000, "offset_ns": -6200000} +{"sample": 59, "t0_client_ns": 1786101152568739000, "t1_server_recv_ns": 1786101152565004000, "t2_server_send_ns": 1786101152565006000, "t3_client_ns": 1786101152573995000, "delay_ns": 5254000, "offset_ns": -6362000} +{"sample": 60, "t0_client_ns": 1786101152595817000, "t1_server_recv_ns": 1786101152591958000, "t2_server_send_ns": 1786101152591959000, "t3_client_ns": 1786101152601103000, "delay_ns": 5285000, "offset_ns": -6501500} +{"sample": 61, "t0_client_ns": 1786101152617289000, "t1_server_recv_ns": 1786101152613324000, "t2_server_send_ns": 1786101152613325000, "t3_client_ns": 1786101152622054000, "delay_ns": 4764000, "offset_ns": -6347000} +{"sample": 62, "t0_client_ns": 1786101152634149000, "t1_server_recv_ns": 1786101152630276000, "t2_server_send_ns": 1786101152630277000, "t3_client_ns": 1786101152639100000, "delay_ns": 4950000, "offset_ns": -6348000} +{"sample": 63, "t0_client_ns": 1786101152655093000, "t1_server_recv_ns": 1786101152935876000, "t2_server_send_ns": 1786101152935879000, "t3_client_ns": 1786101152945106000, "delay_ns": 290010000, "offset_ns": 135778000} +{"sample": 64, "t0_client_ns": 1786101152969868000, "t1_server_recv_ns": 1786101152965979000, "t2_server_send_ns": 1786101152965980000, "t3_client_ns": 1786101152974828000, "delay_ns": 4959000, "offset_ns": -6368500} +{"sample": 65, "t0_client_ns": 1786101152987879000, "t1_server_recv_ns": 1786101152984027000, "t2_server_send_ns": 1786101152984029000, "t3_client_ns": 1786101152992972000, "delay_ns": 5091000, "offset_ns": -6397500} +{"sample": 66, "t0_client_ns": 1786101153008984000, "t1_server_recv_ns": 1786101153005263000, "t2_server_send_ns": 1786101153005264000, "t3_client_ns": 1786101153013989000, "delay_ns": 5004000, "offset_ns": -6223000} +{"sample": 67, "t0_client_ns": 1786101153037872000, "t1_server_recv_ns": 1786101153034098000, "t2_server_send_ns": 1786101153034099000, "t3_client_ns": 1786101153042788000, "delay_ns": 4915000, "offset_ns": -6231500} +{"sample": 68, "t0_client_ns": 1786101153053826000, "t1_server_recv_ns": 1786101153049760000, "t2_server_send_ns": 1786101153049762000, "t3_client_ns": 1786101153058450000, "delay_ns": 4622000, "offset_ns": -6377000} +{"sample": 69, "t0_client_ns": 1786101153083274000, "t1_server_recv_ns": 1786101153079558000, "t2_server_send_ns": 1786101153079559000, "t3_client_ns": 1786101153088265000, "delay_ns": 4990000, "offset_ns": -6211000} +{"sample": 70, "t0_client_ns": 1786101153111032000, "t1_server_recv_ns": 1786101153107305000, "t2_server_send_ns": 1786101153107306000, "t3_client_ns": 1786101153116906000, "delay_ns": 5873000, "offset_ns": -6663500} +{"sample": 71, "t0_client_ns": 1786101153140044000, "t1_server_recv_ns": 1786101153136847000, "t2_server_send_ns": 1786101153136849000, "t3_client_ns": 1786101153145560000, "delay_ns": 5514000, "offset_ns": -5954000} +{"sample": 72, "t0_client_ns": 1786101153160002000, "t1_server_recv_ns": 1786101153156204000, "t2_server_send_ns": 1786101153156205000, "t3_client_ns": 1786101153164934000, "delay_ns": 4931000, "offset_ns": -6263500} +{"sample": 73, "t0_client_ns": 1786101153177015000, "t1_server_recv_ns": 1786101153173270000, "t2_server_send_ns": 1786101153173272000, "t3_client_ns": 1786101153181976000, "delay_ns": 4959000, "offset_ns": -6224500} +{"sample": 74, "t0_client_ns": 1786101153198772000, "t1_server_recv_ns": 1786101153195096000, "t2_server_send_ns": 1786101153195097000, "t3_client_ns": 1786101153203861000, "delay_ns": 5088000, "offset_ns": -6220000} +{"sample": 75, "t0_client_ns": 1786101153225981000, "t1_server_recv_ns": 1786101153222224000, "t2_server_send_ns": 1786101153222225000, "t3_client_ns": 1786101153231109000, "delay_ns": 5127000, "offset_ns": -6320500} +{"sample": 76, "t0_client_ns": 1786101153253746000, "t1_server_recv_ns": 1786101153250802000, "t2_server_send_ns": 1786101153250804000, "t3_client_ns": 1786101153259539000, "delay_ns": 5791000, "offset_ns": -5839500} +{"sample": 77, "t0_client_ns": 1786101153278415000, "t1_server_recv_ns": 1786101153275072000, "t2_server_send_ns": 1786101153275074000, "t3_client_ns": 1786101153283916000, "delay_ns": 5499000, "offset_ns": -6092500} +{"sample": 78, "t0_client_ns": 1786101153294573000, "t1_server_recv_ns": 1786101153291082000, "t2_server_send_ns": 1786101153291085000, "t3_client_ns": 1786101153300361000, "delay_ns": 5785000, "offset_ns": -6383500} +{"sample": 79, "t0_client_ns": 1786101153321074000, "t1_server_recv_ns": 1786101153317599000, "t2_server_send_ns": 1786101153317603000, "t3_client_ns": 1786101153326370000, "delay_ns": 5292000, "offset_ns": -6121000} +{"sample": 80, "t0_client_ns": 1786101153345824000, "t1_server_recv_ns": 1786101153341967000, "t2_server_send_ns": 1786101153341968000, "t3_client_ns": 1786101153351048000, "delay_ns": 5223000, "offset_ns": -6468500} +{"sample": 81, "t0_client_ns": 1786101153370104000, "t1_server_recv_ns": 1786101153366209000, "t2_server_send_ns": 1786101153366211000, "t3_client_ns": 1786101153374962000, "delay_ns": 4856000, "offset_ns": -6323000} +{"sample": 82, "t0_client_ns": 1786101153392730000, "t1_server_recv_ns": 1786101153388902000, "t2_server_send_ns": 1786101153388904000, "t3_client_ns": 1786101153397794000, "delay_ns": 5062000, "offset_ns": -6359000} +{"sample": 83, "t0_client_ns": 1786101153404801000, "t1_server_recv_ns": 1786101153400823000, "t2_server_send_ns": 1786101153400824000, "t3_client_ns": 1786101153410226000, "delay_ns": 5424000, "offset_ns": -6690000} +{"sample": 84, "t0_client_ns": 1786101153430661000, "t1_server_recv_ns": 1786101153426984000, "t2_server_send_ns": 1786101153426985000, "t3_client_ns": 1786101153436331000, "delay_ns": 5669000, "offset_ns": -6511500} +{"sample": 85, "t0_client_ns": 1786101153456272000, "t1_server_recv_ns": 1786101153452325000, "t2_server_send_ns": 1786101153452327000, "t3_client_ns": 1786101153461137000, "delay_ns": 4863000, "offset_ns": -6378500} +{"sample": 86, "t0_client_ns": 1786101153481193000, "t1_server_recv_ns": 1786101153477381000, "t2_server_send_ns": 1786101153477382000, "t3_client_ns": 1786101153486239000, "delay_ns": 5045000, "offset_ns": -6334500} +{"sample": 87, "t0_client_ns": 1786101153496015000, "t1_server_recv_ns": 1786101153492168000, "t2_server_send_ns": 1786101153492169000, "t3_client_ns": 1786101153500874000, "delay_ns": 4858000, "offset_ns": -6276000} +{"sample": 88, "t0_client_ns": 1786101153529409000, "t1_server_recv_ns": 1786101153525588000, "t2_server_send_ns": 1786101153525590000, "t3_client_ns": 1786101153534415000, "delay_ns": 5004000, "offset_ns": -6323000} +{"sample": 89, "t0_client_ns": 1786101153544812000, "t1_server_recv_ns": 1786101153541997000, "t2_server_send_ns": 1786101153541998000, "t3_client_ns": 1786101153550688000, "delay_ns": 5875000, "offset_ns": -5752500} +{"sample": 90, "t0_client_ns": 1786101153569587000, "t1_server_recv_ns": 1786101153565944000, "t2_server_send_ns": 1786101153565946000, "t3_client_ns": 1786101153575731000, "delay_ns": 6142000, "offset_ns": -6714000} +{"sample": 91, "t0_client_ns": 1786101153594870000, "t1_server_recv_ns": 1786101153591149000, "t2_server_send_ns": 1786101153591151000, "t3_client_ns": 1786101153600828000, "delay_ns": 5956000, "offset_ns": -6699000} +{"sample": 92, "t0_client_ns": 1786101153616188000, "t1_server_recv_ns": 1786101153612329000, "t2_server_send_ns": 1786101153612330000, "t3_client_ns": 1786101153621168000, "delay_ns": 4979000, "offset_ns": -6348500} +{"sample": 93, "t0_client_ns": 1786101153638945000, "t1_server_recv_ns": 1786101153635126000, "t2_server_send_ns": 1786101153635127000, "t3_client_ns": 1786101153644467000, "delay_ns": 5521000, "offset_ns": -6579500} +{"sample": 94, "t0_client_ns": 1786101153666247000, "t1_server_recv_ns": 1786101153662780000, "t2_server_send_ns": 1786101153662781000, "t3_client_ns": 1786101153672503000, "delay_ns": 6255000, "offset_ns": -6594500} +{"sample": 95, "t0_client_ns": 1786101153689594000, "t1_server_recv_ns": 1786101153686022000, "t2_server_send_ns": 1786101153686024000, "t3_client_ns": 1786101153694557000, "delay_ns": 4961000, "offset_ns": -6052500} +{"sample": 96, "t0_client_ns": 1786101153716300000, "t1_server_recv_ns": 1786101153712603000, "t2_server_send_ns": 1786101153712604000, "t3_client_ns": 1786101153722436000, "delay_ns": 6135000, "offset_ns": -6764500} +{"sample": 97, "t0_client_ns": 1786101153733434000, "t1_server_recv_ns": 1786101153729633000, "t2_server_send_ns": 1786101153729634000, "t3_client_ns": 1786101153738380000, "delay_ns": 4945000, "offset_ns": -6273500} +{"sample": 98, "t0_client_ns": 1786101153749266000, "t1_server_recv_ns": 1786101153745322000, "t2_server_send_ns": 1786101153745323000, "t3_client_ns": 1786101153754257000, "delay_ns": 4990000, "offset_ns": -6439000} +{"sample": 99, "t0_client_ns": 1786101153771720000, "t1_server_recv_ns": 1786101153767547000, "t2_server_send_ns": 1786101153767548000, "t3_client_ns": 1786101153776258000, "delay_ns": 4537000, "offset_ns": -6441500} +{"sample": 100, "t0_client_ns": 1786101153800970000, "t1_server_recv_ns": 1786101153797162000, "t2_server_send_ns": 1786101153797163000, "t3_client_ns": 1786101153806272000, "delay_ns": 5301000, "offset_ns": -6458500} +{"sample": 101, "t0_client_ns": 1786101153815189000, "t1_server_recv_ns": 1786101153811268000, "t2_server_send_ns": 1786101153811269000, "t3_client_ns": 1786101153823590000, "delay_ns": 8400000, "offset_ns": -8121000} +{"sample": 102, "t0_client_ns": 1786101153846814000, "t1_server_recv_ns": 1786101153842982000, "t2_server_send_ns": 1786101153842983000, "t3_client_ns": 1786101153852077000, "delay_ns": 5262000, "offset_ns": -6463000} +{"sample": 103, "t0_client_ns": 1786101153868985000, "t1_server_recv_ns": 1786101153864998000, "t2_server_send_ns": 1786101153865000000, "t3_client_ns": 1786101153875136000, "delay_ns": 6149000, "offset_ns": -7061500} +{"sample": 104, "t0_client_ns": 1786101153898963000, "t1_server_recv_ns": 1786101153895744000, "t2_server_send_ns": 1786101153895745000, "t3_client_ns": 1786101153907927000, "delay_ns": 8963000, "offset_ns": -7700500} +{"sample": 105, "t0_client_ns": 1786101153928070000, "t1_server_recv_ns": 1786101153924323000, "t2_server_send_ns": 1786101153924324000, "t3_client_ns": 1786101153933312000, "delay_ns": 5241000, "offset_ns": -6367500} +{"sample": 106, "t0_client_ns": 1786101153945885000, "t1_server_recv_ns": 1786101153941966000, "t2_server_send_ns": 1786101153941968000, "t3_client_ns": 1786101153950744000, "delay_ns": 4857000, "offset_ns": -6347500} +{"sample": 107, "t0_client_ns": 1786101153965703000, "t1_server_recv_ns": 1786101153961929000, "t2_server_send_ns": 1786101153961932000, "t3_client_ns": 1786101153970689000, "delay_ns": 4983000, "offset_ns": -6265500} +{"sample": 108, "t0_client_ns": 1786101153994812000, "t1_server_recv_ns": 1786101153991363000, "t2_server_send_ns": 1786101153991367000, "t3_client_ns": 1786101154006463000, "delay_ns": 11647000, "offset_ns": -9272500} +{"sample": 109, "t0_client_ns": 1786101154029578000, "t1_server_recv_ns": 1786101154025859000, "t2_server_send_ns": 1786101154025861000, "t3_client_ns": 1786101154037543000, "delay_ns": 7963000, "offset_ns": -7700500} +{"sample": 110, "t0_client_ns": 1786101154057754000, "t1_server_recv_ns": 1786101154053924000, "t2_server_send_ns": 1786101154053926000, "t3_client_ns": 1786101154062842000, "delay_ns": 5086000, "offset_ns": -6373000} +{"sample": 111, "t0_client_ns": 1786101154078701000, "t1_server_recv_ns": 1786101154074937000, "t2_server_send_ns": 1786101154074938000, "t3_client_ns": 1786101154083743000, "delay_ns": 5041000, "offset_ns": -6284500} +{"sample": 112, "t0_client_ns": 1786101154096012000, "t1_server_recv_ns": 1786101154092122000, "t2_server_send_ns": 1786101154092123000, "t3_client_ns": 1786101154101011000, "delay_ns": 4998000, "offset_ns": -6389000} +{"sample": 113, "t0_client_ns": 1786101154112984000, "t1_server_recv_ns": 1786101154109101000, "t2_server_send_ns": 1786101154109102000, "t3_client_ns": 1786101154118529000, "delay_ns": 5544000, "offset_ns": -6655000} +{"sample": 114, "t0_client_ns": 1786101154137397000, "t1_server_recv_ns": 1786101154134410000, "t2_server_send_ns": 1786101154134411000, "t3_client_ns": 1786101154143246000, "delay_ns": 5848000, "offset_ns": -5911000} +{"sample": 115, "t0_client_ns": 1786101154167645000, "t1_server_recv_ns": 1786101154163977000, "t2_server_send_ns": 1786101154163979000, "t3_client_ns": 1786101154172814000, "delay_ns": 5167000, "offset_ns": -6251500} +{"sample": 116, "t0_client_ns": 1786101154192752000, "t1_server_recv_ns": 1786101154188924000, "t2_server_send_ns": 1786101154188925000, "t3_client_ns": 1786101154197912000, "delay_ns": 5159000, "offset_ns": -6407500} +{"sample": 117, "t0_client_ns": 1786101154222275000, "t1_server_recv_ns": 1786101154218574000, "t2_server_send_ns": 1786101154218576000, "t3_client_ns": 1786101154227816000, "delay_ns": 5539000, "offset_ns": -6470500} +{"sample": 118, "t0_client_ns": 1786101154246892000, "t1_server_recv_ns": 1786101154243119000, "t2_server_send_ns": 1786101154243120000, "t3_client_ns": 1786101154251843000, "delay_ns": 4950000, "offset_ns": -6248000} +{"sample": 119, "t0_client_ns": 1786101154275798000, "t1_server_recv_ns": 1786101154272169000, "t2_server_send_ns": 1786101154272170000, "t3_client_ns": 1786101154281608000, "delay_ns": 5809000, "offset_ns": -6533500} +{"sample": 120, "t0_client_ns": 1786101154287506000, "t1_server_recv_ns": 1786101154283604000, "t2_server_send_ns": 1786101154283605000, "t3_client_ns": 1786101154292411000, "delay_ns": 4904000, "offset_ns": -6354000} +{"sample": 121, "t0_client_ns": 1786101154318600000, "t1_server_recv_ns": 1786101154314851000, "t2_server_send_ns": 1786101154314854000, "t3_client_ns": 1786101154324615000, "delay_ns": 6012000, "offset_ns": -6755000} +{"sample": 122, "t0_client_ns": 1786101154347350000, "t1_server_recv_ns": 1786101154343840000, "t2_server_send_ns": 1786101154343842000, "t3_client_ns": 1786101154352629000, "delay_ns": 5277000, "offset_ns": -6148500} +{"sample": 123, "t0_client_ns": 1786101154372837000, "t1_server_recv_ns": 1786101154369538000, "t2_server_send_ns": 1786101154369540000, "t3_client_ns": 1786101154378946000, "delay_ns": 6107000, "offset_ns": -6352500} +{"sample": 124, "t0_client_ns": 1786101154402983000, "t1_server_recv_ns": 1786101154399230000, "t2_server_send_ns": 1786101154399232000, "t3_client_ns": 1786101154409561000, "delay_ns": 6576000, "offset_ns": -7041000} +{"sample": 125, "t0_client_ns": 1786101154431171000, "t1_server_recv_ns": 1786101154427821000, "t2_server_send_ns": 1786101154427822000, "t3_client_ns": 1786101154436465000, "delay_ns": 5293000, "offset_ns": -5996500} +{"sample": 126, "t0_client_ns": 1786101154461572000, "t1_server_recv_ns": 1786101154457807000, "t2_server_send_ns": 1786101154457808000, "t3_client_ns": 1786101154466613000, "delay_ns": 5040000, "offset_ns": -6285000} +{"sample": 127, "t0_client_ns": 1786101154484121000, "t1_server_recv_ns": 1786101154480144000, "t2_server_send_ns": 1786101154480145000, "t3_client_ns": 1786101154488788000, "delay_ns": 4666000, "offset_ns": -6310000} +{"sample": 128, "t0_client_ns": 1786101154512311000, "t1_server_recv_ns": 1786101154509953000, "t2_server_send_ns": 1786101154509955000, "t3_client_ns": 1786101154520662000, "delay_ns": 8349000, "offset_ns": -6532500} +{"sample": 129, "t0_client_ns": 1786101154542514000, "t1_server_recv_ns": 1786101154538615000, "t2_server_send_ns": 1786101154538617000, "t3_client_ns": 1786101154547286000, "delay_ns": 4770000, "offset_ns": -6284000} +{"sample": 130, "t0_client_ns": 1786101154554523000, "t1_server_recv_ns": 1786101154550674000, "t2_server_send_ns": 1786101154550676000, "t3_client_ns": 1786101154559457000, "delay_ns": 4932000, "offset_ns": -6315000} +{"sample": 131, "t0_client_ns": 1786101154579901000, "t1_server_recv_ns": 1786101154576210000, "t2_server_send_ns": 1786101154576212000, "t3_client_ns": 1786101154584855000, "delay_ns": 4952000, "offset_ns": -6167000} +{"sample": 132, "t0_client_ns": 1786101154604535000, "t1_server_recv_ns": 1786101154600731000, "t2_server_send_ns": 1786101154600732000, "t3_client_ns": 1786101154609509000, "delay_ns": 4973000, "offset_ns": -6290500} +{"sample": 133, "t0_client_ns": 1786101154633585000, "t1_server_recv_ns": 1786101154629785000, "t2_server_send_ns": 1786101154629786000, "t3_client_ns": 1786101154639314000, "delay_ns": 5728000, "offset_ns": -6664000} +{"sample": 134, "t0_client_ns": 1786101154663608000, "t1_server_recv_ns": 1786101154659944000, "t2_server_send_ns": 1786101154659946000, "t3_client_ns": 1786101154668667000, "delay_ns": 5057000, "offset_ns": -6192500} +{"sample": 135, "t0_client_ns": 1786101154688134000, "t1_server_recv_ns": 1786101154684789000, "t2_server_send_ns": 1786101154684791000, "t3_client_ns": 1786101154693569000, "delay_ns": 5433000, "offset_ns": -6061500} +{"sample": 136, "t0_client_ns": 1786101154717655000, "t1_server_recv_ns": 1786101154715323000, "t2_server_send_ns": 1786101154715324000, "t3_client_ns": 1786101154724105000, "delay_ns": 6449000, "offset_ns": -5556500} +{"sample": 137, "t0_client_ns": 1786101154740172000, "t1_server_recv_ns": 1786101154738496000, "t2_server_send_ns": 1786101154738497000, "t3_client_ns": 1786101154752417000, "delay_ns": 12244000, "offset_ns": -7798000} +{"sample": 138, "t0_client_ns": 1786101154766693000, "t1_server_recv_ns": 1786101154762719000, "t2_server_send_ns": 1786101154762720000, "t3_client_ns": 1786101154771570000, "delay_ns": 4876000, "offset_ns": -6412000} +{"sample": 139, "t0_client_ns": 1786101154787986000, "t1_server_recv_ns": 1786101154784057000, "t2_server_send_ns": 1786101154784058000, "t3_client_ns": 1786101154792907000, "delay_ns": 4920000, "offset_ns": -6389000} +{"sample": 140, "t0_client_ns": 1786101154817783000, "t1_server_recv_ns": 1786101154813963000, "t2_server_send_ns": 1786101154813964000, "t3_client_ns": 1786101154823749000, "delay_ns": 5965000, "offset_ns": -6802500} +{"sample": 141, "t0_client_ns": 1786101154846653000, "t1_server_recv_ns": 1786101154842865000, "t2_server_send_ns": 1786101154842866000, "t3_client_ns": 1786101154852251000, "delay_ns": 5597000, "offset_ns": -6586500} +{"sample": 142, "t0_client_ns": 1786101154876010000, "t1_server_recv_ns": 1786101154872261000, "t2_server_send_ns": 1786101154872262000, "t3_client_ns": 1786101154881416000, "delay_ns": 5405000, "offset_ns": -6451500} +{"sample": 143, "t0_client_ns": 1786101154902696000, "t1_server_recv_ns": 1786101154898951000, "t2_server_send_ns": 1786101154898952000, "t3_client_ns": 1786101154907799000, "delay_ns": 5102000, "offset_ns": -6296000} +{"sample": 144, "t0_client_ns": 1786101154931986000, "t1_server_recv_ns": 1786101154929291000, "t2_server_send_ns": 1786101154929292000, "t3_client_ns": 1786101154937828000, "delay_ns": 5841000, "offset_ns": -5615500} +{"sample": 145, "t0_client_ns": 1786101154961138000, "t1_server_recv_ns": 1786101154957154000, "t2_server_send_ns": 1786101154957155000, "t3_client_ns": 1786101154967412000, "delay_ns": 6273000, "offset_ns": -7120500} diff --git a/docs/evidence/mount-latency-20260807/raw/control-create.jsonl b/docs/evidence/mount-latency-20260807/raw/control-create.jsonl new file mode 100644 index 00000000..16af9257 --- /dev/null +++ b/docs/evidence/mount-latency-20260807/raw/control-create.jsonl @@ -0,0 +1,25 @@ +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 1, "path": "control-001/probe.txt", "t_create_ns": 1786101800568243000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 2, "path": "control-002/probe.txt", "t_create_ns": 1786101801579439000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 3, "path": "control-003/probe.txt", "t_create_ns": 1786101802591422000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 4, "path": "control-004/probe.txt", "t_create_ns": 1786101803595768000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 5, "path": "control-005/probe.txt", "t_create_ns": 1786101804605374000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 6, "path": "control-006/probe.txt", "t_create_ns": 1786101805611161000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 7, "path": "control-007/probe.txt", "t_create_ns": 1786101806616160000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 8, "path": "control-008/probe.txt", "t_create_ns": 1786101807624800000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 9, "path": "control-009/probe.txt", "t_create_ns": 1786101808629361000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 10, "path": "control-010/probe.txt", "t_create_ns": 1786101809636131000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 11, "path": "control-011/probe.txt", "t_create_ns": 1786101810643324000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 12, "path": "control-012/probe.txt", "t_create_ns": 1786101811650699000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 13, "path": "control-013/probe.txt", "t_create_ns": 1786101812658739000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 14, "path": "control-014/probe.txt", "t_create_ns": 1786101813663414000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 15, "path": "control-015/probe.txt", "t_create_ns": 1786101814671474000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 16, "path": "control-016/probe.txt", "t_create_ns": 1786101815681426000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 17, "path": "control-017/probe.txt", "t_create_ns": 1786101816687092000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 18, "path": "control-018/probe.txt", "t_create_ns": 1786101817691901000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 19, "path": "control-019/probe.txt", "t_create_ns": 1786101818702810000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 20, "path": "control-020/probe.txt", "t_create_ns": 1786101819706072000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 21, "path": "control-021/probe.txt", "t_create_ns": 1786101820708010000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 22, "path": "control-022/probe.txt", "t_create_ns": 1786101821716549000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 23, "path": "control-023/probe.txt", "t_create_ns": 1786101822718870000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 24, "path": "control-024/probe.txt", "t_create_ns": 1786101823730186000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"kind": "control_create", "run_id": "ctrl20260807", "trial": 25, "path": "control-025/probe.txt", "t_create_ns": 1786101824741212000, "host": "sf-mini", "clock": "CLOCK_REALTIME"} diff --git a/docs/evidence/mount-latency-20260807/raw/control-watch.jsonl b/docs/evidence/mount-latency-20260807/raw/control-watch.jsonl new file mode 100644 index 00000000..9fded81f --- /dev/null +++ b/docs/evidence/mount-latency-20260807/raw/control-watch.jsonl @@ -0,0 +1,29 @@ +{"kind": "watcher_started", "watch_dir": "/Users/khaliqgant/.relayfile-latency-harness/control-dir", "poll_seconds": 0.001, "primed_paths": 0, "started_ns": 1786101797649995000} +{"path": "control-001/probe.txt", "observed_ns": 1786101800569468000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-002/probe.txt", "observed_ns": 1786101801580141000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-003/probe.txt", "observed_ns": 1786101802592843000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-004/probe.txt", "observed_ns": 1786101803596128000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-005/probe.txt.tmp", "observed_ns": 1786101804605098000, "size": 0, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-005/probe.txt", "observed_ns": 1786101804607132000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-006/probe.txt", "observed_ns": 1786101805612090000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-007/probe.txt", "observed_ns": 1786101806617551000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-008/probe.txt", "observed_ns": 1786101807625517000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-009/probe.txt", "observed_ns": 1786101808630974000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-010/probe.txt", "observed_ns": 1786101809637000000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-011/probe.txt", "observed_ns": 1786101810644761000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-012/probe.txt", "observed_ns": 1786101811652319000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-013/probe.txt", "observed_ns": 1786101812660275000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-014/probe.txt.tmp", "observed_ns": 1786101813663354000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-014/probe.txt", "observed_ns": 1786101813665105000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-015/probe.txt", "observed_ns": 1786101814672518000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-016/probe.txt", "observed_ns": 1786101815683775000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-017/probe.txt", "observed_ns": 1786101816689547000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-018/probe.txt", "observed_ns": 1786101817692492000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-019/probe.txt", "observed_ns": 1786101818703801000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-020/probe.txt", "observed_ns": 1786101819706753000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-021/probe.txt", "observed_ns": 1786101820708813000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-022/probe.txt.tmp", "observed_ns": 1786101821716502000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-022/probe.txt", "observed_ns": 1786101821718562000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-023/probe.txt", "observed_ns": 1786101822719982000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-024/probe.txt", "observed_ns": 1786101823731338000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "control-025/probe.txt", "observed_ns": 1786101824743247000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} diff --git a/docs/evidence/mount-latency-20260807/raw/heartbeat-gate-post.jsonl b/docs/evidence/mount-latency-20260807/raw/heartbeat-gate-post.jsonl new file mode 100644 index 00000000..78757215 --- /dev/null +++ b/docs/evidence/mount-latency-20260807/raw/heartbeat-gate-post.jsonl @@ -0,0 +1,14 @@ +{"sampledAtLocalUtc":"2026-08-07T11:29:21.412747+00:00","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:29:08.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:29:32.545417+00:00","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:29:08.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:29:44.194190+00:00","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:29:08.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:29:55.290910+00:00","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:29:08.000Z", "status": "offline", "live": false, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:30:06.536428+00:00","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:29:08.000Z", "status": "offline", "live": false, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:30:17.754069+00:00","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:30:08.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:30:29.210152+00:00","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:30:08.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:30:40.477144+00:00","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:30:08.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:30:51.732124+00:00","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:30:08.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:31:03.127281+00:00","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:30:08.000Z", "status": "offline", "live": false, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:31:14.325223+00:00","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:31:09.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:31:25.377374+00:00","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:31:09.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:31:36.550285+00:00","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:31:09.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:31:47.703594+00:00","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:31:09.000Z", "status": "online", "live": true, "activeAgents": 7}} diff --git a/docs/evidence/mount-latency-20260807/raw/heartbeat-gate-pre.jsonl b/docs/evidence/mount-latency-20260807/raw/heartbeat-gate-pre.jsonl new file mode 100644 index 00000000..696ac446 --- /dev/null +++ b/docs/evidence/mount-latency-20260807/raw/heartbeat-gate-pre.jsonl @@ -0,0 +1,14 @@ +{"sampledAtLocalUtc":"2026-08-07T11:10:48.3NZ","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:09:54.000Z", "status": "offline", "live": false, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:10:59.3NZ","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:11:00.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:11:11.3NZ","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:11:00.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:11:22.3NZ","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:11:00.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:11:33.3NZ","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:11:00.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:11:44.3NZ","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:11:00.000Z", "status": "offline", "live": false, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:11:56.3NZ","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:11:00.000Z", "status": "offline", "live": false, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:12:07.3NZ","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:12:00.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:12:18.3NZ","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:12:00.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:12:30.3NZ","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:12:00.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:12:42.3NZ","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:12:00.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:12:53.3NZ","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:12:00.000Z", "status": "offline", "live": false, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:13:04.3NZ","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:13:00.000Z", "status": "online", "live": true, "activeAgents": 7}} +{"sampledAtLocalUtc":"2026-08-07T11:13:16.3NZ","node":{"present": true, "lastHeartbeatAt": "2026-08-07T11:13:00.000Z", "status": "online", "live": true, "activeAgents": 7}} diff --git a/docs/evidence/mount-latency-20260807/raw/heartbeat-gate-pre.sh b/docs/evidence/mount-latency-20260807/raw/heartbeat-gate-pre.sh new file mode 100755 index 00000000..d2622e49 --- /dev/null +++ b/docs/evidence/mount-latency-20260807/raw/heartbeat-gate-pre.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Heartbeat-advance gate: sf-mini is live iff its OWN lastHeartbeatAt advances. +# Absence from a fleet listing is NOT evidence of offline (the list returns +# nondeterministic subsets), and status/live are registration fields, not +# liveness fields. Only monotonic advance of lastHeartbeatAt counts. +OUT="$1"; DURATION="${2:-150}"; INTERVAL="${3:-10}" +END=$(( $(date +%s) + DURATION )) +while [ "$(date +%s)" -lt "$END" ]; do + TS=$(python3 -c "import datetime;print(datetime.datetime.now(datetime.timezone.utc).isoformat())") + HB=$(timeout 25 agent-relay fleet nodes --name sf-mini --all 2>/dev/null \ + | python3 -c 'import sys,json;d=json.load(sys.stdin);n=[x for x in d.get("nodes",[]) if x.get("name")=="sf-mini"];print(json.dumps({"present":bool(n),"lastHeartbeatAt":n[0].get("lastHeartbeatAt") if n else None,"status":n[0].get("status") if n else None,"live":n[0].get("live") if n else None,"activeAgents":n[0].get("activeAgents") if n else None}) if True else "")' 2>/dev/null) + [ -z "$HB" ] && HB='{"present":false,"lastHeartbeatAt":null,"status":null,"live":null,"error":"query_failed"}' + echo "{\"sampledAtLocalUtc\":\"$TS\",\"node\":$HB}" >> "$OUT" + sleep "$INTERVAL" +done diff --git a/docs/evidence/mount-latency-20260807/raw/mount-watch.jsonl b/docs/evidence/mount-latency-20260807/raw/mount-watch.jsonl new file mode 100644 index 00000000..4098f8af --- /dev/null +++ b/docs/evidence/mount-latency-20260807/raw/mount-watch.jsonl @@ -0,0 +1,278 @@ +{"kind": "watcher_started", "watch_dir": "/Users/khaliqgant/relayfile-latency-mount-20260807", "poll_seconds": 0.001, "primed_paths": 2, "started_ns": 1786101847897568000} +{"path": "trials/run20260807/small-001/probe.txt", "observed_ns": 1786101857076845000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/run20260807/small-002/probe.txt", "observed_ns": 1786101861108313000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/run20260807/small-003/.probe.txt.tmp-1973661880", "observed_ns": 1786101865128068000, "size": 0, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/run20260807/small-003/probe.txt", "observed_ns": 1786101865130497000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/run20260807/small-004/probe.txt", "observed_ns": 1786101869207039000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/run20260807/small-005/probe.txt", "observed_ns": 1786101873281623000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/run20260807/small-006/probe.txt", "observed_ns": 1786101877343195000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/run20260807/small-007/probe.txt", "observed_ns": 1786101881432832000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/run20260807/small-008/.probe.txt.tmp-1261788292", "observed_ns": 1786101885513899000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/run20260807/small-008/probe.txt", "observed_ns": 1786101885516586000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/run20260807/small-009/probe.txt", "observed_ns": 1786101889634997000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/run20260807/small-010/.probe.txt.tmp-2549592552", "observed_ns": 1786101893608284000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/run20260807/small-010/probe.txt", "observed_ns": 1786101893612910000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/run20260807/small-011/probe.txt", "observed_ns": 1786101897686769000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/run20260807/small-012/.probe.txt.tmp-3422675134", "observed_ns": 1786101901763470000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/run20260807/small-012/probe.txt", "observed_ns": 1786101901766345000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-001/probe.txt", "observed_ns": 1786101994108922000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-002/probe.txt", "observed_ns": 1786101997999680000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-003/probe.txt", "observed_ns": 1786102002072841000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-004/.probe.txt.tmp-749829831", "observed_ns": 1786102006154863000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-004/probe.txt", "observed_ns": 1786102006158044000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-005/probe.txt", "observed_ns": 1786102010235641000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-006/probe.txt", "observed_ns": 1786102014301731000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-007/probe.txt", "observed_ns": 1786102018323420000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-008/probe.txt", "observed_ns": 1786102022401378000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-009/.probe.txt.tmp-3124405718", "observed_ns": 1786102026484875000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-009/probe.txt", "observed_ns": 1786102026488460000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-010/probe.txt", "observed_ns": 1786102030669712000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-011/.probe.txt.tmp-3673320926", "observed_ns": 1786102034638425000, "size": 0, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-011/probe.txt", "observed_ns": 1786102034642477000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-012/.probe.txt.tmp-3176617263", "observed_ns": 1786102038699850000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-012/probe.txt", "observed_ns": 1786102038702468000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-013/probe.txt", "observed_ns": 1786102042813956000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-014/probe.txt", "observed_ns": 1786102046812372000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-015/.probe.txt.tmp-4283185647", "observed_ns": 1786102051041613000, "size": null, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-015/probe.txt", "observed_ns": 1786102051045504000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-016/probe.txt", "observed_ns": 1786102054913951000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-017/probe.txt", "observed_ns": 1786102058982481000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-018/.probe.txt.tmp-2313264102", "observed_ns": 1786102063123266000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-018/probe.txt", "observed_ns": 1786102063127057000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-019/.probe.txt.tmp-1606707704", "observed_ns": 1786102067225164000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-019/probe.txt", "observed_ns": 1786102067237645000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/small-020/probe.txt", "observed_ns": 1786102071165891000, "size": 300, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-001/src/module_00.go", "observed_ns": 1786102071263699000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-001/src/module_01.go", "observed_ns": 1786102071284201000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-001/src/module_02.go", "observed_ns": 1786102071294960000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-001/src/module_03.go", "observed_ns": 1786102071307526000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-001/src/.module_04.go.tmp-214638458", "observed_ns": 1786102071319783000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-001/src/module_04.go", "observed_ns": 1786102071323513000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-001/src/module_05.go", "observed_ns": 1786102071336398000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-001/src/module_06.go", "observed_ns": 1786102071352431000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-001/src/module_07.go", "observed_ns": 1786102071374251000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-001/src/module_08.go", "observed_ns": 1786102071392285000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-001/src/module_09.go", "observed_ns": 1786102071399902000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-001/src/module_10.go", "observed_ns": 1786102071417246000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-002/src/module_00.go", "observed_ns": 1786102075327970000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-002/src/module_01.go", "observed_ns": 1786102075352015000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-002/src/.module_02.go.tmp-2664515302", "observed_ns": 1786102075366388000, "size": 0, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-002/src/module_02.go", "observed_ns": 1786102075369878000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-002/src/module_03.go", "observed_ns": 1786102075383039000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-002/src/.module_04.go.tmp-3628111869", "observed_ns": 1786102075395799000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-002/src/module_04.go", "observed_ns": 1786102075398735000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-002/src/module_05.go", "observed_ns": 1786102075413199000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-002/src/module_06.go", "observed_ns": 1786102075438252000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-002/src/module_07.go", "observed_ns": 1786102075449372000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-002/src/module_08.go", "observed_ns": 1786102075458875000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-002/src/module_09.go", "observed_ns": 1786102075464913000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-002/src/module_10.go", "observed_ns": 1786102075483202000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-003/src/module_00.go", "observed_ns": 1786102079402558000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-003/src/module_01.go", "observed_ns": 1786102079426271000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-003/src/module_02.go", "observed_ns": 1786102079443675000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-003/src/module_03.go", "observed_ns": 1786102079457808000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-003/src/module_04.go", "observed_ns": 1786102079471779000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-003/src/module_05.go", "observed_ns": 1786102079487026000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-003/src/module_06.go", "observed_ns": 1786102079498821000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-003/src/module_07.go", "observed_ns": 1786102079511967000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-003/src/module_08.go", "observed_ns": 1786102079526599000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-003/src/module_09.go", "observed_ns": 1786102079539129000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-003/src/module_10.go", "observed_ns": 1786102079550114000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-004/src/module_00.go", "observed_ns": 1786102083407246000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-004/src/module_01.go", "observed_ns": 1786102083432746000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-004/src/module_02.go", "observed_ns": 1786102083455036000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-004/src/module_03.go", "observed_ns": 1786102083471402000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-004/src/module_04.go", "observed_ns": 1786102083486882000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-004/src/module_05.go", "observed_ns": 1786102083504902000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-004/src/module_06.go", "observed_ns": 1786102083519974000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-004/src/module_07.go", "observed_ns": 1786102083534351000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-004/src/.module_08.go.tmp-3253902787", "observed_ns": 1786102083550569000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-004/src/module_08.go", "observed_ns": 1786102083553858000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-004/src/module_09.go", "observed_ns": 1786102083566109000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-004/src/module_10.go", "observed_ns": 1786102083577251000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-005/src/module_00.go", "observed_ns": 1786102087485285000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-005/src/module_01.go", "observed_ns": 1786102087509193000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-005/src/.module_02.go.tmp-1821778106", "observed_ns": 1786102087528447000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-005/src/module_02.go", "observed_ns": 1786102087532242000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-005/src/module_03.go", "observed_ns": 1786102087549636000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-005/src/module_04.go", "observed_ns": 1786102087566802000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-005/src/module_05.go", "observed_ns": 1786102087597442000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-005/src/module_06.go", "observed_ns": 1786102087621327000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-005/src/module_07.go", "observed_ns": 1786102087636889000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-005/src/module_08.go", "observed_ns": 1786102087652147000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-005/src/module_09.go", "observed_ns": 1786102087681525000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-005/src/module_10.go", "observed_ns": 1786102087693939000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-006/src/module_00.go", "observed_ns": 1786102091532041000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-006/src/module_01.go", "observed_ns": 1786102091554369000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-006/src/module_02.go", "observed_ns": 1786102091578853000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-006/src/module_03.go", "observed_ns": 1786102091599077000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-006/src/module_04.go", "observed_ns": 1786102091621195000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-006/src/module_05.go", "observed_ns": 1786102091646979000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-006/src/module_06.go", "observed_ns": 1786102091665781000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-006/src/.module_07.go.tmp-2289625961", "observed_ns": 1786102091677079000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-006/src/module_07.go", "observed_ns": 1786102091681039000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-006/src/module_08.go", "observed_ns": 1786102091691315000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-006/src/module_09.go", "observed_ns": 1786102091705565000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-006/src/module_10.go", "observed_ns": 1786102091716096000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-007/src/module_00.go", "observed_ns": 1786102095563931000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-007/src/module_01.go", "observed_ns": 1786102095594296000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-007/src/module_02.go", "observed_ns": 1786102095617152000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-007/src/module_03.go", "observed_ns": 1786102095635704000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-007/src/module_04.go", "observed_ns": 1786102095657277000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-007/src/module_05.go", "observed_ns": 1786102095675843000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-007/src/module_06.go", "observed_ns": 1786102095689286000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-007/src/module_07.go", "observed_ns": 1786102095705075000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-007/src/module_08.go", "observed_ns": 1786102095721018000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-007/src/.module_09.go.tmp-4173610473", "observed_ns": 1786102095733039000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-007/src/module_09.go", "observed_ns": 1786102095736843000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-007/src/module_10.go", "observed_ns": 1786102095746997000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-008/src/module_00.go", "observed_ns": 1786102099645242000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-008/src/module_01.go", "observed_ns": 1786102099668381000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-008/src/module_02.go", "observed_ns": 1786102099687483000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-008/src/module_03.go", "observed_ns": 1786102099703899000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-008/src/module_04.go", "observed_ns": 1786102099717839000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-008/src/module_05.go", "observed_ns": 1786102099736478000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-008/src/module_06.go", "observed_ns": 1786102099750521000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-008/src/module_07.go", "observed_ns": 1786102099758424000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-008/src/module_08.go", "observed_ns": 1786102099769557000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-008/src/module_09.go", "observed_ns": 1786102099785058000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-008/src/module_10.go", "observed_ns": 1786102099791805000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-009/src/module_00.go", "observed_ns": 1786102103678788000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-009/src/module_01.go", "observed_ns": 1786102103699364000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-009/src/module_02.go", "observed_ns": 1786102103719161000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-009/src/module_03.go", "observed_ns": 1786102103738566000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-009/src/module_04.go", "observed_ns": 1786102103757374000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-009/src/.module_05.go.tmp-4240180216", "observed_ns": 1786102103772260000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-009/src/module_05.go", "observed_ns": 1786102103777530000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-009/src/module_06.go", "observed_ns": 1786102103794451000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-009/src/module_07.go", "observed_ns": 1786102103810015000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-009/src/module_08.go", "observed_ns": 1786102103820907000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-009/src/module_09.go", "observed_ns": 1786102103830850000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-009/src/module_10.go", "observed_ns": 1786102103843171000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-010/src/module_00.go", "observed_ns": 1786102107777900000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-010/src/module_01.go", "observed_ns": 1786102107806429000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-010/src/module_02.go", "observed_ns": 1786102107823554000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-010/src/.module_03.go.tmp-1918979592", "observed_ns": 1786102107844755000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-010/src/module_03.go", "observed_ns": 1786102107850131000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-010/src/module_04.go", "observed_ns": 1786102107869312000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-010/src/module_05.go", "observed_ns": 1786102107891825000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-010/src/module_06.go", "observed_ns": 1786102107905023000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-010/src/module_07.go", "observed_ns": 1786102107931762000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-010/src/module_08.go", "observed_ns": 1786102107949211000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-010/src/module_09.go", "observed_ns": 1786102107996763000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-010/src/module_10.go", "observed_ns": 1786102108042251000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-011/src/module_00.go", "observed_ns": 1786102111832838000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-011/src/module_01.go", "observed_ns": 1786102111861429000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-011/src/module_02.go", "observed_ns": 1786102111882048000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-011/src/module_03.go", "observed_ns": 1786102111904221000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-011/src/module_04.go", "observed_ns": 1786102111921565000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-011/src/module_05.go", "observed_ns": 1786102111945251000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-011/src/module_06.go", "observed_ns": 1786102111961828000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-011/src/module_07.go", "observed_ns": 1786102111977985000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-011/src/module_08.go", "observed_ns": 1786102111994658000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-011/src/module_09.go", "observed_ns": 1786102112006877000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-011/src/module_10.go", "observed_ns": 1786102112020866000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-012/src/module_00.go", "observed_ns": 1786102115906374000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-012/src/module_01.go", "observed_ns": 1786102115934799000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-012/src/module_02.go", "observed_ns": 1786102115955915000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-012/src/module_03.go", "observed_ns": 1786102115978844000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-012/src/module_04.go", "observed_ns": 1786102116003043000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-012/src/module_05.go", "observed_ns": 1786102116018116000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-012/src/module_06.go", "observed_ns": 1786102116033419000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-012/src/module_07.go", "observed_ns": 1786102116048495000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-012/src/module_08.go", "observed_ns": 1786102116065670000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-012/src/module_09.go", "observed_ns": 1786102116081756000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-012/src/module_10.go", "observed_ns": 1786102116093218000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-013/src/module_00.go", "observed_ns": 1786102119915484000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-013/src/module_01.go", "observed_ns": 1786102119940747000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-013/src/module_02.go", "observed_ns": 1786102119967704000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-013/src/module_03.go", "observed_ns": 1786102119986688000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-013/src/module_04.go", "observed_ns": 1786102120097304000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-013/src/module_05.go", "observed_ns": 1786102120114292000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-013/src/module_06.go", "observed_ns": 1786102120140255000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-013/src/module_07.go", "observed_ns": 1786102120157600000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-013/src/module_08.go", "observed_ns": 1786102120178662000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-013/src/module_09.go", "observed_ns": 1786102120200520000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-013/src/module_10.go", "observed_ns": 1786102120225174000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-014/src/module_00.go", "observed_ns": 1786102123977299000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-014/src/module_01.go", "observed_ns": 1786102124007008000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-014/src/module_02.go", "observed_ns": 1786102124033552000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-014/src/module_03.go", "observed_ns": 1786102124056360000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-014/src/module_04.go", "observed_ns": 1786102124076061000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-014/src/module_05.go", "observed_ns": 1786102124097558000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-014/src/module_06.go", "observed_ns": 1786102124120557000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-014/src/module_07.go", "observed_ns": 1786102124134197000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-014/src/module_08.go", "observed_ns": 1786102124147230000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-014/src/module_09.go", "observed_ns": 1786102124162355000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-014/src/module_10.go", "observed_ns": 1786102124173159000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-015/src/module_00.go", "observed_ns": 1786102128035253000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-015/src/module_01.go", "observed_ns": 1786102128069266000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-015/src/module_02.go", "observed_ns": 1786102128095075000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-015/src/module_03.go", "observed_ns": 1786102128114553000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-015/src/module_04.go", "observed_ns": 1786102128144221000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-015/src/module_05.go", "observed_ns": 1786102128165539000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-015/src/module_06.go", "observed_ns": 1786102128179854000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-015/src/module_07.go", "observed_ns": 1786102128194523000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-015/src/module_08.go", "observed_ns": 1786102128207639000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-015/src/module_09.go", "observed_ns": 1786102128222932000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-015/src/module_10.go", "observed_ns": 1786102128234395000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-016/src/module_00.go", "observed_ns": 1786102132074961000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-016/src/module_01.go", "observed_ns": 1786102132106456000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-016/src/module_02.go", "observed_ns": 1786102132132354000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-016/src/.module_03.go.tmp-4263997822", "observed_ns": 1786102132154700000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-016/src/module_03.go", "observed_ns": 1786102132161077000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-016/src/.module_04.go.tmp-1586712585", "observed_ns": 1786102132181665000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-016/src/module_04.go", "observed_ns": 1786102132186865000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-016/src/module_05.go", "observed_ns": 1786102132203371000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-016/src/module_06.go", "observed_ns": 1786102132222279000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-016/src/.module_07.go.tmp-4105602124", "observed_ns": 1786102132240568000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-016/src/module_07.go", "observed_ns": 1786102132247829000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-016/src/module_08.go", "observed_ns": 1786102132265949000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-016/src/module_09.go", "observed_ns": 1786102132277056000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-016/src/module_10.go", "observed_ns": 1786102132294855000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-017/src/module_00.go", "observed_ns": 1786102136147967000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-017/src/module_01.go", "observed_ns": 1786102136184448000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-017/src/module_02.go", "observed_ns": 1786102136214279000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-017/src/module_03.go", "observed_ns": 1786102136238951000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-017/src/module_04.go", "observed_ns": 1786102136266674000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-017/src/.module_05.go.tmp-3446155946", "observed_ns": 1786102136279259000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-017/src/module_05.go", "observed_ns": 1786102136284256000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-017/src/module_06.go", "observed_ns": 1786102136302742000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-017/src/module_07.go", "observed_ns": 1786102136313823000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-017/src/module_08.go", "observed_ns": 1786102136329328000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-017/src/module_09.go", "observed_ns": 1786102136340560000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-017/src/module_10.go", "observed_ns": 1786102136354119000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-018/src/module_00.go", "observed_ns": 1786102140197663000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-018/src/module_01.go", "observed_ns": 1786102140231560000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-018/src/module_02.go", "observed_ns": 1786102140260939000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-018/src/module_03.go", "observed_ns": 1786102140284312000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-018/src/module_04.go", "observed_ns": 1786102140310429000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-018/src/module_05.go", "observed_ns": 1786102140333918000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-018/src/module_06.go", "observed_ns": 1786102140349756000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-018/src/module_07.go", "observed_ns": 1786102140371396000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-018/src/module_08.go", "observed_ns": 1786102140383703000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-018/src/module_09.go", "observed_ns": 1786102140402772000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-018/src/module_10.go", "observed_ns": 1786102140420302000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-019/src/module_00.go", "observed_ns": 1786102144256439000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-019/src/module_01.go", "observed_ns": 1786102144291122000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-019/src/module_02.go", "observed_ns": 1786102144318062000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-019/src/module_03.go", "observed_ns": 1786102144336594000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-019/src/module_04.go", "observed_ns": 1786102144373350000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-019/src/.module_05.go.tmp-2141080661", "observed_ns": 1786102144384924000, "size": null, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-019/src/module_05.go", "observed_ns": 1786102144391160000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-019/src/module_06.go", "observed_ns": 1786102144406090000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-019/src/module_07.go", "observed_ns": 1786102144421835000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-019/src/module_08.go", "observed_ns": 1786102144433492000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-019/src/module_09.go", "observed_ns": 1786102144451712000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-019/src/module_10.go", "observed_ns": 1786102144468047000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-020/src/module_00.go", "observed_ns": 1786102148332044000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-020/src/module_01.go", "observed_ns": 1786102148359564000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-020/src/module_02.go", "observed_ns": 1786102148387317000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-020/src/module_03.go", "observed_ns": 1786102148410273000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-020/src/module_04.go", "observed_ns": 1786102148445656000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-020/src/module_05.go", "observed_ns": 1786102148472352000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-020/src/module_06.go", "observed_ns": 1786102148490755000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-020/src/module_07.go", "observed_ns": 1786102148508403000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-020/src/module_08.go", "observed_ns": 1786102148525715000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-020/src/module_09.go", "observed_ns": 1786102148543367000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} +{"path": "trials/r2/repo-020/src/module_10.go", "observed_ns": 1786102148564890000, "size": 1272, "host": "sf-mini", "clock": "CLOCK_REALTIME"} diff --git a/docs/evidence/mount-latency-20260807/raw/trials-repo.jsonl b/docs/evidence/mount-latency-20260807/raw/trials-repo.jsonl new file mode 100644 index 00000000..6fa563a6 --- /dev/null +++ b/docs/evidence/mount-latency-20260807/raw/trials-repo.jsonl @@ -0,0 +1,20 @@ +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 1, "correlation_id": "r2-repo-001", "paths": ["/trials/r2/repo-001/src/module_00.go", "/trials/r2/repo-001/src/module_01.go", "/trials/r2/repo-001/src/module_02.go", "/trials/r2/repo-001/src/module_03.go", "/trials/r2/repo-001/src/module_04.go", "/trials/r2/repo-001/src/module_05.go", "/trials/r2/repo-001/src/module_06.go", "/trials/r2/repo-001/src/module_07.go", "/trials/r2/repo-001/src/module_08.go", "/trials/r2/repo-001/src/module_09.go", "/trials/r2/repo-001/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102071247427000, "t_ack_ns": 1786102071267326000, "ack_ms": 19.899, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 2, "correlation_id": "r2-repo-002", "paths": ["/trials/r2/repo-002/src/module_00.go", "/trials/r2/repo-002/src/module_01.go", "/trials/r2/repo-002/src/module_02.go", "/trials/r2/repo-002/src/module_03.go", "/trials/r2/repo-002/src/module_04.go", "/trials/r2/repo-002/src/module_05.go", "/trials/r2/repo-002/src/module_06.go", "/trials/r2/repo-002/src/module_07.go", "/trials/r2/repo-002/src/module_08.go", "/trials/r2/repo-002/src/module_09.go", "/trials/r2/repo-002/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102075322223000, "t_ack_ns": 1786102075325698000, "ack_ms": 3.475, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 3, "correlation_id": "r2-repo-003", "paths": ["/trials/r2/repo-003/src/module_00.go", "/trials/r2/repo-003/src/module_01.go", "/trials/r2/repo-003/src/module_02.go", "/trials/r2/repo-003/src/module_03.go", "/trials/r2/repo-003/src/module_04.go", "/trials/r2/repo-003/src/module_05.go", "/trials/r2/repo-003/src/module_06.go", "/trials/r2/repo-003/src/module_07.go", "/trials/r2/repo-003/src/module_08.go", "/trials/r2/repo-003/src/module_09.go", "/trials/r2/repo-003/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102079396937000, "t_ack_ns": 1786102079399503000, "ack_ms": 2.566, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 4, "correlation_id": "r2-repo-004", "paths": ["/trials/r2/repo-004/src/module_00.go", "/trials/r2/repo-004/src/module_01.go", "/trials/r2/repo-004/src/module_02.go", "/trials/r2/repo-004/src/module_03.go", "/trials/r2/repo-004/src/module_04.go", "/trials/r2/repo-004/src/module_05.go", "/trials/r2/repo-004/src/module_06.go", "/trials/r2/repo-004/src/module_07.go", "/trials/r2/repo-004/src/module_08.go", "/trials/r2/repo-004/src/module_09.go", "/trials/r2/repo-004/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102083405735000, "t_ack_ns": 1786102083409018000, "ack_ms": 3.283, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 5, "correlation_id": "r2-repo-005", "paths": ["/trials/r2/repo-005/src/module_00.go", "/trials/r2/repo-005/src/module_01.go", "/trials/r2/repo-005/src/module_02.go", "/trials/r2/repo-005/src/module_03.go", "/trials/r2/repo-005/src/module_04.go", "/trials/r2/repo-005/src/module_05.go", "/trials/r2/repo-005/src/module_06.go", "/trials/r2/repo-005/src/module_07.go", "/trials/r2/repo-005/src/module_08.go", "/trials/r2/repo-005/src/module_09.go", "/trials/r2/repo-005/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102087481664000, "t_ack_ns": 1786102087486780000, "ack_ms": 5.116, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 6, "correlation_id": "r2-repo-006", "paths": ["/trials/r2/repo-006/src/module_00.go", "/trials/r2/repo-006/src/module_01.go", "/trials/r2/repo-006/src/module_02.go", "/trials/r2/repo-006/src/module_03.go", "/trials/r2/repo-006/src/module_04.go", "/trials/r2/repo-006/src/module_05.go", "/trials/r2/repo-006/src/module_06.go", "/trials/r2/repo-006/src/module_07.go", "/trials/r2/repo-006/src/module_08.go", "/trials/r2/repo-006/src/module_09.go", "/trials/r2/repo-006/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102091528496000, "t_ack_ns": 1786102091531483000, "ack_ms": 2.987, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 7, "correlation_id": "r2-repo-007", "paths": ["/trials/r2/repo-007/src/module_00.go", "/trials/r2/repo-007/src/module_01.go", "/trials/r2/repo-007/src/module_02.go", "/trials/r2/repo-007/src/module_03.go", "/trials/r2/repo-007/src/module_04.go", "/trials/r2/repo-007/src/module_05.go", "/trials/r2/repo-007/src/module_06.go", "/trials/r2/repo-007/src/module_07.go", "/trials/r2/repo-007/src/module_08.go", "/trials/r2/repo-007/src/module_09.go", "/trials/r2/repo-007/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102095561677000, "t_ack_ns": 1786102095564958000, "ack_ms": 3.281, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 8, "correlation_id": "r2-repo-008", "paths": ["/trials/r2/repo-008/src/module_00.go", "/trials/r2/repo-008/src/module_01.go", "/trials/r2/repo-008/src/module_02.go", "/trials/r2/repo-008/src/module_03.go", "/trials/r2/repo-008/src/module_04.go", "/trials/r2/repo-008/src/module_05.go", "/trials/r2/repo-008/src/module_06.go", "/trials/r2/repo-008/src/module_07.go", "/trials/r2/repo-008/src/module_08.go", "/trials/r2/repo-008/src/module_09.go", "/trials/r2/repo-008/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102099640174000, "t_ack_ns": 1786102099643927000, "ack_ms": 3.753, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 9, "correlation_id": "r2-repo-009", "paths": ["/trials/r2/repo-009/src/module_00.go", "/trials/r2/repo-009/src/module_01.go", "/trials/r2/repo-009/src/module_02.go", "/trials/r2/repo-009/src/module_03.go", "/trials/r2/repo-009/src/module_04.go", "/trials/r2/repo-009/src/module_05.go", "/trials/r2/repo-009/src/module_06.go", "/trials/r2/repo-009/src/module_07.go", "/trials/r2/repo-009/src/module_08.go", "/trials/r2/repo-009/src/module_09.go", "/trials/r2/repo-009/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102103677495000, "t_ack_ns": 1786102103679990000, "ack_ms": 2.495, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 10, "correlation_id": "r2-repo-010", "paths": ["/trials/r2/repo-010/src/module_00.go", "/trials/r2/repo-010/src/module_01.go", "/trials/r2/repo-010/src/module_02.go", "/trials/r2/repo-010/src/module_03.go", "/trials/r2/repo-010/src/module_04.go", "/trials/r2/repo-010/src/module_05.go", "/trials/r2/repo-010/src/module_06.go", "/trials/r2/repo-010/src/module_07.go", "/trials/r2/repo-010/src/module_08.go", "/trials/r2/repo-010/src/module_09.go", "/trials/r2/repo-010/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102107753983000, "t_ack_ns": 1786102107757588000, "ack_ms": 3.605, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 11, "correlation_id": "r2-repo-011", "paths": ["/trials/r2/repo-011/src/module_00.go", "/trials/r2/repo-011/src/module_01.go", "/trials/r2/repo-011/src/module_02.go", "/trials/r2/repo-011/src/module_03.go", "/trials/r2/repo-011/src/module_04.go", "/trials/r2/repo-011/src/module_05.go", "/trials/r2/repo-011/src/module_06.go", "/trials/r2/repo-011/src/module_07.go", "/trials/r2/repo-011/src/module_08.go", "/trials/r2/repo-011/src/module_09.go", "/trials/r2/repo-011/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102111823415000, "t_ack_ns": 1786102111827833000, "ack_ms": 4.418, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 12, "correlation_id": "r2-repo-012", "paths": ["/trials/r2/repo-012/src/module_00.go", "/trials/r2/repo-012/src/module_01.go", "/trials/r2/repo-012/src/module_02.go", "/trials/r2/repo-012/src/module_03.go", "/trials/r2/repo-012/src/module_04.go", "/trials/r2/repo-012/src/module_05.go", "/trials/r2/repo-012/src/module_06.go", "/trials/r2/repo-012/src/module_07.go", "/trials/r2/repo-012/src/module_08.go", "/trials/r2/repo-012/src/module_09.go", "/trials/r2/repo-012/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102115901087000, "t_ack_ns": 1786102115904072000, "ack_ms": 2.985, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 13, "correlation_id": "r2-repo-013", "paths": ["/trials/r2/repo-013/src/module_00.go", "/trials/r2/repo-013/src/module_01.go", "/trials/r2/repo-013/src/module_02.go", "/trials/r2/repo-013/src/module_03.go", "/trials/r2/repo-013/src/module_04.go", "/trials/r2/repo-013/src/module_05.go", "/trials/r2/repo-013/src/module_06.go", "/trials/r2/repo-013/src/module_07.go", "/trials/r2/repo-013/src/module_08.go", "/trials/r2/repo-013/src/module_09.go", "/trials/r2/repo-013/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102119910688000, "t_ack_ns": 1786102119914683000, "ack_ms": 3.995, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 14, "correlation_id": "r2-repo-014", "paths": ["/trials/r2/repo-014/src/module_00.go", "/trials/r2/repo-014/src/module_01.go", "/trials/r2/repo-014/src/module_02.go", "/trials/r2/repo-014/src/module_03.go", "/trials/r2/repo-014/src/module_04.go", "/trials/r2/repo-014/src/module_05.go", "/trials/r2/repo-014/src/module_06.go", "/trials/r2/repo-014/src/module_07.go", "/trials/r2/repo-014/src/module_08.go", "/trials/r2/repo-014/src/module_09.go", "/trials/r2/repo-014/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102123966022000, "t_ack_ns": 1786102123969750000, "ack_ms": 3.728, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 15, "correlation_id": "r2-repo-015", "paths": ["/trials/r2/repo-015/src/module_00.go", "/trials/r2/repo-015/src/module_01.go", "/trials/r2/repo-015/src/module_02.go", "/trials/r2/repo-015/src/module_03.go", "/trials/r2/repo-015/src/module_04.go", "/trials/r2/repo-015/src/module_05.go", "/trials/r2/repo-015/src/module_06.go", "/trials/r2/repo-015/src/module_07.go", "/trials/r2/repo-015/src/module_08.go", "/trials/r2/repo-015/src/module_09.go", "/trials/r2/repo-015/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102128027144000, "t_ack_ns": 1786102128031376000, "ack_ms": 4.232, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 16, "correlation_id": "r2-repo-016", "paths": ["/trials/r2/repo-016/src/module_00.go", "/trials/r2/repo-016/src/module_01.go", "/trials/r2/repo-016/src/module_02.go", "/trials/r2/repo-016/src/module_03.go", "/trials/r2/repo-016/src/module_04.go", "/trials/r2/repo-016/src/module_05.go", "/trials/r2/repo-016/src/module_06.go", "/trials/r2/repo-016/src/module_07.go", "/trials/r2/repo-016/src/module_08.go", "/trials/r2/repo-016/src/module_09.go", "/trials/r2/repo-016/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102132062723000, "t_ack_ns": 1786102132070151000, "ack_ms": 7.428, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 17, "correlation_id": "r2-repo-017", "paths": ["/trials/r2/repo-017/src/module_00.go", "/trials/r2/repo-017/src/module_01.go", "/trials/r2/repo-017/src/module_02.go", "/trials/r2/repo-017/src/module_03.go", "/trials/r2/repo-017/src/module_04.go", "/trials/r2/repo-017/src/module_05.go", "/trials/r2/repo-017/src/module_06.go", "/trials/r2/repo-017/src/module_07.go", "/trials/r2/repo-017/src/module_08.go", "/trials/r2/repo-017/src/module_09.go", "/trials/r2/repo-017/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102136112789000, "t_ack_ns": 1786102136119579000, "ack_ms": 6.79, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 18, "correlation_id": "r2-repo-018", "paths": ["/trials/r2/repo-018/src/module_00.go", "/trials/r2/repo-018/src/module_01.go", "/trials/r2/repo-018/src/module_02.go", "/trials/r2/repo-018/src/module_03.go", "/trials/r2/repo-018/src/module_04.go", "/trials/r2/repo-018/src/module_05.go", "/trials/r2/repo-018/src/module_06.go", "/trials/r2/repo-018/src/module_07.go", "/trials/r2/repo-018/src/module_08.go", "/trials/r2/repo-018/src/module_09.go", "/trials/r2/repo-018/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102140194581000, "t_ack_ns": 1786102140199857000, "ack_ms": 5.276, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 19, "correlation_id": "r2-repo-019", "paths": ["/trials/r2/repo-019/src/module_00.go", "/trials/r2/repo-019/src/module_01.go", "/trials/r2/repo-019/src/module_02.go", "/trials/r2/repo-019/src/module_03.go", "/trials/r2/repo-019/src/module_04.go", "/trials/r2/repo-019/src/module_05.go", "/trials/r2/repo-019/src/module_06.go", "/trials/r2/repo-019/src/module_07.go", "/trials/r2/repo-019/src/module_08.go", "/trials/r2/repo-019/src/module_09.go", "/trials/r2/repo-019/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102144251114000, "t_ack_ns": 1786102144254661000, "ack_ms": 3.547, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "repo", "run_id": "r2", "trial": 20, "correlation_id": "r2-repo-020", "paths": ["/trials/r2/repo-020/src/module_00.go", "/trials/r2/repo-020/src/module_01.go", "/trials/r2/repo-020/src/module_02.go", "/trials/r2/repo-020/src/module_03.go", "/trials/r2/repo-020/src/module_04.go", "/trials/r2/repo-020/src/module_05.go", "/trials/r2/repo-020/src/module_06.go", "/trials/r2/repo-020/src/module_07.go", "/trials/r2/repo-020/src/module_08.go", "/trials/r2/repo-020/src/module_09.go", "/trials/r2/repo-020/src/module_10.go"], "expected_bytes": 13992, "t_send_ns": 1786102148323961000, "t_ack_ns": 1786102148328098000, "ack_ms": 4.137, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} diff --git a/docs/evidence/mount-latency-20260807/raw/trials-small.jsonl b/docs/evidence/mount-latency-20260807/raw/trials-small.jsonl new file mode 100644 index 00000000..37e5c7f9 --- /dev/null +++ b/docs/evidence/mount-latency-20260807/raw/trials-small.jsonl @@ -0,0 +1,32 @@ +{"kind": "send", "shape": "small", "run_id": "run20260807", "trial": 1, "correlation_id": "run20260807-small-001", "paths": ["/trials/run20260807/small-001/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786101857051448000, "t_ack_ns": 1786101857067612000, "ack_ms": 16.164, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "run20260807", "trial": 2, "correlation_id": "run20260807-small-002", "paths": ["/trials/run20260807/small-002/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786101861106762000, "t_ack_ns": 1786101861109045000, "ack_ms": 2.283, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "run20260807", "trial": 3, "correlation_id": "run20260807-small-003", "paths": ["/trials/run20260807/small-003/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786101865125541000, "t_ack_ns": 1786101865130639000, "ack_ms": 5.098, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "run20260807", "trial": 4, "correlation_id": "run20260807-small-004", "paths": ["/trials/run20260807/small-004/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786101869202858000, "t_ack_ns": 1786101869205182000, "ack_ms": 2.324, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "run20260807", "trial": 5, "correlation_id": "run20260807-small-005", "paths": ["/trials/run20260807/small-005/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786101873278555000, "t_ack_ns": 1786101873281803000, "ack_ms": 3.248, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "run20260807", "trial": 6, "correlation_id": "run20260807-small-006", "paths": ["/trials/run20260807/small-006/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786101877341472000, "t_ack_ns": 1786101877345365000, "ack_ms": 3.893, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "run20260807", "trial": 7, "correlation_id": "run20260807-small-007", "paths": ["/trials/run20260807/small-007/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786101881420825000, "t_ack_ns": 1786101881435349000, "ack_ms": 14.524, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "run20260807", "trial": 8, "correlation_id": "run20260807-small-008", "paths": ["/trials/run20260807/small-008/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786101885510148000, "t_ack_ns": 1786101885514834000, "ack_ms": 4.686, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "run20260807", "trial": 9, "correlation_id": "run20260807-small-009", "paths": ["/trials/run20260807/small-009/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786101889586851000, "t_ack_ns": 1786101889588768000, "ack_ms": 1.917, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "run20260807", "trial": 10, "correlation_id": "run20260807-small-010", "paths": ["/trials/run20260807/small-010/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786101893599654000, "t_ack_ns": 1786101893604453000, "ack_ms": 4.799, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "run20260807", "trial": 11, "correlation_id": "run20260807-small-011", "paths": ["/trials/run20260807/small-011/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786101897679730000, "t_ack_ns": 1786101897686347000, "ack_ms": 6.617, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "run20260807", "trial": 12, "correlation_id": "run20260807-small-012", "paths": ["/trials/run20260807/small-012/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786101901760702000, "t_ack_ns": 1786101901762547000, "ack_ms": 1.845, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 1, "correlation_id": "r2-small-001", "paths": ["/trials/r2/small-001/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786101993961678000, "t_ack_ns": 1786101993977372000, "ack_ms": 15.694, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 2, "correlation_id": "r2-small-002", "paths": ["/trials/r2/small-002/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786101997992744000, "t_ack_ns": 1786101997998005000, "ack_ms": 5.261, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 3, "correlation_id": "r2-small-003", "paths": ["/trials/r2/small-003/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102002071830000, "t_ack_ns": 1786102002075904000, "ack_ms": 4.074, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 4, "correlation_id": "r2-small-004", "paths": ["/trials/r2/small-004/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102006151564000, "t_ack_ns": 1786102006157540000, "ack_ms": 5.976, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 5, "correlation_id": "r2-small-005", "paths": ["/trials/r2/small-005/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102010231089000, "t_ack_ns": 1786102010235355000, "ack_ms": 4.266, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 6, "correlation_id": "r2-small-006", "paths": ["/trials/r2/small-006/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102014302962000, "t_ack_ns": 1786102014305002000, "ack_ms": 2.04, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 7, "correlation_id": "r2-small-007", "paths": ["/trials/r2/small-007/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102018320230000, "t_ack_ns": 1786102018326730000, "ack_ms": 6.5, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 8, "correlation_id": "r2-small-008", "paths": ["/trials/r2/small-008/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102022400387000, "t_ack_ns": 1786102022404079000, "ack_ms": 3.692, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 9, "correlation_id": "r2-small-009", "paths": ["/trials/r2/small-009/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102026479114000, "t_ack_ns": 1786102026485504000, "ack_ms": 6.39, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 10, "correlation_id": "r2-small-010", "paths": ["/trials/r2/small-010/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102030557304000, "t_ack_ns": 1786102030561149000, "ack_ms": 3.845, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 11, "correlation_id": "r2-small-011", "paths": ["/trials/r2/small-011/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102034636159000, "t_ack_ns": 1786102034638049000, "ack_ms": 1.89, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 12, "correlation_id": "r2-small-012", "paths": ["/trials/r2/small-012/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102038678844000, "t_ack_ns": 1786102038680972000, "ack_ms": 2.128, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 13, "correlation_id": "r2-small-013", "paths": ["/trials/r2/small-013/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102042753529000, "t_ack_ns": 1786102042756812000, "ack_ms": 3.283, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 14, "correlation_id": "r2-small-014", "paths": ["/trials/r2/small-014/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102046799720000, "t_ack_ns": 1786102046801756000, "ack_ms": 2.036, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 15, "correlation_id": "r2-small-015", "paths": ["/trials/r2/small-015/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102050876034000, "t_ack_ns": 1786102050877781000, "ack_ms": 1.747, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 16, "correlation_id": "r2-small-016", "paths": ["/trials/r2/small-016/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102054911446000, "t_ack_ns": 1786102054913528000, "ack_ms": 2.082, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 17, "correlation_id": "r2-small-017", "paths": ["/trials/r2/small-017/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102058980967000, "t_ack_ns": 1786102058983065000, "ack_ms": 2.098, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 18, "correlation_id": "r2-small-018", "paths": ["/trials/r2/small-018/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102063054820000, "t_ack_ns": 1786102063056768000, "ack_ms": 1.948, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 19, "correlation_id": "r2-small-019", "paths": ["/trials/r2/small-019/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102067129582000, "t_ack_ns": 1786102067132506000, "ack_ms": 2.924, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} +{"kind": "send", "shape": "small", "run_id": "r2", "trial": 20, "correlation_id": "r2-small-020", "paths": ["/trials/r2/small-020/probe.txt"], "expected_bytes": 300, "t_send_ns": 1786102071164126000, "t_ack_ns": 1786102071166267000, "ack_ms": 2.141, "http_status": 202, "error": null, "host": "khaliqs-macbook-pro", "clock": "CLOCK_REALTIME"} diff --git a/docs/multi-agent-collaboration-assessment.md b/docs/multi-agent-collaboration-assessment.md index 512ab05c..1b03e273 100644 --- a/docs/multi-agent-collaboration-assessment.md +++ b/docs/multi-agent-collaboration-assessment.md @@ -499,10 +499,23 @@ prior — so this is a two-machine, not three-machine, confirmation). 7. **Docs oversell current behavior.** `docs/guides/collaboration.md:54-76` claims conflict-safety the code doesn't have (see gap #1); - `cmd/relayfile-cli/`'s mount help text undersells actual propagation - speed (says "polls... every 30s", actual steady-state is sub-200ms via - websocket). Cheap to fix, should happen alongside #1 so the doc becomes - true rather than requiring a retraction later. + `cmd/relayfile-cli/`'s mount help text describes propagation speed + inaccurately (says "polls... every 30s", when steady-state delivery is by + websocket). **Correction, 2026-08-07:** this finding previously read + "undersells actual propagation speed... actual steady-state is sub-200ms + via websocket". That sub-200ms figure was an inference — the 2026-07-26 + evidence measured a *round trip* (median 315.5 ms, n=12) and halved it. A + direct one-way measurement (`docs/evidence/mount-latency-20260807/`) shows + the real answer is size-dependent, and one of the two cases is not + sub-200ms: a single small file propagates in a median of 20.2 ms + (p95 161.7 ms, n=20), but a repo-sized change set of 11 files / ~14 KB + takes a median of **216.7 ms** (p95 303.9 ms, n=20) — because the receive + path fetches each file with its own server round trip, so cost scales with + file count. Both figures were measured with the server on the sender's own + machine and a Tailscale LAN to the receiver, so they are LAN best cases and + are **not** measurements of the hosted product path. Any replacement help + text should say the speed depends on change-set size rather than quoting a + single number. 8. **No same-file simultaneous co-editing (CRDT/OT).** **Correction to an earlier framing of this finding:** this *was* scoped and explicitly From 761ba4dd9e67de1d402b4e20ae1c4dba578b6956 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Sun, 9 Aug 2026 22:09:36 +0200 Subject: [PATCH 2/3] test(mount): red test pinning .relay/state.json to a single writer /.relay/state.json has two independent writers in the same binary, emitting two disjoint schemas to one path: - internal/mountsync/syncer.go:1608 derives the path; syncer.go:7200 writes a `publicState` (localRoot, states, files, counters, circuit, outbox, credExpiresInSecs). - cmd/relayfile-cli/main.go:10998 writes a `syncStateFile` (providers, daemon, guards, stallReason, incrementalReadNotReadySince). Both are handed the same directory by one closure at main.go:6852-6892, which passes scope.LocalDir as NewSyncer's LocalRoot and as runMountLoopWithAuthLock's localDir. Whichever writes last defines the document every consumer sees; live, they alternate about once a second. The test fails on this commit. It is red, not a regression guard. Co-Authored-By: Claude Opus 5 --- .../relay_state_two_writers_test.go | 209 ++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 cmd/relayfile-cli/relay_state_two_writers_test.go diff --git a/cmd/relayfile-cli/relay_state_two_writers_test.go b/cmd/relayfile-cli/relay_state_two_writers_test.go new file mode 100644 index 00000000..ccff97cf --- /dev/null +++ b/cmd/relayfile-cli/relay_state_two_writers_test.go @@ -0,0 +1,209 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/agentworkforce/relayfile/internal/mountsync" +) + +// newPublicStateWriter builds a real mountsync.Syncer rooted at localDir. +// Its LocalRoot is the same directory the CLI mirror writer is handed in +// cmd/relayfile-cli/main.go:6852-6892, so both writers derive the identical +// /.relay/state.json path — that shared derivation is the defect. +// +// No network call is made: FlushOutboxOnce with an empty outbox goes straight +// to markSyncSuccess -> saveStateWithoutLocalScan -> savePublicState. +func newPublicStateWriter(t *testing.T, localDir string) *mountsync.Syncer { + t.Helper() + client := mountsync.NewHTTPClient("http://127.0.0.1:1", "test-token", &http.Client{ + Timeout: time.Second, + }) + websocketDisabled := false + syncer, err := mountsync.NewSyncer(client, mountsync.SyncerOptions{ + WorkspaceID: "rw_twowriters", + RemoteRoot: "/linear", + LocalRoot: localDir, + WebSocket: &websocketDisabled, + RootCtx: context.Background(), + }) + if err != nil { + t.Fatalf("NewSyncer: %v", err) + } + return syncer +} + +// writeMountsyncPublicState drives WRITER 1 (internal/mountsync/syncer.go:7200, +// path derived at syncer.go:1608). +func writeMountsyncPublicState(t *testing.T, syncer *mountsync.Syncer) { + t.Helper() + if err := syncer.FlushOutboxOnce(context.Background()); err != nil { + t.Fatalf("FlushOutboxOnce (writer 1): %v", err) + } +} + +// writeCLIMirrorState drives WRITER 2 (cmd/relayfile-cli/main.go:10998), with a +// snapshot shaped like the one buildSyncStateSnapshot produces for a live +// mount: providers from the cloud feed, a daemon block, guards, a stall reason. +func writeCLIMirrorState(t *testing.T, localDir string) { + t.Helper() + snapshot := syncStateFile{ + WorkspaceID: "rw_twowriters", + RemoteRoot: "/linear", + Mode: defaultMountMode, + Status: "ready", + // Provider-feed clock. Deliberately old: this is the field an + // operator misread as "the feed moved". + LastEventAt: "2026-08-03T07:26:26.334Z", + IntervalMs: 30000, + Providers: []syncStateProvider{ + {Provider: "linear", Status: "ready", LastEventAt: "2026-08-03T07:26:26.334Z"}, + }, + StallReason: "provider feed frozen", + Daemon: &syncStateDaemon{PID: 63173}, + Guards: &syncStateGuards{ + CircuitOpenEvents: 7, + TombstonesConfirmed: 3, + }, + } + if err := writeMirrorStateFile(localDir, snapshot); err != nil { + t.Fatalf("writeMirrorStateFile (writer 2): %v", err) + } +} + +func readStateDocument(t *testing.T, localDir string) map[string]any { + t.Helper() + payload, err := os.ReadFile(filepath.Join(localDir, ".relay", "state.json")) + if err != nil { + t.Fatalf("read .relay/state.json: %v", err) + } + document := map[string]any{} + if err := json.Unmarshal(payload, &document); err != nil { + t.Fatalf("unmarshal .relay/state.json: %v", err) + } + return document +} + +// TestRelayStateJSONHasExactlyOneWriter is the contract this file exists to +// pin: /.relay/state.json must be a single document with a single +// owner. Today two writers in the same binary emit two disjoint schemas to +// that one path, so whichever wrote last defines what every consumer sees. +// +// The test drives writer 1 then writer 2 then writer 1 again against one +// localDir and asserts, after each write, that the document still satisfies +// BOTH consumer contracts. It fails on HEAD at the first assertion. +func TestRelayStateJSONHasExactlyOneWriter(t *testing.T) { + localDir := t.TempDir() + syncer := newPublicStateWriter(t, localDir) + + // --- writer 1 (mountsync) writes first --------------------------------- + writeMountsyncPublicState(t, syncer) + afterWriter1 := readStateDocument(t, localDir) + if _, ok := afterWriter1["localRoot"]; !ok { + t.Fatalf("precondition failed: writer 1 did not emit its own document; keys=%v", sortedKeys(afterWriter1)) + } + + // --- writer 2 (CLI mirror) writes second -------------------------------- + writeCLIMirrorState(t, localDir) + afterWriter2 := readStateDocument(t, localDir) + + // Writer 1's fields must survive writer 2. They do not: writeMirrorStateFile + // marshals a syncStateFile from scratch, so every publicState-only field is + // dropped from the file on disk. + for _, field := range []string{"localRoot", "states", "counters", "files", "outbox", "credExpiresInSecs"} { + if _, ok := afterWriter2[field]; !ok { + t.Errorf("writer 2 clobbered writer 1: field %q is absent from .relay/state.json after the CLI mirror write (keys now: %v)", + field, sortedKeys(afterWriter2)) + } + } + + // --- writer 1 writes again (this is what happens ~1x/second live) ------- + writeMountsyncPublicState(t, syncer) + afterWriter1Again := readStateDocument(t, localDir) + + // Writer 2's fields must survive writer 1. They do not. + for _, field := range []string{"providers", "daemon", "guards", "stallReason"} { + if _, ok := afterWriter1Again[field]; !ok { + t.Errorf("writer 1 clobbered writer 2: field %q is absent from .relay/state.json after the mountsync write (keys now: %v)", + field, sortedKeys(afterWriter1Again)) + } + } + + // The production readers in this package must keep working across both + // writes. They do not: each reads a field only one writer emits. + // readGuardCounters is the subtle one. It does not return nil here — it + // falls through to writer 1's `counters`/`circuit` block and returns a + // *different* guards document than the one writer 2 persisted. The CLI + // surface silently swaps its data source depending on who wrote last. + guards := readGuardCounters(localDir) + if guards == nil { + t.Errorf("readGuardCounters (main.go:10912) returned nil after a mountsync write") + } else { + if guards.CircuitOpenEvents != 7 { + t.Errorf("readGuardCounters (main.go:10912) circuitOpenEvents = %d, want 7 — it silently switched from writer 2's persisted `guards` block to writer 1's `counters` block", + guards.CircuitOpenEvents) + } + if guards.TombstonesConfirmed != 3 { + t.Errorf("readGuardCounters (main.go:10912) tombstonesConfirmed = %d, want 3 — writer 2's guard telemetry is gone from the file", + guards.TombstonesConfirmed) + } + } + if reason := readPersistedStallReason(localDir); reason != "provider feed frozen" { + t.Errorf("readPersistedStallReason (main.go:8556) = %q, want %q — the stall reason is erased every time writer 1 wins the race", + reason, "provider feed frozen") + } + state, err := readWritebackState(localDir) + if err != nil { + t.Fatalf("readWritebackState: %v", err) + } + if len(state.Providers) == 0 { + t.Errorf("readWritebackState (main.go:5487) sees 0 providers — the TypeScript SDK readiness check (packages/sdk/typescript/src/mount-launcher.ts:355) treats an empty providers list as ready, so this races into a false ready") + } +} + +// TestRelayStateJSONLastEventAtHasOneMeaning pins the field that produced the +// real-world misreport. Both writers emit `lastEventAt`, with different +// semantics: writer 1 means "mountsync's own event clock" and writer 2 means +// "the cloud provider feed". A consumer polling this one field gets a value +// whose meaning changes between reads. +func TestRelayStateJSONLastEventAtHasOneMeaning(t *testing.T) { + localDir := t.TempDir() + syncer := newPublicStateWriter(t, localDir) + + writeCLIMirrorState(t, localDir) + fromProviderFeed, _ := readStateDocument(t, localDir)["lastEventAt"].(string) + if fromProviderFeed != "2026-08-03T07:26:26.334Z" { + t.Fatalf("precondition failed: writer 2 lastEventAt = %q", fromProviderFeed) + } + + writeMountsyncPublicState(t, syncer) + afterMountsync := readStateDocument(t, localDir) + fromMountsync, present := afterMountsync["lastEventAt"].(string) + + if !present { + t.Errorf("lastEventAt vanished from .relay/state.json after a mountsync write: a consumer polling this field sees the provider-feed timestamp %q disappear and reappear as the mount loop and the CLI mirror alternate", + fromProviderFeed) + } + if present && fromMountsync != fromProviderFeed { + t.Errorf("lastEventAt changed meaning without changing name: %q (cloud provider feed, writer 2) then %q (mountsync event clock, writer 1)", + fromProviderFeed, fromMountsync) + } +} + +func sortedKeys(document map[string]any) []string { + keys := make([]string, 0, len(document)) + for key := range document { + keys = append(keys, key) + } + for i := 1; i < len(keys); i++ { + for j := i; j > 0 && keys[j] < keys[j-1]; j-- { + keys[j], keys[j-1] = keys[j-1], keys[j] + } + } + return keys +} From 3e6ada3164ff8db1d9bcd32515946432ff34fde4 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Sun, 9 Aug 2026 22:13:32 +0200 Subject: [PATCH 3/3] test(mount): pin failedWritebacks against the two-writer lost update Separates two claims about the failedWritebacks counter that are easy to conflate: - It is NOT zeroed by the schema alternation. Both writers read it back before writing (internal/mountsync/syncer.go:7020, and the max at cmd/relayfile-cli/main.go:10986). TestFailedWritebacksSurvivesAlternation passes and locks that hardening in. - It IS lost under concurrency. Neither writer holds a lock the other respects: failedWritebacksStateMu lives in cmd/relayfile-cli and internal/mountsync cannot take it. An increment landing between savePublicState's read (syncer.go:7020) and its write (syncer.go:7200) is overwritten with the stale value. TestFailedWritebacksLostUpdateUnderConcurrency loses 1-5 of 200 increments in 4 of 5 runs. It is a race demonstration, so it is inherently non-deterministic; it passed once in five runs. Co-Authored-By: Claude Opus 5 --- cmd/relayfile-cli/relay_state_counter_test.go | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 cmd/relayfile-cli/relay_state_counter_test.go diff --git a/cmd/relayfile-cli/relay_state_counter_test.go b/cmd/relayfile-cli/relay_state_counter_test.go new file mode 100644 index 00000000..57234a3e --- /dev/null +++ b/cmd/relayfile-cli/relay_state_counter_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "sync" + "testing" +) + +// TestFailedWritebacksSurvivesAlternation checks the specific claim that the +// failedWritebacks counter is zeroed when the two writers alternate. +// +// It is NOT zeroed. Both writers deliberately read the counter back before +// writing it: mountsync at internal/mountsync/syncer.go:7020 +// (readPublicFailedWritebacks) and the CLI mirror at +// cmd/relayfile-cli/main.go:10986 (max against the persisted value). This test +// documents that hardening so a future refactor does not remove it. +func TestFailedWritebacksSurvivesAlternation(t *testing.T) { + localDir := t.TempDir() + syncer := newPublicStateWriter(t, localDir) + + for i := 0; i < 5; i++ { + if err := incrementFailedWritebacksInState(localDir); err != nil { + t.Fatalf("increment %d: %v", i, err) + } + } + if got := readPersistedFailedWritebacks(localDir); got != 5 { + t.Fatalf("precondition: counter = %d, want 5", got) + } + + // Alternate the two writers several times. + for i := 0; i < 3; i++ { + writeMountsyncPublicState(t, syncer) + if got := readPersistedFailedWritebacks(localDir); got != 5 { + t.Errorf("after mountsync write %d: counter = %d, want 5 (mountsync dropped the counter)", i, got) + } + writeCLIMirrorState(t, localDir) + if got := readPersistedFailedWritebacks(localDir); got != 5 { + t.Errorf("after CLI mirror write %d: counter = %d, want 5 (CLI mirror dropped the counter)", i, got) + } + } +} + +// TestFailedWritebacksLostUpdateUnderConcurrency probes the real hazard: both +// writers preserve the counter via read-modify-write, but neither holds a lock +// the other respects, so an increment landing between mountsync's read +// (syncer.go:7020) and its write (syncer.go:7200) is silently overwritten with +// the stale value. +// +// failedWritebacksStateMu (cmd/relayfile-cli/main.go) serializes the CLI-side +// increments against each other, but mountsync lives in another package and +// cannot take it. The mutex looks synchronized and is not. +// +// N increments must yield a counter of N. +func TestFailedWritebacksLostUpdateUnderConcurrency(t *testing.T) { + const increments = 200 + + localDir := t.TempDir() + syncer := newPublicStateWriter(t, localDir) + + var wg sync.WaitGroup + stop := make(chan struct{}) + + // mountsync republishing public state, as the live daemon does ~1x/sec. + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + _ = syncer.FlushOutboxOnce(t.Context()) + } + } + }() + + for i := 0; i < increments; i++ { + if err := incrementFailedWritebacksInState(localDir); err != nil { + t.Fatalf("increment %d: %v", i, err) + } + } + + close(stop) + wg.Wait() + + if got := readPersistedFailedWritebacks(localDir); got != increments { + t.Errorf("failedWritebacks = %d after %d increments: %d writeback failures were silently lost. "+ + "failedWritebacksStateMu (cmd/relayfile-cli) does not cross into internal/mountsync, so "+ + "savePublicState's read at syncer.go:7020 / write at syncer.go:7200 overwrites increments "+ + "that land in between", + got, increments, increments-got) + } +}