Differently is an open-source control plane for sandboxes. It runs on any cloud or on bare metal
- and places each sandbox on OpenSandbox, E2B, Daytona, a cloud provider's sandbox service or a plugin,
+ and places each sandbox on OpenSandbox, Daytona, Freestyle or a plugin,
according to placement policy. Applications use one API; quotas, placement decisions and audit live in the control plane.
@@ -300,7 +315,7 @@ const compare = [
Postgres
OpenSandboxDocker · Kubernetes runtime
OpenSandboxFast Sandbox microVMs
-
ProvidersE2B · Daytona · cloud
+
ProvidersDaytona · Freestyle
one API
diff --git a/site/src/styles/landing.css b/site/src/styles/landing.css
index 7b4b956..3f76e8e 100644
--- a/site/src/styles/landing.css
+++ b/site/src/styles/landing.css
@@ -96,7 +96,7 @@ section:not(.hero) { padding: 96px 0; border-top: 1px solid var(--line); }
}
.note { color: var(--muted); font-size: 0.875rem; margin-top: 16px; }
-.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); border-top: 1px solid var(--line); border-left: 1px solid var(--line); }
+.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); border-top: 1px solid var(--line); border-left: 1px solid var(--line); }
.grid article { border-right: 1px solid var(--line); border-bottom: 1px solid var(--line); padding: 28px; display: flex; flex-direction: column; gap: 12px; }
.grid .k { font: 0.75rem var(--mono); color: var(--muted); }
.grid p { color: var(--muted); margin: 0; flex: 1; font-size: 0.9375rem; }
diff --git a/test/e2e/README.md b/test/e2e/README.md
index 80f92ba..de64b54 100644
--- a/test/e2e/README.md
+++ b/test/e2e/README.md
@@ -69,7 +69,9 @@ the pool `TestStoredProviderKeyAndSwitching` registers, and
`DIFFERENTLY_E2E_POOL` names the config's private pool (`opensandbox`). The
server's config must match [`config.yaml`](config.yaml) where the tests
depend on it: the `small` and `medium` profiles, `audit.failClosed: true`
-and a short `scheduler.reconcileIntervalSeconds`. On a Fast Sandbox pool, sandboxes boot templates, not images:
+and a short `scheduler.reconcileIntervalSeconds`. With several server replicas, point
+`DIFFERENTLY_E2E_METRICS_URL` at the scheduler leader's pod: only the
+leader counts sandbox transitions. On a Fast Sandbox pool, sandboxes boot templates, not images:
`DIFFERENTLY_E2E_TEMPLATE` and `DIFFERENTLY_E2E_PYTHON_TEMPLATE` name them,
and the pool's `templateId` must be that template. Tests adapt to what the
pool advertises: where pause or snapshots are not offered (the Kubernetes
@@ -112,9 +114,12 @@ them precisely instead.
| 2026-10-29 | Azure k3s and Colima, as above | same | 14/14 on both; an idle sandbox is stopped (Kubernetes) or paused (Docker) after 60 s and woken by the next exec in about 1.7 s (Kubernetes) or under 0.1 s (Docker) |
| 2026-10-28 | k3s v1.36.5 on an Azure `Standard_D8s_v6`, x86_64; Differently from its Helm chart with embedded PostgreSQL | server, controller and `execd` release-1.2.0, Kubernetes runtime, no snapshot registry | 12/12 passed (pause and snapshots checked as refusals); sandboxes running in about 1.4–2.7 s; an unpullable image retries after OpenSandbox's 60 s pod-ready timeout with the pool healthy |
| 2026-10-29 | same Azure VM; pool `opensandbox-microvm` (`runtime: fastsandbox`), template `tpl-1cbcfa94` (1 vCPU, 1 GiB, guest kernel 6.18.36+) | server, Fast Sandbox and `execd` release-1.2.0 | **Partial run**, stopped when the VM was deallocated. Sandboxes running in about 1.3 s; files and pause/resume passed (pause about 52 s for 1 GiB); sandbox env and a template other than the pool's are refused precisely (the tests now expect that). Fork failed with a bare 422 from OpenSandbox on restore, not yet diagnosed. Port URL, idle wake, memory kept and the desktop stream did not run |
+| 2026-10-29 | Azure, two-node k3s (`Standard_D8s_v6` + `Standard_D16s_v6` KVM node), Postgres Flexible Server, two server replicas | server 1.2.0, Fast Sandbox with a 6.18.36 guest kernel with nftables; pool `runtime: fastsandbox` with OpenWork's Workbot desktop template | 14/14 run passed (desktop, unpullable-image, idle-sweep and switching tests not run); pause about 3–4 min for the 4 GiB desktop, idle wake by a port request 2.3 s |
Not covered yet:
-- the full suite on OpenSandbox's Fast Sandbox (Firecracker microVM) runtime (one partial run, above);
-- several nodes and a node drain on the Kubernetes runtime;
-- external providers.
+- on Fast Sandbox, the four tests not run there (the e2e desktop image,
+ unpullable images, Differently's own idle sweep and provider switching);
+- a node drain, and more than one KVM node;
+- external providers through this suite (Daytona and Freestyle have run
+ with their adapters' live tests instead).
diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
index 9c2c60d..327adc5 100644
--- a/test/e2e/e2e_test.go
+++ b/test/e2e/e2e_test.go
@@ -60,7 +60,9 @@ func env(k, def string) string {
}
func ctx(t *testing.T) context.Context {
- c, cancel := context.WithTimeout(context.Background(), 4*time.Minute)
+ // A Fast Sandbox pause or snapshot writes a full checkpoint, minutes for
+ // a desktop-sized template, and a test may do both.
+ c, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
t.Cleanup(cancel)
return c
}
@@ -270,7 +272,9 @@ func TestExecSemantics(t *testing.T) {
if advertises(t, "isolation.microvm") {
// A Fast Sandbox template fixes the workload: sandbox env is refused
// with a reason that says so, and per-command env still works.
- refused, err := c.CreateSandbox(ctx, api.CreateSandboxRequest{Template: defaultTemplate, Env: map[string]string{"GREETING": "hello"}}, false)
+ // Pinned to the pool under test: on an install with other private
+ // pools the refused sandbox would be retried there.
+ refused, err := c.CreateSandbox(ctx, api.CreateSandboxRequest{Template: defaultTemplate, Env: map[string]string{"GREETING": "hello"}, Placement: &api.Placement{AllowedPools: []string{mainPool}}}, false)
if err != nil {
t.Fatal(err)
}
@@ -637,8 +641,25 @@ func TestAuditTrustAndMetrics(t *testing.T) {
if strings.Contains(string(raw), osbKey) {
t.Fatal("trust report leaks the OpenSandbox API key")
}
- if len(report.Pools) != 1 || report.Pools[0].Adapter != "opensandbox" || !strings.Contains(report.Pools[0].Isolation, "container") || report.Pools[0].CredentialRef != "env:DIFFERENTLY_OPENSANDBOX_API_KEY" {
- t.Fatalf("trust report pools %+v", report.Pools)
+ // The pool under test, among any others the install has: its isolation
+ // as deployed, and its credential only as a reference.
+ isolation := "container"
+ if advertises(t, "isolation.microvm") {
+ isolation = "microVM"
+ }
+ found := false
+ for _, p := range report.Pools {
+ if p.Name != mainPool {
+ continue
+ }
+ found = true
+ ref := strings.SplitN(p.CredentialRef, ":", 2)
+ if p.Adapter != "opensandbox" || !strings.Contains(p.Isolation, isolation) || len(ref) != 2 || !slices.Contains([]string{"env", "file", "secret"}, ref[0]) {
+ t.Fatalf("trust report pool %+v", p)
+ }
+ }
+ if !found {
+ t.Fatalf("trust report has no pool %s: %+v", mainPool, report.Pools)
}
if !strings.Contains(strings.Join(report.Limitations, " "), "Differently does not isolate sandboxes itself") {
t.Fatal("trust report does not state that isolation is the runtime's")
@@ -798,7 +819,9 @@ func TestIdlePauseAndWakeThroughPortURL(t *testing.T) {
before = body
return code == http.StatusOK
})
- eventually(t, 3*time.Minute, "the idle microVM to be paused", func() bool {
+ // A minute idle, then a full checkpoint of disk and memory (about three
+ // minutes for a desktop-sized template).
+ eventually(t, 8*time.Minute, "the idle microVM to be paused", func() bool {
got, err := c.GetSandbox(context.Background(), sb.ID)
return err == nil && got.IdledAt != nil && got.Status == api.StatusPaused
})
diff --git a/test/pause/README.md b/test/pause/README.md
new file mode 100644
index 0000000..0d6aaa0
--- /dev/null
+++ b/test/pause/README.md
@@ -0,0 +1,72 @@
+# Pause runs: where a Fast Sandbox pause spends its time
+
+Scripts that pause and resume a sandbox through Differently over and over,
+split every pause into its phases from the Fast Sandbox components' logs,
+trace the state-root disk and check that the resumed guest kept its state.
+They produced the numbers in
+[docs/research/fast-sandbox-pause-time.md](../../docs/research/fast-sandbox-pause-time.md).
+
+Run them on a **test** install only: they fill pool slots, write a full
+checkpoint per pause and can drain a burstable disk's credits.
+
+## Requirements
+
+- A single-node (or KVM-node) k3s install with Fast Sandbox, as in
+ [docs/deploy/azure-vms.md](../../docs/deploy/azure-vms.md); the scripts run
+ on the KVM node with `sudo` (for `k3s kubectl logs` and `/proc`).
+- A Differently pool on that runtime, and a project key that can use it.
+- Python 3 (standard library only).
+
+```bash
+export DIFFERENTLY_URL=http://
:8080
+export DIFFERENTLY_API_KEY=... # a project key; never write it to a file here
+export DIFFERENTLY_POOL=opensandbox-microvm
+export STATE_ROOT_DEVICE=nvme0n1 # the disk under /var/lib/fast-sandbox
+python3 runset.py before idle:3 in-use:5 # fresh sandbox per scenario, N cycles each
+python3 summarize.py runs/before-*.jsonl # Markdown tables: median and p95
+```
+
+`idle` pauses a desktop as created. `in-use` first starts Xvfb, LibreOffice,
+Chromium and a process holding 384 MiB of random data, writes 300 MiB of
+random data and a copy of `/usr/share/doc` to disk, reads most of `/usr`
+into the page cache, then adds 64 MiB of new files before every later pause
+(`guest-use.sh`).
+
+## What a cycle records
+
+| Field | Source |
+|---|---|
+| `pause_s`, `resume_s` | Differently's API: from the call until the sandbox reads `paused` / `running` |
+| `resume_to_exec_s` | the resume plus the first `exec` that answers |
+| `node.phases.quiesce_and_disk_clone` | fastlet: `dump starting` to `paused for dump` (root filesystem reflink, then the VM's vCPUs stop) |
+| `node.phases.memory_dump` | fastlet: Firecracker's `dumpAPI` (memory and device state written) |
+| `node.phases.checksum_and_manifest` | fastlet: `resumed after dump` to `manifest assembled` (SHA-256 of every staged file, SHA256SUMS, manifest) |
+| `node.phases.upload` | runtime agent: `publish started` to `publish completed`; per file from Garage's request log |
+| `node.phases.metadata_commit` | fastlet `snapshot published` to the controller's `runtime released` |
+| `node.disk.` | `sampler.py`: MB read and written on the state-root disk, busy share, CPU seconds of Garage, the agent, the fastlet and Firecracker |
+| `garage_stored_mb` | growth of Garage's block directory |
+| `state_kept`, `procs_kept`, `holder_ok` | `guest-state.sh` before and after: long-lived processes with their start times, their open files, a checksum of the memory-held buffer, a tmpfs file and a disk file, the boot ID |
+
+Compression happens inside Garage (zstd on blocks it does not already
+hold), so it shows as Garage CPU during the upload, not as its own phase.
+`s3bench.py` uploads a file the way the agent does (one PUT,
+`UNSIGNED-PAYLOAD`) or as a parallel multipart upload, to measure the store
+alone.
+
+## Traps
+
+- **The page cache decides the upload.** The checksum pass reads the whole
+ checkpoint; whether the upload reads it again from disk depends on free
+ memory. Keep other memory users (tmpfs files included) off the node while
+ measuring, and compare runs on the same host.
+- **Burstable disks.** Premium SSD P6 to P20 burst to 170 MB/s on credits;
+ a long run can drain them. `dd ... iflag=direct` before and after shows
+ whether the disk still bursts.
+- **Do not delete a checkpoint a live sandbox came from.** A resumed
+ sandbox needs its checkpoint's manifest to take the next one; Fast Sandbox
+ 1.2.0 then retries the failed pause forever, dumping memory each time.
+ `drop-checkpoints.sh` (and `PAUSE_DROP_CHECKPOINTS=1`) is for test hosts,
+ with no sandbox alive.
+- `punch.py` turns a file's all-zero 64 KiB runs into holes, in place: a
+ stand-in for the sparse-pull patch on cached templates. Run it only with no
+ sandbox alive.
diff --git a/test/pause/cycle.py b/test/pause/cycle.py
new file mode 100644
index 0000000..461b6d7
--- /dev/null
+++ b/test/pause/cycle.py
@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""Pause and resume one sandbox N times through Differently, timing each
+phase from the Fast Sandbox logs and checking the guest kept its state.
+Run on the KVM node (it reads k3s logs and the disk with sudo).
+usage: cycle.py SANDBOX_ID idle|in-use N TAG -> $PAUSE_RUNS/TAG.jsonl"""
+import datetime as dt, json, os, subprocess, sys, time
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, HERE)
+import dly
+
+sid, scenario, n, tag = sys.argv[1], sys.argv[2], int(sys.argv[3]), sys.argv[4]
+RUNS = os.environ.get("PAUSE_RUNS", os.path.join(HERE, "runs"))
+os.makedirs(RUNS, exist_ok=True)
+out = open(os.path.join(RUNS, f"{tag}.jsonl"), "a", buffering=1)
+samples = os.path.join(RUNS, f"{tag}.samples.jsonl")
+sampler = subprocess.Popen(["sudo", "python3", os.path.join(HERE, "sampler.py"), samples, os.environ.get("STATE_ROOT_DEVICE", "nvme0n1")])
+GUEST_USE = open(os.path.join(HERE, "guest-use.sh")).read()
+GUEST_STATE = open(os.path.join(HERE, "guest-state.sh")).read()
+
+def guest(script, *args):
+ r = dly.exe(sid, "cat > /tmp/g.sh <<'GEOF'\n" + script + "\nGEOF\nsh /tmp/g.sh " + " ".join(args), timeout=900)
+ return r.get("stdout", "")
+
+def state():
+ lines = guest(GUEST_STATE).splitlines()
+ state = {l.split()[0]: " ".join(l.split()[1:]) for l in lines if not l.startswith(("proc ", "fd "))}
+ state["_fds"] = [l[3:] for l in lines if l.startswith("fd ")]
+ return state, [l for l in lines if l.startswith("proc ")]
+
+def fsb_name():
+ out = subprocess.run(["sudo", "k3s", "kubectl", "-n", os.environ.get("FSB_DATAPLANE_NAMESPACE", "opensandbox-dataplane"), "get", "sandboxes", "-o", "json"], capture_output=True, text=True).stdout
+ for item in json.loads(out)["items"]:
+ if sid in json.dumps(item.get("metadata", {})) or sid in json.dumps(item.get("spec", {}).get("metadata", {})):
+ return item["metadata"]["name"]
+ return None
+
+def garage_bytes():
+ return int(subprocess.run(["sudo", "du", "-sb", "/var/lib/fast-sandbox/garage/blocks"], capture_output=True, text=True).stdout.split()[0])
+
+try:
+ if scenario == "in-use":
+ print(guest(GUEST_USE, "setup"), file=sys.stderr)
+ for i in range(1, n + 1):
+ rec = {"tag": tag, "scenario": scenario, "cycle": i, "sandbox": sid}
+ if scenario == "in-use" and i > 1:
+ rec["use"] = guest(GUEST_USE, "step", str(i)).strip()
+ before, procs_before = state()
+ name = fsb_name(); rec["fsb"] = name
+ g0 = garage_bytes()
+ since = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+ time.sleep(1)
+ t0 = time.time()
+ code, body = dly.call("POST", f"/v1/sandboxes/{sid}/pause", {})
+ rec["pause_post"] = {"code": code, "s": round(time.time() - t0, 2)}
+ dly.wait(sid, "paused", timeout=1800)
+ rec["pause_s"] = round(time.time() - t0, 2)
+ time.sleep(3)
+ rec["garage_stored_mb"] = round((garage_bytes() - g0) / 1e6)
+ ph = subprocess.run(["sudo", "python3", os.path.join(HERE, "phases.py"), since, name, samples], capture_output=True, text=True)
+ try:
+ rec["node"] = json.loads(ph.stdout)
+ except ValueError:
+ rec["node_error"] = (ph.stdout + ph.stderr)[-500:]
+ time.sleep(5)
+ t1 = time.time()
+ code, body = dly.call("POST", f"/v1/sandboxes/{sid}/resume", {})
+ rec["resume_post"] = {"code": code, "s": round(time.time() - t1, 2)}
+ dly.wait(sid, "running", timeout=600)
+ rec["resume_s"] = round(time.time() - t1, 2)
+ t2 = time.time(); dly.exe(sid, "true"); rec["first_exec_s"] = round(time.time() - t2, 2)
+ rec["resume_to_exec_s"] = round(time.time() - t1, 2)
+ after, procs_after = state()
+ fds_before, fds_after = set(before.pop("_fds")), set(after.pop("_fds"))
+ if fds_before != fds_after:
+ # Which process opened or closed what: apps change their files on their own.
+ rec["fds_changed"] = {"closed": sorted(fds_before - fds_after)[:20], "opened": sorted(fds_after - fds_before)[:20]}
+ rec["state_kept"] = {k: before.get(k) == after.get(k) for k in before}
+ rec["procs_kept"] = procs_before == procs_after
+ if not all(rec["state_kept"].values()) or not rec["procs_kept"]:
+ rec["state_diff"] = {"before": before, "after": after, "procs_before": procs_before, "procs_after": procs_after}
+ rec["holder_ok"] = (after.get("holder_mem") or "x y").split()[0] == (after.get("holder_mem") or "x y").split()[1] if after.get("holder_mem") else None
+ out.write(json.dumps(rec) + "\n")
+ p = (rec.get("node") or {}).get("phases", {})
+ print(f"[{tag}] cycle {i}: pause {rec['pause_s']} s (dump {p.get('memory_dump')}, checksum {p.get('checksum_and_manifest')}, upload {p.get('upload')}, commit {p.get('metadata_commit')}), resume {rec['resume_s']} s, exec after {rec['resume_to_exec_s']} s, kept {rec['procs_kept'] and all(rec['state_kept'].values())}", flush=True)
+ time.sleep(20)
+finally:
+ sampler.terminate()
diff --git a/test/pause/dly.py b/test/pause/dly.py
new file mode 100644
index 0000000..6b98b38
--- /dev/null
+++ b/test/pause/dly.py
@@ -0,0 +1,47 @@
+"""A small Differently API client for the pause runs (standard library only).
+env: DIFFERENTLY_URL (default http://localhost:8080), DIFFERENTLY_API_KEY
+(a project key), DIFFERENTLY_POOL (default opensandbox-microvm)."""
+import json, os, time, urllib.error, urllib.request
+
+BASE = os.environ.get("DIFFERENTLY_URL", "http://localhost:8080").rstrip("/")
+POOL = os.environ.get("DIFFERENTLY_POOL", "opensandbox-microvm")
+
+
+def call(method, path, body=None, timeout=1800):
+ data = None if body is None else json.dumps(body).encode()
+ req = urllib.request.Request(BASE + path, data=data, method=method, headers={
+ "Authorization": "Bearer " + os.environ["DIFFERENTLY_API_KEY"], "Content-Type": "application/json"})
+ try:
+ with urllib.request.urlopen(req, timeout=timeout) as r:
+ raw = r.read()
+ return r.status, (json.loads(raw) if raw else None)
+ except urllib.error.HTTPError as e:
+ raw = e.read()
+ try:
+ return e.code, json.loads(raw)
+ except ValueError:
+ return e.code, raw.decode(errors="replace")
+
+
+def create(ttl=7200):
+ return call("POST", "/v1/sandboxes", {"placement": {"allowedPools": [POOL], "external": "never"},
+ "ttlSeconds": ttl, "idleTimeoutSeconds": -1, "labels": {"purpose": "pause-runs"}})
+
+
+def wait(sid, status, timeout=1800):
+ t0 = time.time()
+ while time.time() - t0 < timeout:
+ code, s = call("GET", "/v1/sandboxes/" + sid)
+ if code == 200 and s.get("status") == status:
+ return s
+ if code == 200 and s.get("status") in ("failed", "terminated"):
+ raise SystemExit(f"{sid} is {s.get('status')}: {s.get('statusReason')}")
+ time.sleep(0.25)
+ raise SystemExit(f"{sid}: timed out waiting for {status}")
+
+
+def exe(sid, cmd, timeout=600):
+ code, r = call("POST", f"/v1/sandboxes/{sid}/exec", {"command": cmd, "timeoutSeconds": timeout}, timeout=timeout + 30)
+ if code != 200:
+ raise SystemExit(f"exec {code}: {r}")
+ return r
diff --git a/test/pause/drop-checkpoints.sh b/test/pause/drop-checkpoints.sh
new file mode 100644
index 0000000..b71b496
--- /dev/null
+++ b/test/pause/drop-checkpoints.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+# Remove consumed checkpoint sets from Fast Sandbox's node image cache (what
+# its cache GC does): entries whose manifest records a checkpoint owner.
+# Templates and golden images stay. Run as root, and only with no sandbox
+# alive: a sandbox resumed from a checkpoint needs that checkpoint's
+# manifest to take its next one.
+cd /var/lib/fast-sandbox/firecracker/images || exit 1
+for d in */; do
+ d=${d%/}
+ if grep -q '"checkpoint"' "$d/manifest.json" 2>/dev/null; then rm -rf "$d" && echo "dropped $d"; fi
+done
+df -h /var/lib/fast-sandbox | tail -1
diff --git a/test/pause/guest-state.sh b/test/pause/guest-state.sh
new file mode 100644
index 0000000..022283e
--- /dev/null
+++ b/test/pause/guest-state.sh
@@ -0,0 +1,22 @@
+# Prints a fingerprint of the guest: long-lived processes (pid, start, name),
+# their open files, and checksums of memory and disk contents.
+pids=$(for p in /proc/[0-9]*; do n=$(cat $p/comm 2>/dev/null); case "$n" in Xvfb|openbox|soffice.bin|chromium|python3|execd|sandbox-init) echo ${p#/proc/};; esac; done)
+for p in $pids; do
+ set -- $(sed 's/^.*) //' /proc/$p/stat 2>/dev/null); st=${20}
+ [ -n "$st" ] && echo "proc $p $(cat /proc/$p/comm) $st"
+done | grep -v " python3 .*" > /tmp/procs.txt
+hp=$(cat /root/holder.pid 2>/dev/null)
+[ -n "$hp" ] && echo "proc $hp holder $(sed 's/^.*) //' /proc/$hp/stat | awk '{print $20}')" >> /tmp/procs.txt
+sort /tmp/procs.txt
+echo "procs_sha $(sort /tmp/procs.txt | sha256sum | cut -c1-16)"
+# open files of the desktop apps (execd's own exec connections change per call)
+fpids=$(for p in $pids $hp; do [ "$(cat /proc/$p/comm 2>/dev/null)" = execd ] || echo $p; done)
+for p in $fpids; do ls -l /proc/$p/fd 2>/dev/null | awk -v p=$p 'NR>1{print p, $NF}'; done | sort > /tmp/fds.txt
+sed 's/^/fd /' /tmp/fds.txt
+echo "fds_count $(wc -l < /tmp/fds.txt)"
+echo "fds_sha $(sha256sum < /tmp/fds.txt | cut -c1-16)"
+if [ -n "$hp" ]; then rm -f /root/holder.check; kill -USR1 $hp; for i in 1 2 3 4 5 6 7 8 9 10; do [ -s /root/holder.check ] && break; sleep 0.5; done
+ echo "holder_mem $(cut -c1-16 /root/holder.sha) $(cut -c1-16 /root/holder.check)"; fi
+[ -f /tmp/marker.bin ] && echo "tmpfs_sha $(sha256sum /tmp/marker.bin | cut -c1-16)"
+[ -f /workspace/data/rand-0.bin ] && echo "disk_sha $(sha256sum /workspace/data/rand-0.bin | cut -c1-16)"
+echo "boot_id $(cat /proc/sys/kernel/random/boot_id)"
diff --git a/test/pause/guest-use.sh b/test/pause/guest-use.sh
new file mode 100644
index 0000000..2f20a85
--- /dev/null
+++ b/test/pause/guest-use.sh
@@ -0,0 +1,40 @@
+# Runs inside the desktop guest. "setup" starts a working desktop session;
+# "step N" adds a little more use before each pause.
+export DISPLAY=:99 HOME=/root
+case "$1" in
+setup)
+ chmod 0755 /run 2>/dev/null
+ mkdir -p /workspace/data /root/chr
+ setsid Xvfb :99 -screen 0 1280x800x24 -nolisten tcp >/dev/null 2>&1 < /dev/null &
+ sleep 1
+ command -v openbox >/dev/null && setsid openbox >/dev/null 2>&1 < /dev/null &
+ setsid soffice --writer --norestore --nologo >/dev/null 2>&1 < /dev/null &
+ setsid chromium --no-sandbox --no-first-run --disable-gpu --user-data-dir=/root/chr \
+ file:///usr/share/doc/ file:///usr/share/common-licenses/ >/dev/null 2>&1 < /dev/null &
+ # an app holding state in memory: 384 MiB of random bytes with its checksum
+ cat > /root/holder.py <<'PY'
+import hashlib, os, signal, time
+buf = bytearray(os.urandom(384 << 20))
+open("/root/holder.sha", "w").write(hashlib.sha256(buf).hexdigest())
+def check(*_):
+ open("/root/holder.check", "w").write(hashlib.sha256(buf).hexdigest())
+signal.signal(signal.SIGUSR1, check)
+while True:
+ time.sleep(3600)
+PY
+ setsid python3 /root/holder.py >/dev/null 2>&1 < /dev/null &
+ echo $! > /root/holder.pid
+ head -c 256M /dev/urandom > /tmp/marker.bin # tmpfs: guest RAM
+ head -c 300M /dev/urandom > /workspace/data/rand-0.bin # user files on disk
+ cp -r /usr/share/doc /workspace/data/doc-0 2>/dev/null
+ find /usr -xdev -type f -size -4M -print0 2>/dev/null | xargs -0 cat > /dev/null 2>&1 # page cache
+ sleep 15
+ ;;
+step)
+ head -c 64M /dev/urandom > /workspace/data/rand-$2.bin
+ cp -r /usr/share/doc/python3* /workspace/data/ 2>/dev/null
+ sync
+ ;;
+esac
+free -m | awk '/Mem/{print "guest mem used MiB", $3, "cache MiB", $6}'
+df -m / | awk 'NR==2{print "guest disk used MiB", $3}'
diff --git a/test/pause/phases.py b/test/pause/phases.py
new file mode 100644
index 0000000..fb4eb92
--- /dev/null
+++ b/test/pause/phases.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""Split one Fast Sandbox pause into phases from its components' logs
+(controller, fastlet, runtime agent, Garage) and the sampler's disk trace.
+usage: phases.py SINCE_ISO FAST_SANDBOX_NAME [SAMPLES.jsonl] (root, on the node)
+Prints one JSON object: timestamps, phase durations (s), bytes, disk traffic."""
+import datetime as dt, json, os, re, subprocess, sys
+
+since, name = sys.argv[1], sys.argv[2]
+samples = sys.argv[3] if len(sys.argv) > 3 else None
+K = ["k3s", "kubectl"]
+SYSTEM = os.environ.get("FSB_SYSTEM_NAMESPACE", "opensandbox-system")
+DATAPLANE = os.environ.get("FSB_DATAPLANE_NAMESPACE", "opensandbox-dataplane")
+YEAR = dt.datetime.now(dt.timezone.utc).year # klog lines carry no year
+
+def logs(ns, target, container=None):
+ cmd = K + ["-n", ns, "logs", target, "--since-time", since, "--timestamps=false"]
+ if container:
+ cmd += ["-c", container]
+ return subprocess.run(cmd, capture_output=True, text=True).stdout.splitlines()
+
+def pod(ns, prefix):
+ out = subprocess.run(K + ["-n", ns, "get", "pods", "-o", "name"], capture_output=True, text=True).stdout.split()
+ return next(p for p in out if p.split("/", 1)[1].startswith(prefix))
+
+KLOG = re.compile(r"^[IWE](\d\d)(\d\d) (\d\d:\d\d:\d\d\.\d+)")
+def kt(line):
+ m = KLOG.match(line)
+ if not m:
+ return None
+ return dt.datetime.strptime(f"{YEAR}-{m[1]}-{m[2]} {m[3][:15]}", "%Y-%m-%d %H:%M:%S.%f").replace(tzinfo=dt.timezone.utc).timestamp()
+
+def kv(line, key):
+ m = re.search(key + r'="?([^" ]+)"?', line)
+ return m[1] if m else None
+
+def dur(s):
+ if s is None:
+ return None
+ total, units = 0.0, {"h": 3600, "m": 60, "s": 1, "ms": 1e-3, "µs": 1e-6, "us": 1e-6, "ns": 1e-9}
+ for num, unit in re.findall(r"([\d.]+)(h|ms|µs|us|ns|m|s)", s):
+ total += float(num) * units[unit]
+ return total
+
+ctl = logs(SYSTEM, "deploy/fast-sandbox-controller")
+fast = logs(DATAPLANE, pod(DATAPLANE, "fsb-pool-fastlet"), "fastlet")
+agent = logs(SYSTEM, pod(SYSTEM, "firecracker-runtime"), "firecracker-agent")
+garage = logs(SYSTEM, "deploy/fsb-garage")
+
+r = {"sandbox": name}
+uid = None
+for l in ctl:
+ if f"Sandbox=\"{DATAPLANE}/{name}\"" in l or f"sandbox=\"{name}\"" in l:
+ uid = uid or kv(l, "sandbox_uid")
+ if "pause checkpoint persisted" in l:
+ r["t_persisted"] = kt(l)
+ elif "sandbox paused; runtime released" in l:
+ r["t_released"] = kt(l)
+req = [kt(l) for l in ctl if "fastpath sandbox pause requested" in l]
+r["sandbox_uid"] = uid
+snap = None
+for l in fast:
+ if uid and f'sandboxID="{uid}"' in l:
+ if "dump starting" in l:
+ r["t_dump_start"], snap = kt(l), kv(l, "snapshotId")
+ elif "paused for dump" in l:
+ r["t_vm_paused"] = kt(l)
+ elif "resumed after dump" in l:
+ r["t_dump_done"] = kt(l)
+ for k in ("pauseAPI", "dumpAPI", "rootfsClone", "rootfsCopy", "resumeAPI"):
+ r[k] = dur(kv(l, k))
+ elif "manifest assembled" in l:
+ r["t_manifest"], r["bytes_logical"] = kt(l), int(kv(l, "sizeBytes"))
+ elif "snapshot published" in l:
+ r["t_published"], r["publish"] = kt(l), dur(kv(l, "publish"))
+if "t_dump_start" in r:
+ before = [t for t in req if t <= r["t_dump_start"] + 0.5]
+ r["t_requested"] = before[-1] if before else None
+for l in agent:
+ if snap and snap in l and "publish started" in l:
+ r["t_upload_start"], digest = kt(l), kv(l, "digest")
+ r["namespace"] = digest[:16]
+ elif r.get("namespace") and r["namespace"] in l and "publish completed" in l:
+ r["t_upload_done"] = kt(l)
+GT = re.compile(r"(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+)Z")
+files = []
+if r.get("namespace"):
+ for l in garage:
+ clean = re.sub(r"\x1b\[[0-9;]*m", "", l)
+ if f"PUT /sandbox-images/publish/{r['namespace']}/" in clean:
+ m = GT.search(clean)
+ t = dt.datetime.strptime(m[1][:26], "%Y-%m-%dT%H:%M:%S.%f").replace(tzinfo=dt.timezone.utc).timestamp()
+ files.append((t, clean.rsplit("/", 1)[1].strip()))
+files.sort()
+ends = [t for t, _ in files[1:]] + [r.get("t_upload_done")]
+r["upload_files"] = {f: round(e - t, 2) for (t, f), e in zip(files, ends) if e}
+
+def d(a, b):
+ return round(r[b] - r[a], 2) if r.get(a) and r.get(b) else None
+r["phases"] = {
+ "request_to_node": d("t_requested", "t_dump_start"),
+ "quiesce_and_disk_clone": d("t_dump_start", "t_vm_paused"),
+ "memory_dump": round(r["dumpAPI"], 2) if r.get("dumpAPI") else None,
+ "checksum_and_manifest": d("t_dump_done", "t_manifest"),
+ "upload": d("t_upload_start", "t_upload_done"),
+ "upload_handoff": round((r.get("t_upload_start") or 0) - (r.get("t_manifest") or 0), 2) if r.get("t_upload_start") else None,
+ "metadata_commit": d("t_published", "t_released"),
+ "node_total": d("t_dump_start", "t_published"),
+}
+if samples and r.get("t_dump_start"):
+ rows = [json.loads(x) for x in open(samples)]
+ marks = [("memory_dump", "t_vm_paused", "t_dump_done"), ("checksum_and_manifest", "t_dump_done", "t_manifest"),
+ ("upload", "t_upload_start", "t_upload_done"), ("metadata_commit", "t_published", "t_released")]
+ traffic = {}
+ for label, a, b in marks:
+ if r.get(a) and r.get(b):
+ sel = [x for x in rows if r[a] < x["t"] <= r[b] + 1]
+ cpu = {}
+ for x in sel:
+ for k, v in x["cpu"].items():
+ cpu[k] = round(cpu.get(k, 0) + v, 1)
+ traffic[label] = {"read_mb": round(sum(x["rd_mb"] for x in sel)), "write_mb": round(sum(x["wr_mb"] for x in sel)),
+ "disk_busy_pct": round(100 * sum(x["busy_ms"] for x in sel) / max(1, 1000 * len(sel))), "cpu_s": cpu}
+ r["disk"] = traffic
+print(json.dumps(r))
diff --git a/test/pause/punch.py b/test/pause/punch.py
new file mode 100644
index 0000000..0ad747c
--- /dev/null
+++ b/test/pause/punch.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""Turn the all-zero 64 KiB runs of a file into holes, in place: what Fast
+Sandbox's pull would write with the sparse-pull patch. Content is unchanged.
+usage (root): punch.py FILE..."""
+import ctypes, os, sys
+libc = ctypes.CDLL("libc.so.6", use_errno=True)
+libc.fallocate.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_long, ctypes.c_long]
+PUNCH = 0x02 | 0x01 # FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE
+CHUNK, BUF = 64 << 10, 4 << 20
+ZERO = bytes(CHUNK)
+
+def punch(fd, off, n):
+ if n and libc.fallocate(fd, PUNCH, off, n) != 0:
+ raise OSError(ctypes.get_errno(), "fallocate")
+
+for path in sys.argv[1:]:
+ st = os.stat(path)
+ fd = os.open(path, os.O_RDWR)
+ run_start, run_len, off = None, 0, 0
+ with os.fdopen(fd, "r+b", buffering=0) as f:
+ while True:
+ buf = f.read(BUF)
+ if not buf:
+ break
+ for i in range(0, len(buf), CHUNK):
+ chunk = buf[i:i + CHUNK]
+ if chunk == ZERO[:len(chunk)]:
+ if run_start is None:
+ run_start, run_len = off + i, 0
+ run_len += len(chunk)
+ elif run_start is not None:
+ punch(fd, run_start, run_len); run_start = None
+ off += len(buf)
+ if run_start is not None:
+ punch(fd, run_start, run_len)
+ os.fsync(fd)
+ after = os.stat(path)
+ print(f"{path}: {st.st_size/2**30:.2f} GiB, allocated {st.st_blocks*512/2**30:.2f} -> {after.st_blocks*512/2**30:.2f} GiB")
diff --git a/test/pause/runset.py b/test/pause/runset.py
new file mode 100644
index 0000000..bfa724a
--- /dev/null
+++ b/test/pause/runset.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+"""Run a set: for each SCENARIO:N, create a fresh sandbox through
+Differently, run cycle.py on it N times, delete it.
+usage: runset.py PREFIX idle:3 in-use:5
+PAUSE_DROP_CHECKPOINTS=1 also clears consumed checkpoints from the node's
+image cache after each sandbox is deleted (test hosts with a small disk)."""
+import os, subprocess, sys, time
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, HERE)
+import dly
+prefix = sys.argv[1]
+for spec in sys.argv[2:]:
+ scenario, n = spec.split(":")
+ t0 = time.time()
+ code, s = dly.create()
+ s = dly.wait(s["id"], "running", timeout=900)
+ print(f"[{prefix}-{scenario}] created {s['id']} in {time.time()-t0:.1f} s", flush=True)
+ time.sleep(20)
+ subprocess.run(["python3", os.path.join(HERE, "cycle.py"), s["id"], scenario, n, f"{prefix}-{scenario}"])
+ dly.call("DELETE", "/v1/sandboxes/" + s["id"])
+ time.sleep(15)
+ if os.environ.get("PAUSE_DROP_CHECKPOINTS") == "1":
+ subprocess.run(["sudo", "sh", os.path.join(HERE, "drop-checkpoints.sh")])
+print(f"[{prefix}] done", flush=True)
diff --git a/test/pause/s3bench.py b/test/pause/s3bench.py
new file mode 100644
index 0000000..6e61b71
--- /dev/null
+++ b/test/pause/s3bench.py
@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Upload one file to an S3 store the way Fast Sandbox's agent does (one PUT,
+UNSIGNED-PAYLOAD) or as a parallel multipart upload. stdlib only.
+env: S3_ENDPOINT (http://host:port), S3_BUCKET, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY
+usage: s3bench.py FILE KEY single | multipart PART_MiB CONCURRENCY | delete"""
+import concurrent.futures as cf, datetime, hashlib, hmac, http.client, os, re, sys, time, urllib.parse
+
+EP = urllib.parse.urlparse(os.environ["S3_ENDPOINT"]); BUCKET = os.environ["S3_BUCKET"]
+AK, SK, REGION = os.environ["AWS_ACCESS_KEY_ID"], os.environ["AWS_SECRET_ACCESS_KEY"], os.environ.get("S3_REGION", "us-east-1")
+
+def sign(method, path, query="", payload="UNSIGNED-PAYLOAD", extra=None):
+ now = datetime.datetime.now(datetime.timezone.utc); amz = now.strftime("%Y%m%dT%H%M%SZ"); day = amz[:8]
+ headers = {"host": EP.netloc, "x-amz-date": amz, "x-amz-content-sha256": payload, **(extra or {})}
+ names = sorted(headers)
+ canon = "\n".join([method, urllib.parse.quote(path), query, "".join(f"{k}:{headers[k]}\n" for k in names), ";".join(names), payload])
+ scope = f"{day}/{REGION}/s3/aws4_request"
+ sts = "\n".join(["AWS4-HMAC-SHA256", amz, scope, hashlib.sha256(canon.encode()).hexdigest()])
+ k = ("AWS4" + SK).encode()
+ for part in (day, REGION, "s3", "aws4_request"):
+ k = hmac.new(k, part.encode(), hashlib.sha256).digest()
+ sig = hmac.new(k, sts.encode(), hashlib.sha256).hexdigest()
+ headers["Authorization"] = f"AWS4-HMAC-SHA256 Credential={AK}/{scope}, SignedHeaders={';'.join(names)}, Signature={sig}"
+ return headers
+
+def request(method, path, query="", body=None, length=None):
+ h = sign(method, path, query)
+ if length is not None:
+ h["Content-Length"] = str(length)
+ c = http.client.HTTPConnection(EP.hostname, EP.port, timeout=3600, blocksize=1 << 20)
+ c.request(method, path + ("?" + query if query else ""), body=body, headers=h)
+ r = c.getresponse(); data = r.read(); c.close()
+ if r.status >= 300:
+ raise SystemExit(f"{method} {path}?{query} -> {r.status} {data[:300]}")
+ return r, data
+
+class Section:
+ def __init__(self, f, off, n): self.f, self.off, self.left = f, off, n; f.seek(off)
+ def read(self, size=-1):
+ if self.left <= 0: return b""
+ size = self.left if size < 0 else min(size, self.left)
+ b = self.f.read(size); self.left -= len(b); return b
+
+def main():
+ path, key, mode = sys.argv[1], sys.argv[2], sys.argv[3]
+ obj = f"/{BUCKET}/{key}"
+ if mode == "delete":
+ request("DELETE", obj); return
+ size = os.path.getsize(path); t0 = time.time()
+ if mode == "single":
+ with open(path, "rb") as f:
+ request("PUT", obj, body=f, length=size)
+ else:
+ part = int(sys.argv[4]) << 20; conc = int(sys.argv[5])
+ _, data = request("POST", obj, "uploads=")
+ upload = re.search(rb"(.*?)", data)[1].decode()
+ def put(i):
+ off = i * part; n = min(part, size - off)
+ with open(path, "rb") as f:
+ r, _ = request("PUT", obj, f"partNumber={i+1}&uploadId={urllib.parse.quote(upload)}", body=Section(f, off, n), length=n)
+ return i + 1, r.getheader("ETag")
+ n = (size + part - 1) // part
+ with cf.ThreadPoolExecutor(conc) as ex:
+ etags = sorted(ex.map(put, range(n)))
+ body = ("" + "".join(f"{i}{e}" for i, e in etags) + "").encode()
+ request("POST", obj, f"uploadId={urllib.parse.quote(upload)}", body=body, length=len(body))
+ dt = time.time() - t0
+ print(f"{mode} {size/1e6:.0f} MB in {dt:.1f} s = {size/1e6/dt:.0f} MB/s")
+
+main()
diff --git a/test/pause/sampler.py b/test/pause/sampler.py
new file mode 100644
index 0000000..ff15211
--- /dev/null
+++ b/test/pause/sampler.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+"""Sample the state-root disk and the pause path's processes once a second.
+Writes JSON lines {t, rd_mb, wr_mb, busy_ms, cpu: {name: seconds}}, each a
+delta over the last second. Run as root: sampler.py OUT [DEVICE]"""
+import json, os, sys, time
+
+out = open(sys.argv[1], "a", buffering=1)
+dev = sys.argv[2] if len(sys.argv) > 2 else "nvme0n1"
+HZ = os.sysconf("SC_CLK_TCK")
+# Process name prefixes (comm is cut at 15 bytes), most specific first.
+NAMES = [("firecracker-run", "agent"), ("firecracker", "firecracker"), ("garage", "garage"), ("fastlet", "fastlet")]
+
+
+def disk():
+ for line in open("/proc/diskstats"):
+ f = line.split()
+ if f[2] == dev:
+ return int(f[5]) * 512, int(f[9]) * 512, int(f[12])
+ return 0, 0, 0
+
+
+def procs():
+ ticks = {}
+ for pid in os.listdir("/proc"):
+ if not pid.isdigit():
+ continue
+ try:
+ comm = open(f"/proc/{pid}/comm").read().strip()
+ label = next((v for k, v in NAMES if comm.startswith(k)), None)
+ if label is None:
+ continue
+ st = open(f"/proc/{pid}/stat").read().rsplit(")", 1)[1].split()
+ ticks[pid] = (label, int(st[11]) + int(st[12]))
+ except (FileNotFoundError, ProcessLookupError, PermissionError, IndexError):
+ pass
+ return ticks
+
+
+pr, pw, pb = disk()
+pc = procs()
+while True:
+ time.sleep(1)
+ r, w, b = disk()
+ c = procs()
+ cpu = {}
+ for pid, (label, t) in c.items():
+ cpu[label] = cpu.get(label, 0) + (t - pc.get(pid, (label, 0))[1]) / HZ # a new process counts from 0
+ out.write(json.dumps({"t": round(time.time(), 3), "rd_mb": round((r - pr) / 1e6, 2), "wr_mb": round((w - pw) / 1e6, 2),
+ "busy_ms": b - pb, "cpu": {k: round(v, 2) for k, v in cpu.items()}}) + "\n")
+ pr, pw, pb, pc = r, w, b, c
diff --git a/test/pause/summarize.py b/test/pause/summarize.py
new file mode 100644
index 0000000..a3ca4de
--- /dev/null
+++ b/test/pause/summarize.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+"""Summarize pause/resume runs (cycle.py output) as Markdown tables.
+usage: summarize.py RUN.jsonl..."""
+import json, math, statistics, sys
+
+def pct(xs, q):
+ xs = sorted(x for x in xs if x is not None)
+ if not xs:
+ return None
+ return xs[max(0, min(len(xs) - 1, math.ceil(q * len(xs)) - 1))]
+
+def fmt(x, unit=" s", nd=1):
+ return "–" if x is None else f"{x:.{nd}f}{unit}"
+
+def at_floor(r):
+ """A burstable disk out of credits: a phase kept the disk busy but moved
+ far less than its burst rate (about 175 MB/s on a P6 or P15)."""
+ node = r.get("node") or {}
+ for ph in ("memory_dump", "checksum_and_manifest", "upload"):
+ d, secs = (node.get("disk") or {}).get(ph) or {}, (node.get("phases") or {}).get(ph)
+ if not secs or secs < 10:
+ continue
+ busy, rate = d.get("disk_busy_pct", 0), (d.get("read_mb", 0) + d.get("write_mb", 0)) / secs
+ if (busy >= 95 and rate < 150) or (busy >= 85 and rate < 80):
+ return True
+ return False
+
+
+runs = {}
+for path in sys.argv[1:]:
+ for line in open(path):
+ r = json.loads(line)
+ tag = r["tag"] + (" (disk at its floor)" if at_floor(r) else "")
+ runs.setdefault(tag, []).append(r)
+
+print("| Run | Cycles | Pause median | Pause p95 | Resume median | Resume p95 | Resume to first exec, median | State kept |")
+print("|---|---:|---:|---:|---:|---:|---:|---|")
+for tag, rs in runs.items():
+ kept = sum(1 for r in rs if r.get("procs_kept") and all(r.get("state_kept", {}).values()))
+ p = [r["pause_s"] for r in rs]; q = [r["resume_s"] for r in rs]; e = [r["resume_to_exec_s"] for r in rs]
+ print(f"| {tag} | {len(rs)} | {fmt(statistics.median(p))} | {fmt(pct(p, .95))} | {fmt(statistics.median(q), nd=2)} | {fmt(pct(q, .95), nd=2)} | {fmt(statistics.median(e), nd=2)} | {kept}/{len(rs)} |")
+
+phases = ["quiesce_and_disk_clone", "memory_dump", "checksum_and_manifest", "upload", "metadata_commit", "node_total"]
+print()
+print("| Run | " + " | ".join(phases) + " |")
+print("|---|" + "---:|" * len(phases))
+for tag, rs in runs.items():
+ cells = []
+ for ph in phases:
+ xs = [(r.get("node") or {}).get("phases", {}).get(ph) for r in rs]
+ cells.append(f"{fmt(statistics.median([x for x in xs if x is not None]) if any(x is not None for x in xs) else None, nd=2)} / {fmt(pct(xs, .95), '', 1)}")
+ print(f"| {tag} | " + " | ".join(cells) + " |")
+
+print()
+print("| Run | Phase | Disk read MB | Disk write MB | MB/s through disk | Busy | Garage CPU s |")
+print("|---|---|---:|---:|---:|---:|---:|")
+for tag, rs in runs.items():
+ for ph in ["memory_dump", "checksum_and_manifest", "upload", "metadata_commit"]:
+ rd = [((r.get("node") or {}).get("disk", {}).get(ph) or {}).get("read_mb") for r in rs]
+ wr = [((r.get("node") or {}).get("disk", {}).get(ph) or {}).get("write_mb") for r in rs]
+ bz = [((r.get("node") or {}).get("disk", {}).get(ph) or {}).get("disk_busy_pct") for r in rs]
+ gc = [(((r.get("node") or {}).get("disk", {}).get(ph) or {}).get("cpu_s") or {}).get("garage") for r in rs]
+ du = [(r.get("node") or {}).get("phases", {}).get(ph) for r in rs]
+ if not any(x is not None for x in rd):
+ continue
+ rate = [((a or 0) + (b or 0)) / d for a, b, d in zip(rd, wr, du) if d]
+ print(f"| {tag} | {ph} | {statistics.median([x for x in rd if x is not None]):.0f} | {statistics.median([x for x in wr if x is not None]):.0f} | {statistics.median(rate) if rate else 0:.0f} | {statistics.median([x for x in bz if x is not None]):.0f}% | {statistics.median([x for x in gc if x is not None]):.0f} |")
+
+print()
+print("| Run | Logical bytes per pause | Garage stored per pause (median) | Upload, rootfs.ext4 / memory.snap (median) |")
+print("|---|---:|---:|---:|")
+for tag, rs in runs.items():
+ lb = [(r.get("node") or {}).get("bytes_logical") for r in rs]
+ gs = [r.get("garage_stored_mb") for r in rs]
+ ur = [((r.get("node") or {}).get("upload_files") or {}).get("rootfs.ext4") for r in rs]
+ um = [((r.get("node") or {}).get("upload_files") or {}).get("memory.snap") for r in rs]
+ def med(xs):
+ xs = [x for x in xs if x is not None]
+ return statistics.median(xs) if xs else None
+ print(f"| {tag} | {fmt(med(lb) / 1e9 if med(lb) else None, ' GB', 2)} | {fmt(med(gs), ' MB', 0)} | {fmt(med(ur))} / {fmt(med(um))} |")
+