diff --git a/benchmarks/ethereum-classic-rpc.yml b/benchmarks/ethereum-classic-rpc.yml index 6add2964..c7122e78 100644 --- a/benchmarks/ethereum-classic-rpc.yml +++ b/benchmarks/ethereum-classic-rpc.yml @@ -4,7 +4,7 @@ slug: ethereum-classic-rpc number: "194" title: Fastest free Ethereum Classic RPC, live no-key endpoint latency seo_title: "Fastest free Ethereum Classic RPC 2026" -seo_description: "{{best_name}} leads free Ethereum Classic RPC at {{best_p50}} (eth_getBlockByNumber p50, 24h). 3 no-key providers measured every 60s from 3 regions." +seo_description: "{{best_name}} leads free Ethereum Classic RPC at {{best_p50}} (eth_getBlockByNumber p50, 24h). 1 no-key provider measured every 60s from 3 regions." subtitle: HTTP round-trip latency for eth_getBlockByNumber against every free, no-key public Ethereum Classic EVM endpoint, audited every 60 seconds from 3 regions. category: RPCs @@ -14,13 +14,13 @@ unit: ms higher_is_better: false seo_intro: | - Ethereum Classic (chain 61) is the original Ethereum chain post-2016 DAO fork, continuing as a Proof-of-Work EVM network with ETC as the gas token. The chain upholds code-is-law immutability and has a dedicated long-term holder community. Three keyless endpoints are probed continuously: dRPC (etc.drpc.org), ETCDesktop (etc.etcdesktop.com), and ETCMC (rpc.etcmc.net). + Ethereum Classic (chain 61) is the original Ethereum chain post-2016 DAO fork, continuing as a Proof-of-Work EVM network with ETC as the gas token. The chain upholds code-is-law immutability and has a dedicated long-term holder community. One keyless endpoint is probed continuously: dRPC (etc.drpc.org). ETCDesktop and ETCMC are no longer responding to probes and have been removed. abstract: | Per-chain member of the RPC latency cluster, extended to Ethereum Classic. We measure the round-trip latency of a single, identical RPC call (`eth_getBlockByNumber`) against every no-key public Ethereum Classic endpoint - that sustains continuous probing, 3 providers at launch, + that sustains continuous probing, 1 provider at launch, every 60 seconds, from us-east, eu-west and Singapore. The cross-chain view lives on the parent `rpc-capabilities` benchmark; this page is the Ethereum Classic-scoped answer with per-region breakdowns as a first-class dimension. @@ -31,16 +31,16 @@ methodology: - "Latency: client-side round-trip delta in milliseconds, exposed as both a gauge and a histogram (buckets 50 ms to 10 s), so percentiles are computed via Prometheus `quantile_over_time` over the last 24 hours." - "Call-result classification: `ok` (HTTP 200 + non-empty result), `http_err`, `jsonrpc_err`, `stale` (more than 20 blocks behind the cross-provider tip), `timeout`." - "This page is part of the per-chain RPC cluster derived from the cross-chain [rpc-capabilities](https://openchainbench.com/benchmarks/rpc-capabilities) benchmark; the identical harness, cadence and exclusion rules apply on every chain." - - "Chain scope: every query on this page is pinned to `chain=\"ethereum-classic\"`. Provider coverage at launch: 3 no-key endpoints." + - "Chain scope: every query on this page is pinned to `chain=\"ethereum-classic\"`. Provider coverage: 1 no-key endpoint (dRPC). ETCDesktop and ETCMC stopped responding to probes and have been removed." findings: - - "{{best_name}} currently leads free Ethereum Classic RPC at {{best_p50}} (`eth_getBlockByNumber` p50, 24h) across 3 measured providers." + - "{{best_name}} currently leads free Ethereum Classic RPC at {{best_p50}} (`eth_getBlockByNumber` p50, 24h), 1 measured provider." faq: - q: "What is the fastest free Ethereum Classic RPC right now?" - a: "{{best_name}} currently leads at {{best_p50}} (`eth_getBlockByNumber` p50 over the last 24h), measured against 3 no-key providers probed every 60 seconds from us-east, eu-west and Singapore." + a: "{{best_name}} currently leads at {{best_p50}} (`eth_getBlockByNumber` p50 over the last 24h), measured against 1 no-key provider probed every 60 seconds from us-east, eu-west and Singapore." - q: "Which Ethereum Classic RPCs work without an API key?" - a: "3 endpoints sustain continuous keyless probing: dRPC, ETCDesktop, and ETCMC. Every endpoint was live-verified before inclusion." + a: "1 endpoint sustains continuous keyless probing: dRPC (etc.drpc.org). ETCDesktop and ETCMC stopped responding to probes and have been removed from the bench." - q: "Does the fastest Ethereum Classic RPC change by region?" a: "Often. The region tabs at the top of the page re-scope every number to a single origin." - q: "How is Ethereum Classic RPC latency measured here?" @@ -84,49 +84,3 @@ providers: - region: ap-southeast p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="drpc", chain="ethereum-classic", region="sgp"}) series: avg_over_time(rpc_latency_milliseconds{provider="drpc", chain="ethereum-classic", region="sgp"}[1h]) - - - slug: etcdesktop - name: ETCDesktop - tag: ETCDesktop community public RPC for Ethereum Classic - formula: "50th percentile over 24h of client-side round-trip latency (ms) for a single `eth_getBlockByNumber` POST sent every 60s from 3 regions to ETCDesktop's no-key Ethereum Classic endpoint." - queries: - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="etcdesktop", chain="ethereum-classic"}) - p90: avg(ocb:rpc_latency_milliseconds:p90_24h{provider="etcdesktop", chain="ethereum-classic"}) - p99: avg(ocb:rpc_latency_milliseconds:p99_24h{provider="etcdesktop", chain="ethereum-classic"}) - mean: avg(ocb:rpc_latency_milliseconds:mean_24h{provider="etcdesktop", chain="ethereum-classic"}) - success: sum(ocb:rpc_call:ok_rate_24h{provider="etcdesktop", chain="ethereum-classic"}) / sum(ocb:rpc_call:rate_24h{provider="etcdesktop", chain="ethereum-classic"}) - sample_size: sum(ocb:rpc_call:increase_24h{provider="etcdesktop", chain="ethereum-classic"}) - series: avg(avg_over_time(rpc_latency_milliseconds{provider="etcdesktop", chain="ethereum-classic"}[1h])) - regions: - - region: us-east - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="etcdesktop", chain="ethereum-classic", region="us-east"}) - series: avg_over_time(rpc_latency_milliseconds{provider="etcdesktop", chain="ethereum-classic", region="us-east"}[1h]) - - region: eu-west - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="etcdesktop", chain="ethereum-classic", region="eu-west"}) - series: avg_over_time(rpc_latency_milliseconds{provider="etcdesktop", chain="ethereum-classic", region="eu-west"}[1h]) - - region: ap-southeast - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="etcdesktop", chain="ethereum-classic", region="sgp"}) - series: avg_over_time(rpc_latency_milliseconds{provider="etcdesktop", chain="ethereum-classic", region="sgp"}[1h]) - - - slug: etcmc - name: ETCMC - tag: ETCMC community public RPC for Ethereum Classic - formula: "50th percentile over 24h of client-side round-trip latency (ms) for a single `eth_getBlockByNumber` POST sent every 60s from 3 regions to ETCMC's no-key Ethereum Classic endpoint." - queries: - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="etcmc", chain="ethereum-classic"}) - p90: avg(ocb:rpc_latency_milliseconds:p90_24h{provider="etcmc", chain="ethereum-classic"}) - p99: avg(ocb:rpc_latency_milliseconds:p99_24h{provider="etcmc", chain="ethereum-classic"}) - mean: avg(ocb:rpc_latency_milliseconds:mean_24h{provider="etcmc", chain="ethereum-classic"}) - success: sum(ocb:rpc_call:ok_rate_24h{provider="etcmc", chain="ethereum-classic"}) / sum(ocb:rpc_call:rate_24h{provider="etcmc", chain="ethereum-classic"}) - sample_size: sum(ocb:rpc_call:increase_24h{provider="etcmc", chain="ethereum-classic"}) - series: avg(avg_over_time(rpc_latency_milliseconds{provider="etcmc", chain="ethereum-classic"}[1h])) - regions: - - region: us-east - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="etcmc", chain="ethereum-classic", region="us-east"}) - series: avg_over_time(rpc_latency_milliseconds{provider="etcmc", chain="ethereum-classic", region="us-east"}[1h]) - - region: eu-west - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="etcmc", chain="ethereum-classic", region="eu-west"}) - series: avg_over_time(rpc_latency_milliseconds{provider="etcmc", chain="ethereum-classic", region="eu-west"}[1h]) - - region: ap-southeast - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="etcmc", chain="ethereum-classic", region="sgp"}) - series: avg_over_time(rpc_latency_milliseconds{provider="etcmc", chain="ethereum-classic", region="sgp"}[1h]) diff --git a/benchmarks/xdc-rpc.yml b/benchmarks/xdc-rpc.yml index e4c64a92..493e78f5 100644 --- a/benchmarks/xdc-rpc.yml +++ b/benchmarks/xdc-rpc.yml @@ -4,7 +4,7 @@ slug: xdc-rpc number: "153" title: Fastest free XDC Network RPC, live no-key endpoint latency seo_title: "Fastest free XDC Network RPC 2026" -seo_description: "{{best_name}} leads free XDC Network RPC at {{best_p50}} (eth_getBlockByNumber p50, 24h). 5 no-key providers measured every 60s from 3 regions." +seo_description: "{{best_name}} leads free XDC Network RPC at {{best_p50}} (eth_getBlockByNumber p50, 24h). 2 no-key providers measured every 60s from 3 regions." subtitle: HTTP round-trip latency for eth_getBlockByNumber against every free, no-key public XDC Network RPC endpoint, audited every 60 seconds from 3 regions. category: RPCs @@ -14,13 +14,13 @@ unit: ms higher_is_better: false seo_intro: | - XDC Network (chain 50) is an EVM-compatible hybrid blockchain targeting enterprise and trade-finance use cases, XDC gas token, delegated proof of stake consensus, designed for ISO 20022-compliant financial applications. The XDC Foundation's rpc.xinfin.network and additional endpoints from Ankr and the community provide keyless access probed every 60 seconds from three regions. + XDC Network (chain 50) is an EVM-compatible hybrid blockchain targeting enterprise and trade-finance use cases, XDC gas token, delegated proof of stake consensus, designed for ISO 20022-compliant financial applications. Two keyless endpoints are probed continuously: Ankr and XDC eRPC (erpc.xinfin.network). XDC Foundation official, XDCrpc, and XDC.org stopped responding to probes and have been removed. abstract: | Per-chain member of the RPC latency cluster, extended to XDC Network. We measure the round-trip latency of a single, identical RPC call (`eth_getBlockByNumber`) against every no-key public XDC Network endpoint - that sustains continuous probing, 5 providers at launch, + that sustains continuous probing, 2 providers, every 60 seconds, from us-east, eu-west and Singapore. The harness classifies every response (ok / http_err / jsonrpc_err / stale / timeout) so the leaderboard rewards sustained availability rather than @@ -34,16 +34,16 @@ methodology: - "Latency: client-side round-trip delta in milliseconds, exposed as both a gauge and a histogram (buckets 50 ms to 10 s), so percentiles are computed via Prometheus `quantile_over_time` over the last 24 hours." - "Call-result classification: `ok` (HTTP 200 + non-empty result), `http_err`, `jsonrpc_err` (HTTP 200 carrying an error body), `stale` (more than 20 blocks behind the cross-provider tip), `timeout`." - "This page is part of the per-chain RPC cluster derived from the cross-chain [rpc-capabilities](https://openchainbench.com/benchmarks/rpc-capabilities) benchmark; the identical harness, cadence and exclusion rules apply on every chain." - - "Chain scope: every query on this page is pinned to `chain=\"xdc\"`. Provider coverage at launch: 5 no-key endpoints (XDC Foundation, Ankr, XDCrpc, XDC eRPC, XDC.org)." + - "Chain scope: every query on this page is pinned to `chain=\"xdc\"`. Provider coverage: 2 no-key endpoints (Ankr, XDC eRPC). XDC Foundation official, XDCrpc and XDC.org stopped responding to probes and have been removed." findings: - - "{{best_name}} currently leads free XDC Network RPC at {{best_p50}} (`eth_getBlockByNumber` p50, 24h) across 5 measured providers." + - "{{best_name}} currently leads free XDC Network RPC at {{best_p50}} (`eth_getBlockByNumber` p50, 24h) across 2 measured providers." faq: - q: "What is the fastest free XDC Network RPC right now?" - a: "{{best_name}} currently leads at {{best_p50}} (`eth_getBlockByNumber` p50 over the last 24h), measured against 5 no-key providers probed every 60 seconds from us-east, eu-west and Singapore. The leaderboard re-sorts continuously against fresh Prometheus samples." + a: "{{best_name}} currently leads at {{best_p50}} (`eth_getBlockByNumber` p50 over the last 24h), measured against 2 no-key providers probed every 60 seconds from us-east, eu-west and Singapore. The leaderboard re-sorts continuously against fresh Prometheus samples." - q: "Which XDC Network RPCs work without an API key?" - a: "5 endpoints sustain continuous keyless probing at launch: XDC Foundation, Ankr, XDCrpc, XDC eRPC, XDC.org. Every listed endpoint was live-verified before inclusion." + a: "2 endpoints sustain continuous keyless probing: Ankr and XDC eRPC (erpc.xinfin.network). XDC Foundation official, XDCrpc and XDC.org stopped responding to probes and have been removed." - q: "Does the fastest XDC Network RPC change by region?" a: "Often. The region tabs at the top of the page re-scope every number to a single origin; pick the one closest to where your requests originate." - q: "How is XDC Network RPC latency measured here?" @@ -65,29 +65,6 @@ dimensions: - { value: sgp, label: Singapore } providers: - - slug: xdc-official - name: XDC Foundation - tag: XDC Foundation's primary public RPC - formula: "50th percentile over 24h of client-side round-trip latency (ms) for a single `eth_getBlockByNumber` POST sent every 60s from 3 regions (us-east + eu-west + sgp) to XDC Foundation's no-key XDC Network endpoint." - queries: - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="xdc-official", chain="xdc"}) - p90: avg(ocb:rpc_latency_milliseconds:p90_24h{provider="xdc-official", chain="xdc"}) - p99: avg(ocb:rpc_latency_milliseconds:p99_24h{provider="xdc-official", chain="xdc"}) - mean: avg(ocb:rpc_latency_milliseconds:mean_24h{provider="xdc-official", chain="xdc"}) - success: sum(ocb:rpc_call:ok_rate_24h{provider="xdc-official", chain="xdc"}) / sum(ocb:rpc_call:rate_24h{provider="xdc-official", chain="xdc"}) - sample_size: sum(ocb:rpc_call:increase_24h{provider="xdc-official", chain="xdc"}) - series: avg(avg_over_time(rpc_latency_milliseconds{provider="xdc-official", chain="xdc"}[1h])) - regions: - - region: us-east - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="xdc-official", chain="xdc", region="us-east"}) - series: avg_over_time(rpc_latency_milliseconds{provider="xdc-official", chain="xdc", region="us-east"}[1h]) - - region: eu-west - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="xdc-official", chain="xdc", region="eu-west"}) - series: avg_over_time(rpc_latency_milliseconds{provider="xdc-official", chain="xdc", region="eu-west"}[1h]) - - region: ap-southeast - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="xdc-official", chain="xdc", region="sgp"}) - series: avg_over_time(rpc_latency_milliseconds{provider="xdc-official", chain="xdc", region="sgp"}[1h]) - - slug: ankr name: Ankr tag: Multi-cloud RPC network @@ -111,29 +88,6 @@ providers: p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="ankr", chain="xdc", region="sgp"}) series: avg_over_time(rpc_latency_milliseconds{provider="ankr", chain="xdc", region="sgp"}[1h]) - - slug: xdcrpc - name: XDCrpc - tag: Community-operated XDC public RPC - formula: "50th percentile over 24h of client-side round-trip latency (ms) for a single `eth_getBlockByNumber` POST sent every 60s from 3 regions (us-east + eu-west + sgp) to XDCrpc's no-key XDC Network endpoint." - queries: - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="xdcrpc", chain="xdc"}) - p90: avg(ocb:rpc_latency_milliseconds:p90_24h{provider="xdcrpc", chain="xdc"}) - p99: avg(ocb:rpc_latency_milliseconds:p99_24h{provider="xdcrpc", chain="xdc"}) - mean: avg(ocb:rpc_latency_milliseconds:mean_24h{provider="xdcrpc", chain="xdc"}) - success: sum(ocb:rpc_call:ok_rate_24h{provider="xdcrpc", chain="xdc"}) / sum(ocb:rpc_call:rate_24h{provider="xdcrpc", chain="xdc"}) - sample_size: sum(ocb:rpc_call:increase_24h{provider="xdcrpc", chain="xdc"}) - series: avg(avg_over_time(rpc_latency_milliseconds{provider="xdcrpc", chain="xdc"}[1h])) - regions: - - region: us-east - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="xdcrpc", chain="xdc", region="us-east"}) - series: avg_over_time(rpc_latency_milliseconds{provider="xdcrpc", chain="xdc", region="us-east"}[1h]) - - region: eu-west - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="xdcrpc", chain="xdc", region="eu-west"}) - series: avg_over_time(rpc_latency_milliseconds{provider="xdcrpc", chain="xdc", region="eu-west"}[1h]) - - region: ap-southeast - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="xdcrpc", chain="xdc", region="sgp"}) - series: avg_over_time(rpc_latency_milliseconds{provider="xdcrpc", chain="xdc", region="sgp"}[1h]) - - slug: xdc-erpc name: XDC eRPC tag: XDC Foundation alternate public RPC gateway @@ -156,26 +110,3 @@ providers: - region: ap-southeast p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="xdc-erpc", chain="xdc", region="sgp"}) series: avg_over_time(rpc_latency_milliseconds{provider="xdc-erpc", chain="xdc", region="sgp"}[1h]) - - - slug: xdc-org - name: XDC.org - tag: XDC Foundation third public RPC mirror - formula: "50th percentile over 24h of client-side round-trip latency (ms) for a single `eth_getBlockByNumber` POST sent every 60s from 3 regions (us-east + eu-west + sgp) to XDC Foundation's rpc.xdc.org endpoint." - queries: - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="xdc-org", chain="xdc"}) - p90: avg(ocb:rpc_latency_milliseconds:p90_24h{provider="xdc-org", chain="xdc"}) - p99: avg(ocb:rpc_latency_milliseconds:p99_24h{provider="xdc-org", chain="xdc"}) - mean: avg(ocb:rpc_latency_milliseconds:mean_24h{provider="xdc-org", chain="xdc"}) - success: sum(ocb:rpc_call:ok_rate_24h{provider="xdc-org", chain="xdc"}) / sum(ocb:rpc_call:rate_24h{provider="xdc-org", chain="xdc"}) - sample_size: sum(ocb:rpc_call:increase_24h{provider="xdc-org", chain="xdc"}) - series: avg(avg_over_time(rpc_latency_milliseconds{provider="xdc-org", chain="xdc"}[1h])) - regions: - - region: us-east - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="xdc-org", chain="xdc", region="us-east"}) - series: avg_over_time(rpc_latency_milliseconds{provider="xdc-org", chain="xdc", region="us-east"}[1h]) - - region: eu-west - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="xdc-org", chain="xdc", region="eu-west"}) - series: avg_over_time(rpc_latency_milliseconds{provider="xdc-org", chain="xdc", region="eu-west"}[1h]) - - region: ap-southeast - p50: avg(ocb:rpc_latency_milliseconds:p50_24h{provider="xdc-org", chain="xdc", region="sgp"}) - series: avg_over_time(rpc_latency_milliseconds{provider="xdc-org", chain="xdc", region="sgp"}[1h]) diff --git a/harnesses/aggregator-head-lag/Dockerfile b/harnesses/aggregator-head-lag/Dockerfile index 74ca8f0f..777b00d3 100644 --- a/harnesses/aggregator-head-lag/Dockerfile +++ b/harnesses/aggregator-head-lag/Dockerfile @@ -21,9 +21,12 @@ FROM debian:bookworm-slim WORKDIR /app -# Install runtime dependencies +# Install runtime dependencies + Chromium for in-process JWE scraping RUN apt-get update && apt-get install -y \ ca-certificates \ + chromium \ + chromium-sandbox \ + fonts-liberation \ && rm -rf /var/lib/apt/lists/* # Copy binary from builder diff --git a/harnesses/aggregator-head-lag/cmd/scrape-cookie/main.go b/harnesses/aggregator-head-lag/cmd/scrape-cookie/main.go index 686fa5d6..f64c3df5 100644 --- a/harnesses/aggregator-head-lag/cmd/scrape-cookie/main.go +++ b/harnesses/aggregator-head-lag/cmd/scrape-cookie/main.go @@ -1,12 +1,9 @@ package main import ( - "bytes" "context" "encoding/json" "fmt" - "io" - "net/http" "net/url" "os" "time" @@ -28,16 +25,66 @@ func main() { defer cancel() ctx, cancel := chromedp.NewContext(allocCtx) defer cancel() - ctx, cancel = context.WithTimeout(ctx, 45*time.Second) + ctx, cancel = context.WithTimeout(ctx, 60*time.Second) defer cancel() cookies := map[string]string{} + // Monkey-patch WebSocket to capture connection_init sent to graph.codex.io + // and the server's first response (ack or error/close). + captureScript := ` +window.__wsCaptures = []; +window.__wsResponses = []; +const _WS = window.WebSocket; +window.WebSocket = function(url, protocols) { + const ws = new _WS(url, protocols); + if (url && url.includes('graph.codex.io')) { + const _send = ws.send.bind(ws); + ws.send = function(data) { + try { + const parsed = JSON.parse(data); + if (parsed.type === 'connection_init') { + window.__wsCaptures.push({url: url, payload: parsed.payload, raw: data}); + console.log('WS_CAPTURE:' + JSON.stringify({url, payload: parsed.payload})); + } + } catch(e) {} + return _send(data); + }; + ws.addEventListener('message', function(evt) { + try { + const parsed = JSON.parse(evt.data); + window.__wsResponses.push({url: url, type: parsed.type, raw: evt.data.slice(0, 200)}); + console.log('WS_RESPONSE:' + JSON.stringify({url, type: parsed.type})); + } catch(e) {} + }); + ws.addEventListener('close', function(evt) { + window.__wsResponses.push({url: url, closeCode: evt.code, reason: evt.reason}); + console.log('WS_CLOSE:' + JSON.stringify({url, code: evt.code, reason: evt.reason})); + }); + } + return ws; +}; +window.WebSocket.prototype = _WS.prototype; +window.WebSocket.CONNECTING = _WS.CONNECTING; +window.WebSocket.OPEN = _WS.OPEN; +window.WebSocket.CLOSING = _WS.CLOSING; +window.WebSocket.CLOSED = _WS.CLOSED; +` + err := chromedp.Run(ctx, + // Inject monkey-patch before page loads + chromedp.ActionFunc(func(ctx context.Context) error { + return chromedp.Evaluate(`void 0`, nil).Do(ctx) // warm up + }), + chromedp.Navigate("about:blank"), + chromedp.Evaluate(captureScript, nil), chromedp.Navigate("https://www.defined.fi/"), chromedp.WaitVisible(`body`, chromedp.ByQuery), - chromedp.Sleep(8*time.Second), + // Re-inject on the loaded page (navigate clears scripts) + chromedp.Evaluate(captureScript, nil), + chromedp.Sleep(20*time.Second), chromedp.ActionFunc(func(ctx context.Context) error { + // Get cookies cookieParams, err := network.GetCookies().Do(ctx) if err != nil { return fmt.Errorf("failed to get cookies: %w", err) @@ -48,41 +95,49 @@ func main() { } return nil }), + // Capture what the browser sent in connection_init + chromedp.ActionFunc(func(ctx context.Context) error { + var captures []map[string]interface{} + err := chromedp.Evaluate(`window.__wsCaptures || []`, &captures).Do(ctx) + if err != nil { + fmt.Fprintf(os.Stderr, "capture eval error: %v\n", err) + return nil + } + if len(captures) == 0 { + fmt.Fprintf(os.Stderr, "\n[WS CAPTURE] No connection_init captured for graph.codex.io\n") + // Try to get captures via console approach + var raw string + chromedp.Evaluate(`JSON.stringify(window.__wsCaptures || [])`, &raw).Do(ctx) + fmt.Fprintf(os.Stderr, "[WS CAPTURE] raw: %s\n", raw) + } + for i, c := range captures { + fmt.Fprintf(os.Stderr, "\n[WS CAPTURE #%d] url=%v\n", i, c["url"]) + payloadBytes, _ := json.MarshalIndent(c["payload"], "", " ") + fmt.Fprintf(os.Stderr, "[WS CAPTURE #%d] connection_init payload:\n%s\n", i, string(payloadBytes)) + fmt.Printf("WS_INIT_PAYLOAD=%s\n", string(payloadBytes)) + } + // Capture server responses (ack / close) + var responses []map[string]interface{} + chromedp.Evaluate(`window.__wsResponses || []`, &responses).Do(ctx) + if len(responses) == 0 { + fmt.Fprintf(os.Stderr, "\n[WS RESPONSE] No server responses captured yet\n") + } + for i, r := range responses { + respBytes, _ := json.MarshalIndent(r, "", " ") + fmt.Fprintf(os.Stderr, "\n[WS RESPONSE #%d]:\n%s\n", i, string(respBytes)) + } + return nil + }), ) if err != nil { fmt.Fprintf(os.Stderr, "chrome error: %v\n", err) os.Exit(1) } - // Try codex_token first (it's a pre-minted token, might be usable directly) - // It's stored as URL-encoded JSON: {"token":""} - if raw, ok := cookies["codex_token"]; ok { - decoded, err := url.QueryUnescape(raw) - if err == nil { - var obj struct { - Token string `json:"token"` - } - if json.Unmarshal([]byte(decoded), &obj) == nil && obj.Token != "" { - fmt.Fprintf(os.Stderr, "\ncodex_token JWT (len=%d): %s...\n", len(obj.Token), obj.Token[:min(60, len(obj.Token))]) - // Test if we can use this token directly for a Codex query - testCodexToken(obj.Token) - } - } - } - - // Also test defined-attestation-token for JWT minting - if attToken, ok := cookies["defined-attestation-token"]; ok { - fmt.Fprintf(os.Stderr, "\nTesting defined-attestation-token for JWT mint...\n") - testJWTMint(attToken, "defined-attestation-token") - } - - // Output fresh defined-attestation-token + // Output cookies if attToken, ok := cookies["defined-attestation-token"]; ok { fmt.Printf("DEFINED_SESSION_COOKIE=%s\n", attToken) - fmt.Printf("COOKIE_NAME=defined-attestation-token\n") } - - // Also output full codex_token JSON decoded token if raw, ok := cookies["codex_token"]; ok { decoded, _ := url.QueryUnescape(raw) var obj struct { @@ -93,57 +148,3 @@ func main() { } } } - -func testJWTMint(cookieValue, cookieName string) { - reqBody := map[string]interface{}{ - "operationName": "CreateApiToken", - "query": "mutation CreateApiToken { createApiTokens(input: { count: 1 }) { token } }", - "variables": map[string]interface{}{}, - } - bodyBytes, _ := json.Marshal(reqBody) - req, _ := http.NewRequest("POST", "https://www.defined.fi/api", bytes.NewBuffer(bodyBytes)) - req.Header.Set("Accept", "application/json") - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Origin", "https://www.defined.fi") - req.Header.Set("Referer", "https://www.defined.fi/") - req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36") - req.AddCookie(&http.Cookie{Name: cookieName, Value: cookieValue}) - - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Do(req) - if err != nil { - fmt.Fprintf(os.Stderr, "mint request failed: %v\n", err) - return - } - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - fmt.Fprintf(os.Stderr, "mint status: %d, body: %s\n", resp.StatusCode, string(body[:min(200, len(body))])) -} - -func testCodexToken(token string) { - // Try using the token directly as a Bearer for Codex GraphQL - reqBody := map[string]interface{}{ - "query": "{ getNetworkStats(networkId: 1) { addressCount } }", - } - bodyBytes, _ := json.Marshal(reqBody) - req, _ := http.NewRequest("POST", "https://graph.codex.io/graphql", bytes.NewBuffer(bodyBytes)) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+token) - - client := &http.Client{Timeout: 10 * time.Second} - resp, err := client.Do(req) - if err != nil { - fmt.Fprintf(os.Stderr, "codex token test failed: %v\n", err) - return - } - defer resp.Body.Close() - body, _ := io.ReadAll(resp.Body) - fmt.Fprintf(os.Stderr, "codex test status: %d, body: %s\n", resp.StatusCode, string(body[:min(200, len(body))])) -} - -func min(a, b int) int { - if a < b { - return a - } - return b -} diff --git a/harnesses/aggregator-head-lag/cmd/script/codex_scraper.go b/harnesses/aggregator-head-lag/cmd/script/codex_scraper.go new file mode 100644 index 00000000..6ec7cac7 --- /dev/null +++ b/harnesses/aggregator-head-lag/cmd/script/codex_scraper.go @@ -0,0 +1,141 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "os" + "strings" + "sync" + "time" + + "github.com/chromedp/cdproto/network" + "github.com/chromedp/chromedp" +) + +// in-process JWE refresher: scrapes defined.fi every 5 min via headless Chrome. +// Result is stored here so GetDefinedJWTToken can use it without hitting the Paris box. +var inProcessJWE struct { + sync.RWMutex + token string + mintedAt time.Time +} + +func getInProcessJWE() (string, time.Time) { + inProcessJWE.RLock() + defer inProcessJWE.RUnlock() + return inProcessJWE.token, inProcessJWE.mintedAt +} + +func setInProcessJWE(token string) { + inProcessJWE.Lock() + defer inProcessJWE.Unlock() + inProcessJWE.token = token + inProcessJWE.mintedAt = time.Now() +} + +// scrapeCodexToken runs headless Chrome, navigates to defined.fi, and extracts the codex_token JWE. +func scrapeCodexToken() (string, error) { + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.Flag("headless", true), + chromedp.Flag("disable-gpu", true), + chromedp.Flag("no-sandbox", true), + chromedp.Flag("disable-dev-shm-usage", true), + chromedp.UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"), + ) + + // Allow overriding Chrome binary path via env (useful for Alpine/Debian containers). + if chromePath := os.Getenv("CHROME_PATH"); chromePath == "" { + // Try common locations + for _, p := range []string{"/usr/bin/chromium", "/usr/bin/chromium-browser", "/usr/bin/google-chrome"} { + if _, err := os.Stat(p); err == nil { + opts = append(opts, chromedp.ExecPath(p)) + break + } + } + } else { + opts = append(opts, chromedp.ExecPath(chromePath)) + } + + allocCtx, allocCancel := chromedp.NewExecAllocator(context.Background(), opts...) + defer allocCancel() + + ctx, cancel := chromedp.NewContext(allocCtx) + defer cancel() + + ctx, cancel = context.WithTimeout(ctx, 90*time.Second) + defer cancel() + + var cookies []*network.Cookie + err := chromedp.Run(ctx, + chromedp.Navigate("https://www.defined.fi/"), + chromedp.WaitVisible(`body`, chromedp.ByQuery), + chromedp.Sleep(20*time.Second), + chromedp.ActionFunc(func(ctx context.Context) error { + var err error + cookies, err = network.GetCookies().Do(ctx) + return err + }), + ) + if err != nil { + return "", fmt.Errorf("chromedp run failed: %w", err) + } + + for _, c := range cookies { + if c.Name == "codex_token" { + decoded, err := url.QueryUnescape(c.Value) + if err != nil { + return "", fmt.Errorf("url unescape failed: %w", err) + } + var obj struct { + Token string `json:"token"` + } + if err := json.Unmarshal([]byte(decoded), &obj); err == nil && obj.Token != "" { + return obj.Token, nil + } + // Fallback: if the cookie value is the JWE directly + if strings.HasPrefix(decoded, "eyJ") { + return decoded, nil + } + } + } + return "", fmt.Errorf("codex_token cookie not found after page load") +} + +// startInProcessScraper launches a background goroutine that refreshes the JWE every 5 min. +// Call once from main. Safe to call even if Chrome is not installed (logs error, no crash). +func startInProcessScraper(stopChan <-chan struct{}) { + go func() { + // Initial delay: let the container fully start before launching Chrome. + select { + case <-stopChan: + return + case <-time.After(10 * time.Second): + } + + refreshInterval := 5 * time.Minute + + for { + fmt.Println("[CODEX-SCRAPER] Scraping defined.fi for fresh JWE...") + tok, err := scrapeCodexToken() + if err != nil { + fmt.Printf("[CODEX-SCRAPER] Scrape failed: %v — retrying in 60s\n", err) + select { + case <-stopChan: + return + case <-time.After(60 * time.Second): + continue + } + } + setInProcessJWE(tok) + fmt.Printf("[CODEX-SCRAPER] Fresh JWE stored (len=%d), next refresh in %v\n", len(tok), refreshInterval) + + select { + case <-stopChan: + return + case <-time.After(refreshInterval): + } + } + }() +} diff --git a/harnesses/aggregator-head-lag/cmd/script/defined_auth.go b/harnesses/aggregator-head-lag/cmd/script/defined_auth.go index 62728c56..987a1f13 100644 --- a/harnesses/aggregator-head-lag/cmd/script/defined_auth.go +++ b/harnesses/aggregator-head-lag/cmd/script/defined_auth.go @@ -130,17 +130,23 @@ func tryTokenService(baseURL string) (string, error) { } // GetDefinedJWTToken returns a cached JWT token or generates a new one if expired. -// Priority: CODEX_JWT env var > direct /api/codex/token (same IP as WS) > sidecar > inline mint. +// Priority: CODEX_JWT env var > in-process scraper (fresh, <5min) > direct /api/codex/token > sidecar > inline mint. func GetDefinedJWTToken(sessionCookie string) (string, error) { if jwt := os.Getenv("CODEX_JWT"); jwt != "" { return jwt, nil } + // In-process scraper: headless Chrome running inside this container, freshest possible. + // JWE expires in ~10 min; scraper refreshes every 5 min so token is always <5 min old. + if tok, mintedAt := getInProcessJWE(); tok != "" && time.Since(mintedAt) < 8*time.Minute { + fmt.Printf("[DEFINED-AUTH] Got token from in-process scraper (age=%v, len=%d)\n", time.Since(mintedAt).Round(time.Second), len(tok)) + return tok, nil + } // Direct mint: JWE minted from this container's IP = same IP used for WS = no 4403. if tok, err := tryDirectCodexToken(sessionCookie); err == nil && tok != "" { fmt.Printf("[DEFINED-AUTH] Got token via direct /api/codex/token (len=%d)\n", len(tok)) return tok, nil } - // Sidecar fallback (Paris box chromedp, auto-refreshes every 25 min) + // Sidecar fallback (Paris box chromedp, auto-refreshes every 25 min — may be stale) if svcURL := os.Getenv("DEFINED_TOKEN_SERVICE_URL"); svcURL != "" { if tok, err := tryTokenService(svcURL); err == nil && tok != "" { fmt.Printf("[DEFINED-AUTH] Got token from sidecar (len=%d)\n", len(tok)) diff --git a/harnesses/aggregator-head-lag/cmd/script/main.go b/harnesses/aggregator-head-lag/cmd/script/main.go index 84444379..a545d832 100644 --- a/harnesses/aggregator-head-lag/cmd/script/main.go +++ b/harnesses/aggregator-head-lag/cmd/script/main.go @@ -38,6 +38,11 @@ func main() { var wg sync.WaitGroup stopChan := make(chan struct{}) + // In-process JWE scraper: headless Chrome inside this container scrapes defined.fi + // every 5 min to keep the Codex token fresh. JWE expires in ~10 min, so 5 min gives + // a comfortable margin. Runs only if Chrome is installed (no-op otherwise). + startInProcessScraper(stopChan) + wg.Add(1) go func() { defer wg.Done() diff --git a/harnesses/solana-exec/cmd/collector/main.go b/harnesses/solana-exec/cmd/collector/main.go index 1f4b2338..eb494555 100644 --- a/harnesses/solana-exec/cmd/collector/main.go +++ b/harnesses/solana-exec/cmd/collector/main.go @@ -45,20 +45,28 @@ func collect(ctx context.Context, db *store.DB, h *helius.Client, plt, feeAccoun } const sigLimit = 100 - // Fetch signatures newer than last seen. Results are newest-first. - sigs, err := h.GetSignaturesForAddress(ctx, feeAccount, sigLimit, cursor.LastSig) - if err != nil { - return fmt.Errorf("get sigs: %w", err) + // Paginate through ALL signatures newer than cursor (newest-first per page). + // Each page uses `before=oldestSigInPreviousPage` to walk backwards until + // we exhaust the window. This guarantees complete coverage regardless of volume. + var sigs []helius.SigEntry + before := "" + for { + batch, err := h.GetSignaturesForAddress(ctx, feeAccount, sigLimit, cursor.LastSig, before) + if err != nil { + return fmt.Errorf("get sigs (before=%s): %w", before, err) + } + sigs = append(sigs, batch...) + if len(batch) < sigLimit { + break // last page + } + before = batch[len(batch)-1].Signature + time.Sleep(300 * time.Millisecond) // respect Helius free-tier rate limit between pages } if len(sigs) == 0 { return nil } - // On incremental polls (cursor set), hitting the cap means we're dropping older - // txs from this window — tx counts will be understated, though fee stats remain - // a representative sample of the most-recent transactions. - // The initial bootstrap (no cursor) always hits the cap; that's expected. - if len(sigs) == sigLimit && cursor.LastSig != "" { - log.Printf("collector: %s: WARNING hit sig limit (%d) — older txs in this poll window dropped; reduce POLL_INTERVAL or increase limit", plt, sigLimit) + if len(sigs) > sigLimit { + log.Printf("collector: %s: paginated %d raw sigs (%d pages)", plt, len(sigs), (len(sigs)+sigLimit-1)/sigLimit) } // Reverse to process oldest-first so cursor is always the true watermark. diff --git a/harnesses/solana-exec/internal/helius/client.go b/harnesses/solana-exec/internal/helius/client.go index b35fd148..45bc6c17 100644 --- a/harnesses/solana-exec/internal/helius/client.go +++ b/harnesses/solana-exec/internal/helius/client.go @@ -34,11 +34,11 @@ type SigEntry struct { Err any `json:"err"` // nil = success } -// GetSignaturesForAddress fetches up to `limit` finalized signatures -// for `address`. Pass `until=""` to get the most recent; pass a sig -// to get only transactions newer than that sig (exclusive). -// Results are returned newest-first. -func (c *Client) GetSignaturesForAddress(ctx context.Context, address string, limit int, until string) ([]SigEntry, error) { +// GetSignaturesForAddress fetches up to `limit` finalized signatures for +// `address`, newest-first. `until` is the exclusive upper bound (cursor); +// `before` is the exclusive lower bound used for pagination (pass "" for +// the first page). +func (c *Client) GetSignaturesForAddress(ctx context.Context, address string, limit int, until, before string) ([]SigEntry, error) { params := map[string]any{ "limit": limit, "commitment": "finalized", @@ -46,6 +46,9 @@ func (c *Client) GetSignaturesForAddress(ctx context.Context, address string, li if until != "" { params["until"] = until } + if before != "" { + params["before"] = before + } body, _ := json.Marshal(map[string]any{ "jsonrpc": "2.0", diff --git a/src/lib/materialize/load.ts b/src/lib/materialize/load.ts index f6271f05..420739a5 100644 --- a/src/lib/materialize/load.ts +++ b/src/lib/materialize/load.ts @@ -807,6 +807,11 @@ async function tryLoadLive( // common AWAITING trigger. Logged at warn level only on the // unfiltered "All" view to avoid spamming logs on filtered views // where a missing provider is expected behavior. + // For gauge-only benches (e.g. validator yield) p90/p99 are not + // defined in the spec. Fall back to p50 so the provider isn't skipped. + if (p90 == null && !q.p90) p90 = p50; + if (p99 == null && !q.p99) p99 = p50; + if (p50 == null || p90 == null || p99 == null) { // Unresponsive cohort member: the latency series is gone from // the window (failed probes record no latency, so a fully dead