From e7f362d226017d05a990020e01e36db1ac3e2c76 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 27 Jul 2026 12:54:40 -0400 Subject: [PATCH 1/2] feat(browser): opt-in OpenTelemetry OTLP metrics export; v1.13.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an opt-in metrics path. When `metrics.enabled` and `metrics.otlpEndpoint` are set, the worker folds each stats window — the same delta counts and raw per-render timing samples `logStats` already computes — into OpenTelemetry counters, histograms, and gauges and pushes them over OTLP/HTTP. Design notes: - Off by default. The OpenTelemetry SDK is dynamically imported only when enabled, so a disabled worker loads no OTel code and pays zero cost. - Single integration point: `RenderWorker.logStats` calls `metrics.record()` once per window (reusing the existing aggregation — no new hot-path work). - Histograms record the raw ms samples (not the pre-aggregated percentiles) so fleet-wide quantiles can be re-derived correctly across workers/pods. - Export is periodic and bounded (fire-and-forget): a slow/unreachable collector never blocks, delays, throws into record(), or fails a render, and the shutdown flush is capped so it can't delay container exit. Co-Authored-By: Claude Opus 4.8 --- package-lock.json | 190 ++++++++++++++++++++- packages/browser/package.json | 6 +- packages/browser/src/Worker.ts | 57 +++++++ packages/browser/src/index.ts | 17 ++ packages/browser/src/metrics.ts | 220 +++++++++++++++++++++++++ packages/browser/src/settings.ts | 38 +++++ packages/browser/test/metrics.test.ts | 84 ++++++++++ packages/browser/test/settings.test.ts | 21 +++ 8 files changed, 631 insertions(+), 2 deletions(-) create mode 100644 packages/browser/src/metrics.ts create mode 100644 packages/browser/test/metrics.test.ts diff --git a/package-lock.json b/package-lock.json index 2f41cde..366bbdb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1863,6 +1863,190 @@ "node": ">= 8" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.221.0.tgz", + "integrity": "sha512-OlanaW1vv7ufTqQ3/fPLI4arGt5ZoM+P8abOMki6uEYnpRazepSWDwDnnw+la7kE26SHVC18//SMccrDvLKOXQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.10.0.tgz", + "integrity": "sha512-/wNZ8twnEQQA4HoHu22+vcsdru6pWPWxW+7w+FlxT6Id7PE/WIbZmVKkte+PF72e0F2dnImFeHD2syyE1Mw6MQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-http": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-http/-/exporter-metrics-otlp-http-0.221.0.tgz", + "integrity": "sha512-sRfCKbOzgy8xZQV2as0RzIZlnCmCseCKZGLfRcrpo2CBngJDr+rPtX0zkG0+oUCV5kfQPUoW3W3C96Ag3Y/Clg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/otlp-exporter-base": "0.221.0", + "@opentelemetry/otlp-transformer": "0.221.0", + "@opentelemetry/resources": "2.10.0", + "@opentelemetry/sdk-metrics": "2.10.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-proto": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-proto/-/exporter-metrics-otlp-proto-0.221.0.tgz", + "integrity": "sha512-YMF4LveY2I3yhw61rn6nmC9FE8U24IZHPeKU1Duc5+sbwjMd8FwZAwba318ImdThCg/HuVQvhm2y6bfgNPnfYg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/exporter-metrics-otlp-http": "0.221.0", + "@opentelemetry/otlp-exporter-base": "0.221.0", + "@opentelemetry/otlp-transformer": "0.221.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.221.0.tgz", + "integrity": "sha512-UFPIq80OH3Ns/oPFHRj14d4DTOxUo+MUFU8hUiCq5jTqFhdeJnfVSANHT+xp92409cA+oxzvlZCe6NM1wvCuBA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/otlp-transformer": "0.221.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.221.0.tgz", + "integrity": "sha512-lg6lkOU08Az23jVcn/0Els9HP+V8PnR4Km6p0KgpTggS0n/WuhnmY64rSh83Of9iR9nD+dpWr6adlcX8KzAwjg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.221.0", + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0", + "@opentelemetry/sdk-logs": "0.221.0", + "@opentelemetry/sdk-metrics": "2.10.0", + "@opentelemetry/sdk-trace": "2.10.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.10.0.tgz", + "integrity": "sha512-q6MMm2zhggzsHVNbabYwut+a6nbuQQe3URUoxaojM/8K1IBfwwPzvxIjNi2/lI1TFe+fMHMW9MWhrtDLEXEnkA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.221.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.221.0.tgz", + "integrity": "sha512-FaDcazjyMp7TZZZAsqbo4IkovP0UegoCu0EBkiNt+qCqvUf7FPAsfcrZ3+ZEkKgXZ/jHafop+JoGPDk3A0SmLg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.221.0", + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.10.0.tgz", + "integrity": "sha512-t6r1VSvXNtSDnPXU1FbZeetJb7yyovHmgu0wRSoftxtE0g2rSNhQZQUy69sRUCL+iioJpX8SN/S6wq6ZtvLySQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace/-/sdk-trace-2.10.0.tgz", + "integrity": "sha512-MfQGq3GRmTh5fM/y+OjaO0vj6+luCB1XO2gfXCalKCfgKw0eHL++sm75DNweC6ohlp+aFvACqeE0fYayqdRaoQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.10.0", + "@opentelemetry/resources": "2.10.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.43.0.tgz", + "integrity": "sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/@phc/format": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@phc/format/-/format-1.0.0.tgz", @@ -8967,9 +9151,13 @@ }, "packages/browser": { "name": "@harperfast/prerender-browser", - "version": "1.12.0", + "version": "1.13.0", "license": "Apache-2.0", "dependencies": { + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/exporter-metrics-otlp-proto": "^0.221.0", + "@opentelemetry/resources": "^2.10.0", + "@opentelemetry/sdk-metrics": "^2.10.0", "mqtt": "^5.10.4", "pino": "^9.9.0", "puppeteer": "^24.7.2", diff --git a/packages/browser/package.json b/packages/browser/package.json index c192512..72c23e1 100644 --- a/packages/browser/package.json +++ b/packages/browser/package.json @@ -1,6 +1,6 @@ { "name": "@harperfast/prerender-browser", - "version": "1.12.0", + "version": "1.13.0", "type": "module", "description": "Headless-browser render library for Harper Prerender: claims render jobs from the @harperfast/prerender queue, renders pages in headless Chrome (Puppeteer), and posts the HTML back. Embedded by a render service and configured entirely via startWorker() options.", "keywords": [ @@ -44,6 +44,10 @@ "prepublishOnly": "npm run build" }, "dependencies": { + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/exporter-metrics-otlp-proto": "^0.221.0", + "@opentelemetry/resources": "^2.10.0", + "@opentelemetry/sdk-metrics": "^2.10.0", "mqtt": "^5.10.4", "pino": "^9.9.0", "puppeteer": "^24.7.2", diff --git a/packages/browser/src/Worker.ts b/packages/browser/src/Worker.ts index e37d465..29db543 100644 --- a/packages/browser/src/Worker.ts +++ b/packages/browser/src/Worker.ts @@ -8,6 +8,7 @@ import { noop } from './util/noop.js'; import { getResourceCache } from './ResourceCache.js'; import { settings } from './settings.js'; import { CpuSampler } from './util/cpu.js'; +import type { MetricsRecorder } from './metrics.js'; export type Renderer = (page: Page, job: RenderJob) => Promise; @@ -29,6 +30,12 @@ type RenderWorkerConfig = { rps?: number; browserLaunchOptions?: LaunchOptions; + + /** + * Optional metrics recorder. When present, each stats window is folded into it (see + * `logStats`) and it is flushed on shutdown. Absent unless the embedder enabled OTLP metrics. + */ + metrics?: MetricsRecorder; }; export default class RenderWorker { @@ -57,6 +64,9 @@ export default class RenderWorker { rps = 10; + // Optional OTLP metrics recorder; null unless the embedder enabled metrics. + private metrics: MetricsRecorder | null = null; + inflight: Set> = new Set(); lastRenderStartTime = Date.now(); @@ -102,6 +112,7 @@ export default class RenderWorker { this.CONCURRENCY = config.maxConcurrency ?? 5; this.BROWSER_MAX_TOTAL_PAGES = config.browserExpirationThreshold ?? 5000; this.renderFn = config.renderer; + this.metrics = config.metrics ?? null; this.browserCleanupInterval = setInterval(() => { this.closeRetiredBrowsers(); @@ -284,6 +295,40 @@ export default class RenderWorker { rssMb: Math.round(mem.rss / 1024 / 1024), resourceCache: cacheStats, }); + + // Fold the same window into OTLP metrics (no-op unless the embedder enabled them). `s` + // still holds the raw per-render sample arrays, so histograms get true observations — + // not the already-summarized percentiles above. + this.metrics?.record( + { + completed: s.completed, + succeeded: s.succeeded, + emptyContent: s.emptyContent, + fromSitemap: s.fromSitemap, + failures: s.failures, + expiredSkipped: s.expiredSkipped, + concurrencyBlocked: s.concurrencyBlocked, + rpsDelayed: s.rpsDelayed, + resultPostFailures: s.resultPostFailures, + browserLaunches: s.browserLaunches, + browserRetirements: s.browserRetirements, + renderTimes: s.renderTimes, + navTtfb: s.navTtfb, + navTotal: s.navTotal, + settle: s.settle, + postProcess: s.postProcess, + }, + { + inflight: this.inflight.size, + concurrency: this.CONCURRENCY, + retiredBrowsers: this.retiredBrowsers.size, + rssBytes: mem.rss, + workerCores: cpu.workerCores, + nodeCores: cpu.nodeCores, + browserCores: cpu.browserCores, + cacheHitRate: cacheStats ? cacheStats.hitRate : null, + } + ); } /** @@ -319,6 +364,18 @@ export default class RenderWorker { } const closing: Promise[] = []; + // Flush buffered metrics before the loop dies (shutdown() is internally guarded — never + // throws). Cap the wait so a down collector's flush retries can't delay container exit; a + // dropped final window is an acceptable trade at shutdown. AbortController cancels the cap + // timer once the flush wins so it doesn't linger on the event loop (mirrors the drain above). + if (this.metrics) { + const metrics = this.metrics; + this.metrics = null; + const ac = new AbortController(); + const flush = metrics.shutdown().finally(() => ac.abort()); + const cap = setTimeout(2000, undefined, { signal: ac.signal }).catch(() => {}); + closing.push(Promise.race([flush, cap])); + } if (this.browser) { closing.push(this.browser.close().catch(noop)); } diff --git a/packages/browser/src/index.ts b/packages/browser/src/index.ts index 54ea232..746cc1e 100644 --- a/packages/browser/src/index.ts +++ b/packages/browser/src/index.ts @@ -24,6 +24,7 @@ import { applySettings, settings, defaultLaunchOptions } from './settings.js'; import type { BrowserOptions } from './settings.js'; import { initResourceCache } from './ResourceCache.js'; import { ErrorHandler } from './errorHandler.js'; +import type { MetricsRecorder } from './metrics.js'; export type StartWorkerOptions = BrowserOptions & { /** Renderer to use instead of the built-in default. */ @@ -54,9 +55,24 @@ export async function startWorker(options: StartWorkerOptions): Promise; +}; + +// Render durations run seconds-to-tens-of-seconds (settle dominates); phases skew smaller. +const RENDER_DURATION_BUCKETS_MS = [ + 250, 500, 1000, 2000, 4000, 6000, 8000, 10000, 12000, 15000, 20000, 30000, 45000, 60000, +]; +const PHASE_DURATION_BUCKETS_MS = [50, 100, 250, 500, 1000, 2000, 4000, 6000, 8000, 10000, 15000, 20000, 30000]; + +/** + * Build a metrics recorder that exports over OTLP/HTTP. Resolves once the meter provider, + * exporter, and instruments are wired. Caller is responsible for `shutdown()` on drain. + */ +export async function createMetrics(options: ResolvedMetrics, context: { workerId: string }): Promise { + // Lazy imports: nothing here loads unless a worker actually enabled metrics. + const { MeterProvider, PeriodicExportingMetricReader, AggregationType } = await import('@opentelemetry/sdk-metrics'); + const { resourceFromAttributes } = await import('@opentelemetry/resources'); + const { OTLPMetricExporter } = await import('@opentelemetry/exporter-metrics-otlp-proto'); + + // Bound a single export (incl. the SDK's internal retries) so a down/slow collector can't + // keep an export — or the final shutdown flush — in flight for the exporter's 10s default. + const exportTimeoutMs = Math.min(options.exportIntervalMs, 8000); + const reader = new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter({ + url: options.otlpEndpoint, + headers: options.headers, + timeoutMillis: exportTimeoutMs, + }), + exportIntervalMillis: options.exportIntervalMs, + exportTimeoutMillis: exportTimeoutMs, + }); + + const provider = new MeterProvider({ + resource: resourceFromAttributes({ + 'service.name': 'prerender-browser', + 'service.instance.id': context.workerId, + 'worker.id': context.workerId, + }), + readers: [reader], + views: [ + { + instrumentName: 'prerender.render.duration', + aggregation: { + type: AggregationType.EXPLICIT_BUCKET_HISTOGRAM, + options: { boundaries: RENDER_DURATION_BUCKETS_MS }, + }, + }, + { + instrumentName: 'prerender.render.phase.duration', + aggregation: { + type: AggregationType.EXPLICIT_BUCKET_HISTOGRAM, + options: { boundaries: PHASE_DURATION_BUCKETS_MS }, + }, + }, + ], + }); + + const meter = provider.getMeter('@harperfast/prerender-browser'); + const counter = (name: string, description: string): Counter => meter.createCounter(name, { description }); + + const completed = counter('prerender.renders.completed', 'Renders completed (any outcome).'); + const succeeded = counter('prerender.renders.succeeded', 'Renders that completed without error.'); + const emptyContent = counter('prerender.renders.empty_content', 'Successful renders that produced empty content.'); + const fromSitemap = counter('prerender.renders.from_sitemap', 'Renders whose job originated from a sitemap.'); + const failures = counter('prerender.renders.failures', 'Failed renders, labelled by failure `type`.'); + const resultPostFailures = counter( + 'prerender.renders.result_post_failures', + 'Renders whose result POST back to Harper failed.' + ); + const expiredSkipped = counter( + 'prerender.jobs.expired_skipped', + 'Claimed jobs skipped because their lease had nearly expired.' + ); + const concurrencyBlocked = counter( + 'prerender.jobs.concurrency_blocked', + 'Times a job start waited on a free concurrency slot.' + ); + const rpsDelayed = counter('prerender.jobs.rps_delayed', 'Times a job start was delayed by the rps limiter.'); + const browserLaunches = counter('prerender.browser.launches', 'Chrome browser launches.'); + const browserRetirements = counter('prerender.browser.retirements', 'Chrome browsers retired and replaced.'); + + const renderDuration = meter.createHistogram('prerender.render.duration', { + unit: 'ms', + description: 'Wall-clock per render.', + }); + const phaseDuration = meter.createHistogram('prerender.render.phase.duration', { + unit: 'ms', + description: 'Per-phase render wall-clock, labelled by `phase`.', + }); + + // Observable gauges report the most recent window's values, stored on each `record()`. + let lastGauges: MetricsGauges | null = null; + const gauge = (name: string, description: string, pick: (g: MetricsGauges) => number | null): void => { + meter.createObservableGauge(name, { description }).addCallback((result) => { + const value = lastGauges ? pick(lastGauges) : null; + if (value !== null && value !== undefined) result.observe(value); + }); + }; + gauge('prerender.renders.inflight', 'Renders in flight at collection time.', (g) => g.inflight); + gauge('prerender.concurrency', 'Configured max concurrent renders.', (g) => g.concurrency); + gauge('prerender.browser.retired_open', 'Retired browsers not yet closed.', (g) => g.retiredBrowsers); + gauge('prerender.process.rss_bytes', 'Worker process resident set size (bytes).', (g) => g.rssBytes); + gauge('prerender.cpu.worker_cores', 'CPU cores used by this worker (Node + its Chrome tree).', (g) => g.workerCores); + gauge('prerender.cpu.node_cores', 'CPU cores used by the Node process.', (g) => g.nodeCores); + gauge('prerender.cpu.browser_cores', 'CPU cores used by the Chrome tree.', (g) => g.browserCores); + gauge('prerender.resource_cache.hit_rate', 'Resource cache hit rate over the window (0–1).', (g) => g.cacheHitRate); + + logger.info( + { event: 'prerender-metrics-started', endpoint: options.otlpEndpoint, exportIntervalMs: options.exportIntervalMs }, + 'OpenTelemetry metrics export enabled' + ); + + const recordPhase = (samples: number[], phase: string): void => { + for (const ms of samples) phaseDuration.record(ms, { phase }); + }; + + return { + record(snapshot, gauges) { + try { + completed.add(snapshot.completed); + succeeded.add(snapshot.succeeded); + emptyContent.add(snapshot.emptyContent); + fromSitemap.add(snapshot.fromSitemap); + // Only emit a failure series for types that actually occurred, to avoid five + // permanently-zero label sets. + for (const [type, n] of Object.entries(snapshot.failures)) { + if (n) failures.add(n, { type }); + } + resultPostFailures.add(snapshot.resultPostFailures); + expiredSkipped.add(snapshot.expiredSkipped); + concurrencyBlocked.add(snapshot.concurrencyBlocked); + rpsDelayed.add(snapshot.rpsDelayed); + browserLaunches.add(snapshot.browserLaunches); + browserRetirements.add(snapshot.browserRetirements); + for (const ms of snapshot.renderTimes) renderDuration.record(ms); + recordPhase(snapshot.navTtfb, 'navTtfb'); + recordPhase(snapshot.navTotal, 'navTotal'); + recordPhase(snapshot.settle, 'settle'); + recordPhase(snapshot.postProcess, 'postProcess'); + lastGauges = gauges; + } catch (err) { + // Telemetry bookkeeping must never disrupt the worker. + logger.debug({ err }, 'metrics record failed'); + } + }, + async shutdown() { + try { + await provider.shutdown(); + } catch (err) { + logger.debug({ err }, 'metrics shutdown failed'); + } + }, + }; +} diff --git a/packages/browser/src/settings.ts b/packages/browser/src/settings.ts index 4a6b8be..0d31ccc 100644 --- a/packages/browser/src/settings.ts +++ b/packages/browser/src/settings.ts @@ -87,6 +87,24 @@ export type BrowserOptions = { browserLaunchOptions?: LaunchOptions; /** On-disk shared sub-resource (script/stylesheet) cache. */ resourceCache?: ResourceCacheOptions; + /** + * Opt-in OpenTelemetry metrics export over OTLP/HTTP. Disabled unless `enabled` is true + * AND `otlpEndpoint` is set — when off, no OpenTelemetry code is loaded (it is imported + * lazily) and there is zero runtime overhead. Export is periodic and fire-and-forget: a + * slow or unreachable collector can never block, delay, or fail a render. See `src/metrics`. + */ + metrics?: MetricsOptions; +}; + +export type MetricsOptions = { + /** Turn metrics export on. Still a no-op unless `otlpEndpoint` is also set. Default false. */ + enabled?: boolean; + /** OTLP/HTTP metrics endpoint, e.g. `http://alloy.monitoring:4318/v1/metrics`. */ + otlpEndpoint?: string; + /** Export/collect interval in ms — defaults to 60000 to match the stats-log window. */ + exportIntervalMs?: number; + /** Extra headers on the OTLP POST (e.g. auth), if the collector needs them. */ + headers?: Record; }; export type ResolvedBackoff = { @@ -123,6 +141,14 @@ export type Settings = { browserLaunchOptions?: LaunchOptions; resourceCache: ResolvedResourceCache; backoff: ResolvedBackoff; + metrics: ResolvedMetrics; +}; + +export type ResolvedMetrics = { + enabled: boolean; + otlpEndpoint: string; + exportIntervalMs: number; + headers: Record; }; const DEFAULT_CHROME_ARGS = [ @@ -200,6 +226,12 @@ const defaults = (): Settings => ({ maxIdleMs: 60000, resultRetries: 3, }, + metrics: { + enabled: false, + otlpEndpoint: '', + exportIntervalMs: 60000, + headers: {}, + }, }); // The live settings (stable reference; mutated in place by applySettings so existing @@ -282,6 +314,12 @@ export const resolveSettings = ( maxIdleMs: options.backoff?.maxIdleMs ?? fresh.backoff.maxIdleMs, resultRetries: options.backoff?.resultRetries ?? fresh.backoff.resultRetries, }; + fresh.metrics = { + enabled: options.metrics?.enabled ?? fresh.metrics.enabled, + otlpEndpoint: options.metrics?.otlpEndpoint ?? fresh.metrics.otlpEndpoint, + exportIntervalMs: options.metrics?.exportIntervalMs ?? fresh.metrics.exportIntervalMs, + headers: options.metrics?.headers ?? fresh.metrics.headers, + }; Object.assign(settings, fresh); return settings; diff --git a/packages/browser/test/metrics.test.ts b/packages/browser/test/metrics.test.ts new file mode 100644 index 0000000..3031f93 --- /dev/null +++ b/packages/browser/test/metrics.test.ts @@ -0,0 +1,84 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { createMetrics } from '../dist/metrics.js'; + +// A refused endpoint (port 1) makes any export fail fast (ECONNREFUSED) so the "export is +// swallowed" path is exercised without a real collector or a slow socket timeout. A short +// interval keeps exportTimeoutMillis small so shutdown()'s final flush can't stall the test. +const REFUSED_ENDPOINT = 'http://127.0.0.1:1/v1/metrics'; +// Short interval → small export timeout, so the shutdown flush against the dead endpoint +// returns fast instead of retrying for the exporter's multi-second default. +const OPTS = { enabled: true, otlpEndpoint: REFUSED_ENDPOINT, exportIntervalMs: 250, headers: {} }; + +const SNAPSHOT = { + completed: 5, + succeeded: 4, + emptyContent: 1, + fromSitemap: 2, + failures: { timeout: 1, protocol: 0, tooManyRedirects: 0, getPageFailed: 0, other: 0 }, + expiredSkipped: 0, + concurrencyBlocked: 3, + rpsDelayed: 0, + resultPostFailures: 1, + browserLaunches: 1, + browserRetirements: 0, + renderTimes: [1200, 9800, 15000], + navTtfb: [600], + navTotal: [1800], + settle: [9000, 9500], + postProcess: [200], +}; + +const GAUGES = { + inflight: 3, + concurrency: 8, + retiredBrowsers: 0, + rssBytes: 123456, + workerCores: 4.2, + nodeCores: null, + browserCores: null, + cacheHitRate: 0.87, +}; + +test('record() and shutdown() never throw, even with the collector unreachable', async () => { + const recorder = await createMetrics(OPTS, { workerId: 'test-worker-1' }); + assert.doesNotThrow(() => recorder.record(SNAPSHOT, GAUGES)); + // A second window must fold in cleanly (cumulative counters, more histogram samples). + assert.doesNotThrow(() => recorder.record(SNAPSHOT, GAUGES)); + // shutdown() flushes once to the refused endpoint; it must resolve, not reject. + await recorder.shutdown(); +}); + +test('record() tolerates an all-zero window and null CPU/cache gauges', async () => { + const recorder = await createMetrics(OPTS, { workerId: 'test-worker-2' }); + const zeroSnapshot = { + completed: 0, + succeeded: 0, + emptyContent: 0, + fromSitemap: 0, + failures: { timeout: 0, protocol: 0, tooManyRedirects: 0, getPageFailed: 0, other: 0 }, + expiredSkipped: 0, + concurrencyBlocked: 0, + rpsDelayed: 0, + resultPostFailures: 0, + browserLaunches: 0, + browserRetirements: 0, + renderTimes: [], + navTtfb: [], + navTotal: [], + settle: [], + postProcess: [], + }; + const nullGauges = { + inflight: 0, + concurrency: 8, + retiredBrowsers: 0, + rssBytes: 0, + workerCores: null, + nodeCores: null, + browserCores: null, + cacheHitRate: null, + }; + assert.doesNotThrow(() => recorder.record(zeroSnapshot, nullGauges)); + await recorder.shutdown(); +}); diff --git a/packages/browser/test/settings.test.ts b/packages/browser/test/settings.test.ts index 5313b0a..c19f507 100644 --- a/packages/browser/test/settings.test.ts +++ b/packages/browser/test/settings.test.ts @@ -121,3 +121,24 @@ test('no host-resolver flag is added when hostResolverRules is omitted', () => { assert.deepEqual(settings.hostResolverRules, {}); assert.ok(!settings.chromeArgs.some((a: string) => a.startsWith('--host-resolver-rules'))); }); + +test('metrics defaults to disabled with a 60s export interval', () => { + applySettings({ harper: HARPER }); + assert.deepEqual(settings.metrics, { + enabled: false, + otlpEndpoint: '', + exportIntervalMs: 60000, + headers: {}, + }); +}); + +test('metrics options override defaults (partial merge)', () => { + applySettings({ + harper: HARPER, + metrics: { enabled: true, otlpEndpoint: 'http://alloy.monitoring:4318/v1/metrics' }, + }); + assert.equal(settings.metrics.enabled, true); + assert.equal(settings.metrics.otlpEndpoint, 'http://alloy.monitoring:4318/v1/metrics'); + assert.equal(settings.metrics.exportIntervalMs, 60000); // untouched default preserved + assert.deepEqual(settings.metrics.headers, {}); +}); From 1ccc4e07f42ad70c18296f0e2f7cd75d22a49445 Mon Sep 17 00:00:00 2001 From: Joe Date: Mon, 27 Jul 2026 13:24:17 -0400 Subject: [PATCH 2/2] fix(browser): flush the final stats window on shutdown before tearing down metrics destroy() shut down the metrics provider without folding in the stats accumulated since the last 60s logStats() tick, so a graceful shutdown/deploy dropped up to a full window of counts/timings from the export. Call logStats() once at the start of destroy() (guarded, before this.metrics is cleared) so the final window is logged and recorded into the OTel instruments ahead of the flush. Addresses gemini-code-assist review on PR #30. Co-Authored-By: Claude Opus 4.8 --- packages/browser/src/Worker.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/browser/src/Worker.ts b/packages/browser/src/Worker.ts index 29db543..7aecfd4 100644 --- a/packages/browser/src/Worker.ts +++ b/packages/browser/src/Worker.ts @@ -363,11 +363,23 @@ export default class RenderWorker { this.browserCleanupInterval = null; } + // Emit one final stats window before tearing down. The interval timer is now cleared, so the + // up-to-one-window of counts/timings accumulated since the last tick would otherwise be lost + // from both the log line and (more importantly) the metrics flush below — logStats() folds it + // into the recorder via record(), while this.metrics is still set. Guarded so a failure here + // (e.g. mid-uncaughtException) can't block the browser cleanup that follows. + try { + this.logStats(); + } catch (err) { + logger.warn({ err }, 'failed to log final stats during destroy'); + } + const closing: Promise[] = []; - // Flush buffered metrics before the loop dies (shutdown() is internally guarded — never - // throws). Cap the wait so a down collector's flush retries can't delay container exit; a - // dropped final window is an acceptable trade at shutdown. AbortController cancels the cap - // timer once the flush wins so it doesn't linger on the event loop (mirrors the drain above). + // Flush the buffered metrics (incl. the final window just recorded above) before the loop + // dies (shutdown() is internally guarded — never throws). Cap the wait so a down collector's + // flush retries can't delay container exit — losing that last export to a dead collector is an + // acceptable trade. AbortController cancels the cap timer once the flush wins so it doesn't + // linger on the event loop (mirrors the drain above). if (this.metrics) { const metrics = this.metrics; this.metrics = null;