From ecf95078acff6ea57ea7209a0c9bce95fd55d1ef Mon Sep 17 00:00:00 2001 From: Tyler Beebe Date: Fri, 13 Nov 2026 17:37:04 +0000 Subject: [PATCH 1/3] feat(telemetry): send upload metrics as OTLP to the CLI telemetry route (TRUNK-23773) The upload metrics went to trunk1's telemetry service (telemetry./v1/flakytests-cli/upload-metrics) as protobuf with the raw key on x-api-token, which only a legacy key could authenticate. They now go to sawmill-api's CLI telemetry route, /v2/cloud/telemetry/v1/logs on the API host, as one OTLP JSON log record, with the run's API key on Authorization: the gateway exchanges a v2 or a legacy key for a JWT, and the route stamps the org from it. The record carries what UploadMetrics did, under the names the route keeps for trunk-analytics-cli (trunk-io/trunk-cloud#5923): whether the upload failed and its failure category, the CLI and upload durations, the quarantine query result and resolution mode, and the repository's host, owner and name. The send keeps its 1 s timeout and stays best-effort; a run with no API key (a fork pull request, whose GitHub Actions login the gateway refuses on this path) sends nothing. The mock server serves the new path and reads the OTLP record back into UploadMetrics, so the existing telemetry tests check what is sent. Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 2 +- api/src/client/mod.rs | 64 +++++------ api/src/lib.rs | 1 + api/src/otlp.rs | 206 ++++++++++++++++++++++++++++++++++ test_utils/src/mock_server.rs | 60 ++++++++-- 5 files changed, 288 insertions(+), 45 deletions(-) create mode 100644 api/src/otlp.rs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e437f6ab..ccee6c9c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -112,7 +112,7 @@ The `tracing` library is used for: `--org-url-slug` (env `TRUNK_ORG_URL_SLUG`) is declared as a non-optional `String` on `UploadArgs`, which makes it a hard requirement for both `upload` and `test`. Keep it that way: the slug is what attributes a run to an organization across every kind of telemetry we collect. - **Sentry**: `setup_logger` in `cli/src/main.rs` attaches it as an `org_url_slug` tag on every `tracing::error!()` event forwarded to Sentry, alongside `command_name` and `repo_root`. Those tags are how Trunk narrows Sentry to a single customer's CI runs when someone reports a failing upload — without the slug, an error report can't be tied back to an organization and we have to ask the user to reproduce. -- **Our own upload telemetry**: the slug is stored on the bundle as `base_props.org` and scopes the `UploadMetrics` we report to the telemetry endpoint at the end of a run (`cli/src/upload_command.rs`, `ApiClient::telemetry_upload_metrics`). Timing, quarantine outcome, and failure-reason metrics are only useful if we know which org they came from. +- **Our own upload telemetry**: at the end of a run the CLI sends one OTLP log record (`api/src/otlp.rs`, `ApiClient::telemetry_upload_metrics`) to sawmill-api's CLI telemetry route, `/v2/cloud/telemetry/v1/logs`, with the run's API key; a run with no key (a fork pull request) sends nothing. The route stamps the org from the key rather than from the slug, and keeps only the attribute names it lists for `trunk-analytics-cli` (trunk-cloud `apps/sawmill-api-rs/src/cli_telemetry.rs`), so a new attribute is dropped until it is added there too. The slug also travels with the upload requests, builds the org-scoped links we print for uploads and tests (`api/src/urls.rs`), and backs the settings-page hint shown on unauthorized errors (`cli/src/error_report.rs`). diff --git a/api/src/client/mod.rs b/api/src/client/mod.rs index 238a3220..bfa643ec 100644 --- a/api/src/client/mod.rs +++ b/api/src/client/mod.rs @@ -50,7 +50,6 @@ impl std::error::Error for ApiErrorEndpoint {} pub struct ApiClient { pub api_host: String, - pub telemetry_host: String, s3_client: Client, trunk_api_client: Client, /// Holds the run's v2 credential: an API token from the start, or a fork pull request's Trunk @@ -60,7 +59,9 @@ pub struct ApiClient { credential_free: bool, /// A credential-free GitHub Actions fork PR run, which on v2 swaps its `GITHUB_TOKEN` for a login. github_fork_pull_request: Option, - telemetry_client: Client, + /// The run's API token, for the CLI telemetry route; `None` without one (a fork pull request + /// has no token, and the gateway refuses its GitHub Actions login there). + telemetry_client: Option, version_path_prefix: String, org_url_slug: String, render_sender: Option>, @@ -88,6 +89,10 @@ impl V2Credential { const TRUNK_TOKEN_HEADER: HeaderName = HeaderName::from_static("x-trunk-token"); fn client(&self) -> anyhow::Result { + self.client_with_timeout(ApiClient::TRUNK_API_TIMEOUT) + } + + fn client_with_timeout(&self, timeout: std::time::Duration) -> anyhow::Result { let (name, value, error) = match self { Self::ApiKey(token) => ( header::AUTHORIZATION, @@ -109,7 +114,7 @@ impl V2Credential { ); headers.append(name, value); Ok(Client::builder() - .timeout(ApiClient::TRUNK_API_TIMEOUT) + .timeout(timeout) .default_headers(headers) .build()?) } @@ -160,18 +165,6 @@ impl ApiClient { let api_host = get_api_host(); tracing::debug!("Using public api address {}", api_host); - let telemetry_host = if api_host.contains("https://") { - format!( - "https://telemetry.{}", - api_host.split("https://").nth(1).unwrap_or_default() - ) - } else { - // If the api_host is not https, we default to the api host - // this happens when the api_host is localhost and we are running - // inside of test environments - api_host.clone() - }; - let mut trunk_api_client_default_headers = HeaderMap::new(); trunk_api_client_default_headers.append( header::CONTENT_TYPE, @@ -199,21 +192,12 @@ impl ApiClient { .transpose()?; let credential_free = allow_forked_pr_uploads && api_token.is_none(); - let mut telemetry_client_default_headers = HeaderMap::new(); - telemetry_client_default_headers.append( - header::CONTENT_TYPE, - HeaderValue::from_static("application/x-protobuf"), - ); - - if let Some(api_token_header_value) = api_token_header_value { - telemetry_client_default_headers - .append(Self::TRUNK_API_TOKEN_HEADER, api_token_header_value); - } - - let telemetry_client = Client::builder() - .timeout(Self::TRUNK_TELEMETRY_TIMEOUT) - .default_headers(telemetry_client_default_headers) - .build()?; + let telemetry_client = api_token + .clone() + .map(|token| { + V2Credential::ApiKey(token).client_with_timeout(Self::TRUNK_TELEMETRY_TIMEOUT) + }) + .transpose()?; let mut s3_client_default_headers = HeaderMap::new(); s3_client_default_headers.append( @@ -231,7 +215,6 @@ impl ApiClient { }; Ok(ApiClient { - telemetry_host, api_host, s3_client, trunk_api_client, @@ -443,18 +426,25 @@ impl ApiClient { &self, request: &message::TelemetryUploadMetricsRequest, ) -> anyhow::Result<()> { + let Some(telemetry_client) = &self.telemetry_client else { + return Ok(()); + }; + let environment = if self.api_host == DEFAULT_ORIGIN { + "production" + } else { + "staging" + }; CallApi { action: || async { if std::env::var("DISABLE_TELEMETRY").is_ok() { return Ok(()); } - let response = self - .telemetry_client - .post(format!( - "{}{}/flakytests-cli/upload-metrics", - self.telemetry_host, self.version_path_prefix + let response = telemetry_client + .post(format!("{}{}", self.api_host, crate::otlp::CLI_TELEMETRY_PATH)) + .json(&crate::otlp::upload_metrics_logs( + &request.upload_metrics, + environment, )) - .body(prost::Message::encode_to_vec(&request.upload_metrics)) .send() .await?; diff --git a/api/src/lib.rs b/api/src/lib.rs index 314be8b3..5f7a72f5 100644 --- a/api/src/lib.rs +++ b/api/src/lib.rs @@ -1,5 +1,6 @@ mod call_api; pub mod client; pub mod message; +pub mod otlp; pub mod quarantine; pub mod urls; diff --git a/api/src/otlp.rs b/api/src/otlp.rs new file mode 100644 index 00000000..5c076211 --- /dev/null +++ b/api/src/otlp.rs @@ -0,0 +1,206 @@ +//! The upload's own telemetry as one OTLP JSON log record, for sawmill-api's CLI telemetry route +//! (`/v2/cloud/telemetry/v1/logs`, TRUNK-23773). The route keeps only the attribute names listed +//! for `trunk-analytics-cli` in trunk-cloud's `apps/sawmill-api-rs/src/cli_telemetry.rs` and stamps +//! the org itself, so a name added here is dropped until it is added there too. + +use proto::upload_metrics::trunk::{ + QuarantineQueryResult, QuarantineResolutionMode, UploadMetrics, +}; +use serde_json::{Value, json}; + +pub const CLI_TELEMETRY_PATH: &str = "/v2/cloud/telemetry/v1/logs"; +pub const SERVICE_NAME: &str = "trunk-analytics-cli"; +pub const COMMAND: &str = "upload"; + +pub const COMMAND_ATTRIBUTE: &str = "trunk.cli.command"; +pub const DURATION_ATTRIBUTE: &str = "trunk.cli.duration_ms"; +pub const FAILED_ATTRIBUTE: &str = "trunk.analytics_cli.upload.failed"; +pub const FAILURE_REASON_ATTRIBUTE: &str = "trunk.analytics_cli.upload.failure_reason"; +pub const UPLOAD_DURATION_ATTRIBUTE: &str = "trunk.analytics_cli.upload.duration_ms"; +pub const QUARANTINE_QUERY_RESULT_ATTRIBUTE: &str = "trunk.analytics_cli.quarantine.query_result"; +pub const QUARANTINE_RESOLUTION_MODE_ATTRIBUTE: &str = + "trunk.analytics_cli.quarantine.resolution_mode"; +pub const REPOSITORY_HOST_ATTRIBUTE: &str = "trunk.vcs.repository.host"; +pub const REPOSITORY_OWNER_ATTRIBUTE: &str = "trunk.vcs.repository.owner"; +pub const REPOSITORY_NAME_ATTRIBUTE: &str = "trunk.vcs.repository.name"; + +/// OTLP's `SEVERITY_NUMBER_INFO` and `SEVERITY_NUMBER_ERROR`. +const SEVERITY_INFO: i32 = 9; +const SEVERITY_ERROR: i32 = 17; + +const QUARANTINE_QUERY_RESULT_PREFIX: &str = "QUARANTINE_QUERY_RESULT_"; +const QUARANTINE_RESOLUTION_MODE_PREFIX: &str = "QUARANTINE_RESOLUTION_MODE_"; + +/// OTLP JSON writes 64-bit integers as decimal strings. +pub fn upload_metrics_logs(metrics: &UploadMetrics, environment: &str) -> Value { + let version = metrics + .client_version + .as_ref() + .map(|version| { + let base = format!("{}.{}.{}", version.major, version.minor, version.patch); + if version.suffix.is_empty() { + base + } else { + format!("{base}-{}", version.suffix) + } + }) + .unwrap_or_default(); + let resource_attributes = [ + string_attribute("service.name", SERVICE_NAME), + string_attribute("service.version", &version), + string_attribute("deployment.environment.name", environment), + string_attribute("os.type", os_type(std::env::consts::OS)), + string_attribute("host.arch", host_arch(std::env::consts::ARCH)), + ]; + + let mut attributes = vec![ + string_attribute(COMMAND_ATTRIBUTE, COMMAND), + bool_attribute(FAILED_ATTRIBUTE, metrics.failed), + string_attribute( + QUARANTINE_QUERY_RESULT_ATTRIBUTE, + &enum_name( + QuarantineQueryResult::try_from(metrics.quarantine_query_result) + .unwrap_or_default() + .as_str_name(), + QUARANTINE_QUERY_RESULT_PREFIX, + ), + ), + string_attribute( + QUARANTINE_RESOLUTION_MODE_ATTRIBUTE, + &enum_name( + QuarantineResolutionMode::try_from(metrics.quarantine_resolution_mode) + .unwrap_or_default() + .as_str_name(), + QUARANTINE_RESOLUTION_MODE_PREFIX, + ), + ), + ]; + if !metrics.failure_reason.is_empty() { + attributes.push(string_attribute( + FAILURE_REASON_ATTRIBUTE, + &metrics.failure_reason, + )); + } + let finished = metrics + .upload_finished_at + .as_ref() + .map(|time| unix_nanos(time.seconds, time.nanos)); + let cli_started = metrics + .cli_started_at + .as_ref() + .map(|time| unix_nanos(time.seconds, time.nanos)); + let upload_started = metrics + .upload_started_at + .as_ref() + .map(|time| unix_nanos(time.seconds, time.nanos)); + for (key, started) in [ + (DURATION_ATTRIBUTE, cli_started), + (UPLOAD_DURATION_ATTRIBUTE, upload_started), + ] { + if let (Some(started), Some(finished)) = (started, finished) { + attributes.push(int_attribute(key, millis_between(started, finished))); + } + } + if let Some(repo) = &metrics.repo { + for (key, value) in [ + (REPOSITORY_HOST_ATTRIBUTE, &repo.host), + (REPOSITORY_OWNER_ATTRIBUTE, &repo.owner), + (REPOSITORY_NAME_ATTRIBUTE, &repo.name), + ] { + if !value.is_empty() { + attributes.push(string_attribute(key, value)); + } + } + } + + let (severity_number, severity_text) = if metrics.failed { + (SEVERITY_ERROR, "ERROR") + } else { + (SEVERITY_INFO, "INFO") + }; + + json!({ + "resourceLogs": [{ + "resource": { "attributes": resource_attributes }, + "scopeLogs": [{ + "scope": { "name": SERVICE_NAME, "version": version }, + "logRecords": [{ + "timeUnixNano": finished.unwrap_or_default().to_string(), + "severityNumber": severity_number, + "severityText": severity_text, + "body": { "stringValue": format!("{SERVICE_NAME} {COMMAND}") }, + "attributes": attributes, + }], + }], + }], + }) +} + +fn enum_name(name: &str, prefix: &str) -> String { + name.strip_prefix(prefix).unwrap_or(name).to_lowercase() +} + +fn unix_nanos(seconds: i64, nanos: i32) -> i128 { + i128::from(seconds) * 1_000_000_000 + i128::from(nanos) +} + +fn millis_between(started: i128, finished: i128) -> i64 { + i64::try_from((finished - started).max(0) / 1_000_000).unwrap_or(i64::MAX) +} + +fn string_attribute(key: &str, value: &str) -> Value { + json!({ "key": key, "value": { "stringValue": value } }) +} + +fn bool_attribute(key: &str, value: bool) -> Value { + json!({ "key": key, "value": { "boolValue": value } }) +} + +fn int_attribute(key: &str, value: i64) -> Value { + json!({ "key": key, "value": { "intValue": value.to_string() } }) +} + +fn os_type(os: &'static str) -> &'static str { + match os { + "macos" => "darwin", + other => other, + } +} + +/// OpenTelemetry semantic-convention names, as the trunk CLI sends them. +fn host_arch(arch: &'static str) -> &'static str { + match arch { + "x86_64" => "amd64", + "aarch64" => "arm64", + other => other, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn enum_names_lose_their_prefix() { + assert_eq!( + enum_name( + "QUARANTINE_QUERY_RESULT_SUCCESS", + QUARANTINE_QUERY_RESULT_PREFIX + ), + "success" + ); + assert_eq!( + enum_name( + "QUARANTINE_RESOLUTION_MODE_TEST_COLLECTION", + QUARANTINE_RESOLUTION_MODE_PREFIX + ), + "test_collection" + ); + } + + #[test] + fn a_clock_that_ran_backwards_reads_as_zero() { + assert_eq!(millis_between(5_000_000, 1_000_000), 0); + assert_eq!(millis_between(0, 2_500_000), 2); + } +} diff --git a/test_utils/src/mock_server.rs b/test_utils/src/mock_server.rs index 680c94fe..475a4a1b 100644 --- a/test_utils/src/mock_server.rs +++ b/test_utils/src/mock_server.rs @@ -19,7 +19,6 @@ use axum::{ response::{IntoResponse, Response}, routing::{MethodRouter, any, post, put}, }; -use prost::Message; use proto::upload_metrics::trunk::UploadMetrics; use tempfile::tempdir; use tokio::{net::TcpListener, spawn}; @@ -166,10 +165,7 @@ impl MockServerBuilder { self.get_quarantining_config_handler, ) .route("/s3upload", self.s3_upload_handler) - .route( - "/v1/flakytests-cli/upload-metrics", - self.telemetry_upload_metrics, - ); + .route(api::otlp::CLI_TELEMETRY_PATH, self.telemetry_upload_metrics); for (path, method_router) in self.routes { app = app.route(path, method_router); @@ -266,8 +262,10 @@ pub async fn telemetry_upload_metrics_handler( State(state): State, bytes: Bytes, ) -> Response { - let upload_metrics = proto::upload_metrics::trunk::UploadMetrics::decode(bytes); - if let Ok(upload_metrics) = upload_metrics { + let upload_metrics = serde_json::from_slice::(&bytes) + .ok() + .and_then(|logs| upload_metrics_from_otlp(&logs)); + if let Some(upload_metrics) = upload_metrics { state .requests .lock() @@ -304,3 +302,51 @@ pub async fn s3_upload_handler( .push(RequestPayload::S3Upload(tar_extract_directory.into_path())); Response::new(String::from("OK")) } + +/// The fields of `UploadMetrics` that `api::otlp::upload_metrics_logs` writes, read back from the +/// OTLP JSON it sent, so the tests check what reaches the wire. `None` unless the body is one +/// `trunk-analytics-cli` record. +pub fn upload_metrics_from_otlp(logs: &serde_json::Value) -> Option { + use api::otlp; + use proto::upload_metrics::trunk::{QuarantineQueryResult, QuarantineResolutionMode, Repo}; + + let resource_logs = &logs["resourceLogs"][0]; + let attribute = |list: &serde_json::Value, key: &str| -> Option { + list.as_array()? + .iter() + .find(|attribute| attribute["key"] == key) + .map(|attribute| attribute["value"].clone()) + }; + if attribute(&resource_logs["resource"]["attributes"], "service.name")?["stringValue"] + != otlp::SERVICE_NAME + { + return None; + } + let record = &resource_logs["scopeLogs"][0]["logRecords"][0]["attributes"]; + let text = |key: &str| { + attribute(record, key) + .and_then(|value| value["stringValue"].as_str().map(str::to_owned)) + .unwrap_or_default() + }; + let upper = |key: &str| text(key).to_uppercase(); + Some(UploadMetrics { + failed: attribute(record, otlp::FAILED_ATTRIBUTE)?["boolValue"].as_bool()?, + failure_reason: text(otlp::FAILURE_REASON_ATTRIBUTE), + quarantine_query_result: QuarantineQueryResult::from_str_name(&format!( + "QUARANTINE_QUERY_RESULT_{}", + upper(otlp::QUARANTINE_QUERY_RESULT_ATTRIBUTE) + ))? + .into(), + quarantine_resolution_mode: QuarantineResolutionMode::from_str_name(&format!( + "QUARANTINE_RESOLUTION_MODE_{}", + upper(otlp::QUARANTINE_RESOLUTION_MODE_ATTRIBUTE) + ))? + .into(), + repo: Some(Repo { + host: text(otlp::REPOSITORY_HOST_ATTRIBUTE), + owner: text(otlp::REPOSITORY_OWNER_ATTRIBUTE), + name: text(otlp::REPOSITORY_NAME_ATTRIBUTE), + }), + ..UploadMetrics::default() + }) +} From 75f2fd8367c1ce6ae7e8d9be2d0902d1f760b53b Mon Sep 17 00:00:00 2001 From: Tyler Beebe Date: Fri, 13 Nov 2026 20:20:24 +0000 Subject: [PATCH 2/3] feat(telemetry): send to public-api's /v2/telemetry/v1/logs (TRUNK-23773) The CLI telemetry route moved from trunk-cloud's sawmill-api into trunk2's public-api (trunk-io/trunk2#8502). Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 2 +- api/src/otlp.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ccee6c9c..3617b654 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -112,7 +112,7 @@ The `tracing` library is used for: `--org-url-slug` (env `TRUNK_ORG_URL_SLUG`) is declared as a non-optional `String` on `UploadArgs`, which makes it a hard requirement for both `upload` and `test`. Keep it that way: the slug is what attributes a run to an organization across every kind of telemetry we collect. - **Sentry**: `setup_logger` in `cli/src/main.rs` attaches it as an `org_url_slug` tag on every `tracing::error!()` event forwarded to Sentry, alongside `command_name` and `repo_root`. Those tags are how Trunk narrows Sentry to a single customer's CI runs when someone reports a failing upload — without the slug, an error report can't be tied back to an organization and we have to ask the user to reproduce. -- **Our own upload telemetry**: at the end of a run the CLI sends one OTLP log record (`api/src/otlp.rs`, `ApiClient::telemetry_upload_metrics`) to sawmill-api's CLI telemetry route, `/v2/cloud/telemetry/v1/logs`, with the run's API key; a run with no key (a fork pull request) sends nothing. The route stamps the org from the key rather than from the slug, and keeps only the attribute names it lists for `trunk-analytics-cli` (trunk-cloud `apps/sawmill-api-rs/src/cli_telemetry.rs`), so a new attribute is dropped until it is added there too. +- **Our own upload telemetry**: at the end of a run the CLI sends one OTLP log record (`api/src/otlp.rs`, `ApiClient::telemetry_upload_metrics`) to public-api's CLI telemetry route, `/v2/telemetry/v1/logs`, with the run's API key; a run with no key (a fork pull request) sends nothing. The route stamps the org from the key rather than from the slug, and keeps only the attribute names it lists for `trunk-analytics-cli` (trunk2 `ts/apps/public-api`), so a new attribute is dropped until it is added there too. The slug also travels with the upload requests, builds the org-scoped links we print for uploads and tests (`api/src/urls.rs`), and backs the settings-page hint shown on unauthorized errors (`cli/src/error_report.rs`). diff --git a/api/src/otlp.rs b/api/src/otlp.rs index 5c076211..1cbbfae1 100644 --- a/api/src/otlp.rs +++ b/api/src/otlp.rs @@ -1,14 +1,14 @@ -//! The upload's own telemetry as one OTLP JSON log record, for sawmill-api's CLI telemetry route -//! (`/v2/cloud/telemetry/v1/logs`, TRUNK-23773). The route keeps only the attribute names listed -//! for `trunk-analytics-cli` in trunk-cloud's `apps/sawmill-api-rs/src/cli_telemetry.rs` and stamps -//! the org itself, so a name added here is dropped until it is added there too. +//! The upload's own telemetry as one OTLP JSON log record, for public-api's CLI telemetry route +//! (`/v2/telemetry/v1/logs`, TRUNK-23773). The route keeps only the attribute names listed for +//! `trunk-analytics-cli` in trunk2's `ts/apps/public-api` and stamps the org itself, so a name +//! added here is dropped until it is added there too. use proto::upload_metrics::trunk::{ QuarantineQueryResult, QuarantineResolutionMode, UploadMetrics, }; use serde_json::{Value, json}; -pub const CLI_TELEMETRY_PATH: &str = "/v2/cloud/telemetry/v1/logs"; +pub const CLI_TELEMETRY_PATH: &str = "/v2/telemetry/v1/logs"; pub const SERVICE_NAME: &str = "trunk-analytics-cli"; pub const COMMAND: &str = "upload"; From 6becd4a274a14e06c22df594021ebfe5dcf13377 Mon Sep 17 00:00:00 2001 From: Tyler Beebe Date: Mon, 16 Nov 2026 17:44:39 +0000 Subject: [PATCH 3/3] feat(telemetry): send upload telemetry as OTLP metrics (TRUNK-23773) Replace the single OTLP log record with one OTLP/HTTP JSON ExportMetricsServiceRequest to public-api's /v2/telemetry/v1/metrics: an upload count and the upload and run duration histograms, delta temporality, with trunk1's telemetry-service labels, duration buckets and boundedLabel rule. The mock server decodes the metrics back so the telemetry tests assert status, failure reason, quarantine result, repo labels and each duration's bucket. Co-Authored-By: Claude Opus 5.5 --- CONTRIBUTING.md | 2 +- api/src/client/mod.rs | 6 +- api/src/otlp.rs | 402 +++++++++++++++++++++++---------- cli/tests/upload.rs | 80 +++---- test_report/tests/telemetry.rs | 25 +- test_utils/src/mock_server.rs | 165 ++++++++++---- 6 files changed, 459 insertions(+), 221 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3617b654..ad980d96 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -112,7 +112,7 @@ The `tracing` library is used for: `--org-url-slug` (env `TRUNK_ORG_URL_SLUG`) is declared as a non-optional `String` on `UploadArgs`, which makes it a hard requirement for both `upload` and `test`. Keep it that way: the slug is what attributes a run to an organization across every kind of telemetry we collect. - **Sentry**: `setup_logger` in `cli/src/main.rs` attaches it as an `org_url_slug` tag on every `tracing::error!()` event forwarded to Sentry, alongside `command_name` and `repo_root`. Those tags are how Trunk narrows Sentry to a single customer's CI runs when someone reports a failing upload — without the slug, an error report can't be tied back to an organization and we have to ask the user to reproduce. -- **Our own upload telemetry**: at the end of a run the CLI sends one OTLP log record (`api/src/otlp.rs`, `ApiClient::telemetry_upload_metrics`) to public-api's CLI telemetry route, `/v2/telemetry/v1/logs`, with the run's API key; a run with no key (a fork pull request) sends nothing. The route stamps the org from the key rather than from the slug, and keeps only the attribute names it lists for `trunk-analytics-cli` (trunk2 `ts/apps/public-api`), so a new attribute is dropped until it is added there too. +- **Our own upload telemetry**: at the end of a run the CLI sends one OTLP metrics request (`api/src/otlp.rs`, `ApiClient::telemetry_upload_metrics`) to public-api's CLI telemetry metrics route, `/v2/telemetry/v1/metrics`, with the run's API key; a run with no key (a fork pull request) sends nothing. It holds an upload count and the upload and run duration histograms (delta, one data point each), with trunk1's telemetry-service labels and buckets, and each label value bounded the way trunk1's `boundedLabel` bounds it. The route stamps the org from the key rather than from the slug, and keeps only the metric and attribute names it lists for `trunk-analytics-cli` (trunk2 `ts/apps/public-api/src/products/telemetry`), so a new metric or attribute is dropped until it is added there too. The slug also travels with the upload requests, builds the org-scoped links we print for uploads and tests (`api/src/urls.rs`), and backs the settings-page hint shown on unauthorized errors (`cli/src/error_report.rs`). diff --git a/api/src/client/mod.rs b/api/src/client/mod.rs index bfa643ec..e87d7e9f 100644 --- a/api/src/client/mod.rs +++ b/api/src/client/mod.rs @@ -439,11 +439,13 @@ impl ApiClient { if std::env::var("DISABLE_TELEMETRY").is_ok() { return Ok(()); } + let now = chrono::Utc::now(); let response = telemetry_client - .post(format!("{}{}", self.api_host, crate::otlp::CLI_TELEMETRY_PATH)) - .json(&crate::otlp::upload_metrics_logs( + .post(format!("{}{}", self.api_host, crate::otlp::CLI_TELEMETRY_METRICS_PATH)) + .json(&crate::otlp::upload_metrics_request( &request.upload_metrics, environment, + crate::otlp::unix_nanos(now.timestamp(), now.timestamp_subsec_nanos() as i32), )) .send() .await?; diff --git a/api/src/otlp.rs b/api/src/otlp.rs index 1cbbfae1..534b7b51 100644 --- a/api/src/otlp.rs +++ b/api/src/otlp.rs @@ -1,38 +1,51 @@ -//! The upload's own telemetry as one OTLP JSON log record, for public-api's CLI telemetry route -//! (`/v2/telemetry/v1/logs`, TRUNK-23773). The route keeps only the attribute names listed for -//! `trunk-analytics-cli` in trunk2's `ts/apps/public-api` and stamps the org itself, so a name -//! added here is dropped until it is added there too. - -use proto::upload_metrics::trunk::{ - QuarantineQueryResult, QuarantineResolutionMode, UploadMetrics, -}; +//! The upload's own telemetry as OTLP JSON metrics, for public-api's CLI telemetry metrics route +//! (`/v2/telemetry/v1/metrics`, TRUNK-23773). The route keeps only the metric and attribute names +//! it lists for `trunk-analytics-cli` in trunk2's `ts/apps/public-api/src/products/telemetry` and +//! stamps the org itself, so a name added here is dropped until it is added there too. Names, +//! labels and buckets match trunk1's telemetry service so its dashboards carry over. + +use proto::upload_metrics::trunk::{QuarantineQueryResult, UploadMetrics}; use serde_json::{Value, json}; -pub const CLI_TELEMETRY_PATH: &str = "/v2/telemetry/v1/logs"; +pub const CLI_TELEMETRY_METRICS_PATH: &str = "/v2/telemetry/v1/metrics"; pub const SERVICE_NAME: &str = "trunk-analytics-cli"; -pub const COMMAND: &str = "upload"; -pub const COMMAND_ATTRIBUTE: &str = "trunk.cli.command"; -pub const DURATION_ATTRIBUTE: &str = "trunk.cli.duration_ms"; -pub const FAILED_ATTRIBUTE: &str = "trunk.analytics_cli.upload.failed"; +pub const UPLOADS_METRIC: &str = "trunk.analytics_cli.uploads"; +pub const UPLOAD_DURATION_METRIC: &str = "trunk.analytics_cli.upload.duration"; +pub const RUN_DURATION_METRIC: &str = "trunk.analytics_cli.run.duration"; +pub const UPLOADS_UNIT: &str = "{upload}"; +pub const DURATION_UNIT: &str = "s"; + +pub const STATUS_ATTRIBUTE: &str = "trunk.analytics_cli.upload.status"; pub const FAILURE_REASON_ATTRIBUTE: &str = "trunk.analytics_cli.upload.failure_reason"; -pub const UPLOAD_DURATION_ATTRIBUTE: &str = "trunk.analytics_cli.upload.duration_ms"; pub const QUARANTINE_QUERY_RESULT_ATTRIBUTE: &str = "trunk.analytics_cli.quarantine.query_result"; -pub const QUARANTINE_RESOLUTION_MODE_ATTRIBUTE: &str = - "trunk.analytics_cli.quarantine.resolution_mode"; -pub const REPOSITORY_HOST_ATTRIBUTE: &str = "trunk.vcs.repository.host"; pub const REPOSITORY_OWNER_ATTRIBUTE: &str = "trunk.vcs.repository.owner"; pub const REPOSITORY_NAME_ATTRIBUTE: &str = "trunk.vcs.repository.name"; -/// OTLP's `SEVERITY_NUMBER_INFO` and `SEVERITY_NUMBER_ERROR`. -const SEVERITY_INFO: i32 = 9; -const SEVERITY_ERROR: i32 = 17; +pub const STATUS_SUCCESS: &str = "success"; +pub const STATUS_FAILED: &str = "failed"; +pub const UNKNOWN_LABEL: &str = "unknown"; +pub const INVALID_LABEL: &str = "invalid"; + +/// trunk1's `DURATION_BUCKETS_SECONDS`. +pub const DURATION_BOUNDS_SECONDS: [f64; 13] = [ + 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0, +]; + +/// OTLP's `AGGREGATION_TEMPORALITY_DELTA`. +const DELTA: i32 = 1; -const QUARANTINE_QUERY_RESULT_PREFIX: &str = "QUARANTINE_QUERY_RESULT_"; -const QUARANTINE_RESOLUTION_MODE_PREFIX: &str = "QUARANTINE_RESOLUTION_MODE_"; +const LABEL_MAX_CHARS: usize = 32; +/// GitHub's own limits. +const OWNER_MAX_CHARS: usize = 39; +const NAME_MAX_CHARS: usize = 100; /// OTLP JSON writes 64-bit integers as decimal strings. -pub fn upload_metrics_logs(metrics: &UploadMetrics, environment: &str) -> Value { +pub fn upload_metrics_request( + metrics: &UploadMetrics, + environment: &str, + now_unix_nanos: i128, +) -> Value { let version = metrics .client_version .as_ref() @@ -53,111 +66,158 @@ pub fn upload_metrics_logs(metrics: &UploadMetrics, environment: &str) -> Value string_attribute("host.arch", host_arch(std::env::consts::ARCH)), ]; - let mut attributes = vec![ - string_attribute(COMMAND_ATTRIBUTE, COMMAND), - bool_attribute(FAILED_ATTRIBUTE, metrics.failed), + let status = bounded_label( + if metrics.failed { + STATUS_FAILED + } else { + STATUS_SUCCESS + }, + LABEL_MAX_CHARS, + ); + let repo = metrics.repo.clone().unwrap_or_default(); + let quarantine_query_result = + QuarantineQueryResult::try_from(metrics.quarantine_query_result).unwrap_or_default(); + let upload_attributes = [ + string_attribute(STATUS_ATTRIBUTE, &status), + string_attribute( + FAILURE_REASON_ATTRIBUTE, + &bounded_label(&metrics.failure_reason, LABEL_MAX_CHARS), + ), string_attribute( QUARANTINE_QUERY_RESULT_ATTRIBUTE, - &enum_name( - QuarantineQueryResult::try_from(metrics.quarantine_query_result) - .unwrap_or_default() - .as_str_name(), - QUARANTINE_QUERY_RESULT_PREFIX, + &bounded_label( + quarantine_query_result_label(quarantine_query_result), + LABEL_MAX_CHARS, ), ), string_attribute( - QUARANTINE_RESOLUTION_MODE_ATTRIBUTE, - &enum_name( - QuarantineResolutionMode::try_from(metrics.quarantine_resolution_mode) - .unwrap_or_default() - .as_str_name(), - QUARANTINE_RESOLUTION_MODE_PREFIX, - ), + REPOSITORY_OWNER_ATTRIBUTE, + &bounded_label(&repo.owner, OWNER_MAX_CHARS), + ), + string_attribute( + REPOSITORY_NAME_ATTRIBUTE, + &bounded_label(&repo.name, NAME_MAX_CHARS), ), ]; - if !metrics.failure_reason.is_empty() { - attributes.push(string_attribute( - FAILURE_REASON_ATTRIBUTE, - &metrics.failure_reason, - )); - } - let finished = metrics - .upload_finished_at - .as_ref() - .map(|time| unix_nanos(time.seconds, time.nanos)); - let cli_started = metrics - .cli_started_at - .as_ref() - .map(|time| unix_nanos(time.seconds, time.nanos)); - let upload_started = metrics - .upload_started_at - .as_ref() - .map(|time| unix_nanos(time.seconds, time.nanos)); - for (key, started) in [ - (DURATION_ATTRIBUTE, cli_started), - (UPLOAD_DURATION_ATTRIBUTE, upload_started), + + let [cli_started, upload_started, upload_finished] = [ + &metrics.cli_started_at, + &metrics.upload_started_at, + &metrics.upload_finished_at, + ] + .map(|time| { + time.as_ref() + .map(|time| unix_nanos(time.seconds, time.nanos)) + }); + let start_time = cli_started.unwrap_or(now_unix_nanos).to_string(); + let time = now_unix_nanos.to_string(); + + let mut otlp_metrics = vec![json!({ + "name": UPLOADS_METRIC, + "unit": UPLOADS_UNIT, + "sum": { + "aggregationTemporality": DELTA, + "isMonotonic": true, + "dataPoints": [{ + "attributes": upload_attributes, + "startTimeUnixNano": start_time, + "timeUnixNano": time, + "asInt": "1", + }], + }, + })]; + for (name, started) in [ + (UPLOAD_DURATION_METRIC, upload_started), + (RUN_DURATION_METRIC, cli_started), ] { - if let (Some(started), Some(finished)) = (started, finished) { - attributes.push(int_attribute(key, millis_between(started, finished))); - } - } - if let Some(repo) = &metrics.repo { - for (key, value) in [ - (REPOSITORY_HOST_ATTRIBUTE, &repo.host), - (REPOSITORY_OWNER_ATTRIBUTE, &repo.owner), - (REPOSITORY_NAME_ATTRIBUTE, &repo.name), - ] { - if !value.is_empty() { - attributes.push(string_attribute(key, value)); - } - } + let Some(seconds) = duration_seconds(started, upload_finished) else { + continue; + }; + let mut bucket_counts = vec!["0"; DURATION_BOUNDS_SECONDS.len() + 1]; + bucket_counts[bucket_index(seconds, &DURATION_BOUNDS_SECONDS)] = "1"; + otlp_metrics.push(json!({ + "name": name, + "unit": DURATION_UNIT, + "histogram": { + "aggregationTemporality": DELTA, + "dataPoints": [{ + "attributes": [string_attribute(STATUS_ATTRIBUTE, &status)], + "startTimeUnixNano": start_time, + "timeUnixNano": time, + "count": "1", + "sum": seconds, + "min": seconds, + "max": seconds, + "explicitBounds": DURATION_BOUNDS_SECONDS, + "bucketCounts": bucket_counts, + }], + }, + })); } - let (severity_number, severity_text) = if metrics.failed { - (SEVERITY_ERROR, "ERROR") - } else { - (SEVERITY_INFO, "INFO") - }; - json!({ - "resourceLogs": [{ + "resourceMetrics": [{ "resource": { "attributes": resource_attributes }, - "scopeLogs": [{ + "scopeMetrics": [{ "scope": { "name": SERVICE_NAME, "version": version }, - "logRecords": [{ - "timeUnixNano": finished.unwrap_or_default().to_string(), - "severityNumber": severity_number, - "severityText": severity_text, - "body": { "stringValue": format!("{SERVICE_NAME} {COMMAND}") }, - "attributes": attributes, - }], + "metrics": otlp_metrics, }], }], }) } -fn enum_name(name: &str, prefix: &str) -> String { - name.strip_prefix(prefix).unwrap_or(name).to_lowercase() +/// trunk1's `quarantineQueryResultToLabel`. +pub fn quarantine_query_result_label(result: QuarantineQueryResult) -> &'static str { + match result { + QuarantineQueryResult::Unspecified => "unspecified", + QuarantineQueryResult::Success => "success", + QuarantineQueryResult::Failure => "failure", + QuarantineQueryResult::Disabled => "disabled", + QuarantineQueryResult::Skipped => "skipped", + QuarantineQueryResult::Cached => "cached", + } } -fn unix_nanos(seconds: i64, nanos: i32) -> i128 { - i128::from(seconds) * 1_000_000_000 + i128::from(nanos) +/// trunk1's `boundedLabel`: the first `max_chars` characters, or `invalid` when they leave the +/// label charset. An empty value is `unknown`. +pub fn bounded_label(value: &str, max_chars: usize) -> String { + if value.is_empty() { + return UNKNOWN_LABEL.into(); + } + let trimmed: String = value.chars().take(max_chars).collect(); + if trimmed + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '+' | '/' | '-')) + { + trimmed + } else { + INVALID_LABEL.into() + } } -fn millis_between(started: i128, finished: i128) -> i64 { - i64::try_from((finished - started).max(0) / 1_000_000).unwrap_or(i64::MAX) +/// OTLP bucket `i` counts values in `(bounds[i - 1], bounds[i]]`; the last one counts values +/// above every bound. +pub fn bucket_index(value: f64, bounds: &[f64]) -> usize { + bounds + .iter() + .position(|bound| value <= *bound) + .unwrap_or(bounds.len()) } -fn string_attribute(key: &str, value: &str) -> Value { - json!({ "key": key, "value": { "stringValue": value } }) +fn duration_seconds(started: Option, finished: Option) -> Option { + let (started, finished) = (started?, finished?); + if started < 0 || finished < started { + return None; + } + Some((finished - started) as f64 / 1e9) } -fn bool_attribute(key: &str, value: bool) -> Value { - json!({ "key": key, "value": { "boolValue": value } }) +pub fn unix_nanos(seconds: i64, nanos: i32) -> i128 { + i128::from(seconds) * 1_000_000_000 + i128::from(nanos) } -fn int_attribute(key: &str, value: i64) -> Value { - json!({ "key": key, "value": { "intValue": value.to_string() } }) +fn string_attribute(key: &str, value: &str) -> Value { + json!({ "key": key, "value": { "stringValue": value } }) } fn os_type(os: &'static str) -> &'static str { @@ -178,29 +238,143 @@ fn host_arch(arch: &'static str) -> &'static str { #[cfg(test)] mod tests { + use proto::upload_metrics::trunk::Repo; + use super::*; + const SECOND: i64 = 1_000_000_000; + #[test] - fn enum_names_lose_their_prefix() { + fn a_value_on_a_bound_lands_in_that_bound_s_bucket() { + let bounds = DURATION_BOUNDS_SECONDS; + assert_eq!(bucket_index(0.0, &bounds), 0); + assert_eq!(bucket_index(0.05, &bounds), 0); + assert_eq!(bucket_index(0.050_001, &bounds), 1); + assert_eq!(bucket_index(1.0, &bounds), 4); + assert_eq!(bucket_index(1.000_001, &bounds), 5); + assert_eq!(bucket_index(600.0, &bounds), 12); + assert_eq!(bucket_index(600.000_001, &bounds), 13); + } + + #[test] + fn labels_are_bounded_like_trunk1() { + assert_eq!(bounded_label("", 32), "unknown"); + assert_eq!(bounded_label("400_bad_request", 32), "400_bad_request"); + assert_eq!(bounded_label("a.b_c+d/e-F9", 32), "a.b_c+d/e-F9"); + assert_eq!(bounded_label("has space", 32), "invalid"); + assert_eq!(bounded_label("émoji", 32), "invalid"); + assert_eq!(bounded_label(&"a".repeat(33), 32), "a".repeat(32)); assert_eq!( - enum_name( - "QUARANTINE_QUERY_RESULT_SUCCESS", - QUARANTINE_QUERY_RESULT_PREFIX - ), - "success" + bounded_label(&format!("{} x", "a".repeat(32)), 32), + "a".repeat(32) ); + assert_eq!(bounded_label(&"o".repeat(40), 39), "o".repeat(39)); + } + + #[test] + fn missing_or_backwards_durations_are_skipped() { + assert_eq!(duration_seconds(None, Some(1)), None); + assert_eq!(duration_seconds(Some(1), None), None); + assert_eq!(duration_seconds(Some(-1), Some(1)), None); + assert_eq!(duration_seconds(Some(5), Some(1)), None); + assert_eq!(duration_seconds(Some(0), Some(2_500_000_000)), Some(2.5)); + } + + #[test] + fn a_failed_upload_sends_a_count_and_both_durations() { + let now: i64 = 1_700_000_100 * SECOND; + let metrics = UploadMetrics { + repo: Some(Repo { + host: "github.com".into(), + owner: "my-org".into(), + name: "my repo".into(), + }), + cli_started_at: Some(chrono::DateTime::from_timestamp_nanos(now - 12 * SECOND).into()), + upload_started_at: Some(chrono::DateTime::from_timestamp_nanos(now - SECOND).into()), + upload_finished_at: Some(chrono::DateTime::from_timestamp_nanos(now).into()), + failed: true, + failure_reason: "connection".into(), + quarantine_query_result: QuarantineQueryResult::Cached.into(), + ..UploadMetrics::default() + }; + let body = upload_metrics_request(&metrics, "staging", now.into()); + let metrics = body["resourceMetrics"][0]["scopeMetrics"][0]["metrics"] + .as_array() + .unwrap(); + assert_eq!(metrics.len(), 3); + + let uploads = &metrics[0]; + assert_eq!(uploads["name"], UPLOADS_METRIC); + assert_eq!(uploads["unit"], UPLOADS_UNIT); + assert_eq!(uploads["sum"]["aggregationTemporality"], 1); + assert_eq!(uploads["sum"]["isMonotonic"], true); + let point = &uploads["sum"]["dataPoints"][0]; + assert_eq!(point["asInt"], "1"); + assert_eq!(point["timeUnixNano"], now.to_string()); + assert_eq!(point["startTimeUnixNano"], (now - 12 * SECOND).to_string()); + let attributes: Vec<(&str, &str)> = point["attributes"] + .as_array() + .unwrap() + .iter() + .map(|a| { + ( + a["key"].as_str().unwrap(), + a["value"]["stringValue"].as_str().unwrap(), + ) + }) + .collect(); assert_eq!( - enum_name( - "QUARANTINE_RESOLUTION_MODE_TEST_COLLECTION", - QUARANTINE_RESOLUTION_MODE_PREFIX - ), - "test_collection" + attributes, + [ + (STATUS_ATTRIBUTE, "failed"), + (FAILURE_REASON_ATTRIBUTE, "connection"), + (QUARANTINE_QUERY_RESULT_ATTRIBUTE, "cached"), + (REPOSITORY_OWNER_ATTRIBUTE, "my-org"), + (REPOSITORY_NAME_ATTRIBUTE, "invalid"), + ] ); + + for (metric, name, seconds, bucket) in [ + (&metrics[1], UPLOAD_DURATION_METRIC, 1.0, 4), + (&metrics[2], RUN_DURATION_METRIC, 12.0, 8), + ] { + assert_eq!(metric["name"], name); + assert_eq!(metric["unit"], DURATION_UNIT); + assert_eq!(metric["histogram"]["aggregationTemporality"], 1); + let point = &metric["histogram"]["dataPoints"][0]; + assert_eq!(point["count"], "1"); + assert_eq!(point["sum"], seconds); + assert_eq!(point["min"], seconds); + assert_eq!(point["max"], seconds); + let counts = point["bucketCounts"].as_array().unwrap(); + assert_eq!(counts.len(), DURATION_BOUNDS_SECONDS.len() + 1); + for (index, count) in counts.iter().enumerate() { + assert_eq!(count, if index == bucket { "1" } else { "0" }); + } + } } #[test] - fn a_clock_that_ran_backwards_reads_as_zero() { - assert_eq!(millis_between(5_000_000, 1_000_000), 0); - assert_eq!(millis_between(0, 2_500_000), 2); + fn a_duration_without_its_start_is_not_sent() { + let metrics = UploadMetrics { + upload_finished_at: Some(chrono::DateTime::from_timestamp_nanos(5).into()), + ..UploadMetrics::default() + }; + let body = upload_metrics_request(&metrics, "production", 10); + let metrics = body["resourceMetrics"][0]["scopeMetrics"][0]["metrics"] + .as_array() + .unwrap(); + assert_eq!(metrics.len(), 1); + assert_eq!(metrics[0]["name"], UPLOADS_METRIC); + let values: Vec<&str> = metrics[0]["sum"]["dataPoints"][0]["attributes"] + .as_array() + .unwrap() + .iter() + .map(|a| a["value"]["stringValue"].as_str().unwrap()) + .collect(); + assert_eq!( + values, + ["success", "unknown", "unspecified", "unknown", "unknown"] + ); } } diff --git a/cli/tests/upload.rs b/cli/tests/upload.rs index 09bb297d..0276c4fb 100644 --- a/cli/tests/upload.rs +++ b/cli/tests/upload.rs @@ -7,6 +7,7 @@ use api::message::{ CreateBundleUploadRequest, CreateBundleUploadResponse, GetQuarantineConfigRequest, GetQuarantineConfigResponse, }; +use api::otlp::DURATION_BOUNDS_SECONDS; use assert_cmd::{Command, cargo}; use assert_matches::assert_matches; use axum::body::{Body, Bytes}; @@ -49,7 +50,6 @@ use lazy_static::lazy_static; use predicates::prelude::*; use pretty_assertions::assert_eq; use prost::Message; -use proto::upload_metrics::trunk::QuarantineQueryResult; use tempfile::tempdir; #[cfg(target_os = "macos")] use test_utils::inputs::unpack_archive_to_dir; @@ -57,7 +57,7 @@ use test_utils::{ inputs::get_test_file_path, mock_server::{ MOCK_REPO_ID, MOCK_TEST_COLLECTION_ID, MockServerBuilder, RequestPayload, - SharedMockServerState, + SharedMockServerState, TelemetryUploadMetrics, }, }; use trunk_analytics_cli::exit_code::ExitCode; @@ -602,18 +602,6 @@ async fn upload_bundle_records_quarantine_resolution_mode() { QuarantineResolutionMode::TestCollection, "the resolution mode must be persisted in meta.json" ); - - let upload_metrics = requests - .iter() - .find_map(|request| match request { - RequestPayload::TelemetryUploadMetrics(metrics) => Some(metrics), - _ => None, - }) - .expect("telemetry must have been sent"); - assert_eq!( - upload_metrics.quarantine_resolution_mode, - i32::from(proto::upload_metrics::trunk::QuarantineResolutionMode::TestCollection) - ); } #[tokio::test(flavor = "multi_thread")] @@ -2115,6 +2103,19 @@ async fn is_not_ok_on_bad_request() { .stdout(predicate::str::contains("error")); } +fn assert_durations_are_bucketed(metrics: &TelemetryUploadMetrics) { + for duration in [&metrics.upload_duration, &metrics.run_duration] { + let duration = duration.as_ref().expect("both durations must be sent"); + assert_eq!(duration.status, metrics.status); + assert_eq!(duration.explicit_bounds, DURATION_BOUNDS_SECONDS); + assert!(duration.is_in_its_bucket(), "{duration:?}"); + } + assert!( + metrics.run_duration.as_ref().unwrap().seconds + >= metrics.upload_duration.as_ref().unwrap().seconds + ); +} + #[tokio::test(flavor = "multi_thread")] async fn telemetry_upload_metrics_on_upload_failure() { let temp_dir = tempdir().unwrap(); @@ -2140,16 +2141,12 @@ async fn telemetry_upload_metrics_on_upload_failure() { let telemetry_request = assert_matches!(requests.last().unwrap(), RequestPayload::TelemetryUploadMetrics(ur) => ur); - let telemetry_request_repo = telemetry_request.repo.clone().unwrap(); - assert!(telemetry_request.failed); - assert_eq!(telemetry_request_repo.host, "github.com"); - assert_eq!(telemetry_request_repo.owner, "trunk-io"); - assert_eq!(telemetry_request_repo.name, "analytics-cli"); + assert_eq!(telemetry_request.status, "failed"); + assert_eq!(telemetry_request.repo_owner, "trunk-io"); + assert_eq!(telemetry_request.repo_name, "analytics-cli"); assert_eq!(telemetry_request.failure_reason, "400_bad_request"); - assert_eq!( - QuarantineQueryResult::try_from(telemetry_request.quarantine_query_result).unwrap(), - QuarantineQueryResult::Disabled - ); + assert_eq!(telemetry_request.quarantine_query_result, "disabled"); + assert_durations_are_bucketed(telemetry_request); // HINT: View CLI output with `cargo test -- --nocapture` println!("{assert}"); @@ -2175,16 +2172,12 @@ async fn telemetry_upload_metrics_on_upload_success() { let telemetry_request = assert_matches!(requests.last().unwrap(), RequestPayload::TelemetryUploadMetrics(ur) => ur); - let telemetry_request_repo = telemetry_request.repo.clone().unwrap(); - assert!(!telemetry_request.failed); - assert_eq!(telemetry_request_repo.host, "github.com"); - assert_eq!(telemetry_request_repo.owner, "trunk-io"); - assert_eq!(telemetry_request_repo.name, "analytics-cli"); - assert_eq!(telemetry_request.failure_reason, ""); - assert_eq!( - QuarantineQueryResult::try_from(telemetry_request.quarantine_query_result).unwrap(), - QuarantineQueryResult::Disabled - ); + assert_eq!(telemetry_request.status, "success"); + assert_eq!(telemetry_request.repo_owner, "trunk-io"); + assert_eq!(telemetry_request.repo_name, "analytics-cli"); + assert_eq!(telemetry_request.failure_reason, "unknown"); + assert_eq!(telemetry_request.quarantine_query_result, "disabled"); + assert_durations_are_bucketed(telemetry_request); // HINT: View CLI output with `cargo test -- --nocapture` println!("{assert}"); @@ -2208,14 +2201,11 @@ async fn telemetry_upload_metrics_quarantine_query_result_success() { .iter() .rev() .find_map(|req| match req { - RequestPayload::TelemetryUploadMetrics(ur) => Some(ur.quarantine_query_result), + RequestPayload::TelemetryUploadMetrics(ur) => Some(ur.quarantine_query_result.clone()), _ => None, }) .expect("telemetry upload metrics request"); - assert_eq!( - QuarantineQueryResult::try_from(quarantine_query_result).unwrap(), - QuarantineQueryResult::Success - ); + assert_eq!(quarantine_query_result, "success"); } #[tokio::test(flavor = "multi_thread")] @@ -2242,14 +2232,11 @@ async fn telemetry_upload_metrics_quarantine_query_result_failure() { .iter() .rev() .find_map(|req| match req { - RequestPayload::TelemetryUploadMetrics(ur) => Some(ur.quarantine_query_result), + RequestPayload::TelemetryUploadMetrics(ur) => Some(ur.quarantine_query_result.clone()), _ => None, }) .expect("telemetry upload metrics request"); - assert_eq!( - QuarantineQueryResult::try_from(quarantine_query_result).unwrap(), - QuarantineQueryResult::Failure - ); + assert_eq!(quarantine_query_result, "failure"); } #[tokio::test(flavor = "multi_thread")] @@ -2291,14 +2278,11 @@ async fn telemetry_upload_metrics_quarantine_query_result_skipped() { .iter() .rev() .find_map(|req| match req { - RequestPayload::TelemetryUploadMetrics(ur) => Some(ur.quarantine_query_result), + RequestPayload::TelemetryUploadMetrics(ur) => Some(ur.quarantine_query_result.clone()), _ => None, }) .expect("telemetry upload metrics request"); - assert_eq!( - QuarantineQueryResult::try_from(quarantine_query_result).unwrap(), - QuarantineQueryResult::Skipped - ); + assert_eq!(quarantine_query_result, "skipped"); } #[tokio::test(flavor = "multi_thread")] diff --git a/test_report/tests/telemetry.rs b/test_report/tests/telemetry.rs index 1b97a651..7299f511 100644 --- a/test_report/tests/telemetry.rs +++ b/test_report/tests/telemetry.rs @@ -12,29 +12,22 @@ use constants::{ TRUNK_REPO_HEAD_BRANCH_ENV, TRUNK_REPO_HEAD_SHA_ENV, TRUNK_REPO_URL_ENV, TRUNK_USE_UNCLONED_REPO_ENV, }; -use proto::upload_metrics::trunk::QuarantineQueryResult; use serial_test::serial; use tempfile::tempdir; use test_report::report::{MutTestReport, Status}; use test_utils::mock_git_repo::setup_repo_with_commit; use test_utils::mock_server::{MockServerBuilder, RequestPayload, SharedMockServerState}; -fn assert_telemetry_quarantine_query_result( - requests: &[RequestPayload], - expected: QuarantineQueryResult, -) { +fn assert_telemetry_quarantine_query_result(requests: &[RequestPayload], expected: &str) { let quarantine_query_result = requests .iter() .rev() .find_map(|req| match req { - RequestPayload::TelemetryUploadMetrics(ur) => Some(ur.quarantine_query_result), + RequestPayload::TelemetryUploadMetrics(ur) => Some(ur.quarantine_query_result.clone()), _ => None, }) .expect("expected telemetry upload metrics request"); - assert_eq!( - QuarantineQueryResult::try_from(quarantine_query_result).unwrap(), - expected - ); + assert_eq!(quarantine_query_result, expected); } fn set_mock_api_env(api_host: &str) { @@ -110,7 +103,7 @@ async fn telemetry_query_result_success_on_publish() { .unwrap(); let requests = state.requests.lock().unwrap().clone(); - assert_telemetry_quarantine_query_result(&requests, QuarantineQueryResult::Success); + assert_telemetry_quarantine_query_result(&requests, "success"); } #[tokio::test(flavor = "multi_thread")] @@ -133,7 +126,7 @@ async fn telemetry_query_result_skipped_without_lookup_on_publish() { .unwrap(); let requests = state.requests.lock().unwrap().clone(); - assert_telemetry_quarantine_query_result(&requests, QuarantineQueryResult::Skipped); + assert_telemetry_quarantine_query_result(&requests, "skipped"); } #[tokio::test(flavor = "multi_thread")] @@ -185,7 +178,7 @@ async fn telemetry_query_result_cached_on_publish() { "second report should use disk cache during suite" ); // Checks the last telemetry call - assert_telemetry_quarantine_query_result(&requests, QuarantineQueryResult::Cached); + assert_telemetry_quarantine_query_result(&requests, "cached"); } #[tokio::test(flavor = "multi_thread")] @@ -233,7 +226,7 @@ async fn telemetry_query_result_disabled_on_publish() { .unwrap(); let requests = state.requests.lock().unwrap().clone(); - assert_telemetry_quarantine_query_result(&requests, QuarantineQueryResult::Disabled); + assert_telemetry_quarantine_query_result(&requests, "disabled"); } #[tokio::test(flavor = "multi_thread")] @@ -261,7 +254,7 @@ async fn telemetry_query_result_disabled_by_env_on_publish() { .unwrap(); let requests = state.requests.lock().unwrap().clone(); - assert_telemetry_quarantine_query_result(&requests, QuarantineQueryResult::Disabled); + assert_telemetry_quarantine_query_result(&requests, "disabled"); } #[tokio::test(flavor = "multi_thread")] @@ -310,5 +303,5 @@ async fn telemetry_query_result_failure_on_publish() { .unwrap(); let requests = state.requests.lock().unwrap().clone(); - assert_telemetry_quarantine_query_result(&requests, QuarantineQueryResult::Failure); + assert_telemetry_quarantine_query_result(&requests, "failure"); } diff --git a/test_utils/src/mock_server.rs b/test_utils/src/mock_server.rs index 475a4a1b..e626a390 100644 --- a/test_utils/src/mock_server.rs +++ b/test_utils/src/mock_server.rs @@ -19,7 +19,6 @@ use axum::{ response::{IntoResponse, Response}, routing::{MethodRouter, any, post, put}, }; -use proto::upload_metrics::trunk::UploadMetrics; use tempfile::tempdir; use tokio::{net::TcpListener, spawn}; @@ -31,7 +30,46 @@ pub enum RequestPayload { CreateBundleUpload(CreateBundleUploadRequest), GetQuarantineConfig(GetQuarantineConfigRequest), S3Upload(PathBuf), - TelemetryUploadMetrics(UploadMetrics), + TelemetryUploadMetrics(TelemetryUploadMetrics), +} + +/// What one OTLP metrics request from the CLI says, read back from the wire. +#[derive(Debug, Clone, PartialEq)] +pub struct TelemetryUploadMetrics { + pub status: String, + pub failure_reason: String, + pub quarantine_query_result: String, + pub repo_owner: String, + pub repo_name: String, + pub upload_duration: Option, + pub run_duration: Option, +} + +/// One duration histogram's single observation. +#[derive(Debug, Clone, PartialEq)] +pub struct DurationObservation { + pub status: String, + pub seconds: f64, + /// The index of the one bucket counting it. + pub bucket: usize, + pub explicit_bounds: Vec, +} + +impl DurationObservation { + /// Whether `bucket` is the OTLP bucket for `seconds`: bucket `i` counts values in + /// `(bounds[i - 1], bounds[i]]`. + pub fn is_in_its_bucket(&self) -> bool { + let lower = self + .bucket + .checked_sub(1) + .map_or(f64::NEG_INFINITY, |index| self.explicit_bounds[index]); + let upper = self + .explicit_bounds + .get(self.bucket) + .copied() + .unwrap_or(f64::INFINITY); + lower < self.seconds && self.seconds <= upper + } } pub struct RespError { @@ -165,7 +203,10 @@ impl MockServerBuilder { self.get_quarantining_config_handler, ) .route("/s3upload", self.s3_upload_handler) - .route(api::otlp::CLI_TELEMETRY_PATH, self.telemetry_upload_metrics); + .route( + api::otlp::CLI_TELEMETRY_METRICS_PATH, + self.telemetry_upload_metrics, + ); for (path, method_router) in self.routes { app = app.route(path, method_router); @@ -264,7 +305,7 @@ pub async fn telemetry_upload_metrics_handler( ) -> Response { let upload_metrics = serde_json::from_slice::(&bytes) .ok() - .and_then(|logs| upload_metrics_from_otlp(&logs)); + .and_then(|request| telemetry_upload_metrics_from_otlp(&request)); if let Some(upload_metrics) = upload_metrics { state .requests @@ -303,50 +344,94 @@ pub async fn s3_upload_handler( Response::new(String::from("OK")) } -/// The fields of `UploadMetrics` that `api::otlp::upload_metrics_logs` writes, read back from the -/// OTLP JSON it sent, so the tests check what reaches the wire. `None` unless the body is one -/// `trunk-analytics-cli` record. -pub fn upload_metrics_from_otlp(logs: &serde_json::Value) -> Option { +/// The CLI's OTLP metrics request read back, so the tests check what reaches the wire. `None` +/// unless the body is one `trunk-analytics-cli` resource whose metrics have the names, types, +/// units and temporality `api::otlp` promises, each with one data point. +pub fn telemetry_upload_metrics_from_otlp( + request: &serde_json::Value, +) -> Option { use api::otlp; - use proto::upload_metrics::trunk::{QuarantineQueryResult, QuarantineResolutionMode, Repo}; + use serde_json::Value; - let resource_logs = &logs["resourceLogs"][0]; - let attribute = |list: &serde_json::Value, key: &str| -> Option { + let attribute = |list: &Value, key: &str| -> Option { list.as_array()? .iter() - .find(|attribute| attribute["key"] == key) - .map(|attribute| attribute["value"].clone()) + .find(|attribute| attribute["key"] == key)?["value"]["stringValue"] + .as_str() + .map(str::to_owned) }; - if attribute(&resource_logs["resource"]["attributes"], "service.name")?["stringValue"] - != otlp::SERVICE_NAME + let resource_metrics = request["resourceMetrics"].as_array()?; + let [resource_metrics] = resource_metrics.as_slice() else { + return None; + }; + if attribute(&resource_metrics["resource"]["attributes"], "service.name")? != otlp::SERVICE_NAME { return None; } - let record = &resource_logs["scopeLogs"][0]["logRecords"][0]["attributes"]; - let text = |key: &str| { - attribute(record, key) - .and_then(|value| value["stringValue"].as_str().map(str::to_owned)) - .unwrap_or_default() + let metrics = resource_metrics["scopeMetrics"][0]["metrics"].as_array()?; + let metric = |name: &str| metrics.iter().find(|metric| metric["name"] == name); + let single_point = |data: &Value| -> Option { + if data["aggregationTemporality"] != 1 { + return None; + } + match data["dataPoints"].as_array()?.as_slice() { + [point] => Some(point.clone()), + _ => None, + } }; - let upper = |key: &str| text(key).to_uppercase(); - Some(UploadMetrics { - failed: attribute(record, otlp::FAILED_ATTRIBUTE)?["boolValue"].as_bool()?, - failure_reason: text(otlp::FAILURE_REASON_ATTRIBUTE), - quarantine_query_result: QuarantineQueryResult::from_str_name(&format!( - "QUARANTINE_QUERY_RESULT_{}", - upper(otlp::QUARANTINE_QUERY_RESULT_ATTRIBUTE) - ))? - .into(), - quarantine_resolution_mode: QuarantineResolutionMode::from_str_name(&format!( - "QUARANTINE_RESOLUTION_MODE_{}", - upper(otlp::QUARANTINE_RESOLUTION_MODE_ATTRIBUTE) - ))? - .into(), - repo: Some(Repo { - host: text(otlp::REPOSITORY_HOST_ATTRIBUTE), - owner: text(otlp::REPOSITORY_OWNER_ATTRIBUTE), - name: text(otlp::REPOSITORY_NAME_ATTRIBUTE), - }), - ..UploadMetrics::default() + + let uploads = metric(otlp::UPLOADS_METRIC)?; + if uploads["unit"] != otlp::UPLOADS_UNIT || uploads["sum"]["isMonotonic"] != true { + return None; + } + let point = single_point(&uploads["sum"])?; + if point["asInt"] != "1" { + return None; + } + let labels = &point["attributes"]; + + let duration = |name: &str| -> Option> { + let Some(histogram) = metric(name) else { + return Some(None); + }; + if histogram["unit"] != otlp::DURATION_UNIT { + return None; + } + let point = single_point(&histogram["histogram"])?; + let seconds = point["sum"].as_f64()?; + if point["count"] != "1" + || point["min"].as_f64()? != seconds + || point["max"].as_f64()? != seconds + { + return None; + } + let explicit_bounds = point["explicitBounds"] + .as_array()? + .iter() + .map(Value::as_f64) + .collect::>>()?; + let counts = point["bucketCounts"].as_array()?; + if counts.len() != explicit_bounds.len() + 1 + || counts.iter().filter(|count| *count == "1").count() != 1 + || counts.iter().any(|count| *count != "0" && *count != "1") + { + return None; + } + Some(Some(DurationObservation { + status: attribute(&point["attributes"], otlp::STATUS_ATTRIBUTE)?, + seconds, + bucket: counts.iter().position(|count| count == "1")?, + explicit_bounds, + })) + }; + + Some(TelemetryUploadMetrics { + status: attribute(labels, otlp::STATUS_ATTRIBUTE)?, + failure_reason: attribute(labels, otlp::FAILURE_REASON_ATTRIBUTE)?, + quarantine_query_result: attribute(labels, otlp::QUARANTINE_QUERY_RESULT_ATTRIBUTE)?, + repo_owner: attribute(labels, otlp::REPOSITORY_OWNER_ATTRIBUTE)?, + repo_name: attribute(labels, otlp::REPOSITORY_NAME_ATTRIBUTE)?, + upload_duration: duration(otlp::UPLOAD_DURATION_METRIC)?, + run_duration: duration(otlp::RUN_DURATION_METRIC)?, }) }