diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e437f6ab..ad980d96 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -112,7 +112,7 @@ The `tracing` library is used for: `--org-url-slug` (env `TRUNK_ORG_URL_SLUG`) is declared as a non-optional `String` on `UploadArgs`, which makes it a hard requirement for both `upload` and `test`. Keep it that way: the slug is what attributes a run to an organization across every kind of telemetry we collect. - **Sentry**: `setup_logger` in `cli/src/main.rs` attaches it as an `org_url_slug` tag on every `tracing::error!()` event forwarded to Sentry, alongside `command_name` and `repo_root`. Those tags are how Trunk narrows Sentry to a single customer's CI runs when someone reports a failing upload — without the slug, an error report can't be tied back to an organization and we have to ask the user to reproduce. -- **Our own upload telemetry**: the slug is stored on the bundle as `base_props.org` and scopes the `UploadMetrics` we report to the telemetry endpoint at the end of a run (`cli/src/upload_command.rs`, `ApiClient::telemetry_upload_metrics`). Timing, quarantine outcome, and failure-reason metrics are only useful if we know which org they came from. +- **Our own upload telemetry**: at the end of a run the CLI sends one OTLP metrics request (`api/src/otlp.rs`, `ApiClient::telemetry_upload_metrics`) to public-api's CLI telemetry metrics route, `/v2/telemetry/v1/metrics`, with the run's API key; a run with no key (a fork pull request) sends nothing. It holds an upload count and the upload and run duration histograms (delta, one data point each), with trunk1's telemetry-service labels and buckets, and each label value bounded the way trunk1's `boundedLabel` bounds it. The route stamps the org from the key rather than from the slug, and keeps only the metric and attribute names it lists for `trunk-analytics-cli` (trunk2 `ts/apps/public-api/src/products/telemetry`), so a new metric or attribute is dropped until it is added there too. The slug also travels with the upload requests, builds the org-scoped links we print for uploads and tests (`api/src/urls.rs`), and backs the settings-page hint shown on unauthorized errors (`cli/src/error_report.rs`). diff --git a/api/src/client/mod.rs b/api/src/client/mod.rs index 238a3220..e87d7e9f 100644 --- a/api/src/client/mod.rs +++ b/api/src/client/mod.rs @@ -50,7 +50,6 @@ impl std::error::Error for ApiErrorEndpoint {} pub struct ApiClient { pub api_host: String, - pub telemetry_host: String, s3_client: Client, trunk_api_client: Client, /// Holds the run's v2 credential: an API token from the start, or a fork pull request's Trunk @@ -60,7 +59,9 @@ pub struct ApiClient { credential_free: bool, /// A credential-free GitHub Actions fork PR run, which on v2 swaps its `GITHUB_TOKEN` for a login. github_fork_pull_request: Option, - telemetry_client: Client, + /// The run's API token, for the CLI telemetry route; `None` without one (a fork pull request + /// has no token, and the gateway refuses its GitHub Actions login there). + telemetry_client: Option, version_path_prefix: String, org_url_slug: String, render_sender: Option>, @@ -88,6 +89,10 @@ impl V2Credential { const TRUNK_TOKEN_HEADER: HeaderName = HeaderName::from_static("x-trunk-token"); fn client(&self) -> anyhow::Result { + self.client_with_timeout(ApiClient::TRUNK_API_TIMEOUT) + } + + fn client_with_timeout(&self, timeout: std::time::Duration) -> anyhow::Result { let (name, value, error) = match self { Self::ApiKey(token) => ( header::AUTHORIZATION, @@ -109,7 +114,7 @@ impl V2Credential { ); headers.append(name, value); Ok(Client::builder() - .timeout(ApiClient::TRUNK_API_TIMEOUT) + .timeout(timeout) .default_headers(headers) .build()?) } @@ -160,18 +165,6 @@ impl ApiClient { let api_host = get_api_host(); tracing::debug!("Using public api address {}", api_host); - let telemetry_host = if api_host.contains("https://") { - format!( - "https://telemetry.{}", - api_host.split("https://").nth(1).unwrap_or_default() - ) - } else { - // If the api_host is not https, we default to the api host - // this happens when the api_host is localhost and we are running - // inside of test environments - api_host.clone() - }; - let mut trunk_api_client_default_headers = HeaderMap::new(); trunk_api_client_default_headers.append( header::CONTENT_TYPE, @@ -199,21 +192,12 @@ impl ApiClient { .transpose()?; let credential_free = allow_forked_pr_uploads && api_token.is_none(); - let mut telemetry_client_default_headers = HeaderMap::new(); - telemetry_client_default_headers.append( - header::CONTENT_TYPE, - HeaderValue::from_static("application/x-protobuf"), - ); - - if let Some(api_token_header_value) = api_token_header_value { - telemetry_client_default_headers - .append(Self::TRUNK_API_TOKEN_HEADER, api_token_header_value); - } - - let telemetry_client = Client::builder() - .timeout(Self::TRUNK_TELEMETRY_TIMEOUT) - .default_headers(telemetry_client_default_headers) - .build()?; + let telemetry_client = api_token + .clone() + .map(|token| { + V2Credential::ApiKey(token).client_with_timeout(Self::TRUNK_TELEMETRY_TIMEOUT) + }) + .transpose()?; let mut s3_client_default_headers = HeaderMap::new(); s3_client_default_headers.append( @@ -231,7 +215,6 @@ impl ApiClient { }; Ok(ApiClient { - telemetry_host, api_host, s3_client, trunk_api_client, @@ -443,18 +426,27 @@ impl ApiClient { &self, request: &message::TelemetryUploadMetricsRequest, ) -> anyhow::Result<()> { + let Some(telemetry_client) = &self.telemetry_client else { + return Ok(()); + }; + let environment = if self.api_host == DEFAULT_ORIGIN { + "production" + } else { + "staging" + }; CallApi { action: || async { if std::env::var("DISABLE_TELEMETRY").is_ok() { return Ok(()); } - let response = self - .telemetry_client - .post(format!( - "{}{}/flakytests-cli/upload-metrics", - self.telemetry_host, self.version_path_prefix + let now = chrono::Utc::now(); + let response = telemetry_client + .post(format!("{}{}", self.api_host, crate::otlp::CLI_TELEMETRY_METRICS_PATH)) + .json(&crate::otlp::upload_metrics_request( + &request.upload_metrics, + environment, + crate::otlp::unix_nanos(now.timestamp(), now.timestamp_subsec_nanos() as i32), )) - .body(prost::Message::encode_to_vec(&request.upload_metrics)) .send() .await?; diff --git a/api/src/lib.rs b/api/src/lib.rs index 314be8b3..5f7a72f5 100644 --- a/api/src/lib.rs +++ b/api/src/lib.rs @@ -1,5 +1,6 @@ mod call_api; pub mod client; pub mod message; +pub mod otlp; pub mod quarantine; pub mod urls; diff --git a/api/src/otlp.rs b/api/src/otlp.rs new file mode 100644 index 00000000..534b7b51 --- /dev/null +++ b/api/src/otlp.rs @@ -0,0 +1,380 @@ +//! The upload's own telemetry as OTLP JSON metrics, for public-api's CLI telemetry metrics route +//! (`/v2/telemetry/v1/metrics`, TRUNK-23773). The route keeps only the metric and attribute names +//! it lists for `trunk-analytics-cli` in trunk2's `ts/apps/public-api/src/products/telemetry` and +//! stamps the org itself, so a name added here is dropped until it is added there too. Names, +//! labels and buckets match trunk1's telemetry service so its dashboards carry over. + +use proto::upload_metrics::trunk::{QuarantineQueryResult, UploadMetrics}; +use serde_json::{Value, json}; + +pub const CLI_TELEMETRY_METRICS_PATH: &str = "/v2/telemetry/v1/metrics"; +pub const SERVICE_NAME: &str = "trunk-analytics-cli"; + +pub const UPLOADS_METRIC: &str = "trunk.analytics_cli.uploads"; +pub const UPLOAD_DURATION_METRIC: &str = "trunk.analytics_cli.upload.duration"; +pub const RUN_DURATION_METRIC: &str = "trunk.analytics_cli.run.duration"; +pub const UPLOADS_UNIT: &str = "{upload}"; +pub const DURATION_UNIT: &str = "s"; + +pub const STATUS_ATTRIBUTE: &str = "trunk.analytics_cli.upload.status"; +pub const FAILURE_REASON_ATTRIBUTE: &str = "trunk.analytics_cli.upload.failure_reason"; +pub const QUARANTINE_QUERY_RESULT_ATTRIBUTE: &str = "trunk.analytics_cli.quarantine.query_result"; +pub const REPOSITORY_OWNER_ATTRIBUTE: &str = "trunk.vcs.repository.owner"; +pub const REPOSITORY_NAME_ATTRIBUTE: &str = "trunk.vcs.repository.name"; + +pub const STATUS_SUCCESS: &str = "success"; +pub const STATUS_FAILED: &str = "failed"; +pub const UNKNOWN_LABEL: &str = "unknown"; +pub const INVALID_LABEL: &str = "invalid"; + +/// trunk1's `DURATION_BUCKETS_SECONDS`. +pub const DURATION_BOUNDS_SECONDS: [f64; 13] = [ + 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0, 600.0, +]; + +/// OTLP's `AGGREGATION_TEMPORALITY_DELTA`. +const DELTA: i32 = 1; + +const LABEL_MAX_CHARS: usize = 32; +/// GitHub's own limits. +const OWNER_MAX_CHARS: usize = 39; +const NAME_MAX_CHARS: usize = 100; + +/// OTLP JSON writes 64-bit integers as decimal strings. +pub fn upload_metrics_request( + metrics: &UploadMetrics, + environment: &str, + now_unix_nanos: i128, +) -> Value { + let version = metrics + .client_version + .as_ref() + .map(|version| { + let base = format!("{}.{}.{}", version.major, version.minor, version.patch); + if version.suffix.is_empty() { + base + } else { + format!("{base}-{}", version.suffix) + } + }) + .unwrap_or_default(); + let resource_attributes = [ + string_attribute("service.name", SERVICE_NAME), + string_attribute("service.version", &version), + string_attribute("deployment.environment.name", environment), + string_attribute("os.type", os_type(std::env::consts::OS)), + string_attribute("host.arch", host_arch(std::env::consts::ARCH)), + ]; + + let status = bounded_label( + if metrics.failed { + STATUS_FAILED + } else { + STATUS_SUCCESS + }, + LABEL_MAX_CHARS, + ); + let repo = metrics.repo.clone().unwrap_or_default(); + let quarantine_query_result = + QuarantineQueryResult::try_from(metrics.quarantine_query_result).unwrap_or_default(); + let upload_attributes = [ + string_attribute(STATUS_ATTRIBUTE, &status), + string_attribute( + FAILURE_REASON_ATTRIBUTE, + &bounded_label(&metrics.failure_reason, LABEL_MAX_CHARS), + ), + string_attribute( + QUARANTINE_QUERY_RESULT_ATTRIBUTE, + &bounded_label( + quarantine_query_result_label(quarantine_query_result), + LABEL_MAX_CHARS, + ), + ), + string_attribute( + REPOSITORY_OWNER_ATTRIBUTE, + &bounded_label(&repo.owner, OWNER_MAX_CHARS), + ), + string_attribute( + REPOSITORY_NAME_ATTRIBUTE, + &bounded_label(&repo.name, NAME_MAX_CHARS), + ), + ]; + + let [cli_started, upload_started, upload_finished] = [ + &metrics.cli_started_at, + &metrics.upload_started_at, + &metrics.upload_finished_at, + ] + .map(|time| { + time.as_ref() + .map(|time| unix_nanos(time.seconds, time.nanos)) + }); + let start_time = cli_started.unwrap_or(now_unix_nanos).to_string(); + let time = now_unix_nanos.to_string(); + + let mut otlp_metrics = vec![json!({ + "name": UPLOADS_METRIC, + "unit": UPLOADS_UNIT, + "sum": { + "aggregationTemporality": DELTA, + "isMonotonic": true, + "dataPoints": [{ + "attributes": upload_attributes, + "startTimeUnixNano": start_time, + "timeUnixNano": time, + "asInt": "1", + }], + }, + })]; + for (name, started) in [ + (UPLOAD_DURATION_METRIC, upload_started), + (RUN_DURATION_METRIC, cli_started), + ] { + let Some(seconds) = duration_seconds(started, upload_finished) else { + continue; + }; + let mut bucket_counts = vec!["0"; DURATION_BOUNDS_SECONDS.len() + 1]; + bucket_counts[bucket_index(seconds, &DURATION_BOUNDS_SECONDS)] = "1"; + otlp_metrics.push(json!({ + "name": name, + "unit": DURATION_UNIT, + "histogram": { + "aggregationTemporality": DELTA, + "dataPoints": [{ + "attributes": [string_attribute(STATUS_ATTRIBUTE, &status)], + "startTimeUnixNano": start_time, + "timeUnixNano": time, + "count": "1", + "sum": seconds, + "min": seconds, + "max": seconds, + "explicitBounds": DURATION_BOUNDS_SECONDS, + "bucketCounts": bucket_counts, + }], + }, + })); + } + + json!({ + "resourceMetrics": [{ + "resource": { "attributes": resource_attributes }, + "scopeMetrics": [{ + "scope": { "name": SERVICE_NAME, "version": version }, + "metrics": otlp_metrics, + }], + }], + }) +} + +/// trunk1's `quarantineQueryResultToLabel`. +pub fn quarantine_query_result_label(result: QuarantineQueryResult) -> &'static str { + match result { + QuarantineQueryResult::Unspecified => "unspecified", + QuarantineQueryResult::Success => "success", + QuarantineQueryResult::Failure => "failure", + QuarantineQueryResult::Disabled => "disabled", + QuarantineQueryResult::Skipped => "skipped", + QuarantineQueryResult::Cached => "cached", + } +} + +/// trunk1's `boundedLabel`: the first `max_chars` characters, or `invalid` when they leave the +/// label charset. An empty value is `unknown`. +pub fn bounded_label(value: &str, max_chars: usize) -> String { + if value.is_empty() { + return UNKNOWN_LABEL.into(); + } + let trimmed: String = value.chars().take(max_chars).collect(); + if trimmed + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '+' | '/' | '-')) + { + trimmed + } else { + INVALID_LABEL.into() + } +} + +/// OTLP bucket `i` counts values in `(bounds[i - 1], bounds[i]]`; the last one counts values +/// above every bound. +pub fn bucket_index(value: f64, bounds: &[f64]) -> usize { + bounds + .iter() + .position(|bound| value <= *bound) + .unwrap_or(bounds.len()) +} + +fn duration_seconds(started: Option, finished: Option) -> Option { + let (started, finished) = (started?, finished?); + if started < 0 || finished < started { + return None; + } + Some((finished - started) as f64 / 1e9) +} + +pub fn unix_nanos(seconds: i64, nanos: i32) -> i128 { + i128::from(seconds) * 1_000_000_000 + i128::from(nanos) +} + +fn string_attribute(key: &str, value: &str) -> Value { + json!({ "key": key, "value": { "stringValue": value } }) +} + +fn os_type(os: &'static str) -> &'static str { + match os { + "macos" => "darwin", + other => other, + } +} + +/// OpenTelemetry semantic-convention names, as the trunk CLI sends them. +fn host_arch(arch: &'static str) -> &'static str { + match arch { + "x86_64" => "amd64", + "aarch64" => "arm64", + other => other, + } +} + +#[cfg(test)] +mod tests { + use proto::upload_metrics::trunk::Repo; + + use super::*; + + const SECOND: i64 = 1_000_000_000; + + #[test] + fn a_value_on_a_bound_lands_in_that_bound_s_bucket() { + let bounds = DURATION_BOUNDS_SECONDS; + assert_eq!(bucket_index(0.0, &bounds), 0); + assert_eq!(bucket_index(0.05, &bounds), 0); + assert_eq!(bucket_index(0.050_001, &bounds), 1); + assert_eq!(bucket_index(1.0, &bounds), 4); + assert_eq!(bucket_index(1.000_001, &bounds), 5); + assert_eq!(bucket_index(600.0, &bounds), 12); + assert_eq!(bucket_index(600.000_001, &bounds), 13); + } + + #[test] + fn labels_are_bounded_like_trunk1() { + assert_eq!(bounded_label("", 32), "unknown"); + assert_eq!(bounded_label("400_bad_request", 32), "400_bad_request"); + assert_eq!(bounded_label("a.b_c+d/e-F9", 32), "a.b_c+d/e-F9"); + assert_eq!(bounded_label("has space", 32), "invalid"); + assert_eq!(bounded_label("émoji", 32), "invalid"); + assert_eq!(bounded_label(&"a".repeat(33), 32), "a".repeat(32)); + assert_eq!( + bounded_label(&format!("{} x", "a".repeat(32)), 32), + "a".repeat(32) + ); + assert_eq!(bounded_label(&"o".repeat(40), 39), "o".repeat(39)); + } + + #[test] + fn missing_or_backwards_durations_are_skipped() { + assert_eq!(duration_seconds(None, Some(1)), None); + assert_eq!(duration_seconds(Some(1), None), None); + assert_eq!(duration_seconds(Some(-1), Some(1)), None); + assert_eq!(duration_seconds(Some(5), Some(1)), None); + assert_eq!(duration_seconds(Some(0), Some(2_500_000_000)), Some(2.5)); + } + + #[test] + fn a_failed_upload_sends_a_count_and_both_durations() { + let now: i64 = 1_700_000_100 * SECOND; + let metrics = UploadMetrics { + repo: Some(Repo { + host: "github.com".into(), + owner: "my-org".into(), + name: "my repo".into(), + }), + cli_started_at: Some(chrono::DateTime::from_timestamp_nanos(now - 12 * SECOND).into()), + upload_started_at: Some(chrono::DateTime::from_timestamp_nanos(now - SECOND).into()), + upload_finished_at: Some(chrono::DateTime::from_timestamp_nanos(now).into()), + failed: true, + failure_reason: "connection".into(), + quarantine_query_result: QuarantineQueryResult::Cached.into(), + ..UploadMetrics::default() + }; + let body = upload_metrics_request(&metrics, "staging", now.into()); + let metrics = body["resourceMetrics"][0]["scopeMetrics"][0]["metrics"] + .as_array() + .unwrap(); + assert_eq!(metrics.len(), 3); + + let uploads = &metrics[0]; + assert_eq!(uploads["name"], UPLOADS_METRIC); + assert_eq!(uploads["unit"], UPLOADS_UNIT); + assert_eq!(uploads["sum"]["aggregationTemporality"], 1); + assert_eq!(uploads["sum"]["isMonotonic"], true); + let point = &uploads["sum"]["dataPoints"][0]; + assert_eq!(point["asInt"], "1"); + assert_eq!(point["timeUnixNano"], now.to_string()); + assert_eq!(point["startTimeUnixNano"], (now - 12 * SECOND).to_string()); + let attributes: Vec<(&str, &str)> = point["attributes"] + .as_array() + .unwrap() + .iter() + .map(|a| { + ( + a["key"].as_str().unwrap(), + a["value"]["stringValue"].as_str().unwrap(), + ) + }) + .collect(); + assert_eq!( + attributes, + [ + (STATUS_ATTRIBUTE, "failed"), + (FAILURE_REASON_ATTRIBUTE, "connection"), + (QUARANTINE_QUERY_RESULT_ATTRIBUTE, "cached"), + (REPOSITORY_OWNER_ATTRIBUTE, "my-org"), + (REPOSITORY_NAME_ATTRIBUTE, "invalid"), + ] + ); + + for (metric, name, seconds, bucket) in [ + (&metrics[1], UPLOAD_DURATION_METRIC, 1.0, 4), + (&metrics[2], RUN_DURATION_METRIC, 12.0, 8), + ] { + assert_eq!(metric["name"], name); + assert_eq!(metric["unit"], DURATION_UNIT); + assert_eq!(metric["histogram"]["aggregationTemporality"], 1); + let point = &metric["histogram"]["dataPoints"][0]; + assert_eq!(point["count"], "1"); + assert_eq!(point["sum"], seconds); + assert_eq!(point["min"], seconds); + assert_eq!(point["max"], seconds); + let counts = point["bucketCounts"].as_array().unwrap(); + assert_eq!(counts.len(), DURATION_BOUNDS_SECONDS.len() + 1); + for (index, count) in counts.iter().enumerate() { + assert_eq!(count, if index == bucket { "1" } else { "0" }); + } + } + } + + #[test] + fn a_duration_without_its_start_is_not_sent() { + let metrics = UploadMetrics { + upload_finished_at: Some(chrono::DateTime::from_timestamp_nanos(5).into()), + ..UploadMetrics::default() + }; + let body = upload_metrics_request(&metrics, "production", 10); + let metrics = body["resourceMetrics"][0]["scopeMetrics"][0]["metrics"] + .as_array() + .unwrap(); + assert_eq!(metrics.len(), 1); + assert_eq!(metrics[0]["name"], UPLOADS_METRIC); + let values: Vec<&str> = metrics[0]["sum"]["dataPoints"][0]["attributes"] + .as_array() + .unwrap() + .iter() + .map(|a| a["value"]["stringValue"].as_str().unwrap()) + .collect(); + assert_eq!( + values, + ["success", "unknown", "unspecified", "unknown", "unknown"] + ); + } +} diff --git a/cli/tests/upload.rs b/cli/tests/upload.rs index 09bb297d..0276c4fb 100644 --- a/cli/tests/upload.rs +++ b/cli/tests/upload.rs @@ -7,6 +7,7 @@ use api::message::{ CreateBundleUploadRequest, CreateBundleUploadResponse, GetQuarantineConfigRequest, GetQuarantineConfigResponse, }; +use api::otlp::DURATION_BOUNDS_SECONDS; use assert_cmd::{Command, cargo}; use assert_matches::assert_matches; use axum::body::{Body, Bytes}; @@ -49,7 +50,6 @@ use lazy_static::lazy_static; use predicates::prelude::*; use pretty_assertions::assert_eq; use prost::Message; -use proto::upload_metrics::trunk::QuarantineQueryResult; use tempfile::tempdir; #[cfg(target_os = "macos")] use test_utils::inputs::unpack_archive_to_dir; @@ -57,7 +57,7 @@ use test_utils::{ inputs::get_test_file_path, mock_server::{ MOCK_REPO_ID, MOCK_TEST_COLLECTION_ID, MockServerBuilder, RequestPayload, - SharedMockServerState, + SharedMockServerState, TelemetryUploadMetrics, }, }; use trunk_analytics_cli::exit_code::ExitCode; @@ -602,18 +602,6 @@ async fn upload_bundle_records_quarantine_resolution_mode() { QuarantineResolutionMode::TestCollection, "the resolution mode must be persisted in meta.json" ); - - let upload_metrics = requests - .iter() - .find_map(|request| match request { - RequestPayload::TelemetryUploadMetrics(metrics) => Some(metrics), - _ => None, - }) - .expect("telemetry must have been sent"); - assert_eq!( - upload_metrics.quarantine_resolution_mode, - i32::from(proto::upload_metrics::trunk::QuarantineResolutionMode::TestCollection) - ); } #[tokio::test(flavor = "multi_thread")] @@ -2115,6 +2103,19 @@ async fn is_not_ok_on_bad_request() { .stdout(predicate::str::contains("error")); } +fn assert_durations_are_bucketed(metrics: &TelemetryUploadMetrics) { + for duration in [&metrics.upload_duration, &metrics.run_duration] { + let duration = duration.as_ref().expect("both durations must be sent"); + assert_eq!(duration.status, metrics.status); + assert_eq!(duration.explicit_bounds, DURATION_BOUNDS_SECONDS); + assert!(duration.is_in_its_bucket(), "{duration:?}"); + } + assert!( + metrics.run_duration.as_ref().unwrap().seconds + >= metrics.upload_duration.as_ref().unwrap().seconds + ); +} + #[tokio::test(flavor = "multi_thread")] async fn telemetry_upload_metrics_on_upload_failure() { let temp_dir = tempdir().unwrap(); @@ -2140,16 +2141,12 @@ async fn telemetry_upload_metrics_on_upload_failure() { let telemetry_request = assert_matches!(requests.last().unwrap(), RequestPayload::TelemetryUploadMetrics(ur) => ur); - let telemetry_request_repo = telemetry_request.repo.clone().unwrap(); - assert!(telemetry_request.failed); - assert_eq!(telemetry_request_repo.host, "github.com"); - assert_eq!(telemetry_request_repo.owner, "trunk-io"); - assert_eq!(telemetry_request_repo.name, "analytics-cli"); + assert_eq!(telemetry_request.status, "failed"); + assert_eq!(telemetry_request.repo_owner, "trunk-io"); + assert_eq!(telemetry_request.repo_name, "analytics-cli"); assert_eq!(telemetry_request.failure_reason, "400_bad_request"); - assert_eq!( - QuarantineQueryResult::try_from(telemetry_request.quarantine_query_result).unwrap(), - QuarantineQueryResult::Disabled - ); + assert_eq!(telemetry_request.quarantine_query_result, "disabled"); + assert_durations_are_bucketed(telemetry_request); // HINT: View CLI output with `cargo test -- --nocapture` println!("{assert}"); @@ -2175,16 +2172,12 @@ async fn telemetry_upload_metrics_on_upload_success() { let telemetry_request = assert_matches!(requests.last().unwrap(), RequestPayload::TelemetryUploadMetrics(ur) => ur); - let telemetry_request_repo = telemetry_request.repo.clone().unwrap(); - assert!(!telemetry_request.failed); - assert_eq!(telemetry_request_repo.host, "github.com"); - assert_eq!(telemetry_request_repo.owner, "trunk-io"); - assert_eq!(telemetry_request_repo.name, "analytics-cli"); - assert_eq!(telemetry_request.failure_reason, ""); - assert_eq!( - QuarantineQueryResult::try_from(telemetry_request.quarantine_query_result).unwrap(), - QuarantineQueryResult::Disabled - ); + assert_eq!(telemetry_request.status, "success"); + assert_eq!(telemetry_request.repo_owner, "trunk-io"); + assert_eq!(telemetry_request.repo_name, "analytics-cli"); + assert_eq!(telemetry_request.failure_reason, "unknown"); + assert_eq!(telemetry_request.quarantine_query_result, "disabled"); + assert_durations_are_bucketed(telemetry_request); // HINT: View CLI output with `cargo test -- --nocapture` println!("{assert}"); @@ -2208,14 +2201,11 @@ async fn telemetry_upload_metrics_quarantine_query_result_success() { .iter() .rev() .find_map(|req| match req { - RequestPayload::TelemetryUploadMetrics(ur) => Some(ur.quarantine_query_result), + RequestPayload::TelemetryUploadMetrics(ur) => Some(ur.quarantine_query_result.clone()), _ => None, }) .expect("telemetry upload metrics request"); - assert_eq!( - QuarantineQueryResult::try_from(quarantine_query_result).unwrap(), - QuarantineQueryResult::Success - ); + assert_eq!(quarantine_query_result, "success"); } #[tokio::test(flavor = "multi_thread")] @@ -2242,14 +2232,11 @@ async fn telemetry_upload_metrics_quarantine_query_result_failure() { .iter() .rev() .find_map(|req| match req { - RequestPayload::TelemetryUploadMetrics(ur) => Some(ur.quarantine_query_result), + RequestPayload::TelemetryUploadMetrics(ur) => Some(ur.quarantine_query_result.clone()), _ => None, }) .expect("telemetry upload metrics request"); - assert_eq!( - QuarantineQueryResult::try_from(quarantine_query_result).unwrap(), - QuarantineQueryResult::Failure - ); + assert_eq!(quarantine_query_result, "failure"); } #[tokio::test(flavor = "multi_thread")] @@ -2291,14 +2278,11 @@ async fn telemetry_upload_metrics_quarantine_query_result_skipped() { .iter() .rev() .find_map(|req| match req { - RequestPayload::TelemetryUploadMetrics(ur) => Some(ur.quarantine_query_result), + RequestPayload::TelemetryUploadMetrics(ur) => Some(ur.quarantine_query_result.clone()), _ => None, }) .expect("telemetry upload metrics request"); - assert_eq!( - QuarantineQueryResult::try_from(quarantine_query_result).unwrap(), - QuarantineQueryResult::Skipped - ); + assert_eq!(quarantine_query_result, "skipped"); } #[tokio::test(flavor = "multi_thread")] diff --git a/test_report/tests/telemetry.rs b/test_report/tests/telemetry.rs index 1b97a651..7299f511 100644 --- a/test_report/tests/telemetry.rs +++ b/test_report/tests/telemetry.rs @@ -12,29 +12,22 @@ use constants::{ TRUNK_REPO_HEAD_BRANCH_ENV, TRUNK_REPO_HEAD_SHA_ENV, TRUNK_REPO_URL_ENV, TRUNK_USE_UNCLONED_REPO_ENV, }; -use proto::upload_metrics::trunk::QuarantineQueryResult; use serial_test::serial; use tempfile::tempdir; use test_report::report::{MutTestReport, Status}; use test_utils::mock_git_repo::setup_repo_with_commit; use test_utils::mock_server::{MockServerBuilder, RequestPayload, SharedMockServerState}; -fn assert_telemetry_quarantine_query_result( - requests: &[RequestPayload], - expected: QuarantineQueryResult, -) { +fn assert_telemetry_quarantine_query_result(requests: &[RequestPayload], expected: &str) { let quarantine_query_result = requests .iter() .rev() .find_map(|req| match req { - RequestPayload::TelemetryUploadMetrics(ur) => Some(ur.quarantine_query_result), + RequestPayload::TelemetryUploadMetrics(ur) => Some(ur.quarantine_query_result.clone()), _ => None, }) .expect("expected telemetry upload metrics request"); - assert_eq!( - QuarantineQueryResult::try_from(quarantine_query_result).unwrap(), - expected - ); + assert_eq!(quarantine_query_result, expected); } fn set_mock_api_env(api_host: &str) { @@ -110,7 +103,7 @@ async fn telemetry_query_result_success_on_publish() { .unwrap(); let requests = state.requests.lock().unwrap().clone(); - assert_telemetry_quarantine_query_result(&requests, QuarantineQueryResult::Success); + assert_telemetry_quarantine_query_result(&requests, "success"); } #[tokio::test(flavor = "multi_thread")] @@ -133,7 +126,7 @@ async fn telemetry_query_result_skipped_without_lookup_on_publish() { .unwrap(); let requests = state.requests.lock().unwrap().clone(); - assert_telemetry_quarantine_query_result(&requests, QuarantineQueryResult::Skipped); + assert_telemetry_quarantine_query_result(&requests, "skipped"); } #[tokio::test(flavor = "multi_thread")] @@ -185,7 +178,7 @@ async fn telemetry_query_result_cached_on_publish() { "second report should use disk cache during suite" ); // Checks the last telemetry call - assert_telemetry_quarantine_query_result(&requests, QuarantineQueryResult::Cached); + assert_telemetry_quarantine_query_result(&requests, "cached"); } #[tokio::test(flavor = "multi_thread")] @@ -233,7 +226,7 @@ async fn telemetry_query_result_disabled_on_publish() { .unwrap(); let requests = state.requests.lock().unwrap().clone(); - assert_telemetry_quarantine_query_result(&requests, QuarantineQueryResult::Disabled); + assert_telemetry_quarantine_query_result(&requests, "disabled"); } #[tokio::test(flavor = "multi_thread")] @@ -261,7 +254,7 @@ async fn telemetry_query_result_disabled_by_env_on_publish() { .unwrap(); let requests = state.requests.lock().unwrap().clone(); - assert_telemetry_quarantine_query_result(&requests, QuarantineQueryResult::Disabled); + assert_telemetry_quarantine_query_result(&requests, "disabled"); } #[tokio::test(flavor = "multi_thread")] @@ -310,5 +303,5 @@ async fn telemetry_query_result_failure_on_publish() { .unwrap(); let requests = state.requests.lock().unwrap().clone(); - assert_telemetry_quarantine_query_result(&requests, QuarantineQueryResult::Failure); + assert_telemetry_quarantine_query_result(&requests, "failure"); } diff --git a/test_utils/src/mock_server.rs b/test_utils/src/mock_server.rs index 680c94fe..e626a390 100644 --- a/test_utils/src/mock_server.rs +++ b/test_utils/src/mock_server.rs @@ -19,8 +19,6 @@ use axum::{ response::{IntoResponse, Response}, routing::{MethodRouter, any, post, put}, }; -use prost::Message; -use proto::upload_metrics::trunk::UploadMetrics; use tempfile::tempdir; use tokio::{net::TcpListener, spawn}; @@ -32,7 +30,46 @@ pub enum RequestPayload { CreateBundleUpload(CreateBundleUploadRequest), GetQuarantineConfig(GetQuarantineConfigRequest), S3Upload(PathBuf), - TelemetryUploadMetrics(UploadMetrics), + TelemetryUploadMetrics(TelemetryUploadMetrics), +} + +/// What one OTLP metrics request from the CLI says, read back from the wire. +#[derive(Debug, Clone, PartialEq)] +pub struct TelemetryUploadMetrics { + pub status: String, + pub failure_reason: String, + pub quarantine_query_result: String, + pub repo_owner: String, + pub repo_name: String, + pub upload_duration: Option, + pub run_duration: Option, +} + +/// One duration histogram's single observation. +#[derive(Debug, Clone, PartialEq)] +pub struct DurationObservation { + pub status: String, + pub seconds: f64, + /// The index of the one bucket counting it. + pub bucket: usize, + pub explicit_bounds: Vec, +} + +impl DurationObservation { + /// Whether `bucket` is the OTLP bucket for `seconds`: bucket `i` counts values in + /// `(bounds[i - 1], bounds[i]]`. + pub fn is_in_its_bucket(&self) -> bool { + let lower = self + .bucket + .checked_sub(1) + .map_or(f64::NEG_INFINITY, |index| self.explicit_bounds[index]); + let upper = self + .explicit_bounds + .get(self.bucket) + .copied() + .unwrap_or(f64::INFINITY); + lower < self.seconds && self.seconds <= upper + } } pub struct RespError { @@ -167,7 +204,7 @@ impl MockServerBuilder { ) .route("/s3upload", self.s3_upload_handler) .route( - "/v1/flakytests-cli/upload-metrics", + api::otlp::CLI_TELEMETRY_METRICS_PATH, self.telemetry_upload_metrics, ); @@ -266,8 +303,10 @@ pub async fn telemetry_upload_metrics_handler( State(state): State, bytes: Bytes, ) -> Response { - let upload_metrics = proto::upload_metrics::trunk::UploadMetrics::decode(bytes); - if let Ok(upload_metrics) = upload_metrics { + let upload_metrics = serde_json::from_slice::(&bytes) + .ok() + .and_then(|request| telemetry_upload_metrics_from_otlp(&request)); + if let Some(upload_metrics) = upload_metrics { state .requests .lock() @@ -304,3 +343,95 @@ pub async fn s3_upload_handler( .push(RequestPayload::S3Upload(tar_extract_directory.into_path())); Response::new(String::from("OK")) } + +/// The CLI's OTLP metrics request read back, so the tests check what reaches the wire. `None` +/// unless the body is one `trunk-analytics-cli` resource whose metrics have the names, types, +/// units and temporality `api::otlp` promises, each with one data point. +pub fn telemetry_upload_metrics_from_otlp( + request: &serde_json::Value, +) -> Option { + use api::otlp; + use serde_json::Value; + + let attribute = |list: &Value, key: &str| -> Option { + list.as_array()? + .iter() + .find(|attribute| attribute["key"] == key)?["value"]["stringValue"] + .as_str() + .map(str::to_owned) + }; + let resource_metrics = request["resourceMetrics"].as_array()?; + let [resource_metrics] = resource_metrics.as_slice() else { + return None; + }; + if attribute(&resource_metrics["resource"]["attributes"], "service.name")? != otlp::SERVICE_NAME + { + return None; + } + let metrics = resource_metrics["scopeMetrics"][0]["metrics"].as_array()?; + let metric = |name: &str| metrics.iter().find(|metric| metric["name"] == name); + let single_point = |data: &Value| -> Option { + if data["aggregationTemporality"] != 1 { + return None; + } + match data["dataPoints"].as_array()?.as_slice() { + [point] => Some(point.clone()), + _ => None, + } + }; + + let uploads = metric(otlp::UPLOADS_METRIC)?; + if uploads["unit"] != otlp::UPLOADS_UNIT || uploads["sum"]["isMonotonic"] != true { + return None; + } + let point = single_point(&uploads["sum"])?; + if point["asInt"] != "1" { + return None; + } + let labels = &point["attributes"]; + + let duration = |name: &str| -> Option> { + let Some(histogram) = metric(name) else { + return Some(None); + }; + if histogram["unit"] != otlp::DURATION_UNIT { + return None; + } + let point = single_point(&histogram["histogram"])?; + let seconds = point["sum"].as_f64()?; + if point["count"] != "1" + || point["min"].as_f64()? != seconds + || point["max"].as_f64()? != seconds + { + return None; + } + let explicit_bounds = point["explicitBounds"] + .as_array()? + .iter() + .map(Value::as_f64) + .collect::>>()?; + let counts = point["bucketCounts"].as_array()?; + if counts.len() != explicit_bounds.len() + 1 + || counts.iter().filter(|count| *count == "1").count() != 1 + || counts.iter().any(|count| *count != "0" && *count != "1") + { + return None; + } + Some(Some(DurationObservation { + status: attribute(&point["attributes"], otlp::STATUS_ATTRIBUTE)?, + seconds, + bucket: counts.iter().position(|count| count == "1")?, + explicit_bounds, + })) + }; + + Some(TelemetryUploadMetrics { + status: attribute(labels, otlp::STATUS_ATTRIBUTE)?, + failure_reason: attribute(labels, otlp::FAILURE_REASON_ATTRIBUTE)?, + quarantine_query_result: attribute(labels, otlp::QUARANTINE_QUERY_RESULT_ATTRIBUTE)?, + repo_owner: attribute(labels, otlp::REPOSITORY_OWNER_ATTRIBUTE)?, + repo_name: attribute(labels, otlp::REPOSITORY_NAME_ATTRIBUTE)?, + upload_duration: duration(otlp::UPLOAD_DURATION_METRIC)?, + run_duration: duration(otlp::RUN_DURATION_METRIC)?, + }) +}