diff --git a/.sqlx/query-7750b48b515319bd59f2059030516d266695be9468841cbd89814bc604120668.json b/.sqlx/query-7750b48b515319bd59f2059030516d266695be9468841cbd89814bc604120668.json new file mode 100644 index 0000000..03a529d --- /dev/null +++ b/.sqlx/query-7750b48b515319bd59f2059030516d266695be9468841cbd89814bc604120668.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT variant, kind, hour, count\n FROM experiment_counters\n WHERE experiment_id = $1 AND hour >= $2\n ORDER BY variant, hour\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "variant", + "type_info": "Text", + "origin": { + "Table": { + "table": "experiment_counters", + "name": "variant" + } + } + }, + { + "ordinal": 1, + "name": "kind", + "type_info": "Text", + "origin": { + "Table": { + "table": "experiment_counters", + "name": "kind" + } + } + }, + { + "ordinal": 2, + "name": "hour", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "experiment_counters", + "name": "hour" + } + } + }, + { + "ordinal": 3, + "name": "count", + "type_info": "Int8", + "origin": { + "Table": { + "table": "experiment_counters", + "name": "count" + } + } + } + ], + "parameters": { + "Left": [ + "Uuid", + "Timestamptz" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "7750b48b515319bd59f2059030516d266695be9468841cbd89814bc604120668" +} diff --git a/README.md b/README.md index cf9efc1..b6c714f 100644 --- a/README.md +++ b/README.md @@ -284,8 +284,11 @@ An experiment binds a flag to a conversion metric; the flag's variants become the arms and the traffic you already serve becomes the sample. Each arm gets a rate, a 95 % Wilson interval drawn on a shared scale, and a two-proportion z-test against the control — in words, with the exact numbers printed beside -every bar. The lifecycle is one-way (draft → running → stopped) because a -reopened measurement window would average two populations into an answer about +every bar. Above the table, the trailing week as hourly conversion-rate lines, +read straight off the pre-aggregated cells: hours with no traffic break the +line rather than plotting 0 %, because those are different facts. The +lifecycle is one-way (draft → running → stopped) because a reopened +measurement window would average two populations into an answer about neither, and a stopped experiment's results stay exactly as they ended. ### The rest of it diff --git a/crates/api/src/openapi.rs b/crates/api/src/openapi.rs index da98d60..d949045 100644 --- a/crates/api/src/openapi.rs +++ b/crates/api/src/openapi.rs @@ -107,6 +107,8 @@ use crate::routes::{ flagforge_storage::models::Experiment, flagforge_storage::models::ExperimentState, flagforge_storage::models::CounterKind, + flagforge_storage::models::VariantSeries, + flagforge_storage::models::SeriesPoint, flagforge_core::ExperimentSpec, flagforge_core::VariantCounts, flagforge_core::VariantResult, diff --git a/crates/api/src/routes/experiments.rs b/crates/api/src/routes/experiments.rs index 089d6f5..b7bd8e4 100644 --- a/crates/api/src/routes/experiments.rs +++ b/crates/api/src/routes/experiments.rs @@ -44,6 +44,11 @@ pub struct UpdateExperiment { pub description: Option>, } +/// How far back the results time series reaches. A week of hourly points is +/// 168 per arm — small on the wire — and the totals above the series already +/// cover all time, so a longer window would add bytes, not information. +const SERIES_WINDOW: chrono::TimeDelta = chrono::TimeDelta::days(7); + /// An experiment next to its judged results, one entry per variant. #[derive(Debug, Serialize, utoipa::ToSchema)] pub struct ExperimentResults { @@ -53,6 +58,10 @@ pub struct ExperimentResults { /// appears in the counters but not on the flag — possible after a variant /// rename — is appended rather than hidden. pub results: Vec, + /// Hourly tallies over the trailing week, same variant order as `results`. + /// Hours nobody recorded are absent, not zero: the difference between "no + /// traffic" and "0 % converting" matters to a chart. + pub series: Vec, } #[utoipa::path( @@ -391,11 +400,35 @@ pub async fn results( let experiment = experiments::find_experiment(&state.pool, environment.id, &experiment_key).await?; let counts = experiments::experiment_counts(&state.pool, experiment.id).await?; + let since = chrono::Utc::now() - SERIES_WINDOW; + let series = experiments::hourly_counts(&state.pool, experiment.id, since).await?; let judged = flagforge_core::results(&experiment.control_variant, &in_flag_order(&experiment, counts)); - Ok(Json(ExperimentResults { experiment, results: judged })) + Ok(Json(ExperimentResults { + results: judged, + series: series_in_flag_order(&experiment, series), + experiment, + })) +} + +/// The series counterpart of [`in_flag_order`]: flag-variant order first, +/// extras appended. Arms without any point in the window are absent rather +/// than zero-filled — the chart treats no traffic as a gap, not as 0 %. +fn series_in_flag_order( + experiment: &Experiment, + mut series: Vec, +) -> Vec { + let mut ordered = Vec::with_capacity(series.len()); + for variant in &experiment.variants { + if let Some(index) = series.iter().position(|s| s.variant == variant.key) { + ordered.push(series.swap_remove(index)); + } + } + series.sort_by(|a, b| a.variant.cmp(&b.variant)); + ordered.extend(series); + ordered } /// Orders counts by the flag's variant list, zero-filling arms that have no diff --git a/crates/api/src/seed.rs b/crates/api/src/seed.rs index 9f894d3..7356ff7 100644 --- a/crates/api/src/seed.rs +++ b/crates/api/src/seed.rs @@ -278,9 +278,11 @@ pub async fn run( /// operator actually encounters, on the first screen they open. /// /// The counters are deterministic rather than random (seeding must be -/// reproducible), with a small modular wobble so the hourly cells do not all -/// carry the same number: the treatment converts at ~13%, the control at ~9%, -/// enough for the z-test to reach a verdict on a week of data. +/// reproducible), but shaped like traffic rather than like a formula: a daily +/// cosine cycle peaking mid-afternoon, with a small hash-style jitter on both +/// volume and rate so the results chart shows a plausible week, not a +/// sawtooth. The treatment converts at ~13.5%, the control at ~9%, enough for +/// the z-test to reach a verdict on a week of data. async fn seed_experiment( pool: &PgPool, organization_id: Uuid, @@ -333,10 +335,18 @@ async fn seed_experiment( let mut deltas = Vec::new(); for hour in 0..HOURS { let at = now - TimeDelta::hours(hour); - let exposures = 40 + (hour % 9) as u32; - for (variant, conversions) in - [("on", 5 + (hour % 3) as u32), ("off", 3 + (hour % 3) as u32)] - { + // Daily cycle: quiet around 02:00, peaking around 14:00. + let hour_of_day = ((24 + 14 - (hour % 24)) % 24) as f64; + let cycle = 1.0 + (hour_of_day * std::f64::consts::PI / 12.0).cos(); + let exposures = (26.0 + 14.0 * cycle) as u32 + (hour * 37 % 7) as u32; + + for (variant, rate) in [("on", 0.135), ("off", 0.091)] { + // ±1.6 points of deterministic jitter, uncorrelated between + // arms so the two lines do not move in lockstep. + let salt = if variant == "on" { 53 } else { 71 }; + let jitter = ((hour * salt % 9) as f64 - 4.0) * 0.004; + let conversions = (f64::from(exposures) * (rate + jitter)).round() as u32; + deltas.push(CounterDelta { experiment_key: EXPERIMENT_KEY.to_owned(), variant: variant.to_owned(), diff --git a/crates/api/tests/experiments.rs b/crates/api/tests/experiments.rs index 0adc14d..d58a825 100644 --- a/crates/api/tests/experiments.rs +++ b/crates/api/tests/experiments.rs @@ -228,6 +228,64 @@ async fn duplicate_cells_in_one_batch_are_summed_not_a_server_error(pool: PgPool assert_eq!(results["results"][0]["exposures"], 6, "{results}"); } +#[sqlx::test(migrations = "../../migrations")] +async fn the_results_series_buckets_by_hour_and_windows_a_week(pool: PgPool) { + let app = TestApp::new(pool); + let tenant = app.bootstrap("Acme", "ada@acme.test").await; + app.create_flag(&tenant.token, "checkout.v2").await; + + app.post(EXPERIMENTS, Some(&tenant.token), checkout_experiment()) + .await + .expect(StatusCode::CREATED); + app.post(&format!("{EXPERIMENTS}/checkout-cta/start"), Some(&tenant.token), json!({})) + .await + .expect(StatusCode::OK); + + // Two events inside one hour, one in the next, and one far outside the + // window — the first two must merge, the last must not appear at all. + let now = chrono::Utc::now(); + let hour_a = now - chrono::TimeDelta::hours(2); + let hour_a_later = hour_a + chrono::TimeDelta::minutes(20); + let hour_b = now - chrono::TimeDelta::hours(1); + let ancient = now - chrono::TimeDelta::days(30); + let events = json!({ + "events": [ + {"experiment_key": "checkout-cta", "variant": "on", "kind": "exposure", + "count": 3, "at": hour_a}, + {"experiment_key": "checkout-cta", "variant": "on", "kind": "exposure", + "count": 2, "at": hour_a_later}, + {"experiment_key": "checkout-cta", "variant": "on", "kind": "conversion", + "count": 1, "at": hour_b}, + {"experiment_key": "checkout-cta", "variant": "on", "kind": "exposure", + "count": 99, "at": ancient}, + ] + }); + app.post("/api/v1/events", Some(&tenant.sdk_key), events).await.expect(StatusCode::ACCEPTED); + + let body = app + .get(&format!("{EXPERIMENTS}/checkout-cta/results"), Some(&tenant.token)) + .await + .expect(StatusCode::OK); + + let series = body["series"].as_array().expect("series array"); + assert_eq!(series.len(), 1, "only arms with points in the window appear: {body}"); + assert_eq!(series[0]["variant"], "on"); + + let points = series[0]["points"].as_array().expect("points"); + assert_eq!(points.len(), 2, "two hours touched inside the window: {body}"); + assert_eq!(points[0]["exposures"], 5, "same-hour events must merge into one point"); + assert_eq!(points[0]["conversions"], 0); + assert_eq!(points[1]["conversions"], 1); + // Oldest first, and truncated to the hour. + let first = points[0]["hour"].as_str().unwrap(); + let second = points[1]["hour"].as_str().unwrap(); + assert!(first < second, "points must come oldest first: {first} vs {second}"); + assert!(first.contains(":00:00"), "hours must be truncated: {first}"); + + // The totals still count everything, window or not. + assert_eq!(body["results"][0]["exposures"], 104, "{body}"); +} + #[sqlx::test(migrations = "../../migrations")] async fn events_require_an_sdk_key(pool: PgPool) { let app = TestApp::new(pool); diff --git a/crates/storage/src/experiments.rs b/crates/storage/src/experiments.rs index 7bd13f8..ce4a136 100644 --- a/crates/storage/src/experiments.rs +++ b/crates/storage/src/experiments.rs @@ -8,7 +8,9 @@ use sqlx::PgPool; use uuid::Uuid; use crate::error::{FoundExt, Result, StorageError}; -use crate::models::{CounterDelta, CounterKind, Experiment, ExperimentState}; +use crate::models::{ + CounterDelta, CounterKind, Experiment, ExperimentState, SeriesPoint, VariantSeries, +}; /// The columns every experiment read selects: the row itself plus the flag's /// key and variants, which every consumer needs (see [`Experiment`]). @@ -385,6 +387,49 @@ pub async fn experiment_counts(pool: &PgPool, experiment_id: Uuid) -> Result, +) -> Result> { + let rows = sqlx::query!( + r#" + SELECT variant, kind, hour, count + FROM experiment_counters + WHERE experiment_id = $1 AND hour >= $2 + ORDER BY variant, hour + "#, + experiment_id, + since, + ) + .fetch_all(pool) + .await?; + + let mut series: Vec = Vec::new(); + for row in rows { + if series.last().is_none_or(|s| s.variant != row.variant) { + series.push(VariantSeries { variant: row.variant.clone(), points: Vec::new() }); + } + let points = &mut series.last_mut().expect("just ensured").points; + if points.last().is_none_or(|p| p.hour != row.hour) { + points.push(SeriesPoint { hour: row.hour, exposures: 0, conversions: 0 }); + } + let point = points.last_mut().expect("just ensured"); + let count = u64::try_from(row.count).unwrap_or(0); + match row.kind.as_str() { + "exposure" => point.exposures += count, + _ => point.conversions += count, + } + } + Ok(series) +} + /// The running experiments of one environment, for snapshot loading. pub(crate) async fn load_running_experiments( pool: &PgPool, diff --git a/crates/storage/src/models.rs b/crates/storage/src/models.rs index 29940e9..5761673 100644 --- a/crates/storage/src/models.rs +++ b/crates/storage/src/models.rs @@ -268,6 +268,22 @@ impl CounterKind { } } +/// One hour of one variant's tallies, for the results time series. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, ToSchema)] +pub struct SeriesPoint { + pub hour: DateTime, + pub exposures: u64, + pub conversions: u64, +} + +/// One variant's hourly history. Hours nobody recorded are simply absent — +/// the consumer decides whether a gap is a zero or a break in the line. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, ToSchema)] +pub struct VariantSeries { + pub variant: String, + pub points: Vec, +} + /// One increment on its way into `experiment_counters`. #[derive(Debug, Clone, PartialEq, Eq)] pub struct CounterDelta { diff --git a/crates/web/src/api.rs b/crates/web/src/api.rs index 0ba008c..39394c2 100644 --- a/crates/web/src/api.rs +++ b/crates/web/src/api.rs @@ -226,11 +226,29 @@ pub mod models { pub version: i64, } + /// One hour of one variant's tallies. + #[derive(Debug, Clone, Copy, PartialEq, Deserialize)] + pub struct SeriesPoint { + pub hour: chrono::DateTime, + pub exposures: u64, + pub conversions: u64, + } + + /// One variant's hourly history over the server's window. Absent hours are + /// gaps — no traffic — which the chart renders as breaks, not zeroes. + #[derive(Debug, Clone, PartialEq, Deserialize)] + pub struct VariantSeries { + pub variant: String, + pub points: Vec, + } + /// An experiment next to its judged results. #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct ExperimentResults { pub experiment: Experiment, pub results: Vec, + #[serde(default)] + pub series: Vec, } #[derive(Debug, Clone, PartialEq, Deserialize)] diff --git a/crates/web/src/pages/experiments.rs b/crates/web/src/pages/experiments.rs index 79f6cb0..6487ef1 100644 --- a/crates/web/src/pages/experiments.rs +++ b/crates/web/src/pages/experiments.rs @@ -457,6 +457,8 @@ fn ResultsPanel( + {chart(&loaded.series, &position_of)} +
@@ -520,6 +522,219 @@ fn ResultsPanel( } } +/// Chart geometry, in viewBox units. Width scales to the card; the aspect is +/// fixed so stroke widths and markers keep their size. +const CHART_W: f64 = 720.0; +const CHART_H: f64 = 132.0; +const PLOT_TOP: f64 = 10.0; +const PLOT_BOTTOM: f64 = 118.0; + +/// The hourly conversion-rate lines, one per arm, colour-keyed like the +/// meters below. Hours with no exposures break the line rather than plotting +/// 0 % — no traffic is a gap, not a rate — and every marker carries its exact +/// numbers, so the chart is the shape while the figures stay one hover away. +fn chart( + series: &[crate::api::models::VariantSeries], + position_of: &impl Fn(&str) -> usize, +) -> Option { + struct Dot { + x: f64, + y: f64, + title: String, + } + struct Line { + colour: usize, + variant: String, + segments: Vec, + dots: Vec, + } + + /// One plottable hour: exposures were nonzero, so a rate exists. + struct Rated { + hour: chrono::DateTime, + rate: f64, + conversions: u64, + exposures: u64, + } + + let rated: Vec<(&str, Vec)> = series + .iter() + .map(|s| { + let points = s + .points + .iter() + .filter(|p| p.exposures > 0) + .map(|p| Rated { + hour: p.hour, + rate: p.conversions.min(p.exposures) as f64 / p.exposures as f64, + conversions: p.conversions, + exposures: p.exposures, + }) + .collect::>(); + (s.variant.as_str(), points) + }) + .collect(); + + // A line needs two points somewhere; a single lonely hour is the table's + // job, not a chart's. + if rated.iter().map(|(_, p)| p.len()).sum::() < 2 { + return None; + } + + let hours: Vec = + rated.iter().flat_map(|(_, p)| p.iter().map(|r| r.hour.timestamp())).collect(); + let (min_t, max_t) = + (*hours.iter().min().expect("nonempty"), *hours.iter().max().expect("nonempty")); + let span = ((max_t - min_t) as f64).max(3600.0); + + let top_rate = rated.iter().flat_map(|(_, p)| p.iter().map(|r| r.rate)).fold(0.0_f64, f64::max); + let y_scale = ((top_rate / 0.05).ceil() * 0.05).max(0.05); + + let x_of = |t: i64| ((t - min_t) as f64 / span) * (CHART_W - 12.0) + 6.0; + let y_of = |rate: f64| PLOT_BOTTOM - (rate / y_scale) * (PLOT_BOTTOM - PLOT_TOP); + + let lines: Vec = rated + .iter() + .filter(|(_, points)| !points.is_empty()) + .map(|(variant, points)| { + let mut segments = Vec::new(); + let mut current = String::new(); + let mut dots = Vec::new(); + let mut previous: Option = None; + + for point in points { + let t = point.hour.timestamp(); + let (x, y) = (x_of(t), y_of(point.rate)); + // Contiguous hours extend the line; a gap starts a new one. + let joined = previous.is_some_and(|p| t - p == 3600); + if joined { + current.push_str(&format!(" L{x:.1} {y:.1}")); + } else { + if !current.is_empty() { + segments.push(std::mem::take(&mut current)); + } + current = format!("M{x:.1} {y:.1}"); + } + previous = Some(t); + dots.push(Dot { + x, + y, + title: format!( + "{variant} · {}: {:.1} % ({}/{})", + point.hour.format("%b %d %H:00"), + point.rate * 100.0, + point.conversions, + point.exposures + ), + }); + } + if !current.is_empty() { + segments.push(current); + } + Line { colour: position_of(variant), variant: (*variant).to_owned(), segments, dots } + }) + .collect(); + + let legend = lines + .iter() + .map(|line| { + view! { + + + {line.variant.clone()} + + } + }) + .collect_view(); + + Some( + view! { +
+
+ {legend} +
+ + {format!( + "Hourly conversion rate · trailing week · scale 0–{:.0} %", + y_scale * 100.0 + )} + +
+ + {[0.0_f64, 0.5, 1.0] + .into_iter() + .map(|fraction| { + let y = y_of(fraction * y_scale); + view! { + + } + }) + .collect_view()} + {lines + .into_iter() + .map(|line| { + let stroke = format!("var(--bar-{})", line.colour); + view! { + {line + .segments + .into_iter() + .map(|d| { + view! { + + } + }) + .collect_view()} + {line + .dots + .into_iter() + .map(|dot| { + // Invisible, and wider than the line: + // a hover target per hour without 168 + // dots of ink per arm. + view! { + + {dot.title} + + } + }) + .collect_view()} + } + }) + .collect_view()} + +
+ } + .into_any(), + ) +} + #[component] fn ArmRow(arm: VariantResult, is_control: bool, position: usize, scale: f64) -> impl IntoView { let rate_text = arm.rate.map(|rate| format!("{:.1} %", rate * 100.0)); diff --git a/docs/screenshots/experiments.png b/docs/screenshots/experiments.png index 344a85f..f5df003 100644 Binary files a/docs/screenshots/experiments.png and b/docs/screenshots/experiments.png differ