feat(embeddings): AlphaEarth's years on the time slider (embeddings.addCube) (#1720)

crates/engine/src/cmd/stac.rs

@@ -1160,34 +1160,7 @@ fn add_cube(s: &mut Session, p: &Value) -> CmdResult {
                 }
             }
         }
-        // The slider over the cube's dates: on over them when it is off; when it is on, taking
-        // them in (its own dates and the cube's, or its span widened to theirs), so every file
-        // can be reached.
-        let slider = match s.map().time.as_ref() {
-            None => {
-                let d: Vec<Value> = dates.iter().map(|d| json!(d)).collect();
-                json!({"dates": d})
-            }
-            Some(t) if !t.dates.is_empty() => {
-                let mut all: Vec<f64> = t.dates.iter().chain(&dates).copied().collect();
-                all.sort_by(f64::total_cmp);
-                all.dedup();
-                if all.len() > rustgis_map::time::MAX_STEPS {
-                    undo(s);
-                    return Err(format!(
-                        "the time slider's dates and the cube's make {}, more than its {} steps: take a shorter period",
-                        all.len(),
-                        rustgis_map::time::MAX_STEPS
-                    ));
-                }
-                json!({"dates": all, "date": t.current})
-            }
-            Some(t) => {
-                let (first, last) = (dates.first().copied().unwrap_or(t.start), dates.last().copied().unwrap_or(t.end));
-                json!({"start": t.start.min(first), "end": t.end.max(last), "date": t.current})
-            }
-        };
-        if let Err(e) = super::time::set(s, &slider) {
+        if let Err(e) = cube_slider(s, &mut dates, &json!({})) {
             undo(s);
             return Err(e);
         }
@@ -1206,6 +1179,78 @@ fn add_cube(s: &mut Session, p: &Value) -> CmdResult {
     }
 }
 
+/// Turns the time slider on over a data cube's dates (sorted and deduplicated here), or, when it
+/// is on, takes them in (its own dates and the cube's, or its span widened to theirs), so every
+/// file can be reached. `fresh`: what else a slider turned on here is set to (AlphaEarth's years
+/// shown as years). Refused, the slider unchanged, when the dates are more than its steps: a cube
+/// left with files the slider can't reach would say nothing of them.
+pub(crate) fn cube_slider(s: &mut Session, dates: &mut Vec<f64>, fresh: &Value) -> Result<(), String> {
+    dates.sort_by(f64::total_cmp);
+    dates.dedup();
+    if dates.len() > rustgis_map::time::MAX_STEPS {
+        return Err(format!("{} dates; the time slider takes {} at most", dates.len(), rustgis_map::time::MAX_STEPS));
+    }
+    let slider = match s.map().time.as_ref() {
+        None => {
+            let mut q = if fresh.is_object() { fresh.clone() } else { json!({}) };
+            q["dates"] = json!(dates);
+            q
+        }
+        Some(t) if !t.dates.is_empty() => {
+            let mut all: Vec<f64> = t.dates.iter().chain(dates.iter()).copied().collect();
+            all.sort_by(f64::total_cmp);
+            all.dedup();
+            if all.len() > rustgis_map::time::MAX_STEPS {
+                return Err(format!(
+                    "the time slider's dates and the cube's make {}, more than its {} steps: take a shorter period",
+                    all.len(),
+                    rustgis_map::time::MAX_STEPS
+                ));
+            }
+            json!({"dates": all, "date": t.current})
+        }
+        Some(t) => {
+            let (first, last) = (dates.first().copied().unwrap_or(t.start), dates.last().copied().unwrap_or(t.end));
+            let (a, z) = (t.start.min(first), t.end.max(last));
+            // Its steps over the span widened (an hourly slider over years): past what it lists,
+            // the cube's later dates couldn't be reached.
+            let n = rustgis_data::time::steps(a, z, t.unit, t.interval, rustgis_map::time::MAX_STEPS + 1).len();
+            if n > rustgis_map::time::MAX_STEPS {
+                return Err(format!(
+                    "the time slider's steps over the cube's period would be more than its {}: make its step longer (time.set unit) or take a shorter period",
+                    rustgis_map::time::MAX_STEPS
+                ));
+            }
+            // Each of the cube's dates shown at some step: a step longer than the dates' spacing
+            // (two years over yearly files) shows the later of two in one window, never the other.
+            let mut w = t.clone();
+            (w.start, w.end) = (a, z);
+            let mut shown = vec![false; dates.len()];
+            for step in w.steps() {
+                w.current = step;
+                let (from, to) = w.window();
+                let i = dates.partition_point(|d| *d < to);
+                if let Some(seen) = i.checked_sub(1).filter(|&j| dates.get(j).is_some_and(|d| *d >= from)).and_then(|j| shown.get_mut(j)) {
+                    *seen = true;
+                }
+            }
+            let unseen: Vec<String> =
+                dates.iter().zip(&shown).filter(|(_, s)| !**s).map(|(d, _)| rustgis_data::time::format(*d, "YYYY-MM-DD")).collect();
+            if !unseen.is_empty() {
+                let more = if unseen.len() > 5 { format!(" and {} more", unseen.len() - 5) } else { String::new() };
+                return Err(format!(
+                    "the time slider's step never shows {} of the cube's {} dates on their own ({}{more}): make its step shorter (time.set unit, interval) or turn it off first",
+                    unseen.len(),
+                    dates.len(),
+                    unseen.iter().take(5).cloned().collect::<Vec<_>>().join(", ")
+                ));
+            }
+            json!({"start": a, "end": z, "date": t.current})
+        }
+    };
+    super::time::set(s, &slider).map(|_| ())
+}
+
 /// The files a cube's range is sampled from, at most, and the most pixels a side read from each.
 #[cfg(not(target_arch = "wasm32"))]
 const SAMPLE_FILES: usize = 4;

crates/engine/src/cmd/tiles.rs

@@ -48,6 +48,7 @@ pub const NETWORK: &[&str] = &[
     "portal.files",
     "embeddings.search",
     "embeddings.add",
+    "embeddings.addCube",
     "earthdata.collections",
     "earthdata.granules",
     "nldi.flowline",
@@ -982,9 +983,14 @@ fn set(s: &mut Session, p: &Value) -> CmdResult {
         t.name = r.chars().take(120).collect();
     }
     if let Some(url) = new_url {
-        // A dated layer keeps the look at every date: its template takes it too.
+        // A dated layer keeps the look at every date: its template takes it too, and a data
+        // cube's file of each date (the slider would bring the old look back at the next step).
+        let look = cog::split(&url).1;
         if let Some(template) = t.time_url.as_mut() {
-            *template = cog::with_look(template, &cog::split(&url).1);
+            *template = cog::with_look(template, &look);
+        }
+        for (_, u) in &mut t.time_urls {
+            *u = cog::with_look(u, &look);
         }
         t.url = url;
     }

crates/tiles/src/embeddings.rs

@@ -17,6 +17,8 @@
 //! Nothing here waits on the network: searches and reads take a callback (the app's, off the
 //! frame), with blocking forms for the command line.
 
+use std::collections::BTreeMap;
+
 use rustgis_data::source::{Batch, Request, parquet::Vectors};
 use rustgis_data::{Dataset, Feature, Field, FieldType, Geometry, GeometryType, Value};
 use rustgis_geom::{Coord, Extent};
@@ -273,6 +275,14 @@ fn year_of(t: i64) -> i32 {
 /// The tiles of a year in what the index read, nearest the box's centre first (at most
 /// [`MAX_TILES`]).
 pub fn aef_tiles(fields: &[Field], batch: &Batch, year: i32, bbox: [f64; 4]) -> Vec<Tile> {
+    let mut out = aef_rows(fields, batch, bbox, |y| y == year);
+    out.dedup_by(|a, b| a.1.url == b.1.url);
+    out.into_iter().take(MAX_TILES).map(|(_, t)| t).collect()
+}
+
+/// Every tile of the years `keep` takes in what the index read, each titled with its own year,
+/// with its squared distance from the box's centre, nearest first.
+fn aef_rows(fields: &[Field], batch: &Batch, bbox: [f64; 4], keep: impl Fn(i32) -> bool) -> Vec<(f64, Tile)> {
     let col = |n: &str| fields.iter().position(|f| f.name == n);
     let (href, epsg, date, id) = (col(AEF_HREF), col("proj:epsg"), col("datetime"), col("id"));
     let text = |f: &Feature, c: Option<usize>| c.and_then(|c| f.attrs.get(c)).map(|v| v.to_string()).unwrap_or_default();
@@ -281,12 +291,12 @@ pub fn aef_tiles(fields: &[Field], batch: &Batch, year: i32, bbox: [f64; 4]) ->
         .features
         .iter()
         .filter_map(|f| {
-            let y = date.and_then(|c| f.attrs.get(c)).and_then(|v| match v {
+            let year = date.and_then(|c| f.attrs.get(c)).and_then(|v| match v {
                 Value::Int(t) => Some(year_of(*t)),
                 Value::Text(t) => t.get(..4).and_then(|y| y.parse().ok()),
                 _ => None,
             })?;
-            if y != year {
+            if !keep(year) {
                 return None;
             }
             let url = https(&text(f, href));
@@ -327,8 +337,113 @@ pub fn aef_tiles(fields: &[Field], batch: &Batch, year: i32, bbox: [f64; 4]) ->
         })
         .collect();
     out.sort_by(|a, b| a.0.total_cmp(&b.0));
-    out.dedup_by(|a, b| a.1.url == b.1.url);
-    out.into_iter().take(MAX_TILES).map(|(_, t)| t).collect()
+    out
+}
+
+/// One AlphaEarth tile over the years. Each year's export names its files afresh (a prefix of
+/// its own: `xdwdu82ch9yffuj03-0000008192-0000000000.tiff` in 2017,
+/// `x1i7lpngfblt9wshv-0000008192-0000000000.tiff` in 2024), so a tile's years are matched by
+/// place: its coordinate system and footprint.
+#[derive(Clone, Debug, PartialEq)]
+pub struct Series {
+    /// The tile as a card shows it: its latest year's footprint, titled with its years.
+    pub tile: Tile,
+    /// Each year's file.
+    pub years: BTreeMap<i32, String>,
+}
+
+/// What a search over the years found.
+#[derive(Clone, Debug, Default, PartialEq)]
+pub struct SeriesFound {
+    /// The tiles, nearest the box's centre first (at most [`MAX_TILES`]).
+    pub series: Vec<Series>,
+    /// Tiles past [`MAX_TILES`], left out.
+    pub dropped: usize,
+    /// Rows left out because their tile already had another file that year.
+    pub duplicates: usize,
+    /// Whether the index was read whole for the box.
+    pub complete: bool,
+}
+
+/// How far apart (degrees, about 10 m) two years' footprint corners may be and still be one
+/// tile's: far above the noise of the index's coordinates, far below a tile's 82 km, so no
+/// rounding boundary splits a tile in two.
+const SAME_PLACE: f64 = 1e-4;
+
+/// Whether two rows are one tile's: the same coordinate system, and footprints whose corners
+/// agree to [`SAME_PLACE`].
+fn same_place(a: &Tile, b: &Tile) -> bool {
+    fn crs(t: &Tile) -> &str {
+        t.detail.split(" · ").next().unwrap_or("")
+    }
+    a.bbox.iter().zip(b.bbox.iter()).all(|(x, y)| (x - y).abs() <= SAME_PLACE) && crs(a) == crs(b)
+}
+
+/// The tiles of the years `first` to `last` in what the index read, each with its file a year,
+/// nearest the box's centre first.
+pub fn aef_series(fields: &[Field], batch: &Batch, (first, last): (i32, i32), bbox: [f64; 4]) -> SeriesFound {
+    let mut found = SeriesFound { complete: batch.complete, ..Default::default() };
+    // Each tile's place as first seen: matched against, so a tile's latest footprint taking over
+    // its card doesn't move what later rows are matched to. A read holds at most MAX_INDEX_ROWS
+    // rows, so comparing each with the tiles so far is cheap.
+    let mut places: Vec<Tile> = Vec::new();
+    for (_, t) in aef_rows(fields, batch, bbox, |y| (first..=last).contains(&y)) {
+        let Some(year) = t.year else { continue };
+        let i = match places.iter().position(|p| same_place(p, &t)) {
+            Some(i) => i,
+            None => {
+                places.push(t.clone());
+                found.series.push(Series { tile: t.clone(), years: BTreeMap::new() });
+                found.series.len() - 1
+            }
+        };
+        let Some(s) = found.series.get_mut(i) else { continue };
+        match s.years.get(&year) {
+            Some(u) if *u != t.url => found.duplicates += 1,
+            Some(_) => {}
+            None => {
+                if s.tile.year.is_none_or(|y| year > y) {
+                    s.tile = t.clone();
+                }
+                s.years.insert(year, t.url);
+            }
+        }
+    }
+    for s in &mut found.series {
+        let (a, z) = (s.years.keys().next().copied().unwrap_or(first), s.years.keys().next_back().copied().unwrap_or(last));
+        let zone = s.tile.title.split(" · ").nth(1).map(|z| format!(" · {z}")).unwrap_or_default();
+        let crs = s.tile.detail.split(" · ").next().unwrap_or("").to_string();
+        s.tile.title = if a == z { format!("AlphaEarth {a}{zone}") } else { format!("AlphaEarth {a}–{z}{zone}") };
+        s.tile.detail = format!("{crs} · 64 bands · 10 m · {} of {} years", s.years.len(), last - first + 1);
+        s.tile.year = None;
+    }
+    found.dropped = found.series.len().saturating_sub(MAX_TILES);
+    found.series.truncate(MAX_TILES);
+    found
+}
+
+/// The first moment of a year, in milliseconds since 1970 (UTC): a time slider's step for it.
+pub fn year_start_ms(year: i32) -> f64 {
+    // Howard Hinnant's days_from_civil for 1 January, which it counts as month 11 of the year
+    // before (March is its month 1).
+    let y = i64::from(year) - 1;
+    let era = y.div_euclid(400);
+    let yoe = y - era * 400;
+    let doy = (153 * 10 + 2) / 5;
+    let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
+    let days = era * 146_097 + doe - 719_468;
+    days as f64 * 86_400_000.0
+}
+
+/// An AlphaEarth file's place, as a layer's name gives it: its UTM zone's folder and its tile's
+/// row and column (`23S 0000008192-0000000000` from `…/2024/23S/x1i7…-0000008192-0000000000.tiff`).
+pub fn aef_place(url: &str) -> Option<String> {
+    let path = url.split(['?', '#']).next().unwrap_or(url);
+    let mut parts = path.rsplit('/');
+    let file = parts.next()?.trim_end_matches(".tiff").trim_end_matches(".tif");
+    let zone = parts.next().filter(|z| !z.is_empty())?;
+    let (_, place) = file.split_once('-')?;
+    Some(format!("{zone} {place}"))
 }
 
 /// What a search found: the tiles, and whether the index was read whole for the box (a box
@@ -364,6 +479,30 @@ pub fn search_aef_blocking(bbox: [f64; 4], year: i32) -> Result<Found, String> {
     Ok((aef_tiles(&fields, &batch, year, bbox), batch.complete))
 }
 
+/// Finds AlphaEarth's tiles in a box over the years `first` to `last`, each with its file a year:
+/// the same one read of the index as [`search_aef`] (its rows hold every year).
+pub fn search_aef_series(bbox: [f64; 4], years: (i32, i32), done: Box<dyn FnOnce(Result<SeriesFound, String>) + Send>) {
+    let bbox = match check_bbox(bbox) {
+        Ok(b) => b,
+        Err(e) => return done(Err(e)),
+    };
+    rustgis_data::source::query_url(
+        AEF_INDEX,
+        vec![AEF_HREF.into()],
+        aef_request(bbox),
+        Box::new(move |r| done(r.map(|(fields, batch)| aef_series(&fields, &batch, years, bbox)).map_err(|e| e.to_string()))),
+    );
+}
+
+/// [`search_aef_series`], waiting for it (the command line).
+#[cfg(not(target_arch = "wasm32"))]
+pub fn search_aef_series_blocking(bbox: [f64; 4], years: (i32, i32)) -> Result<SeriesFound, String> {
+    let bbox = check_bbox(bbox)?;
+    let (fields, batch) =
+        rustgis_data::source::query_url_blocking(AEF_INDEX, vec![AEF_HREF.into()], aef_request(bbox)).map_err(|e| e.to_string())?;
+    Ok(aef_series(&fields, &batch, years, bbox))
+}
+
 // Earth Index.
 
 /// Where Earth Index's files are.
@@ -801,6 +940,68 @@ mod tests {
         assert!(aef_tiles(&[], &batch, 2021, [0.0, 0.0, 1.0, 1.0]).is_empty());
     }
 
+    /// A tile's years are matched by place, not by name (each year's export names its files
+    /// afresh); another place is another tile, a second file for a tile's year is counted.
+    #[test]
+    fn index_rows_to_series() {
+        let fields = vec![
+            Field::new("id", FieldType::Text),
+            Field::new("datetime", FieldType::Integer),
+            Field::new(AEF_HREF, FieldType::Text),
+            Field::new("proj:epsg", FieldType::Text),
+        ];
+        let row = |x: f64, year: i32, href: &str| Feature {
+            id: 1,
+            geometry: Some(Geometry::Polygon(vec![vec![vec![
+                Coord::new(x, -21.0),
+                Coord::new(x + 0.8, -21.0),
+                Coord::new(x + 0.8, -20.7),
+                Coord::new(x, -20.7),
+                Coord::new(x, -21.0),
+            ]]])),
+            attrs: vec![
+                Value::Text("1".into()),
+                Value::Int(year_start_ms(year) as i64),
+                Value::Text(href.into()),
+                Value::Text("EPSG:32723".into()),
+            ],
+        };
+        let base = "s3://us-west-2.opendata.source.coop/tge-labs/aef/v1/annual";
+        let batch = Batch {
+            features: vec![
+                // A footprint a hair off another year's, across a 1e-5° rounding boundary: the
+                // same tile still.
+                row(-43.400_005_1, 2024, &format!("{base}/2024/23S/xb-0000008192-0000000000.tiff")),
+                row(-43.4, 2017, &format!("{base}/2017/23S/xa-0000008192-0000000000.tiff")),
+                row(-43.4, 2025, &format!("{base}/2025/23S/xc-0000008192-0000000000.tiff")),
+                // The same tile and year twice: the first kept, the other counted.
+                row(-43.4, 2025, &format!("{base}/2025/23S/xd-0000008192-0000000000.tiff")),
+                // Another place; and a year outside those asked for.
+                row(-44.2, 2020, &format!("{base}/2020/23S/xe-0000000000-0000000000.tiff")),
+                row(-43.4, 2016, &format!("{base}/2016/23S/xf-0000008192-0000000000.tiff")),
+            ],
+            matched: 6,
+            complete: false,
+            skipped: 0,
+        };
+        let f = aef_series(&fields, &batch, (2017, 2025), [-43.0, -20.9, -42.9, -20.8]);
+        assert_eq!((f.series.len(), f.duplicates, f.dropped, f.complete), (2, 1, 0, false));
+        let s = &f.series[0];
+        assert_eq!(s.years.keys().copied().collect::<Vec<_>>(), vec![2017, 2024, 2025]);
+        assert_eq!(s.years[&2025], "https://data.source.coop/tge-labs/aef/v1/annual/2025/23S/xc-0000008192-0000000000.tiff");
+        assert_eq!((s.tile.title.as_str(), s.tile.year), ("AlphaEarth 2017–2025 · UTM 23S", None));
+        assert_eq!(s.tile.detail, "EPSG:32723 · 64 bands · 10 m · 3 of 9 years");
+        assert_eq!(s.tile.id, "xc-0000008192-0000000000", "the latest year's file stands for the tile");
+        assert_eq!(f.series[1].tile.title, "AlphaEarth 2020 · UTM 23S");
+        assert_eq!(aef_place(&s.years[&2017]).as_deref(), Some("23S 0000008192-0000000000"));
+        assert_eq!(aef_place("https://h/x.tiff#bands=1"), None);
+        assert_eq!(year_start_ms(1970), 0.0);
+        assert_eq!(year_start_ms(2017), 1_483_228_800_000.0);
+        assert_eq!(year_start_ms(2024), 1_704_067_200_000.0);
+        assert_eq!(year_of(year_start_ms(2000) as i64), 2000);
+        assert_eq!(aef_series(&[], &batch, (2017, 2025), [0.0, 0.0, 1.0, 1.0]).series, Vec::new());
+    }
+
     #[test]
     fn principal_component_colours() {
         // Two clusters along one direction: their first component tells them apart.