From 31c4b5effe5ecbe64b0aea53215c60384ba713b7 Mon Sep 17 00:00:00 2001 From: oglassdev <23040613+oglassdev@users.noreply.github.com> Date: Fri, 20 Nov 2026 14:55:30 -0500 Subject: [PATCH 1/5] feat(assets): collect unused revisions, download blobs in parallel, push without Gradle and serve dev packs off localhost - management: each SetAssetHead deletes the project's revisions that are not the head, one of its 10 newest ready revisions, under an hour old, or pinned by an environment's active, unfinished or 10 newest deployments, then every blob no remaining revision holds from the database and the release store. A per-project advisory lock, shared by uploads and deployments, keeps collection from racing them. ReleaseStore gains delete. - core removes asset revisions and blobs no resident or loading deployment pins; chunk-jvm keeps only the revision it launches. - core, chunk-jvm and chunk assets pull download four blobs at once. - chunk assets push builds only the assets revision, without Gradle. - chunk dev takes --pack-bind and --pack-url for the pack server. - dashboard: deployments show their asset revision, and projects list their revisions. Closes #328 --- Cargo.lock | 3 + crates/chunk-build/src/assets.rs | 40 ++ crates/chunk-build/src/assets/tests.rs | 22 + crates/chunk-cli/Cargo.toml | 1 + crates/chunk-cli/README.md | 20 +- crates/chunk-cli/src/local.rs | 9 +- crates/chunk-cli/src/local/packs.rs | 34 +- crates/chunk-cli/src/local/packs/tests.rs | 14 +- crates/chunk-cli/src/platform/assets.rs | 54 +- crates/chunk-environment/Cargo.toml | 1 + crates/chunk-environment/README.md | 13 +- crates/chunk-environment/src/managed.rs | 18 +- .../chunk-environment/src/managed/assets.rs | 68 ++- .../chunk-environment/src/managed/retire.rs | 32 ++ .../src/managed/tests/assets.rs | 18 + crates/chunk-jvm/Cargo.toml | 1 + crates/chunk-jvm/README.md | 8 +- crates/chunk-jvm/src/assets.rs | 39 +- crates/chunk-jvm/src/cache.rs | 2 +- packages/dashboard/src/components/release.tsx | 5 +- .../src/gen/chunk/management/v1/assets_pb.ts | 485 ++++++++++++++++++ .../gen/chunk/management/v1/deployments_pb.ts | 35 +- .../gen/chunk/management/v1/environment_pb.ts | 108 +++- packages/dashboard/src/lib/client.ts | 2 + packages/dashboard/src/lib/format.test.ts | 10 + packages/dashboard/src/lib/format.ts | 13 + packages/dashboard/src/lib/queries.ts | 18 + .../src/routes/environment/deployments.tsx | 12 +- packages/dashboard/src/routes/project.tsx | 41 +- packages/management/README.md | 7 + packages/management/src/assets/retention.ts | 117 +++++ packages/management/src/assets/service.ts | 39 +- packages/management/src/assets/store.ts | 9 + packages/management/src/deployments/store.ts | 2 + .../management/src/releases/local-store.ts | 5 + packages/management/src/releases/s3-store.ts | 6 + packages/management/src/releases/store.ts | 2 + packages/management/test/assets.test.ts | 37 ++ 38 files changed, 1217 insertions(+), 133 deletions(-) create mode 100644 packages/dashboard/src/gen/chunk/management/v1/assets_pb.ts create mode 100644 packages/dashboard/src/lib/format.test.ts create mode 100644 packages/management/src/assets/retention.ts diff --git a/Cargo.lock b/Cargo.lock index db30ee07..6354835d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -624,6 +624,7 @@ dependencies = [ "clap", "cliclack", "dirs", + "futures-util", "http-body-util", "hyper", "hyper-util", @@ -723,6 +724,7 @@ dependencies = [ "chunk-service", "chunk-store", "flate2", + "futures-util", "http-body", "http-body-util", "hyper", @@ -773,6 +775,7 @@ dependencies = [ "chunk-contract", "chunk-proto", "chunk-service", + "futures-util", "prost", "rustix 1.1.4", "serde_json", diff --git a/crates/chunk-build/src/assets.rs b/crates/chunk-build/src/assets.rs index 46374aaf..d9fee8b6 100644 --- a/crates/chunk-build/src/assets.rs +++ b/crates/chunk-build/src/assets.rs @@ -2,6 +2,7 @@ //! assets from. use std::{ + collections::BTreeSet, fs, io::{self, Read, Write}, path::{Path, PathBuf}, @@ -113,6 +114,20 @@ impl Store { Ok(revision) } + /// Removes every revision but `kept`, their materialized directories, and the blobs none of `kept` holds. Hidden + /// entries, such as writes in progress, stay. Directories materialized from removed blobs keep their hard links. + /// Returns how many entries it removed. + /// # Errors + /// Reports filesystem failures; whatever it removed before one stays removed. + pub fn retain(&self, kept: &[&AssetRevision]) -> io::Result { + let ids: BTreeSet = kept.iter().map(|revision| revision.id()).collect(); + let blobs: BTreeSet<&str> = kept.iter().flat_map(|revision| revision.blobs().into_keys()).collect(); + let revision = |name: &str| name.strip_suffix(".json").is_some_and(|id| ids.contains(id)); + Ok(remove_unless(&self.root.join("blobs"), |name| blobs.contains(name))? + + remove_unless(&self.root.join("revisions"), revision)? + + remove_unless(&self.root.join("apps"), |name| ids.contains(name))?) + } + fn blob(&self, sha256: &str) -> io::Result { check_digest(sha256)?; Ok(self.root.join("blobs").join(sha256)) @@ -245,6 +260,31 @@ fn sync_parent(path: &Path) -> io::Result<()> { Ok(()) } +/// Removes the entries of `directory` that are not hidden and whose names `keep` rejects, returning how many. +fn remove_unless(directory: &Path, keep: impl Fn(&str) -> bool) -> io::Result { + let entries = match fs::read_dir(directory) { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(0), + Err(error) => return Err(error), + }; + let mut removed = 0; + for entry in entries { + let entry = entry?; + let name = entry.file_name(); + if name.to_str().is_some_and(|name| name.starts_with('.') || keep(name)) { + continue; + } + let path = entry.path(); + let result = if entry.file_type()?.is_dir() { fs::remove_dir_all(&path) } else { fs::remove_file(&path) }; + match result { + Ok(()) => removed += 1, + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + Ok(removed) +} + fn check_digest(value: &str) -> io::Result<()> { if value.len() != 64 || !value.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) { return Err(io::Error::new(io::ErrorKind::InvalidInput, format!("invalid SHA-256 {value:?}"))); diff --git a/crates/chunk-build/src/assets/tests.rs b/crates/chunk-build/src/assets/tests.rs index e4a5a545..6b94ba06 100644 --- a/crates/chunk-build/src/assets/tests.rs +++ b/crates/chunk-build/src/assets/tests.rs @@ -117,3 +117,25 @@ fn incomplete_materialized_directories_are_rebuilt() { }); assert_eq!(fs::read(directory.join("worlds/lobby.polar")).unwrap(), POLAR); } + +#[test] +fn retaining_revisions_removes_every_other_revision_and_blob() { + let project_directory = tempfile::tempdir().unwrap(); + let root = project_directory.path(); + project(root); + let store = Store::new(root.join("store")); + let old = build_revision(root, &store).unwrap(); + materialize(&store, &old, "arena").unwrap(); + write(root, "assets/config/game.json", b"changed"); + let new = build_revision(root, &store).unwrap(); + let unfinished = store.root().join("blobs/.write-unfinished"); + fs::write(&unfinished, b"").unwrap(); + + assert_eq!(store.retain(&[&new]).unwrap(), 3); + assert!(new.blobs().keys().all(|sha256| store.contains(sha256).unwrap())); + let changed = &old.shared["config/game.json"].sha256; + assert!(!store.contains(changed).unwrap()); + assert!(store.read_revision(&old.id()).is_err() && !store.root().join("apps").join(old.id()).exists()); + assert_eq!(store.read_revision(&new.id()).unwrap(), new); + assert!(unfinished.exists()); +} diff --git a/crates/chunk-cli/Cargo.toml b/crates/chunk-cli/Cargo.toml index 1d299e21..79bba077 100644 --- a/crates/chunk-cli/Cargo.toml +++ b/crates/chunk-cli/Cargo.toml @@ -25,6 +25,7 @@ chunk-environment = { path = "../chunk-environment", default-features = false } chunk-management = { path = "../chunk-management" } chunk-proto = { path = "../chunk-proto" } clap = { version = "4", features = ["derive", "env"] } +futures-util = "0.3" url = { version = "2", features = ["serde"] } dirs = "6" serde = { version = "1", features = ["derive"] } diff --git a/crates/chunk-cli/README.md b/crates/chunk-cli/README.md index 917062a0..216d25a0 100644 --- a/crates/chunk-cli/README.md +++ b/crates/chunk-cli/README.md @@ -79,7 +79,10 @@ State lives in `PROJECT/.chunk/local`, or `--state DIR`: the backend's data, pub `control/nodes/`, and `control.json`, which `chunk players` and `chunk nodes` read to reach control. One `chunk dev` runs per project at a time. `--java PATH` overrides the Java executable Gradle selected; it must satisfy the release's Java version. `--offline-logins` admits players without Mojang authentication, under offline-mode UUIDs; use it only for -local testing. `--connection-timeout-seconds` (default 10, from 1 to 3600) sets the gateway's login deadline. +local testing. `--connection-timeout-seconds` (default 10, from 1 to 3600) sets the gateway's login deadline. Players' +clients download resource packs over HTTP from an ephemeral loopback port, or from `--pack-bind ADDR`; when they reach +it at another address, such as from other machines, `--pack-url URL` names it (required when `--pack-bind` listens on +every address, such as `0.0.0.0:25568`). Backend functions and the apps' generated `Vars` read `chunk.toml`'s top-level `[vars]`, or with `--env NAME` its `[env.NAME.vars]` over them, key by key. Actions read their secrets from `.dev.vars` in the project root: `NAME=value` @@ -203,13 +206,14 @@ environment holds up to 256 secrets. `chunk secrets list` shows names and versio An asset revision is an immutable manifest of the project's worlds, resource packs and files by content digest, which every build writes beside its release; a deployment pins one release and one revision, so assets change without -rebuilding the release. `chunk assets push` builds the project as `chunk dev` does, uploads the blobs the project lacks, -and moves the project's head to the new revision. It records the revision as the base in `.chunk/assets.json`, and -refuses when someone else moved the head since, until you pull; `--force` replaces the head anyway. `chunk assets pull` -fetches the head and updates each local file whose source is one file (plain files, `.polar` worlds and `.zip` packs): a -file you haven't changed since the base is replaced or deleted, and a file you changed is left as it is and reported as -a conflict, which your next push publishes. Pack directories are listed but never pulled. `chunk assets deploy --env E` -deploys the environment's active release with the head revision, or the one `--revision ID` names. +rebuilding the release. `chunk assets push` builds only the project's assets, without Gradle, into the store `chunk dev` +uses, uploads the blobs the project lacks, and moves the project's head to the new revision. It records the revision as +the base in `.chunk/assets.json`, and refuses when someone else moved the head since, until you pull; `--force` replaces +the head anyway. `chunk assets pull` fetches the head and updates each local file whose source is one file (plain files, +`.polar` worlds and `.zip` packs): a file you haven't changed since the base is replaced or deleted, and a file you +changed is left as it is and reported as a conflict, which your next push publishes. Pack directories are listed but +never pulled. `chunk assets deploy --env E` deploys the environment's active release with the head revision, or the one +`--revision ID` names. `chunk domains add HOSTNAME --env E` claims a custom hostname and prints its state and the DNS records to create: a TXT ownership proof and an SRV record that routes players to the environment. A hostname routes only once it is verified: diff --git a/crates/chunk-cli/src/local.rs b/crates/chunk-cli/src/local.rs index 36c51acd..15da84a7 100644 --- a/crates/chunk-cli/src/local.rs +++ b/crates/chunk-cli/src/local.rs @@ -39,6 +39,13 @@ pub(crate) struct Options { /// Control address, shared by every release. #[arg(long, default_value = "127.0.0.1:25567")] control_bind: SocketAddr, + /// Where the resource pack server listens (defaults to an ephemeral loopback port). + #[arg(long, default_value = "127.0.0.1:0")] + pack_bind: SocketAddr, + /// The URL players' clients download packs from, such as `http://192.168.1.20:25568`, when they can't reach + /// --pack-bind's address; required when it listens on every address. + #[arg(long, value_name = "URL")] + pack_url: Option, /// Print plain progress lines instead of the terminal UI (automatic when stdout is not a terminal). #[arg(long)] plain: bool, @@ -160,7 +167,7 @@ async fn serve( let required = built.java.version; reporter.running("Java", format!("Checking Java {required}+")); let assets = built.asset_store.clone(); - let packs = packs::Packs::start(assets.clone()).await?; + let packs = packs::Packs::start(assets.clone(), options.pack_bind, options.pack_url.as_ref()).await?; let pack_url_prefix = Some(packs.url_prefix().to_owned()); let staged = stage(&project, built, options.java.as_deref(), pack_url_prefix.as_deref(), &stop).await?; packs.serve(&staged.control.assets); diff --git a/crates/chunk-cli/src/local/packs.rs b/crates/chunk-cli/src/local/packs.rs index 59c3d988..c3ea8614 100644 --- a/crates/chunk-cli/src/local/packs.rs +++ b/crates/chunk-cli/src/local/packs.rs @@ -1,10 +1,10 @@ -//! Serves the resource packs of the releases dev staged to its players' clients, over plain HTTP/1.1 on loopback. +//! Serves the resource packs of the releases dev staged to its players' clients, over plain HTTP/1.1. use std::{ collections::BTreeSet, convert::Infallible, io, - net::Ipv4Addr, + net::SocketAddr, path::PathBuf, sync::{Arc, Mutex, PoisonError}, time::Duration, @@ -23,10 +23,13 @@ pub(super) struct Packs { } impl Packs { - /// Starts serving packs from the asset store at `store` on an ephemeral loopback port. - pub async fn start(store: PathBuf) -> io::Result { - let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?; - let url_prefix = format!("http://{}/packs/", listener.local_addr()?); + /// Starts serving packs from the asset store at `store` on `bind`, to clients that reach it at `public`, or else + /// at the address it listens on. + pub async fn start(store: PathBuf, bind: SocketAddr, public: Option<&url::Url>) -> io::Result { + let listener = TcpListener::bind(bind) + .await + .map_err(|error| io::Error::new(error.kind(), format!("cannot serve packs on {bind}: {error}")))?; + let url_prefix = url_prefix(listener.local_addr()?, public)?; let allowed = Arc::default(); let server = tokio::spawn(serve(listener, store.join("blobs"), Arc::clone(&allowed))).abort_handle(); Ok(Self { url_prefix, allowed, server }) @@ -49,6 +52,25 @@ impl Drop for Packs { } } +/// Where clients download packs: `public`'s path followed by `packs/`, or else the listening address's. +pub(super) fn url_prefix(listening: SocketAddr, public: Option<&url::Url>) -> io::Result { + let Some(public) = public else { + if listening.ip().is_unspecified() { + return Err(io::Error::other(format!( + "--pack-bind {listening} listens on every address; set --pack-url to the one players' clients reach" + ))); + } + return Ok(format!("http://{listening}/packs/")); + }; + if !["http", "https"].contains(&public.scheme()) || public.host().is_none() { + return Err(io::Error::other(format!("--pack-url {public} must be an http or https URL with a host"))); + } + if public.query().is_some() || public.fragment().is_some() { + return Err(io::Error::other(format!("--pack-url {public} cannot have a query or fragment"))); + } + Ok(format!("{}/packs/", public.as_str().trim_end_matches('/'))) +} + async fn serve(listener: TcpListener, blobs: PathBuf, allowed: Arc>>) { let mut connections = JoinSet::new(); loop { diff --git a/crates/chunk-cli/src/local/packs/tests.rs b/crates/chunk-cli/src/local/packs/tests.rs index 659e249a..cbebf4aa 100644 --- a/crates/chunk-cli/src/local/packs/tests.rs +++ b/crates/chunk-cli/src/local/packs/tests.rs @@ -1,7 +1,7 @@ use sha2::{Digest, Sha256}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use super::Packs; +use super::{Packs, url_prefix}; #[tokio::test] async fn only_the_packs_of_staged_releases_are_served() { @@ -11,7 +11,7 @@ async fn only_the_packs_of_staged_releases_are_served() { let (pack, other): (&[u8], &[u8]) = (b"PK pack", b"a world"); store.insert(&sha256(pack), pack).unwrap(); store.insert(&sha256(other), other).unwrap(); - let packs = Packs::start(directory.path().into()).await.unwrap(); + let packs = Packs::start(directory.path().into(), "127.0.0.1:0".parse().unwrap(), None).await.unwrap(); let blob = chunk_contract::PackBlob { sha256: sha256(pack), sha1: "0".repeat(40), size: 7 }; let assets = chunk_control::DeploymentAssets { packs: [("base".into(), blob)].into(), @@ -36,3 +36,13 @@ async fn only_the_packs_of_staged_releases_are_served() { assert!(served.starts_with("HTTP/1.1 200") && served.ends_with("PK pack"), "{served}"); assert!(get(sha256(other)).await.starts_with("HTTP/1.1 404")); } + +#[test] +fn packs_are_served_at_the_public_url_or_else_the_listening_address() { + let listening = "127.0.0.1:4000".parse().unwrap(); + assert_eq!(url_prefix(listening, None).unwrap(), "http://127.0.0.1:4000/packs/"); + let public = url::Url::parse("https://packs.example.com/dev/").unwrap(); + assert_eq!(url_prefix(listening, Some(&public)).unwrap(), "https://packs.example.com/dev/packs/"); + assert!(url_prefix("0.0.0.0:4000".parse().unwrap(), None).is_err()); + assert!(url_prefix(listening, Some(&url::Url::parse("ftp://host/").unwrap())).is_err()); +} diff --git a/crates/chunk-cli/src/platform/assets.rs b/crates/chunk-cli/src/platform/assets.rs index 0deba7c0..90ab2d1f 100644 --- a/crates/chunk-cli/src/platform/assets.rs +++ b/crates/chunk-cli/src/platform/assets.rs @@ -3,7 +3,7 @@ //! runs an environment's release against a revision. use std::{ - collections::BTreeMap, + collections::{BTreeMap, BTreeSet}, fs, io::{self, Read, Write as _}, path::{Component, Path, PathBuf}, @@ -19,6 +19,7 @@ use chunk_management::{ }, }; use clap::{Args, Subcommand}; +use futures_util::TryStreamExt as _; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use tokio_util::sync::CancellationToken; @@ -28,11 +29,13 @@ use super::{ deploy::{self, mebibytes}, resources::{table, time}, }; -use crate::building::{self, BuildMode, progress::Progress}; use plan::{Entry, Pull, Push}; mod plan; +/// How many blobs a pull downloads at once. +const CONCURRENT_DOWNLOADS: usize = 4; + #[derive(Args)] pub(crate) struct Assets { #[command(subcommand)] @@ -192,19 +195,25 @@ pub(super) async fn upload_revision( async fn push(session: &Session, path: PathBuf, selector: &ProjectArg, force: bool) -> io::Result<()> { let project = session.project(selector).await?; + let root = path.canonicalize()?; chunk_service::run(|stop| async move { - let local = building::prepare(&building::Options { project: path, output: None, frozen: false })?; cliclack::log::info("Building assets…")?; - let built = building::execute(&local, BuildMode::Dev, stop.clone(), Progress::default()).await?; + let store = dev_store(&root); + let (project_root, target) = (root.clone(), store.clone()); + let built = tokio::task::spawn_blocking(move || chunk_build::assets::build_revision(&project_root, &target)); + let revision = &tokio::select! { + built = built => built.map_err(io::Error::other)??, + () = stop.cancelled() => return Err(io::Error::new(io::ErrorKind::Interrupted, "push stopped")), + }; let client = &session.client; - let (revision, id) = (&built.assets, built.assets.id()); + let id = revision.id(); let head = head_id(client, &project.id).await?; - let base = read_base(&local.root, &project.id)?; + let base = read_base(&root, &project.id)?; match plan::check_push(&head, base.as_deref(), &id, force) { Push::UpToDate => { // Uploads whatever of the head the platform lost; normally nothing. - upload_revision(client, &project.id, revision, &built.asset_store).await?; - write_base(&local.root, &project.id, &id)?; + upload_revision(client, &project.id, revision, store.root()).await?; + write_base(&root, &project.id, &id)?; return cliclack::log::info(format!("Nothing to push; the head is already revision {id}")); } Push::Moved => { @@ -219,11 +228,11 @@ async fn push(session: &Session, path: PathBuf, selector: &ProjectArg, force: bo let previous = if head.is_empty() { AssetRevision::default() } else { - manifest(client, &Store::new(&built.asset_store), &project.id, &head).await? + manifest(client, &store, &project.id, &head).await? }; let moved = tokio::select! { moved = async { - upload_revision(client, &project.id, revision, &built.asset_store).await?; + upload_revision(client, &project.id, revision, store.root()).await?; let request = SetAssetHeadRequest { project_id: project.id.clone(), revision_id: id.clone(), @@ -242,7 +251,7 @@ async fn push(session: &Session, path: PathBuf, selector: &ProjectArg, force: bo () = stop.cancelled() => return Err(io::Error::new(io::ErrorKind::Interrupted, "push stopped")), }; moved?; - write_base(&local.root, &project.id, &id)?; + write_base(&root, &project.id, &id)?; let mut message = format!("Pushed assets revision {id}"); for line in plan::summary(&plan::entries(&previous), &plan::entries(revision)) { message.push_str("\n "); @@ -298,7 +307,7 @@ async fn pull_head( let mut changes = Vec::new(); let mut conflicts = Vec::new(); let mut skipped = Vec::new(); - for entry in base.keys().chain(heads.keys()).collect::>() { + for entry in base.keys().chain(heads.keys()).collect::>() { let (before, after) = (base.get(entry).copied(), heads.get(entry).copied()); if before == after { continue; @@ -321,16 +330,23 @@ async fn pull_head( } } } - let blobs: BTreeMap<_, _> = - response.downloads.iter().map(|download| (download.sha256.as_str(), download)).collect(); + let urls: BTreeMap<_, _> = + response.downloads.iter().map(|download| (download.sha256.as_str(), download.url.as_str())).collect(); + let (sizes, mut missing) = (head.blobs(), BTreeSet::new()); for sha256 in changes.iter().filter_map(|change| change.write) { - if store.contains(sha256)? { - continue; + if !store.contains(sha256)? { + missing.insert(sha256); } - let url = - blobs.get(sha256).ok_or_else(|| io::Error::other(format!("The platform gave no download for {sha256}")))?; - download(client, &store, sha256, head.blobs()[sha256], &url.url).await?; } + futures_util::stream::iter(missing.into_iter().map(Ok)) + .try_for_each_concurrent(CONCURRENT_DOWNLOADS, |sha256| { + let (url, size, store) = (urls.get(sha256).copied(), sizes[sha256], &store); + async move { + let url = url.ok_or_else(|| io::Error::other(format!("The platform gave no download for {sha256}")))?; + download(client, store, sha256, size, url).await + } + }) + .await?; let (mut updated, mut deleted) = (Vec::new(), Vec::new()); for change in changes { let shown = format!("{}: {}", change.entry, change.path.strip_prefix(root).unwrap_or(&change.path).display()); diff --git a/crates/chunk-environment/Cargo.toml b/crates/chunk-environment/Cargo.toml index d1c5b41f..62785085 100644 --- a/crates/chunk-environment/Cargo.toml +++ b/crates/chunk-environment/Cargo.toml @@ -23,6 +23,7 @@ chunk-proto = { path = "../chunk-proto" } chunk-proxy = { path = "../chunk-proxy", default-features = false } chunk-service = { path = "../chunk-service" } chunk-store = { path = "../chunk-store" } +futures-util = "0.3" http-body = "1" prost = "0.14.3" serde = { version = "1", features = ["derive"] } diff --git a/crates/chunk-environment/README.md b/crates/chunk-environment/README.md index 1796d7e4..a181f94b 100644 --- a/crates/chunk-environment/README.md +++ b/crates/chunk-environment/README.md @@ -63,12 +63,13 @@ elsewhere takes effect only when core starts again. Core waits for management be non-zero if stopped before management answered. It downloads each release archive, unpacks it under `$CHUNK_STATE/releases/`, and verifies it with the same checks as `chunk build`. It checks the asset revision the deployment pins against the worlds and packs the release declares, and downloads each blob that JVMs read and -`$CHUNK_STATE/assets/` lacks, verified against its digest, then writes the revision there; packs reach players' clients -from management, never through core, and no blob is deleted yet. Only then does it make the deployment the backend's and -control's current release, and report it `ACTIVE`. A deployment it rejects is reported `FAILED`, and the previous -deployment keeps serving. The gateway starts with the first active deployment. Core retires the versions it no longer -needs, stopping their JVMs first, and removes their unpacked releases. JVMs run on machines core asks management for -(`EnsureCapacity`), each running [`chunk-jvm`](../chunk-jvm/README.md). A core that management fences stops. +`$CHUNK_STATE/assets/` lacks, four at a time, verified against its digest, then writes the revision there; packs reach +players' clients from management, never through core. Once no deployment resident in the backend, nor the one loading, +pins a revision, core removes it and the blobs no kept revision holds. Only then does it make the deployment the +backend's and control's current release, and report it `ACTIVE`. A deployment it rejects is reported `FAILED`, and the +previous deployment keeps serving. The gateway starts with the first active deployment. Core retires the versions it no +longer needs, stopping their JVMs first, and removes their unpacked releases. JVMs run on machines core asks management +for (`EnsureCapacity`), each running [`chunk-jvm`](../chunk-jvm/README.md). A core that management fences stops. Core reports its status about every 15 seconds, and at once when it changes. Each report carries the server-list status the gateway last answered for each hostname, which the [edge](../chunk-edge/README.md) answers pings with while the diff --git a/crates/chunk-environment/src/managed.rs b/crates/chunk-environment/src/managed.rs index 58ed18c4..4af9bbe0 100644 --- a/crates/chunk-environment/src/managed.rs +++ b/crates/chunk-environment/src/managed.rs @@ -81,6 +81,9 @@ pub(crate) struct Managed<'a> { releases: release::Store, /// Where each deployment's asset blobs are downloaded to. assets: chunk_build::assets::Store, + /// Held shared while a load fills `assets`, and exclusively while unused revisions are removed from it. Holds the + /// revisions the last removal kept. + asset_loads: tokio::sync::RwLock>>, /// Where the activation management has not yet accepted is recorded. activation: PathBuf, core: &'a Core, @@ -123,6 +126,8 @@ struct Deployments { unacknowledged: Option, /// The deployment being loaded. loading: Option, + /// The asset revision `loading` pins, when it names one. + loading_assets: Option, /// The latest deployment this core rejected, and why. rejected: Option<(String, String)>, /// How replaced deployments drain, from the latest desired state this process received. @@ -172,7 +177,9 @@ struct Work<'a> { impl Drop for Work<'_> { fn drop(&mut self) { - lock(self.deployments).loading = None; + let mut deployments = lock(self.deployments); + deployments.loading = None; + deployments.loading_assets = None; } } @@ -206,6 +213,7 @@ impl<'a> Managed<'a> { environment: registration.environment, releases: release::Store::new(state, core.archives().clone()), assets: core.asset_store(), + asset_loads: tokio::sync::RwLock::default(), activation: state.join("managed.json"), core, gateway, @@ -430,7 +438,10 @@ impl<'a> Managed<'a> { } fn start(&self, desired: v1::AttachResponse) -> Work<'_> { - lock(&self.deployments).loading = Some(desired.deployment_id.clone()); + let mut deployments = lock(&self.deployments); + deployments.loading = Some(desired.deployment_id.clone()); + deployments.loading_assets = desired.assets.as_ref().map(|assets| assets.revision_id.clone()); + drop(deployments); let cancel = self.stopping.child_token(); Work { revision: desired.revision, @@ -508,6 +519,7 @@ impl<'a> Managed<'a> { if backend.check_deployment(id).await.is_err() { return false; } + let _loading = self.asset_loads.read().await; match assets::restore(&self.client, &self.assets, desired.assets.as_ref(), cancel).await { Ok(restored) => restored.is_some(), Err(error) => { @@ -525,10 +537,12 @@ impl<'a> Managed<'a> { let Some(loaded) = release::load(&self.client, &self.releases, artifact, cancel).await? else { return Ok(false); }; + let loading = self.asset_loads.read().await; let revision = assets::load(&self.client, &self.assets, desired.assets.as_ref(), loaded.assets(), cancel); let Some(revision) = revision.await? else { return Ok(false); }; + drop(loading); let deployment = &desired.deployment_id; if cancel.is_cancelled() { return Ok(false); diff --git a/crates/chunk-environment/src/managed/assets.rs b/crates/chunk-environment/src/managed/assets.rs index 7de02da7..d9739498 100644 --- a/crates/chunk-environment/src/managed/assets.rs +++ b/crates/chunk-environment/src/managed/assets.rs @@ -1,12 +1,20 @@ -//! Loading the asset revision a deployment pins into core's asset store, blob by blob, before the deployment activates. +//! Loading the asset revision a deployment pins into core's asset store before the deployment activates, and removing +//! the revisions no deployment pins any more. use super::release::{self, Staged}; use chunk_build::assets::Store; use chunk_contract::{ASSET_REVISION_VERSION, AssetContract, AssetRevision}; use chunk_management::{Client, v1}; -use std::{collections::BTreeMap, fs, io}; +use futures_util::TryStreamExt as _; +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, io, +}; use tokio_util::sync::CancellationToken; +/// How many blobs a load downloads at once. +const CONCURRENT_DOWNLOADS: usize = 4; + /// Checks the revision `artifact` names against the release's declarations in `contract`, then downloads into `store` /// each blob of it that JVMs read and `store` lacks, verified, and the revision itself. Without an artifact, the /// deployment pins the empty revision. Returns `None` once `cancel` stops it before its downloads finish. @@ -20,6 +28,7 @@ pub(super) async fn load( let Some(artifact) = artifact else { let revision = AssetRevision { version: ASSET_REVISION_VERSION, ..AssetRevision::default() }; contract.check(&revision).map_err(io::Error::other)?; + store.write_revision(&revision)?; return Ok(Some(revision)); }; let revision = decode(artifact)?; @@ -36,7 +45,9 @@ pub(super) async fn restore( cancel: &CancellationToken, ) -> io::Result> { let Some(artifact) = artifact else { - return Ok(Some(AssetRevision { version: ASSET_REVISION_VERSION, ..AssetRevision::default() })); + let revision = AssetRevision { version: ASSET_REVISION_VERSION, ..AssetRevision::default() }; + store.write_revision(&revision)?; + return Ok(Some(revision)); }; download(client, store, artifact, decode(artifact)?, cancel).await } @@ -57,25 +68,33 @@ async fn download( cancel: &CancellationToken, ) -> io::Result> { fs::create_dir_all(store.root())?; + let mut missing = Vec::new(); for (sha256, size) in jvm_blobs(&revision) { - if store.contains(sha256)? { - continue; + if !store.contains(sha256)? { + missing.push(Ok((sha256, size))); } - let staged = Staged(store.root().join(format!(".{}.blob", uuid::Uuid::new_v4()))); - let url = format!("{}{sha256}", artifact.blob_url_prefix); - tokio::select! { - fetched = release::fetch(|| client.download_blob(&url), size, "asset blob", &staged.0) => fetched?, - () = cancel.cancelled() => return Ok(None), - } - let (store, sha256) = (store.clone(), sha256.to_owned()); - tokio::task::spawn_blocking(move || store.insert_file(&sha256, &staged.0)) - .await - .map_err(io::Error::other)??; + } + let fetched = futures_util::stream::iter(missing) + .try_for_each_concurrent(CONCURRENT_DOWNLOADS, |(sha256, size)| { + fetch_blob(client, store, &artifact.blob_url_prefix, sha256, size) + }); + tokio::select! { + fetched = fetched => fetched?, + () = cancel.cancelled() => return Ok(None), } store.write_revision(&revision)?; Ok(Some(revision)) } +/// Downloads the blob into a hidden file beside the store, then inserts it, which verifies its SHA-256. +async fn fetch_blob(client: &Client, store: &Store, url_prefix: &str, sha256: &str, size: u64) -> io::Result<()> { + let staged = Staged(store.root().join(format!(".{}.blob", uuid::Uuid::new_v4()))); + let url = format!("{url_prefix}{sha256}"); + release::fetch(|| client.download_blob(&url), size, "asset blob", &staged.0).await?; + let (store, sha256) = (store.clone(), sha256.to_owned()); + tokio::task::spawn_blocking(move || store.insert_file(&sha256, &staged.0)).await.map_err(io::Error::other)? +} + /// Every blob some app's JVMs read: each app's worlds and files, and the shared files. Players' clients download packs /// from management, never through core. fn jvm_blobs(revision: &AssetRevision) -> BTreeMap<&str, u64> { @@ -94,3 +113,22 @@ pub(super) async fn sweep(store: &Store) -> io::Result<()> { |path: &std::path::PathBuf| path.file_name().is_some_and(|name| name.as_encoded_bytes().starts_with(b".")); release::remove(unfinished.into_iter().filter(hidden).collect()).await } + +/// Removes from `store` every revision but `kept`, and the blobs none of them holds, returning how many entries it +/// removed. Removes nothing and returns `None` while the store lacks one of `kept`, whose blobs it can't tell. +pub(super) async fn prune(store: &Store, kept: BTreeSet) -> io::Result> { + let store = store.clone(); + tokio::task::spawn_blocking(move || { + let mut revisions = Vec::new(); + for id in &kept { + match store.read_revision(id) { + Ok(revision) => revisions.push(revision), + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error), + } + } + store.retain(&revisions.iter().collect::>()).map(Some) + }) + .await + .map_err(io::Error::other)? +} diff --git a/crates/chunk-environment/src/managed/retire.rs b/crates/chunk-environment/src/managed/retire.rs index f2edbf38..5c45fcc4 100644 --- a/crates/chunk-environment/src/managed/retire.rs +++ b/crates/chunk-environment/src/managed/retire.rs @@ -76,6 +76,7 @@ impl Managed<'_> { tick.tick().await; self.retire().await; self.remove_unused_releases().await; + self.remove_unused_assets().await; } } @@ -195,4 +196,35 @@ impl Managed<'_> { tracing::warn!(%error, "unused releases not removed"); } } + + /// Removes the asset revisions and blobs that neither a deployment resident in the backend, whose JVMs read them, + /// nor the one loading pins, whenever that set changes. Skipped while a load fills the store, and loads wait + /// meanwhile, so none reuses a blob being removed. + async fn remove_unused_assets(&self) { + let (Ok(control), Some(backend)) = (self.core.control(), self.core.backend()) else { return }; + let Ok(mut last_kept) = self.asset_loads.try_write() else { return }; + // Read before control, so a load that finishes meanwhile has made its deployment control's by then. + let mut kept: BTreeSet = lock(&self.deployments).loading_assets.iter().cloned().collect(); + let resident = match backend.deployments().await { + Ok(resident) => resident, + Err(error) => return tracing::warn!(%error, "resident deployments unknown"), + }; + for id in &resident { + match control.release(id.as_str()) { + Ok(release) => kept.extend(release.map(|release| release.assets.revision_id.clone())), + Err(error) => return tracing::warn!(%error, "asset revisions in use unknown"), + } + } + kept.remove(""); + if last_kept.as_ref() == Some(&kept) { + return; + } + match super::assets::prune(&self.assets, kept.clone()).await { + Ok(None) => return, + Ok(Some(0)) => {} + Ok(Some(removed)) => tracing::info!(removed, kept = kept.len(), "unused asset revisions removed"), + Err(error) => tracing::warn!(%error, "unused asset revisions not removed"), + } + *last_kept = Some(kept); + } } diff --git a/crates/chunk-environment/src/managed/tests/assets.rs b/crates/chunk-environment/src/managed/tests/assets.rs index 14f0f7d4..0396e124 100644 --- a/crates/chunk-environment/src/managed/tests/assets.rs +++ b/crates/chunk-environment/src/managed/tests/assets.rs @@ -37,3 +37,21 @@ async fn a_revision_and_its_missing_blobs_load_unless_it_lacks_a_declared_world( let renamed = AssetArtifact { revision_id: "0".repeat(64), ..artifact }; assert!(assets::load(&client, &store, Some(&renamed), &declared("koth"), &cancel).await.is_err()); } + +#[tokio::test] +async fn pruning_keeps_everything_until_the_store_holds_every_kept_revision() { + let directory = tempfile::tempdir().unwrap(); + let store = Store::new(directory.path()); + let blob = |bytes: &[u8]| AssetBlob { sha256: format!("{:x}", Sha256::digest(bytes)), size: bytes.len() as u64 }; + let mut old = AssetRevision { version: chunk_contract::ASSET_REVISION_VERSION, ..AssetRevision::default() }; + old.shared.insert("old.txt".into(), blob(b"old")); + store.insert(&blob(b"old").sha256, b"old").unwrap(); + store.write_revision(&old).unwrap(); + let loading = AssetRevision { version: chunk_contract::ASSET_REVISION_VERSION, ..AssetRevision::default() }; + + assert_eq!(assets::prune(&store, BTreeSet::from([loading.id()])).await.unwrap(), None); + assert!(store.contains(&blob(b"old").sha256).unwrap()); + store.write_revision(&loading).unwrap(); + assert_eq!(assets::prune(&store, BTreeSet::from([loading.id()])).await.unwrap(), Some(2)); + assert!(!store.contains(&blob(b"old").sha256).unwrap()); +} diff --git a/crates/chunk-jvm/Cargo.toml b/crates/chunk-jvm/Cargo.toml index de3cdbeb..03bf7407 100644 --- a/crates/chunk-jvm/Cargo.toml +++ b/crates/chunk-jvm/Cargo.toml @@ -12,6 +12,7 @@ chunk-build = { path = "../chunk-build", default-features = false } chunk-contract = { path = "../chunk-contract" } chunk-proto = { path = "../chunk-proto" } chunk-service = { path = "../chunk-service" } +futures-util = "0.3" prost = "0.14.3" rustix = { version = "1", features = ["process"] } serde_json = "1" diff --git a/crates/chunk-jvm/README.md b/crates/chunk-jvm/README.md index 6bb0550f..5a9ab0a7 100644 --- a/crates/chunk-jvm/README.md +++ b/crates/chunk-jvm/README.md @@ -4,10 +4,10 @@ The runner on a JVM machine: the main process of the `chunk-jvm` image, which [management](../../packages/management/README.md) starts when core asks for JVM capacity. It asks core what its host runs (`chunk:launch`), downloads that release's archive from core (`chunk:archive`), checks its size and SHA-256, and unpacks and verifies it with [`chunk-build`](../chunk-build/README.md). It fetches each blob of the deployment's asset -revision that the app reads and its cache lacks (`chunk:asset-read`), checking each one's size and SHA-256, and -materializes the app's asset directory. It then starts the app's JVM as a child process and stays in front of it: it -forwards SIGTERM, SIGINT and SIGQUIT, sends SIGKILL once the stop grace runs out after SIGTERM or SIGINT, and reaps -orphaned processes when it runs as PID 1. +revision that the app reads and its cache lacks (`chunk:asset-read`), four at a time, checking each one's size and +SHA-256, materializes the app's asset directory, and removes every other revision and blob from its cache. It then +starts the app's JVM as a child process and stays in front of it: it forwards SIGTERM, SIGINT and SIGQUIT, sends SIGKILL +once the stop grace runs out after SIGTERM or SIGINT, and reaps orphaned processes when it runs as PID 1. Each run of the runner makes up a new boot ID, and core binds the host to the first boot it sees. A stopped machine is replaced, not restarted: a restarted runner exits with code 77. diff --git a/crates/chunk-jvm/src/assets.rs b/crates/chunk-jvm/src/assets.rs index b8749564..75ed47e1 100644 --- a/crates/chunk-jvm/src/assets.rs +++ b/crates/chunk-jvm/src/assets.rs @@ -4,10 +4,14 @@ use crate::{Failure, cache::Cache, fetch::Core}; use chunk_contract::AssetRevision; use chunk_proto::sync::v1::JvmLaunch; +use futures_util::TryStreamExt as _; use std::path::PathBuf; +/// How many blobs are downloaded at once. +const CONCURRENT_DOWNLOADS: usize = 4; + /// Fetches each blob of `launch`'s app that the cache lacks, checked against its size and digest, and returns the -/// app's materialized directory. +/// app's materialized directory. Every other revision and blob leaves the cache, since this host runs only `launch`. pub(crate) async fn prepare(core: &Core, boot: &str, launch: &JvmLaunch, cache: &Cache) -> Result { let assets = launch.assets.as_ref().ok_or_else(|| Failure::verify("core named no asset revision"))?; let revision = AssetRevision::decode(&assets.manifest).map_err(Failure::verify)?; @@ -16,17 +20,32 @@ pub(crate) async fn prepare(core: &Core, boot: &str, launch: &JvmLaunch, cache: } let store = cache.assets(); let unreadable = |error| Failure::io(format!("cannot store the asset blobs: {error}")); + let mut missing = Vec::new(); for (sha256, size) in revision.app_blobs(&launch.app) { - if store.contains(sha256).map_err(unreadable)? { - continue; + if !store.contains(sha256).map_err(unreadable)? { + missing.push(Ok((sha256, size))); } - let mut staging = cache.blob_staging()?; - core.download_blob(boot, sha256, size, staging.as_file_mut()).await?; - store.insert_file(sha256, staging.path()).map_err(unreadable)?; } + futures_util::stream::iter(missing) + .try_for_each_concurrent(CONCURRENT_DOWNLOADS, |(sha256, size)| async move { + let mut staging = cache.blob_staging()?; + core.download_blob(boot, sha256, size, staging.as_file_mut()).await?; + let (store, sha256) = (store.clone(), sha256.to_owned()); + tokio::task::spawn_blocking(move || store.insert_file(&sha256, staging.path())) + .await + .map_err(Failure::io)? + .map_err(unreadable) + }) + .await?; let (store, app) = (store.clone(), launch.app.clone()); - tokio::task::spawn_blocking(move || chunk_build::assets::materialize(&store, &revision, &app)) - .await - .map_err(Failure::io)? - .map_err(|error| Failure::io(format!("cannot materialize the app's assets: {error}"))) + tokio::task::spawn_blocking(move || { + let directory = chunk_build::assets::materialize(&store, &revision, &app) + .map_err(|error| Failure::io(format!("cannot materialize the app's assets: {error}")))?; + if let Err(error) = store.write_revision(&revision).and_then(|_| store.retain(&[&revision])) { + tracing::warn!(%error, "unused asset blobs not removed"); + } + Ok(directory) + }) + .await + .map_err(Failure::io)? } diff --git a/crates/chunk-jvm/src/cache.rs b/crates/chunk-jvm/src/cache.rs index 8be56b57..a753098b 100644 --- a/crates/chunk-jvm/src/cache.rs +++ b/crates/chunk-jvm/src/cache.rs @@ -1,6 +1,6 @@ //! Verified release installs under `/releases/`, AOT caches under //! `/aot//.aot`, and asset blobs in the store at `/assets`, which may outlive the -//! machine. An AOT cache is kept outside its release, whose verification hashes every file there. +//! machine and holds only the revision the last launch ran. An AOT cache is kept outside its release, whose verification hashes every file there. use crate::Failure; use chunk_build::{Installed, VerifiedRelease, assets::Store, install_trusted_release, installed_release}; diff --git a/packages/dashboard/src/components/release.tsx b/packages/dashboard/src/components/release.tsx index 6dd473a0..957d34d4 100644 --- a/packages/dashboard/src/components/release.tsx +++ b/packages/dashboard/src/components/release.tsx @@ -22,10 +22,11 @@ const styles = stylex.create({ export function Release({ deploymentId }: { deploymentId: string }) { const deployment = useDeployment(deploymentId); if (!deploymentId) return Nothing deployed; - return ; + return ; } -export function ReleaseId({ id }: { id: string }) { +/** A release or asset revision ID, truncated to fit, in full on hover. */ +export function ShortId({ id }: { id: string }) { return ( {id} diff --git a/packages/dashboard/src/gen/chunk/management/v1/assets_pb.ts b/packages/dashboard/src/gen/chunk/management/v1/assets_pb.ts new file mode 100644 index 00000000..65e49400 --- /dev/null +++ b/packages/dashboard/src/gen/chunk/management/v1/assets_pb.ts @@ -0,0 +1,485 @@ +// @generated by protoc-gen-es v2.15.0 with parameter "target=ts,import_extension=ts" +// @generated from file chunk/management/v1/assets.proto (package chunk.management.v1, syntax proto3) +/* eslint-disable */ + +import type { Message } from "@bufbuild/protobuf"; +import type { GenEnum, GenFile, GenMessage, GenService } from "@bufbuild/protobuf/codegenv2"; +import { enumDesc, fileDesc, messageDesc, serviceDesc } from "@bufbuild/protobuf/codegenv2"; +import type { Timestamp } from "@bufbuild/protobuf/wkt"; +import { file_google_protobuf_timestamp } from "@bufbuild/protobuf/wkt"; + +import type { UploadTarget } from "./deployments_pb.ts"; +import { file_chunk_management_v1_deployments } from "./deployments_pb.ts"; + +/** + * Describes the file chunk/management/v1/assets.proto. + */ +export const file_chunk_management_v1_assets: GenFile /*@__PURE__*/ = fileDesc( + "CiBjaHVuay9tYW5hZ2VtZW50L3YxL2Fzc2V0cy5wcm90bxITY2h1bmsubWFuYWdlbWVudC52MSK+AQoNQXNzZXRSZXZpc2lvbhIKCgJpZBgBIAEoCRISCgpwcm9qZWN0X2lkGAIgASgJEjYKBXN0YXRlGAMgASgOMicuY2h1bmsubWFuYWdlbWVudC52MS5Bc3NldFJldmlzaW9uU3RhdGUSEAoIbWFuaWZlc3QYBCABKAwSEgoKc2l6ZV9ieXRlcxgFIAEoBBIvCgtjcmVhdGVfdGltZRgGIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAiTwoKQmxvYlVwbG9hZBIOCgZzaGEyNTYYASABKAkSMQoGdXBsb2FkGAIgASgLMiEuY2h1bmsubWFuYWdlbWVudC52MS5VcGxvYWRUYXJnZXQiKwoMQmxvYkRvd25sb2FkEg4KBnNoYTI1NhgBIAEoCRILCgN1cmwYAiABKAkiOwoTVXBsb2FkQXNzZXRzUmVxdWVzdBISCgpwcm9qZWN0X2lkGAEgASgJEhAKCG1hbmlmZXN0GAIgASgMIn4KFFVwbG9hZEFzc2V0c1Jlc3BvbnNlEjQKCHJldmlzaW9uGAEgASgLMiIuY2h1bmsubWFuYWdlbWVudC52MS5Bc3NldFJldmlzaW9uEjAKB3VwbG9hZHMYAiADKAsyHy5jaHVuay5tYW5hZ2VtZW50LnYxLkJsb2JVcGxvYWQiRQoaQ29tcGxldGVBc3NldFVwbG9hZFJlcXVlc3QSEgoKcHJvamVjdF9pZBgBIAEoCRITCgtyZXZpc2lvbl9pZBgCIAEoCSJTChtDb21wbGV0ZUFzc2V0VXBsb2FkUmVzcG9uc2USNAoIcmV2aXNpb24YASABKAsyIi5jaHVuay5tYW5hZ2VtZW50LnYxLkFzc2V0UmV2aXNpb24iVQoXR2V0QXNzZXRSZXZpc2lvblJlcXVlc3QSEgoKcHJvamVjdF9pZBgBIAEoCRITCgtyZXZpc2lvbl9pZBgCIAEoCRIRCglkb3dubG9hZHMYAyABKAgihgEKGEdldEFzc2V0UmV2aXNpb25SZXNwb25zZRI0CghyZXZpc2lvbhgBIAEoCzIiLmNodW5rLm1hbmFnZW1lbnQudjEuQXNzZXRSZXZpc2lvbhI0Cglkb3dubG9hZHMYAiADKAsyIS5jaHVuay5tYW5hZ2VtZW50LnYxLkJsb2JEb3dubG9hZCJWChlMaXN0QXNzZXRSZXZpc2lvbnNSZXF1ZXN0EhIKCnByb2plY3RfaWQYASABKAkSEQoJcGFnZV9zaXplGAIgASgFEhIKCnBhZ2VfdG9rZW4YAyABKAkifQoaTGlzdEFzc2V0UmV2aXNpb25zUmVzcG9uc2USNQoJcmV2aXNpb25zGAEgAygLMiIuY2h1bmsubWFuYWdlbWVudC52MS5Bc3NldFJldmlzaW9uEg8KB2hlYWRfaWQYAiABKAkSFwoPbmV4dF9wYWdlX3Rva2VuGAMgASgJIlgKE1NldEFzc2V0SGVhZFJlcXVlc3QSEgoKcHJvamVjdF9pZBgBIAEoCRITCgtyZXZpc2lvbl9pZBgCIAEoCRIYChBleHBlY3RlZF9oZWFkX2lkGAMgASgJIjAKFFNldEFzc2V0SGVhZFJlc3BvbnNlEhgKEHByZXZpb3VzX2hlYWRfaWQYASABKAkqfgoSQXNzZXRSZXZpc2lvblN0YXRlEiQKIEFTU0VUX1JFVklTSU9OX1NUQVRFX1VOU1BFQ0lGSUVEEAASIgoeQVNTRVRfUkVWSVNJT05fU1RBVEVfVVBMT0FESU5HEAESHgoaQVNTRVRfUkVWSVNJT05fU1RBVEVfUkVBRFkQAjK6BAoMQXNzZXRTZXJ2aWNlEmMKDFVwbG9hZEFzc2V0cxIoLmNodW5rLm1hbmFnZW1lbnQudjEuVXBsb2FkQXNzZXRzUmVxdWVzdBopLmNodW5rLm1hbmFnZW1lbnQudjEuVXBsb2FkQXNzZXRzUmVzcG9uc2USeAoTQ29tcGxldGVBc3NldFVwbG9hZBIvLmNodW5rLm1hbmFnZW1lbnQudjEuQ29tcGxldGVBc3NldFVwbG9hZFJlcXVlc3QaMC5jaHVuay5tYW5hZ2VtZW50LnYxLkNvbXBsZXRlQXNzZXRVcGxvYWRSZXNwb25zZRJvChBHZXRBc3NldFJldmlzaW9uEiwuY2h1bmsubWFuYWdlbWVudC52MS5HZXRBc3NldFJldmlzaW9uUmVxdWVzdBotLmNodW5rLm1hbmFnZW1lbnQudjEuR2V0QXNzZXRSZXZpc2lvblJlc3BvbnNlEnUKEkxpc3RBc3NldFJldmlzaW9ucxIuLmNodW5rLm1hbmFnZW1lbnQudjEuTGlzdEFzc2V0UmV2aXNpb25zUmVxdWVzdBovLmNodW5rLm1hbmFnZW1lbnQudjEuTGlzdEFzc2V0UmV2aXNpb25zUmVzcG9uc2USYwoMU2V0QXNzZXRIZWFkEiguY2h1bmsubWFuYWdlbWVudC52MS5TZXRBc3NldEhlYWRSZXF1ZXN0GikuY2h1bmsubWFuYWdlbWVudC52MS5TZXRBc3NldEhlYWRSZXNwb25zZWIGcHJvdG8z", + [file_chunk_management_v1_deployments, file_google_protobuf_timestamp], +); + +/** + * @generated from message chunk.management.v1.AssetRevision + */ +export type AssetRevision = Message<"chunk.management.v1.AssetRevision"> & { + /** + * The lowercase hex SHA-256 of `manifest`. + * + * @generated from field: string id = 1; + */ + id: string; + + /** + * @generated from field: string project_id = 2; + */ + projectId: string; + + /** + * @generated from field: chunk.management.v1.AssetRevisionState state = 3; + */ + state: AssetRevisionState; + + /** + * The revision's canonical JSON manifest, as `chunk build` writes it. + * + * @generated from field: bytes manifest = 4; + */ + manifest: Uint8Array; + + /** + * The distinct bytes of its blobs. + * + * @generated from field: uint64 size_bytes = 5; + */ + sizeBytes: bigint; + + /** + * @generated from field: google.protobuf.Timestamp create_time = 6; + */ + createTime?: Timestamp | undefined; +}; + +/** + * Describes the message chunk.management.v1.AssetRevision. + * Use `create(AssetRevisionSchema)` to create a new message. + */ +export const AssetRevisionSchema: GenMessage /*@__PURE__*/ = messageDesc( + file_chunk_management_v1_assets, + 0, +); + +/** + * @generated from message chunk.management.v1.BlobUpload + */ +export type BlobUpload = Message<"chunk.management.v1.BlobUpload"> & { + /** + * Lowercase hex SHA-256 of the blob. + * + * @generated from field: string sha256 = 1; + */ + sha256: string; + + /** + * @generated from field: chunk.management.v1.UploadTarget upload = 2; + */ + upload?: UploadTarget | undefined; +}; + +/** + * Describes the message chunk.management.v1.BlobUpload. + * Use `create(BlobUploadSchema)` to create a new message. + */ +export const BlobUploadSchema: GenMessage /*@__PURE__*/ = messageDesc(file_chunk_management_v1_assets, 1); + +/** + * @generated from message chunk.management.v1.BlobDownload + */ +export type BlobDownload = Message<"chunk.management.v1.BlobDownload"> & { + /** + * @generated from field: string sha256 = 1; + */ + sha256: string; + + /** + * Fetch with HTTP GET; it may expire. + * + * @generated from field: string url = 2; + */ + url: string; +}; + +/** + * Describes the message chunk.management.v1.BlobDownload. + * Use `create(BlobDownloadSchema)` to create a new message. + */ +export const BlobDownloadSchema: GenMessage /*@__PURE__*/ = messageDesc( + file_chunk_management_v1_assets, + 2, +); + +/** + * @generated from message chunk.management.v1.UploadAssetsRequest + */ +export type UploadAssetsRequest = Message<"chunk.management.v1.UploadAssetsRequest"> & { + /** + * @generated from field: string project_id = 1; + */ + projectId: string; + + /** + * The revision's canonical JSON manifest, at most 4 MiB. + * + * @generated from field: bytes manifest = 2; + */ + manifest: Uint8Array; +}; + +/** + * Describes the message chunk.management.v1.UploadAssetsRequest. + * Use `create(UploadAssetsRequestSchema)` to create a new message. + */ +export const UploadAssetsRequestSchema: GenMessage /*@__PURE__*/ = messageDesc( + file_chunk_management_v1_assets, + 3, +); + +/** + * @generated from message chunk.management.v1.UploadAssetsResponse + */ +export type UploadAssetsResponse = Message<"chunk.management.v1.UploadAssetsResponse"> & { + /** + * @generated from field: chunk.management.v1.AssetRevision revision = 1; + */ + revision?: AssetRevision | undefined; + + /** + * The blobs to upload; empty when the project holds them all. + * + * @generated from field: repeated chunk.management.v1.BlobUpload uploads = 2; + */ + uploads: BlobUpload[]; +}; + +/** + * Describes the message chunk.management.v1.UploadAssetsResponse. + * Use `create(UploadAssetsResponseSchema)` to create a new message. + */ +export const UploadAssetsResponseSchema: GenMessage /*@__PURE__*/ = messageDesc( + file_chunk_management_v1_assets, + 4, +); + +/** + * @generated from message chunk.management.v1.CompleteAssetUploadRequest + */ +export type CompleteAssetUploadRequest = Message<"chunk.management.v1.CompleteAssetUploadRequest"> & { + /** + * @generated from field: string project_id = 1; + */ + projectId: string; + + /** + * @generated from field: string revision_id = 2; + */ + revisionId: string; +}; + +/** + * Describes the message chunk.management.v1.CompleteAssetUploadRequest. + * Use `create(CompleteAssetUploadRequestSchema)` to create a new message. + */ +export const CompleteAssetUploadRequestSchema: GenMessage /*@__PURE__*/ = messageDesc( + file_chunk_management_v1_assets, + 5, +); + +/** + * @generated from message chunk.management.v1.CompleteAssetUploadResponse + */ +export type CompleteAssetUploadResponse = Message<"chunk.management.v1.CompleteAssetUploadResponse"> & { + /** + * @generated from field: chunk.management.v1.AssetRevision revision = 1; + */ + revision?: AssetRevision | undefined; +}; + +/** + * Describes the message chunk.management.v1.CompleteAssetUploadResponse. + * Use `create(CompleteAssetUploadResponseSchema)` to create a new message. + */ +export const CompleteAssetUploadResponseSchema: GenMessage /*@__PURE__*/ = messageDesc( + file_chunk_management_v1_assets, + 6, +); + +/** + * @generated from message chunk.management.v1.GetAssetRevisionRequest + */ +export type GetAssetRevisionRequest = Message<"chunk.management.v1.GetAssetRevisionRequest"> & { + /** + * @generated from field: string project_id = 1; + */ + projectId: string; + + /** + * Empty for the project's head. + * + * @generated from field: string revision_id = 2; + */ + revisionId: string; + + /** + * Also return a download URL for every blob. + * + * @generated from field: bool downloads = 3; + */ + downloads: boolean; +}; + +/** + * Describes the message chunk.management.v1.GetAssetRevisionRequest. + * Use `create(GetAssetRevisionRequestSchema)` to create a new message. + */ +export const GetAssetRevisionRequestSchema: GenMessage /*@__PURE__*/ = messageDesc( + file_chunk_management_v1_assets, + 7, +); + +/** + * @generated from message chunk.management.v1.GetAssetRevisionResponse + */ +export type GetAssetRevisionResponse = Message<"chunk.management.v1.GetAssetRevisionResponse"> & { + /** + * Unset when the revision is empty and the project has no head. + * + * @generated from field: chunk.management.v1.AssetRevision revision = 1; + */ + revision?: AssetRevision | undefined; + + /** + * @generated from field: repeated chunk.management.v1.BlobDownload downloads = 2; + */ + downloads: BlobDownload[]; +}; + +/** + * Describes the message chunk.management.v1.GetAssetRevisionResponse. + * Use `create(GetAssetRevisionResponseSchema)` to create a new message. + */ +export const GetAssetRevisionResponseSchema: GenMessage /*@__PURE__*/ = messageDesc( + file_chunk_management_v1_assets, + 8, +); + +/** + * @generated from message chunk.management.v1.ListAssetRevisionsRequest + */ +export type ListAssetRevisionsRequest = Message<"chunk.management.v1.ListAssetRevisionsRequest"> & { + /** + * @generated from field: string project_id = 1; + */ + projectId: string; + + /** + * @generated from field: int32 page_size = 2; + */ + pageSize: number; + + /** + * @generated from field: string page_token = 3; + */ + pageToken: string; +}; + +/** + * Describes the message chunk.management.v1.ListAssetRevisionsRequest. + * Use `create(ListAssetRevisionsRequestSchema)` to create a new message. + */ +export const ListAssetRevisionsRequestSchema: GenMessage /*@__PURE__*/ = messageDesc( + file_chunk_management_v1_assets, + 9, +); + +/** + * @generated from message chunk.management.v1.ListAssetRevisionsResponse + */ +export type ListAssetRevisionsResponse = Message<"chunk.management.v1.ListAssetRevisionsResponse"> & { + /** + * Without manifests. + * + * @generated from field: repeated chunk.management.v1.AssetRevision revisions = 1; + */ + revisions: AssetRevision[]; + + /** + * The project's head; empty when nothing was pushed. + * + * @generated from field: string head_id = 2; + */ + headId: string; + + /** + * @generated from field: string next_page_token = 3; + */ + nextPageToken: string; +}; + +/** + * Describes the message chunk.management.v1.ListAssetRevisionsResponse. + * Use `create(ListAssetRevisionsResponseSchema)` to create a new message. + */ +export const ListAssetRevisionsResponseSchema: GenMessage /*@__PURE__*/ = messageDesc( + file_chunk_management_v1_assets, + 10, +); + +/** + * @generated from message chunk.management.v1.SetAssetHeadRequest + */ +export type SetAssetHeadRequest = Message<"chunk.management.v1.SetAssetHeadRequest"> & { + /** + * @generated from field: string project_id = 1; + */ + projectId: string; + + /** + * @generated from field: string revision_id = 2; + */ + revisionId: string; + + /** + * The head the caller's revision was made from; empty when the project has + * no head yet. + * + * @generated from field: string expected_head_id = 3; + */ + expectedHeadId: string; +}; + +/** + * Describes the message chunk.management.v1.SetAssetHeadRequest. + * Use `create(SetAssetHeadRequestSchema)` to create a new message. + */ +export const SetAssetHeadRequestSchema: GenMessage /*@__PURE__*/ = messageDesc( + file_chunk_management_v1_assets, + 11, +); + +/** + * @generated from message chunk.management.v1.SetAssetHeadResponse + */ +export type SetAssetHeadResponse = Message<"chunk.management.v1.SetAssetHeadResponse"> & { + /** + * The head before the call. + * + * @generated from field: string previous_head_id = 1; + */ + previousHeadId: string; +}; + +/** + * Describes the message chunk.management.v1.SetAssetHeadResponse. + * Use `create(SetAssetHeadResponseSchema)` to create a new message. + */ +export const SetAssetHeadResponseSchema: GenMessage /*@__PURE__*/ = messageDesc( + file_chunk_management_v1_assets, + 12, +); + +/** + * @generated from enum chunk.management.v1.AssetRevisionState + */ +export enum AssetRevisionState { + /** + * @generated from enum value: ASSET_REVISION_STATE_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * @generated from enum value: ASSET_REVISION_STATE_UPLOADING = 1; + */ + UPLOADING = 1, + + /** + * @generated from enum value: ASSET_REVISION_STATE_READY = 2; + */ + READY = 2, +} + +/** + * Describes the enum chunk.management.v1.AssetRevisionState. + */ +export const AssetRevisionStateSchema: GenEnum /*@__PURE__*/ = enumDesc( + file_chunk_management_v1_assets, + 0, +); + +/** + * Asset revisions, for the CLI and dashboards. A revision is an immutable + * manifest of a project's worlds, resource packs and files by content digest, + * which `chunk build` writes beside a release. A deployment pins one release + * and one revision. Blobs are stored once per project, so a revision that + * changes one map uploads one blob. A project's head is the revision `chunk + * platform assets push` last published and `pull` fetches. + * + * @generated from service chunk.management.v1.AssetService + */ +export const AssetService: GenService<{ + /** + * Declares a revision and returns where to upload each of its blobs the + * project does not hold yet. Idempotent by revision ID, which management + * derives from the manifest; then call CompleteAssetUpload. + * + * @generated from rpc chunk.management.v1.AssetService.UploadAssets + */ + uploadAssets: { + methodKind: "unary"; + input: typeof UploadAssetsRequestSchema; + output: typeof UploadAssetsResponseSchema; + }; + /** + * Verifies every blob of the revision against its digests, including each + * pack's SHA-1, and makes the revision deployable. Fails with + * FAILED_PRECONDITION while a blob is missing. Idempotent. + * + * @generated from rpc chunk.management.v1.AssetService.CompleteAssetUpload + */ + completeAssetUpload: { + methodKind: "unary"; + input: typeof CompleteAssetUploadRequestSchema; + output: typeof CompleteAssetUploadResponseSchema; + }; + /** + * Returns a READY revision's manifest, and with `downloads`, a URL for each + * of its blobs. + * + * @generated from rpc chunk.management.v1.AssetService.GetAssetRevision + */ + getAssetRevision: { + methodKind: "unary"; + input: typeof GetAssetRevisionRequestSchema; + output: typeof GetAssetRevisionResponseSchema; + }; + /** + * Lists a project's READY revisions, newest first. + * + * @generated from rpc chunk.management.v1.AssetService.ListAssetRevisions + */ + listAssetRevisions: { + methodKind: "unary"; + input: typeof ListAssetRevisionsRequestSchema; + output: typeof ListAssetRevisionsResponseSchema; + }; + /** + * Moves the project's head to a READY revision if the head is still + * `expected_head_id`; otherwise fails with ABORTED, leaving it as it is. + * + * @generated from rpc chunk.management.v1.AssetService.SetAssetHead + */ + setAssetHead: { + methodKind: "unary"; + input: typeof SetAssetHeadRequestSchema; + output: typeof SetAssetHeadResponseSchema; + }; +}> /*@__PURE__*/ = serviceDesc(file_chunk_management_v1_assets, 0); diff --git a/packages/dashboard/src/gen/chunk/management/v1/deployments_pb.ts b/packages/dashboard/src/gen/chunk/management/v1/deployments_pb.ts index a24d62bb..72ae23f6 100644 --- a/packages/dashboard/src/gen/chunk/management/v1/deployments_pb.ts +++ b/packages/dashboard/src/gen/chunk/management/v1/deployments_pb.ts @@ -15,7 +15,7 @@ import { file_chunk_management_v1_common } from "./common_pb.ts"; * Describes the file chunk/management/v1/deployments.proto. */ export const file_chunk_management_v1_deployments: GenFile /*@__PURE__*/ = fileDesc( - "CiVjaHVuay9tYW5hZ2VtZW50L3YxL2RlcGxveW1lbnRzLnByb3RvEhNjaHVuay5tYW5hZ2VtZW50LnYxIsABCgdSZWxlYXNlEgoKAmlkGAEgASgJEhIKCnByb2plY3RfaWQYAiABKAkSMAoFc3RhdGUYAyABKA4yIS5jaHVuay5tYW5hZ2VtZW50LnYxLlJlbGVhc2VTdGF0ZRIWCg5hcmNoaXZlX3NoYTI1NhgEIAEoCRIaChJhcmNoaXZlX3NpemVfYnl0ZXMYBSABKAQSLwoLY3JlYXRlX3RpbWUYBiABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wIqUCCgpEZXBsb3ltZW50EgoKAmlkGAEgASgJEhYKDmVudmlyb25tZW50X2lkGAIgASgJEhIKCnJlbGVhc2VfaWQYAyABKAkSMwoFc3RhdGUYBCABKA4yJC5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveW1lbnRTdGF0ZRI3Cgd0cmlnZ2VyGAUgASgOMiYuY2h1bmsubWFuYWdlbWVudC52MS5EZXBsb3ltZW50VHJpZ2dlchIPCgdtZXNzYWdlGAYgASgJEi8KC2NyZWF0ZV90aW1lGAcgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIvCgt1cGRhdGVfdGltZRgIIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAiIwoDQXBwEgoKAmlkGAEgASgJEhAKCHNlc3Npb25zGAIgAygJIs0BCgxVcGxvYWRUYXJnZXQSCwoDdXJsGAEgASgJEg4KBm1ldGhvZBgCIAEoCRI/CgdoZWFkZXJzGAMgAygLMi4uY2h1bmsubWFuYWdlbWVudC52MS5VcGxvYWRUYXJnZXQuSGVhZGVyc0VudHJ5Ei8KC2V4cGlyZV90aW1lGAQgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBouCgxIZWFkZXJzRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4ASJyChRVcGxvYWRSZWxlYXNlUmVxdWVzdBISCgpwcm9qZWN0X2lkGAEgASgJEhIKCnJlbGVhc2VfaWQYAiABKAkSFgoOYXJjaGl2ZV9zaGEyNTYYAyABKAkSGgoSYXJjaGl2ZV9zaXplX2J5dGVzGAQgASgEInkKFVVwbG9hZFJlbGVhc2VSZXNwb25zZRItCgdyZWxlYXNlGAEgASgLMhwuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlEjEKBnVwbG9hZBgCIAEoCzIhLmNodW5rLm1hbmFnZW1lbnQudjEuVXBsb2FkVGFyZ2V0IkYKHENvbXBsZXRlUmVsZWFzZVVwbG9hZFJlcXVlc3QSEgoKcHJvamVjdF9pZBgBIAEoCRISCgpyZWxlYXNlX2lkGAIgASgJIk4KHUNvbXBsZXRlUmVsZWFzZVVwbG9hZFJlc3BvbnNlEi0KB3JlbGVhc2UYASABKAsyHC5jaHVuay5tYW5hZ2VtZW50LnYxLlJlbGVhc2UiZgoNRGVwbG95UmVxdWVzdBISCgpyZXF1ZXN0X2lkGAEgASgJEhYKDmVudmlyb25tZW50X2lkGAIgASgJEhIKCnJlbGVhc2VfaWQYAyABKAkSFQoNc3RvcF9wcmV2aW91cxgEIAEoCCJFCg5EZXBsb3lSZXNwb25zZRIzCgpkZXBsb3ltZW50GAEgASgLMh8uY2h1bmsubWFuYWdlbWVudC52MS5EZXBsb3ltZW50ImIKDlByb21vdGVSZXF1ZXN0EhIKCnJlcXVlc3RfaWQYASABKAkSHQoVc291cmNlX2Vudmlyb25tZW50X2lkGAIgASgJEh0KFXRhcmdldF9lbnZpcm9ubWVudF9pZBgDIAEoCSJGCg9Qcm9tb3RlUmVzcG9uc2USMwoKZGVwbG95bWVudBgBIAEoCzIfLmNodW5rLm1hbmFnZW1lbnQudjEuRGVwbG95bWVudCJUCg9Sb2xsYmFja1JlcXVlc3QSEgoKcmVxdWVzdF9pZBgBIAEoCRIWCg5lbnZpcm9ubWVudF9pZBgCIAEoCRIVCg1kZXBsb3ltZW50X2lkGAMgASgJIkcKEFJvbGxiYWNrUmVzcG9uc2USMwoKZGVwbG95bWVudBgBIAEoCzIfLmNodW5rLm1hbmFnZW1lbnQudjEuRGVwbG95bWVudCItChRHZXREZXBsb3ltZW50UmVxdWVzdBIVCg1kZXBsb3ltZW50X2lkGAEgASgJIkwKFUdldERlcGxveW1lbnRSZXNwb25zZRIzCgpkZXBsb3ltZW50GAEgASgLMh8uY2h1bmsubWFuYWdlbWVudC52MS5EZXBsb3ltZW50IlcKFkxpc3REZXBsb3ltZW50c1JlcXVlc3QSFgoOZW52aXJvbm1lbnRfaWQYASABKAkSEQoJcGFnZV9zaXplGAIgASgFEhIKCnBhZ2VfdG9rZW4YAyABKAkiaAoXTGlzdERlcGxveW1lbnRzUmVzcG9uc2USNAoLZGVwbG95bWVudHMYASADKAsyHy5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveW1lbnQSFwoPbmV4dF9wYWdlX3Rva2VuGAIgASgJIj0KD0xpc3RBcHBzUmVxdWVzdBIWCg5lbnZpcm9ubWVudF9pZBgBIAEoCRISCgpyZWxlYXNlX2lkGAIgASgJIjoKEExpc3RBcHBzUmVzcG9uc2USJgoEYXBwcxgBIAMoCzIYLmNodW5rLm1hbmFnZW1lbnQudjEuQXBwKmMKDFJlbGVhc2VTdGF0ZRIdChlSRUxFQVNFX1NUQVRFX1VOU1BFQ0lGSUVEEAASGwoXUkVMRUFTRV9TVEFURV9VUExPQURJTkcQARIXChNSRUxFQVNFX1NUQVRFX1JFQURZEAIqtAEKEURlcGxveW1lbnRUcmlnZ2VyEiIKHkRFUExPWU1FTlRfVFJJR0dFUl9VTlNQRUNJRklFRBAAEh0KGURFUExPWU1FTlRfVFJJR0dFUl9ERVBMT1kQARIeChpERVBMT1lNRU5UX1RSSUdHRVJfUFJPTU9URRACEh8KG0RFUExPWU1FTlRfVFJJR0dFUl9ST0xMQkFDSxADEhsKF0RFUExPWU1FTlRfVFJJR0dFUl9GT1JLEAQyrAYKEURlcGxveW1lbnRTZXJ2aWNlEmYKDVVwbG9hZFJlbGVhc2USKS5jaHVuay5tYW5hZ2VtZW50LnYxLlVwbG9hZFJlbGVhc2VSZXF1ZXN0GiouY2h1bmsubWFuYWdlbWVudC52MS5VcGxvYWRSZWxlYXNlUmVzcG9uc2USfgoVQ29tcGxldGVSZWxlYXNlVXBsb2FkEjEuY2h1bmsubWFuYWdlbWVudC52MS5Db21wbGV0ZVJlbGVhc2VVcGxvYWRSZXF1ZXN0GjIuY2h1bmsubWFuYWdlbWVudC52MS5Db21wbGV0ZVJlbGVhc2VVcGxvYWRSZXNwb25zZRJRCgZEZXBsb3kSIi5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveVJlcXVlc3QaIy5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveVJlc3BvbnNlElQKB1Byb21vdGUSIy5jaHVuay5tYW5hZ2VtZW50LnYxLlByb21vdGVSZXF1ZXN0GiQuY2h1bmsubWFuYWdlbWVudC52MS5Qcm9tb3RlUmVzcG9uc2USVwoIUm9sbGJhY2sSJC5jaHVuay5tYW5hZ2VtZW50LnYxLlJvbGxiYWNrUmVxdWVzdBolLmNodW5rLm1hbmFnZW1lbnQudjEuUm9sbGJhY2tSZXNwb25zZRJmCg1HZXREZXBsb3ltZW50EikuY2h1bmsubWFuYWdlbWVudC52MS5HZXREZXBsb3ltZW50UmVxdWVzdBoqLmNodW5rLm1hbmFnZW1lbnQudjEuR2V0RGVwbG95bWVudFJlc3BvbnNlEmwKD0xpc3REZXBsb3ltZW50cxIrLmNodW5rLm1hbmFnZW1lbnQudjEuTGlzdERlcGxveW1lbnRzUmVxdWVzdBosLmNodW5rLm1hbmFnZW1lbnQudjEuTGlzdERlcGxveW1lbnRzUmVzcG9uc2USVwoITGlzdEFwcHMSJC5jaHVuay5tYW5hZ2VtZW50LnYxLkxpc3RBcHBzUmVxdWVzdBolLmNodW5rLm1hbmFnZW1lbnQudjEuTGlzdEFwcHNSZXNwb25zZWIGcHJvdG8z", + "CiVjaHVuay9tYW5hZ2VtZW50L3YxL2RlcGxveW1lbnRzLnByb3RvEhNjaHVuay5tYW5hZ2VtZW50LnYxIsABCgdSZWxlYXNlEgoKAmlkGAEgASgJEhIKCnByb2plY3RfaWQYAiABKAkSMAoFc3RhdGUYAyABKA4yIS5jaHVuay5tYW5hZ2VtZW50LnYxLlJlbGVhc2VTdGF0ZRIWCg5hcmNoaXZlX3NoYTI1NhgEIAEoCRIaChJhcmNoaXZlX3NpemVfYnl0ZXMYBSABKAQSLwoLY3JlYXRlX3RpbWUYBiABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wIsACCgpEZXBsb3ltZW50EgoKAmlkGAEgASgJEhYKDmVudmlyb25tZW50X2lkGAIgASgJEhIKCnJlbGVhc2VfaWQYAyABKAkSMwoFc3RhdGUYBCABKA4yJC5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveW1lbnRTdGF0ZRI3Cgd0cmlnZ2VyGAUgASgOMiYuY2h1bmsubWFuYWdlbWVudC52MS5EZXBsb3ltZW50VHJpZ2dlchIPCgdtZXNzYWdlGAYgASgJEi8KC2NyZWF0ZV90aW1lGAcgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIvCgt1cGRhdGVfdGltZRgIIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASGQoRYXNzZXRfcmV2aXNpb25faWQYCSABKAkiIwoDQXBwEgoKAmlkGAEgASgJEhAKCHNlc3Npb25zGAIgAygJIs0BCgxVcGxvYWRUYXJnZXQSCwoDdXJsGAEgASgJEg4KBm1ldGhvZBgCIAEoCRI/CgdoZWFkZXJzGAMgAygLMi4uY2h1bmsubWFuYWdlbWVudC52MS5VcGxvYWRUYXJnZXQuSGVhZGVyc0VudHJ5Ei8KC2V4cGlyZV90aW1lGAQgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBouCgxIZWFkZXJzRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4ASJyChRVcGxvYWRSZWxlYXNlUmVxdWVzdBISCgpwcm9qZWN0X2lkGAEgASgJEhIKCnJlbGVhc2VfaWQYAiABKAkSFgoOYXJjaGl2ZV9zaGEyNTYYAyABKAkSGgoSYXJjaGl2ZV9zaXplX2J5dGVzGAQgASgEInkKFVVwbG9hZFJlbGVhc2VSZXNwb25zZRItCgdyZWxlYXNlGAEgASgLMhwuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlEjEKBnVwbG9hZBgCIAEoCzIhLmNodW5rLm1hbmFnZW1lbnQudjEuVXBsb2FkVGFyZ2V0IkYKHENvbXBsZXRlUmVsZWFzZVVwbG9hZFJlcXVlc3QSEgoKcHJvamVjdF9pZBgBIAEoCRISCgpyZWxlYXNlX2lkGAIgASgJIk4KHUNvbXBsZXRlUmVsZWFzZVVwbG9hZFJlc3BvbnNlEi0KB3JlbGVhc2UYASABKAsyHC5jaHVuay5tYW5hZ2VtZW50LnYxLlJlbGVhc2UigQEKDURlcGxveVJlcXVlc3QSEgoKcmVxdWVzdF9pZBgBIAEoCRIWCg5lbnZpcm9ubWVudF9pZBgCIAEoCRISCgpyZWxlYXNlX2lkGAMgASgJEhUKDXN0b3BfcHJldmlvdXMYBCABKAgSGQoRYXNzZXRfcmV2aXNpb25faWQYBSABKAkiRQoORGVwbG95UmVzcG9uc2USMwoKZGVwbG95bWVudBgBIAEoCzIfLmNodW5rLm1hbmFnZW1lbnQudjEuRGVwbG95bWVudCJiCg5Qcm9tb3RlUmVxdWVzdBISCgpyZXF1ZXN0X2lkGAEgASgJEh0KFXNvdXJjZV9lbnZpcm9ubWVudF9pZBgCIAEoCRIdChV0YXJnZXRfZW52aXJvbm1lbnRfaWQYAyABKAkiRgoPUHJvbW90ZVJlc3BvbnNlEjMKCmRlcGxveW1lbnQYASABKAsyHy5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveW1lbnQiVAoPUm9sbGJhY2tSZXF1ZXN0EhIKCnJlcXVlc3RfaWQYASABKAkSFgoOZW52aXJvbm1lbnRfaWQYAiABKAkSFQoNZGVwbG95bWVudF9pZBgDIAEoCSJHChBSb2xsYmFja1Jlc3BvbnNlEjMKCmRlcGxveW1lbnQYASABKAsyHy5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveW1lbnQiLQoUR2V0RGVwbG95bWVudFJlcXVlc3QSFQoNZGVwbG95bWVudF9pZBgBIAEoCSJMChVHZXREZXBsb3ltZW50UmVzcG9uc2USMwoKZGVwbG95bWVudBgBIAEoCzIfLmNodW5rLm1hbmFnZW1lbnQudjEuRGVwbG95bWVudCJXChZMaXN0RGVwbG95bWVudHNSZXF1ZXN0EhYKDmVudmlyb25tZW50X2lkGAEgASgJEhEKCXBhZ2Vfc2l6ZRgCIAEoBRISCgpwYWdlX3Rva2VuGAMgASgJImgKF0xpc3REZXBsb3ltZW50c1Jlc3BvbnNlEjQKC2RlcGxveW1lbnRzGAEgAygLMh8uY2h1bmsubWFuYWdlbWVudC52MS5EZXBsb3ltZW50EhcKD25leHRfcGFnZV90b2tlbhgCIAEoCSI9Cg9MaXN0QXBwc1JlcXVlc3QSFgoOZW52aXJvbm1lbnRfaWQYASABKAkSEgoKcmVsZWFzZV9pZBgCIAEoCSI6ChBMaXN0QXBwc1Jlc3BvbnNlEiYKBGFwcHMYASADKAsyGC5jaHVuay5tYW5hZ2VtZW50LnYxLkFwcCpjCgxSZWxlYXNlU3RhdGUSHQoZUkVMRUFTRV9TVEFURV9VTlNQRUNJRklFRBAAEhsKF1JFTEVBU0VfU1RBVEVfVVBMT0FESU5HEAESFwoTUkVMRUFTRV9TVEFURV9SRUFEWRACKrQBChFEZXBsb3ltZW50VHJpZ2dlchIiCh5ERVBMT1lNRU5UX1RSSUdHRVJfVU5TUEVDSUZJRUQQABIdChlERVBMT1lNRU5UX1RSSUdHRVJfREVQTE9ZEAESHgoaREVQTE9ZTUVOVF9UUklHR0VSX1BST01PVEUQAhIfChtERVBMT1lNRU5UX1RSSUdHRVJfUk9MTEJBQ0sQAxIbChdERVBMT1lNRU5UX1RSSUdHRVJfRk9SSxAEMqwGChFEZXBsb3ltZW50U2VydmljZRJmCg1VcGxvYWRSZWxlYXNlEikuY2h1bmsubWFuYWdlbWVudC52MS5VcGxvYWRSZWxlYXNlUmVxdWVzdBoqLmNodW5rLm1hbmFnZW1lbnQudjEuVXBsb2FkUmVsZWFzZVJlc3BvbnNlEn4KFUNvbXBsZXRlUmVsZWFzZVVwbG9hZBIxLmNodW5rLm1hbmFnZW1lbnQudjEuQ29tcGxldGVSZWxlYXNlVXBsb2FkUmVxdWVzdBoyLmNodW5rLm1hbmFnZW1lbnQudjEuQ29tcGxldGVSZWxlYXNlVXBsb2FkUmVzcG9uc2USUQoGRGVwbG95EiIuY2h1bmsubWFuYWdlbWVudC52MS5EZXBsb3lSZXF1ZXN0GiMuY2h1bmsubWFuYWdlbWVudC52MS5EZXBsb3lSZXNwb25zZRJUCgdQcm9tb3RlEiMuY2h1bmsubWFuYWdlbWVudC52MS5Qcm9tb3RlUmVxdWVzdBokLmNodW5rLm1hbmFnZW1lbnQudjEuUHJvbW90ZVJlc3BvbnNlElcKCFJvbGxiYWNrEiQuY2h1bmsubWFuYWdlbWVudC52MS5Sb2xsYmFja1JlcXVlc3QaJS5jaHVuay5tYW5hZ2VtZW50LnYxLlJvbGxiYWNrUmVzcG9uc2USZgoNR2V0RGVwbG95bWVudBIpLmNodW5rLm1hbmFnZW1lbnQudjEuR2V0RGVwbG95bWVudFJlcXVlc3QaKi5jaHVuay5tYW5hZ2VtZW50LnYxLkdldERlcGxveW1lbnRSZXNwb25zZRJsCg9MaXN0RGVwbG95bWVudHMSKy5jaHVuay5tYW5hZ2VtZW50LnYxLkxpc3REZXBsb3ltZW50c1JlcXVlc3QaLC5jaHVuay5tYW5hZ2VtZW50LnYxLkxpc3REZXBsb3ltZW50c1Jlc3BvbnNlElcKCExpc3RBcHBzEiQuY2h1bmsubWFuYWdlbWVudC52MS5MaXN0QXBwc1JlcXVlc3QaJS5jaHVuay5tYW5hZ2VtZW50LnYxLkxpc3RBcHBzUmVzcG9uc2ViBnByb3RvMw", [file_chunk_management_v1_common, file_google_protobuf_timestamp], ); @@ -109,6 +109,13 @@ export type Deployment = Message<"chunk.management.v1.Deployment"> & { * @generated from field: google.protobuf.Timestamp update_time = 8; */ updateTime?: Timestamp | undefined; + + /** + * The asset revision the deployment pins. + * + * @generated from field: string asset_revision_id = 9; + */ + assetRevisionId: string; }; /** @@ -314,6 +321,13 @@ export type DeployRequest = Message<"chunk.management.v1.DeployRequest"> & { * @generated from field: bool stop_previous = 4; */ stopPrevious: boolean; + + /** + * A READY asset revision of the environment's project. + * + * @generated from field: string asset_revision_id = 5; + */ + assetRevisionId: string; }; /** @@ -407,7 +421,7 @@ export type RollbackRequest = Message<"chunk.management.v1.RollbackRequest"> & { environmentId: string; /** - * The deployment whose release to restore; empty for the one active before the current one. + * The deployment whose release and asset revision to restore; empty for the one active before the current one. * * @generated from field: string deployment_id = 3; */ @@ -648,8 +662,8 @@ export const DeploymentTriggerSchema: GenEnum /*@__PURE__*/ = /** * Releases and deployments, for the CLI and dashboards. A release is an - * immutable archive from `chunk build`; a deployment makes one release an - * environment's desired state. Environments receive deployments through + * immutable archive from `chunk build`; a deployment makes one release and one + * asset revision (see AssetService) an environment's desired state. Environments receive deployments through * EnvironmentService.Attach and report progress through ReportStatus. * * @generated from service chunk.management.v1.DeploymentService @@ -682,7 +696,9 @@ export const DeploymentService: GenService<{ output: typeof CompleteReleaseUploadResponseSchema; }; /** - * Deploys a release of the environment's project. Returns once recorded; the + * Deploys a release and an asset revision of the environment's project. The + * revision must hold every world and pack the release declares, or the call + * fails with FAILED_PRECONDITION. Returns once recorded; the * deployment becomes ACTIVE when the environment reports serving it, and a * later deployment supersedes an unfinished one. Idempotent by request_id. * @@ -694,8 +710,9 @@ export const DeploymentService: GenService<{ output: typeof DeployResponseSchema; }; /** - * Deploys the release active in one environment to another environment of - * the same project. Only the release moves, never data or secrets. + * Deploys the release and asset revision active in one environment to + * another environment of the same project. Only they move, never data or + * secrets. * Idempotent by request_id. * * @generated from rpc chunk.management.v1.DeploymentService.Promote @@ -706,8 +723,8 @@ export const DeploymentService: GenService<{ output: typeof PromoteResponseSchema; }; /** - * Deploys the release of an earlier deployment again. Idempotent by - * request_id. + * Deploys the release and asset revision of an earlier deployment again. + * Idempotent by request_id. * * @generated from rpc chunk.management.v1.DeploymentService.Rollback */ diff --git a/packages/dashboard/src/gen/chunk/management/v1/environment_pb.ts b/packages/dashboard/src/gen/chunk/management/v1/environment_pb.ts index e83ef576..e43b62a7 100644 --- a/packages/dashboard/src/gen/chunk/management/v1/environment_pb.ts +++ b/packages/dashboard/src/gen/chunk/management/v1/environment_pb.ts @@ -15,7 +15,7 @@ import { file_chunk_management_v1_common } from "./common_pb.ts"; * Describes the file chunk/management/v1/environment.proto. */ export const file_chunk_management_v1_environment: GenFile /*@__PURE__*/ = fileDesc( - "CiVjaHVuay9tYW5hZ2VtZW50L3YxL2Vudmlyb25tZW50LnByb3RvEhNjaHVuay5tYW5hZ2VtZW50LnYxInIKDUF0dGFjaFJlcXVlc3QSEwoLaW5zdGFuY2VfaWQYASABKAkSDwoHdmVyc2lvbhgCIAEoCRIMCgRjb3JlGAMgASgIEg0KBWVwb2NoGAQgASgEEh4KFnJlc3RvcmVkX2RlcGxveW1lbnRfaWQYBSABKAkipgMKDkF0dGFjaFJlc3BvbnNlEhAKCHJldmlzaW9uGAEgASgEEhYKDmVudmlyb25tZW50X2lkGAIgASgJEhIKCnByb2plY3RfaWQYAyABKAkSFQoNZGVwbG95bWVudF9pZBgEIAEoCRI1CgdyZWxlYXNlGAUgASgLMiQuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlQXJ0aWZhY3QSMQoHc2VjcmV0cxgGIAMoCzIgLmNodW5rLm1hbmFnZW1lbnQudjEuU2VjcmV0VmFsdWUSMwoJbG9nX3N0b3JlGAcgASgLMiAuY2h1bmsubWFuYWdlbWVudC52MS5PYmplY3RTdG9yZRItCgdyZXN0b3JlGAggASgLMhwuY2h1bmsubWFuYWdlbWVudC52MS5SZXN0b3JlEg0KBWxlYXNlGAkgASgEEhgKEGVudmlyb25tZW50X25hbWUYCiABKAkSMQoFZHJhaW4YCyABKAsyIi5jaHVuay5tYW5hZ2VtZW50LnYxLkRyYWluU2V0dGluZ3MSFQoNc3RvcF9wcmV2aW91cxgMIAEoCCJWCg9SZWxlYXNlQXJ0aWZhY3QSEgoKcmVsZWFzZV9pZBgBIAEoCRILCgN1cmwYAiABKAkSDgoGc2hhMjU2GAMgASgJEhIKCnNpemVfYnl0ZXMYBCABKAQiOwoLU2VjcmV0VmFsdWUSDAoEbmFtZRgBIAEoCRIPCgd2ZXJzaW9uGAIgASgEEg0KBXZhbHVlGAMgASgMIskBCgtPYmplY3RTdG9yZRIQCghlbmRwb2ludBgBIAEoCRIOCgZyZWdpb24YAiABKAkSDgoGYnVja2V0GAMgASgJEg4KBnByZWZpeBgEIAEoCRIVCg1hY2Nlc3Nfa2V5X2lkGAUgASgJEhkKEXNlY3JldF9hY2Nlc3Nfa2V5GAYgASgJEhUKDXNlc3Npb25fdG9rZW4YByABKAkSLwoLZXhwaXJlX3RpbWUYCCABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wIm8KB1Jlc3RvcmUSMAoGc291cmNlGAEgASgLMiAuY2h1bmsubWFuYWdlbWVudC52MS5PYmplY3RTdG9yZRITCgtzbmFwc2hvdF9pZBgCIAEoCRIdChVzb3VyY2VfZW52aXJvbm1lbnRfaWQYAyABKAkizAIKE1JlcG9ydFN0YXR1c1JlcXVlc3QSMAoMb2JzZXJ2ZV90aW1lGAEgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIZChFnYXRld2F5X2FkZHJlc3NlcxgCIAMoCRIWCg5vbmxpbmVfcGxheWVycxgDIAEoDRIuCgVwaW5ncxgEIAMoCzIfLmNodW5rLm1hbmFnZW1lbnQudjEuUGluZ1N0YXR1cxI7CgpkZXBsb3ltZW50GAUgASgLMicuY2h1bmsubWFuYWdlbWVudC52MS5EZXBsb3ltZW50UHJvZ3Jlc3MSGAoQcmVhZHlfdG9fc3VzcGVuZBgGIAEoCBINCgVsZWFzZRgHIAEoBBIQCghzZXF1ZW5jZRgIIAEoBBIYChBkZXNpcmVkX3JldmlzaW9uGAkgASgEEg4KBmxvZ2lucxgKIAEoBCIzCgpQaW5nU3RhdHVzEhAKCGhvc3RuYW1lGAEgASgJEhMKC3N0YXR1c19qc29uGAIgASgJInEKEkRlcGxveW1lbnRQcm9ncmVzcxIVCg1kZXBsb3ltZW50X2lkGAEgASgJEjMKBXN0YXRlGAIgASgOMiQuY2h1bmsubWFuYWdlbWVudC52MS5EZXBsb3ltZW50U3RhdGUSDwoHbWVzc2FnZRgDIAEoCSIWChRSZXBvcnRTdGF0dXNSZXNwb25zZSJ1ChNTZXRXYWtlQWxhcm1SZXF1ZXN0EhIKCmdlbmVyYXRpb24YASABKAQSLAoIZHVlX3RpbWUYAiABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEg0KBWVwb2NoGAMgASgEEg0KBWxlYXNlGAQgASgEImcKFFNldFdha2VBbGFybVJlc3BvbnNlEhIKCmdlbmVyYXRpb24YASABKAQSLAoIZHVlX3RpbWUYAiABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEg0KBWVwb2NoGAMgASgEIrwBChVFbnN1cmVDYXBhY2l0eVJlcXVlc3QSEgoKcmVxdWVzdF9pZBgBIAEoCRIvCgh3b3JrbG9hZBgCIAEoDjIdLmNodW5rLm1hbmFnZW1lbnQudjEuV29ya2xvYWQSFwoPbWFjaGluZV9wcm9maWxlGAMgASgJEhIKCnJlbGVhc2VfaWQYBCABKAkSDgoGYXBwX2lkGAUgASgJEg0KBWxlYXNlGAYgASgEEhIKCmNyZWRlbnRpYWwYByABKAkiSQoWRW5zdXJlQ2FwYWNpdHlSZXNwb25zZRIvCghjYXBhY2l0eRgBIAEoCzIdLmNodW5rLm1hbmFnZW1lbnQudjEuQ2FwYWNpdHkidgoIQ2FwYWNpdHkSEgoKcmVxdWVzdF9pZBgBIAEoCRIxCgVzdGF0ZRgCIAEoDjIiLmNodW5rLm1hbmFnZW1lbnQudjEuQ2FwYWNpdHlTdGF0ZRISCgptYWNoaW5lX2lkGAMgASgJEg8KB21lc3NhZ2UYBCABKAkiOwoWUmVsZWFzZUNhcGFjaXR5UmVxdWVzdBISCgpyZXF1ZXN0X2lkGAEgASgJEg0KBWxlYXNlGAIgASgEIkoKF1JlbGVhc2VDYXBhY2l0eVJlc3BvbnNlEi8KCGNhcGFjaXR5GAEgASgLMh0uY2h1bmsubWFuYWdlbWVudC52MS5DYXBhY2l0eSJHChJSZXBvcnRVc2FnZVJlcXVlc3QSMQoHcmVjb3JkcxgBIAMoCzIgLmNodW5rLm1hbmFnZW1lbnQudjEuVXNhZ2VSZWNvcmQipAEKC1VzYWdlUmVjb3JkEgoKAmlkGAEgASgJEi4KCnN0YXJ0X3RpbWUYAiABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEiwKCGVuZF90aW1lGAMgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIWCg5wbGF5ZXJfc2Vjb25kcxgEIAEoBBITCgtpbnN0YW5jZV9pZBgFIAEoCSIVChNSZXBvcnRVc2FnZVJlc3BvbnNlIkMKEVJlcG9ydExvZ3NSZXF1ZXN0Ei4KB2VudHJpZXMYASADKAsyHS5jaHVuay5tYW5hZ2VtZW50LnYxLkxvZ0VudHJ5IhQKElJlcG9ydExvZ3NSZXNwb25zZSJKChRSZXBvcnRNZXRyaWNzUmVxdWVzdBIyCgdzYW1wbGVzGAEgAygLMiEuY2h1bmsubWFuYWdlbWVudC52MS5NZXRyaWNTYW1wbGUi2AEKDE1ldHJpY1NhbXBsZRIMCgRuYW1lGAEgASgJEj0KBmxhYmVscxgCIAMoCzItLmNodW5rLm1hbmFnZW1lbnQudjEuTWV0cmljU2FtcGxlLkxhYmVsc0VudHJ5EigKBHRpbWUYAyABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEg0KBXZhbHVlGAQgASgBEhMKC2luc3RhbmNlX2lkGAUgASgJGi0KC0xhYmVsc0VudHJ5EgsKA2tleRgBIAEoCRINCgV2YWx1ZRgCIAEoCToCOAEiFwoVUmVwb3J0TWV0cmljc1Jlc3BvbnNlIkwKF1JlcG9ydEZhaWxlZEF1dGhSZXF1ZXN0EjEKCGZhaWx1cmVzGAEgAygLMh8uY2h1bmsubWFuYWdlbWVudC52MS5GYWlsZWRBdXRoIk4KCkZhaWxlZEF1dGgSFgoOY2xpZW50X2FkZHJlc3MYASABKAkSKAoEdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAiGgoYUmVwb3J0RmFpbGVkQXV0aFJlc3BvbnNlKmEKCFdvcmtsb2FkEhgKFFdPUktMT0FEX1VOU1BFQ0lGSUVEEAASEAoMV09SS0xPQURfSlZNEAESFAoQV09SS0xPQURfR0FURVdBWRACIgQIAxADKg1XT1JLTE9BRF9FWEVDKsABCg1DYXBhY2l0eVN0YXRlEh4KGkNBUEFDSVRZX1NUQVRFX1VOU1BFQ0lGSUVEEAASHwobQ0FQQUNJVFlfU1RBVEVfUFJPVklTSU9OSU5HEAESGAoUQ0FQQUNJVFlfU1RBVEVfUkVBRFkQAhIZChVDQVBBQ0lUWV9TVEFURV9GQUlMRUQQAxIbChdDQVBBQ0lUWV9TVEFURV9SRUxFQVNFRBAEEhwKGENBUEFDSVRZX1NUQVRFX1JFTEVBU0lORxAFMqYHChJFbnZpcm9ubWVudFNlcnZpY2USUwoGQXR0YWNoEiIuY2h1bmsubWFuYWdlbWVudC52MS5BdHRhY2hSZXF1ZXN0GiMuY2h1bmsubWFuYWdlbWVudC52MS5BdHRhY2hSZXNwb25zZTABEmMKDFJlcG9ydFN0YXR1cxIoLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0U3RhdHVzUmVxdWVzdBopLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0U3RhdHVzUmVzcG9uc2USYwoMU2V0V2FrZUFsYXJtEiguY2h1bmsubWFuYWdlbWVudC52MS5TZXRXYWtlQWxhcm1SZXF1ZXN0GikuY2h1bmsubWFuYWdlbWVudC52MS5TZXRXYWtlQWxhcm1SZXNwb25zZRJpCg5FbnN1cmVDYXBhY2l0eRIqLmNodW5rLm1hbmFnZW1lbnQudjEuRW5zdXJlQ2FwYWNpdHlSZXF1ZXN0GisuY2h1bmsubWFuYWdlbWVudC52MS5FbnN1cmVDYXBhY2l0eVJlc3BvbnNlEmwKD1JlbGVhc2VDYXBhY2l0eRIrLmNodW5rLm1hbmFnZW1lbnQudjEuUmVsZWFzZUNhcGFjaXR5UmVxdWVzdBosLmNodW5rLm1hbmFnZW1lbnQudjEuUmVsZWFzZUNhcGFjaXR5UmVzcG9uc2USYAoLUmVwb3J0VXNhZ2USJy5jaHVuay5tYW5hZ2VtZW50LnYxLlJlcG9ydFVzYWdlUmVxdWVzdBooLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0VXNhZ2VSZXNwb25zZRJdCgpSZXBvcnRMb2dzEiYuY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRMb2dzUmVxdWVzdBonLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0TG9nc1Jlc3BvbnNlEmYKDVJlcG9ydE1ldHJpY3MSKS5jaHVuay5tYW5hZ2VtZW50LnYxLlJlcG9ydE1ldHJpY3NSZXF1ZXN0GiouY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRNZXRyaWNzUmVzcG9uc2USbwoQUmVwb3J0RmFpbGVkQXV0aBIsLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0RmFpbGVkQXV0aFJlcXVlc3QaLS5jaHVuay5tYW5hZ2VtZW50LnYxLlJlcG9ydEZhaWxlZEF1dGhSZXNwb25zZWIGcHJvdG8z", + "CiVjaHVuay9tYW5hZ2VtZW50L3YxL2Vudmlyb25tZW50LnByb3RvEhNjaHVuay5tYW5hZ2VtZW50LnYxInIKDUF0dGFjaFJlcXVlc3QSEwoLaW5zdGFuY2VfaWQYASABKAkSDwoHdmVyc2lvbhgCIAEoCRIMCgRjb3JlGAMgASgIEg0KBWVwb2NoGAQgASgEEh4KFnJlc3RvcmVkX2RlcGxveW1lbnRfaWQYBSABKAki8wMKDkF0dGFjaFJlc3BvbnNlEhAKCHJldmlzaW9uGAEgASgEEhYKDmVudmlyb25tZW50X2lkGAIgASgJEhIKCnByb2plY3RfaWQYAyABKAkSFQoNZGVwbG95bWVudF9pZBgEIAEoCRI1CgdyZWxlYXNlGAUgASgLMiQuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlQXJ0aWZhY3QSMQoHc2VjcmV0cxgGIAMoCzIgLmNodW5rLm1hbmFnZW1lbnQudjEuU2VjcmV0VmFsdWUSMwoJbG9nX3N0b3JlGAcgASgLMiAuY2h1bmsubWFuYWdlbWVudC52MS5PYmplY3RTdG9yZRItCgdyZXN0b3JlGAggASgLMhwuY2h1bmsubWFuYWdlbWVudC52MS5SZXN0b3JlEg0KBWxlYXNlGAkgASgEEhgKEGVudmlyb25tZW50X25hbWUYCiABKAkSMQoFZHJhaW4YCyABKAsyIi5jaHVuay5tYW5hZ2VtZW50LnYxLkRyYWluU2V0dGluZ3MSFQoNc3RvcF9wcmV2aW91cxgMIAEoCBIyCgZhc3NldHMYDSABKAsyIi5jaHVuay5tYW5hZ2VtZW50LnYxLkFzc2V0QXJ0aWZhY3QSFwoPcGFja191cmxfcHJlZml4GA4gASgJIlYKD1JlbGVhc2VBcnRpZmFjdBISCgpyZWxlYXNlX2lkGAEgASgJEgsKA3VybBgCIAEoCRIOCgZzaGEyNTYYAyABKAkSEgoKc2l6ZV9ieXRlcxgEIAEoBCJPCg1Bc3NldEFydGlmYWN0EhMKC3JldmlzaW9uX2lkGAEgASgJEhAKCG1hbmlmZXN0GAIgASgMEhcKD2Jsb2JfdXJsX3ByZWZpeBgDIAEoCSI7CgtTZWNyZXRWYWx1ZRIMCgRuYW1lGAEgASgJEg8KB3ZlcnNpb24YAiABKAQSDQoFdmFsdWUYAyABKAwiyQEKC09iamVjdFN0b3JlEhAKCGVuZHBvaW50GAEgASgJEg4KBnJlZ2lvbhgCIAEoCRIOCgZidWNrZXQYAyABKAkSDgoGcHJlZml4GAQgASgJEhUKDWFjY2Vzc19rZXlfaWQYBSABKAkSGQoRc2VjcmV0X2FjY2Vzc19rZXkYBiABKAkSFQoNc2Vzc2lvbl90b2tlbhgHIAEoCRIvCgtleHBpcmVfdGltZRgIIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAibwoHUmVzdG9yZRIwCgZzb3VyY2UYASABKAsyIC5jaHVuay5tYW5hZ2VtZW50LnYxLk9iamVjdFN0b3JlEhMKC3NuYXBzaG90X2lkGAIgASgJEh0KFXNvdXJjZV9lbnZpcm9ubWVudF9pZBgDIAEoCSLMAgoTUmVwb3J0U3RhdHVzUmVxdWVzdBIwCgxvYnNlcnZlX3RpbWUYASABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEhkKEWdhdGV3YXlfYWRkcmVzc2VzGAIgAygJEhYKDm9ubGluZV9wbGF5ZXJzGAMgASgNEi4KBXBpbmdzGAQgAygLMh8uY2h1bmsubWFuYWdlbWVudC52MS5QaW5nU3RhdHVzEjsKCmRlcGxveW1lbnQYBSABKAsyJy5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveW1lbnRQcm9ncmVzcxIYChByZWFkeV90b19zdXNwZW5kGAYgASgIEg0KBWxlYXNlGAcgASgEEhAKCHNlcXVlbmNlGAggASgEEhgKEGRlc2lyZWRfcmV2aXNpb24YCSABKAQSDgoGbG9naW5zGAogASgEIjMKClBpbmdTdGF0dXMSEAoIaG9zdG5hbWUYASABKAkSEwoLc3RhdHVzX2pzb24YAiABKAkicQoSRGVwbG95bWVudFByb2dyZXNzEhUKDWRlcGxveW1lbnRfaWQYASABKAkSMwoFc3RhdGUYAiABKA4yJC5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveW1lbnRTdGF0ZRIPCgdtZXNzYWdlGAMgASgJIhYKFFJlcG9ydFN0YXR1c1Jlc3BvbnNlInUKE1NldFdha2VBbGFybVJlcXVlc3QSEgoKZ2VuZXJhdGlvbhgBIAEoBBIsCghkdWVfdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASDQoFZXBvY2gYAyABKAQSDQoFbGVhc2UYBCABKAQiZwoUU2V0V2FrZUFsYXJtUmVzcG9uc2USEgoKZ2VuZXJhdGlvbhgBIAEoBBIsCghkdWVfdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASDQoFZXBvY2gYAyABKAQivAEKFUVuc3VyZUNhcGFjaXR5UmVxdWVzdBISCgpyZXF1ZXN0X2lkGAEgASgJEi8KCHdvcmtsb2FkGAIgASgOMh0uY2h1bmsubWFuYWdlbWVudC52MS5Xb3JrbG9hZBIXCg9tYWNoaW5lX3Byb2ZpbGUYAyABKAkSEgoKcmVsZWFzZV9pZBgEIAEoCRIOCgZhcHBfaWQYBSABKAkSDQoFbGVhc2UYBiABKAQSEgoKY3JlZGVudGlhbBgHIAEoCSJJChZFbnN1cmVDYXBhY2l0eVJlc3BvbnNlEi8KCGNhcGFjaXR5GAEgASgLMh0uY2h1bmsubWFuYWdlbWVudC52MS5DYXBhY2l0eSJ2CghDYXBhY2l0eRISCgpyZXF1ZXN0X2lkGAEgASgJEjEKBXN0YXRlGAIgASgOMiIuY2h1bmsubWFuYWdlbWVudC52MS5DYXBhY2l0eVN0YXRlEhIKCm1hY2hpbmVfaWQYAyABKAkSDwoHbWVzc2FnZRgEIAEoCSI7ChZSZWxlYXNlQ2FwYWNpdHlSZXF1ZXN0EhIKCnJlcXVlc3RfaWQYASABKAkSDQoFbGVhc2UYAiABKAQiSgoXUmVsZWFzZUNhcGFjaXR5UmVzcG9uc2USLwoIY2FwYWNpdHkYASABKAsyHS5jaHVuay5tYW5hZ2VtZW50LnYxLkNhcGFjaXR5IkcKElJlcG9ydFVzYWdlUmVxdWVzdBIxCgdyZWNvcmRzGAEgAygLMiAuY2h1bmsubWFuYWdlbWVudC52MS5Vc2FnZVJlY29yZCKkAQoLVXNhZ2VSZWNvcmQSCgoCaWQYASABKAkSLgoKc3RhcnRfdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASLAoIZW5kX3RpbWUYAyABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEhYKDnBsYXllcl9zZWNvbmRzGAQgASgEEhMKC2luc3RhbmNlX2lkGAUgASgJIhUKE1JlcG9ydFVzYWdlUmVzcG9uc2UiQwoRUmVwb3J0TG9nc1JlcXVlc3QSLgoHZW50cmllcxgBIAMoCzIdLmNodW5rLm1hbmFnZW1lbnQudjEuTG9nRW50cnkiFAoSUmVwb3J0TG9nc1Jlc3BvbnNlIkoKFFJlcG9ydE1ldHJpY3NSZXF1ZXN0EjIKB3NhbXBsZXMYASADKAsyIS5jaHVuay5tYW5hZ2VtZW50LnYxLk1ldHJpY1NhbXBsZSLYAQoMTWV0cmljU2FtcGxlEgwKBG5hbWUYASABKAkSPQoGbGFiZWxzGAIgAygLMi0uY2h1bmsubWFuYWdlbWVudC52MS5NZXRyaWNTYW1wbGUuTGFiZWxzRW50cnkSKAoEdGltZRgDIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASDQoFdmFsdWUYBCABKAESEwoLaW5zdGFuY2VfaWQYBSABKAkaLQoLTGFiZWxzRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4ASIXChVSZXBvcnRNZXRyaWNzUmVzcG9uc2UiTAoXUmVwb3J0RmFpbGVkQXV0aFJlcXVlc3QSMQoIZmFpbHVyZXMYASADKAsyHy5jaHVuay5tYW5hZ2VtZW50LnYxLkZhaWxlZEF1dGgiTgoKRmFpbGVkQXV0aBIWCg5jbGllbnRfYWRkcmVzcxgBIAEoCRIoCgR0aW1lGAIgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcCIaChhSZXBvcnRGYWlsZWRBdXRoUmVzcG9uc2UqYQoIV29ya2xvYWQSGAoUV09SS0xPQURfVU5TUEVDSUZJRUQQABIQCgxXT1JLTE9BRF9KVk0QARIUChBXT1JLTE9BRF9HQVRFV0FZEAIiBAgDEAMqDVdPUktMT0FEX0VYRUMqwAEKDUNhcGFjaXR5U3RhdGUSHgoaQ0FQQUNJVFlfU1RBVEVfVU5TUEVDSUZJRUQQABIfChtDQVBBQ0lUWV9TVEFURV9QUk9WSVNJT05JTkcQARIYChRDQVBBQ0lUWV9TVEFURV9SRUFEWRACEhkKFUNBUEFDSVRZX1NUQVRFX0ZBSUxFRBADEhsKF0NBUEFDSVRZX1NUQVRFX1JFTEVBU0VEEAQSHAoYQ0FQQUNJVFlfU1RBVEVfUkVMRUFTSU5HEAUypgcKEkVudmlyb25tZW50U2VydmljZRJTCgZBdHRhY2gSIi5jaHVuay5tYW5hZ2VtZW50LnYxLkF0dGFjaFJlcXVlc3QaIy5jaHVuay5tYW5hZ2VtZW50LnYxLkF0dGFjaFJlc3BvbnNlMAESYwoMUmVwb3J0U3RhdHVzEiguY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRTdGF0dXNSZXF1ZXN0GikuY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRTdGF0dXNSZXNwb25zZRJjCgxTZXRXYWtlQWxhcm0SKC5jaHVuay5tYW5hZ2VtZW50LnYxLlNldFdha2VBbGFybVJlcXVlc3QaKS5jaHVuay5tYW5hZ2VtZW50LnYxLlNldFdha2VBbGFybVJlc3BvbnNlEmkKDkVuc3VyZUNhcGFjaXR5EiouY2h1bmsubWFuYWdlbWVudC52MS5FbnN1cmVDYXBhY2l0eVJlcXVlc3QaKy5jaHVuay5tYW5hZ2VtZW50LnYxLkVuc3VyZUNhcGFjaXR5UmVzcG9uc2USbAoPUmVsZWFzZUNhcGFjaXR5EisuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlQ2FwYWNpdHlSZXF1ZXN0GiwuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlQ2FwYWNpdHlSZXNwb25zZRJgCgtSZXBvcnRVc2FnZRInLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0VXNhZ2VSZXF1ZXN0GiguY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRVc2FnZVJlc3BvbnNlEl0KClJlcG9ydExvZ3MSJi5jaHVuay5tYW5hZ2VtZW50LnYxLlJlcG9ydExvZ3NSZXF1ZXN0GicuY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRMb2dzUmVzcG9uc2USZgoNUmVwb3J0TWV0cmljcxIpLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0TWV0cmljc1JlcXVlc3QaKi5jaHVuay5tYW5hZ2VtZW50LnYxLlJlcG9ydE1ldHJpY3NSZXNwb25zZRJvChBSZXBvcnRGYWlsZWRBdXRoEiwuY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRGYWlsZWRBdXRoUmVxdWVzdBotLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0RmFpbGVkQXV0aFJlc3BvbnNlYgZwcm90bzM", [file_chunk_management_v1_common, file_google_protobuf_timestamp], ); @@ -162,6 +162,22 @@ export type AttachResponse = Message<"chunk.management.v1.AttachResponse"> & { * @generated from field: bool stop_previous = 12; */ stopPrevious: boolean; + + /** + * The asset revision deployment_id pins; unset with the release. + * + * @generated from field: chunk.management.v1.AssetArtifact assets = 13; + */ + assets?: AssetArtifact | undefined; + + /** + * Where players' clients download the environment's resource packs: this + * prefix followed by a pack's lowercase hex SHA-256. It stays valid while + * the environment exists, and needs no other credentials. + * + * @generated from field: string pack_url_prefix = 14; + */ + packUrlPrefix: string; }; /** @@ -214,6 +230,46 @@ export const ReleaseArtifactSchema: GenMessage /*@__PURE__*/ = 2, ); +/** + * An asset revision to download, blob by blob. + * + * @generated from message chunk.management.v1.AssetArtifact + */ +export type AssetArtifact = Message<"chunk.management.v1.AssetArtifact"> & { + /** + * The lowercase hex SHA-256 of `manifest`. + * + * @generated from field: string revision_id = 1; + */ + revisionId: string; + + /** + * The revision's canonical JSON manifest. + * + * @generated from field: bytes manifest = 2; + */ + manifest: Uint8Array; + + /** + * Fetch a blob with HTTP GET at this prefix followed by its lowercase hex + * SHA-256, sending the environment's bearer token. Only blobs of revisions + * the environment's deployments pin are served; the response may redirect to + * a presigned URL. + * + * @generated from field: string blob_url_prefix = 3; + */ + blobUrlPrefix: string; +}; + +/** + * Describes the message chunk.management.v1.AssetArtifact. + * Use `create(AssetArtifactSchema)` to create a new message. + */ +export const AssetArtifactSchema: GenMessage /*@__PURE__*/ = messageDesc( + file_chunk_management_v1_environment, + 3, +); + /** * @generated from message chunk.management.v1.SecretValue */ @@ -240,7 +296,7 @@ export type SecretValue = Message<"chunk.management.v1.SecretValue"> & { */ export const SecretValueSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 3, + 4, ); /** @@ -300,7 +356,7 @@ export type ObjectStore = Message<"chunk.management.v1.ObjectStore"> & { */ export const ObjectStoreSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 4, + 5, ); /** @@ -333,7 +389,7 @@ export type Restore = Message<"chunk.management.v1.Restore"> & { * Describes the message chunk.management.v1.Restore. * Use `create(RestoreSchema)` to create a new message. */ -export const RestoreSchema: GenMessage /*@__PURE__*/ = messageDesc(file_chunk_management_v1_environment, 5); +export const RestoreSchema: GenMessage /*@__PURE__*/ = messageDesc(file_chunk_management_v1_environment, 6); /** * @generated from message chunk.management.v1.ReportStatusRequest @@ -428,7 +484,7 @@ export type ReportStatusRequest = Message<"chunk.management.v1.ReportStatusReque */ export const ReportStatusRequestSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 6, + 7, ); /** @@ -454,7 +510,7 @@ export type PingStatus = Message<"chunk.management.v1.PingStatus"> & { */ export const PingStatusSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 7, + 8, ); /** @@ -490,7 +546,7 @@ export type DeploymentProgress = Message<"chunk.management.v1.DeploymentProgress */ export const DeploymentProgressSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 8, + 9, ); /** @@ -504,7 +560,7 @@ export type ReportStatusResponse = Message<"chunk.management.v1.ReportStatusResp */ export const ReportStatusResponseSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 9, + 10, ); /** @@ -546,7 +602,7 @@ export type SetWakeAlarmRequest = Message<"chunk.management.v1.SetWakeAlarmReque */ export const SetWakeAlarmRequestSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 10, + 11, ); /** @@ -578,7 +634,7 @@ export type SetWakeAlarmResponse = Message<"chunk.management.v1.SetWakeAlarmResp */ export const SetWakeAlarmResponseSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 11, + 12, ); /** @@ -639,7 +695,7 @@ export type EnsureCapacityRequest = Message<"chunk.management.v1.EnsureCapacityR */ export const EnsureCapacityRequestSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 12, + 13, ); /** @@ -658,7 +714,7 @@ export type EnsureCapacityResponse = Message<"chunk.management.v1.EnsureCapacity */ export const EnsureCapacityResponseSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 13, + 14, ); /** @@ -694,7 +750,7 @@ export type Capacity = Message<"chunk.management.v1.Capacity"> & { * Describes the message chunk.management.v1.Capacity. * Use `create(CapacitySchema)` to create a new message. */ -export const CapacitySchema: GenMessage /*@__PURE__*/ = messageDesc(file_chunk_management_v1_environment, 14); +export const CapacitySchema: GenMessage /*@__PURE__*/ = messageDesc(file_chunk_management_v1_environment, 15); /** * @generated from message chunk.management.v1.ReleaseCapacityRequest @@ -721,7 +777,7 @@ export type ReleaseCapacityRequest = Message<"chunk.management.v1.ReleaseCapacit */ export const ReleaseCapacityRequestSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 15, + 16, ); /** @@ -740,7 +796,7 @@ export type ReleaseCapacityResponse = Message<"chunk.management.v1.ReleaseCapaci */ export const ReleaseCapacityResponseSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 16, + 17, ); /** @@ -759,7 +815,7 @@ export type ReportUsageRequest = Message<"chunk.management.v1.ReportUsageRequest */ export const ReportUsageRequestSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 17, + 18, ); /** @@ -811,7 +867,7 @@ export type UsageRecord = Message<"chunk.management.v1.UsageRecord"> & { */ export const UsageRecordSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 18, + 19, ); /** @@ -825,7 +881,7 @@ export type ReportUsageResponse = Message<"chunk.management.v1.ReportUsageRespon */ export const ReportUsageResponseSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 19, + 20, ); /** @@ -844,7 +900,7 @@ export type ReportLogsRequest = Message<"chunk.management.v1.ReportLogsRequest"> */ export const ReportLogsRequestSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 20, + 21, ); /** @@ -858,7 +914,7 @@ export type ReportLogsResponse = Message<"chunk.management.v1.ReportLogsResponse */ export const ReportLogsResponseSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 21, + 22, ); /** @@ -877,7 +933,7 @@ export type ReportMetricsRequest = Message<"chunk.management.v1.ReportMetricsReq */ export const ReportMetricsRequestSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 22, + 23, ); /** @@ -918,7 +974,7 @@ export type MetricSample = Message<"chunk.management.v1.MetricSample"> & { */ export const MetricSampleSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 23, + 24, ); /** @@ -932,7 +988,7 @@ export type ReportMetricsResponse = Message<"chunk.management.v1.ReportMetricsRe */ export const ReportMetricsResponseSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 24, + 25, ); /** @@ -951,7 +1007,7 @@ export type ReportFailedAuthRequest = Message<"chunk.management.v1.ReportFailedA */ export const ReportFailedAuthRequestSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 25, + 26, ); /** @@ -977,7 +1033,7 @@ export type FailedAuth = Message<"chunk.management.v1.FailedAuth"> & { */ export const FailedAuthSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 26, + 27, ); /** @@ -991,7 +1047,7 @@ export type ReportFailedAuthResponse = Message<"chunk.management.v1.ReportFailed */ export const ReportFailedAuthResponseSchema: GenMessage /*@__PURE__*/ = messageDesc( file_chunk_management_v1_environment, - 27, + 28, ); /** diff --git a/packages/dashboard/src/lib/client.ts b/packages/dashboard/src/lib/client.ts index 05da4615..70eae879 100644 --- a/packages/dashboard/src/lib/client.ts +++ b/packages/dashboard/src/lib/client.ts @@ -3,6 +3,7 @@ import { createConnectTransport } from "@connectrpc/connect-web"; import { QueryClient } from "@tanstack/react-query"; import { useState } from "react"; +import { AssetService } from "../gen/chunk/management/v1/assets_pb.ts"; import { AuthService } from "../gen/chunk/management/v1/auth_pb.ts"; import { DeploymentService } from "../gen/chunk/management/v1/deployments_pb.ts"; import { DomainService } from "../gen/chunk/management/v1/domains_pb.ts"; @@ -44,6 +45,7 @@ export const api = { auth: createClient(AuthService, transport), projects: createClient(ProjectService, transport), deployments: createClient(DeploymentService, transport), + assets: createClient(AssetService, transport), secrets: createClient(SecretService, transport), domains: createClient(DomainService, transport), logs: createClient(LogService, transport), diff --git a/packages/dashboard/src/lib/format.test.ts b/packages/dashboard/src/lib/format.test.ts new file mode 100644 index 00000000..7398cf82 --- /dev/null +++ b/packages/dashboard/src/lib/format.test.ts @@ -0,0 +1,10 @@ +import { expect, test } from "vitest"; + +import { formatBytes } from "./format.ts"; + +test("byte counts read in binary units", () => { + expect(formatBytes(0n)).toBe("0 B"); + expect(formatBytes(1023n)).toBe("1023 B"); + expect(formatBytes(1536n)).toBe("1.5 KiB"); + expect(formatBytes(5n * 1024n ** 3n)).toBe("5.0 GiB"); +}); diff --git a/packages/dashboard/src/lib/format.ts b/packages/dashboard/src/lib/format.ts index cf2d6def..50f2687b 100644 --- a/packages/dashboard/src/lib/format.ts +++ b/packages/dashboard/src/lib/format.ts @@ -77,4 +77,17 @@ export function timeAgo(timestamp: Timestamp | undefined) { return "just now"; } +const byteUnits = ["B", "KiB", "MiB", "GiB", "TiB"]; + +/** A byte count in binary units, such as `1.5 MiB`. */ +export function formatBytes(bytes: bigint) { + let value = Number(bytes); + let unit = 0; + while (value >= 1024 && unit < byteUnits.length - 1) { + value /= 1024; + unit++; + } + return `${unit === 0 ? value : value.toFixed(1)} ${byteUnits[unit]}`; +} + export const clock = new Intl.DateTimeFormat("en", { hour12: false, timeStyle: "medium" }); diff --git a/packages/dashboard/src/lib/queries.ts b/packages/dashboard/src/lib/queries.ts index c8053ac0..2a435991 100644 --- a/packages/dashboard/src/lib/queries.ts +++ b/packages/dashboard/src/lib/queries.ts @@ -73,6 +73,24 @@ export const useDeployments = (environmentId: string) => refetchInterval: refreshMs, }); +/** The project's ready asset revisions, newest first, and its head's ID. */ +export const useAssetRevisions = (projectId: string) => + useQuery({ + queryKey: ["assets", projectId], + queryFn: async () => { + let headId = ""; + const revisions = await allPages( + (pageToken) => api.assets.listAssetRevisions({ projectId, pageToken }), + (response) => { + headId = response.headId; + return response.revisions; + }, + ); + return { revisions, headId }; + }, + refetchInterval: refreshMs, + }); + export const useSecrets = (environmentId: string) => useQuery({ queryKey: ["secrets", environmentId], diff --git a/packages/dashboard/src/routes/environment/deployments.tsx b/packages/dashboard/src/routes/environment/deployments.tsx index 66f4d2f3..dcdfe758 100644 --- a/packages/dashboard/src/routes/environment/deployments.tsx +++ b/packages/dashboard/src/routes/environment/deployments.tsx @@ -6,7 +6,7 @@ import { ConfirmDialog } from "../../components/confirm-dialog.tsx"; import { DeployGuideLink } from "../../components/deploy-guide.tsx"; import { ErrorText } from "../../components/page.tsx"; import { listStyles, Panel, PanelNote } from "../../components/panel.tsx"; -import { ReleaseId } from "../../components/release.tsx"; +import { ShortId } from "../../components/release.tsx"; import { Status } from "../../components/status.tsx"; import { Button } from "../../components/ui/button.tsx"; import { fieldStyles } from "../../components/ui/input.tsx"; @@ -86,9 +86,14 @@ export function Deployments() {
  • - + {triggerLabels[deployment.trigger]}

    + {deployment.assetRevisionId && ( +

    + Assets +

    + )} {deployment.message &&

    {deployment.message}

    }
    @@ -175,7 +180,8 @@ function RollbackDialog({ title="Roll back" description={ <> - Deploys release {deployment.releaseId} to {environment.name} again. + Deploys release {deployment.releaseId} with asset revision{" "} + {deployment.assetRevisionId} to {environment.name} again. } confirmLabel="Roll back" diff --git a/packages/dashboard/src/routes/project.tsx b/packages/dashboard/src/routes/project.tsx index 676ee00d..5c846f56 100644 --- a/packages/dashboard/src/routes/project.tsx +++ b/packages/dashboard/src/routes/project.tsx @@ -4,12 +4,12 @@ import { useParams } from "@tanstack/react-router"; import { ListLink } from "../components/list-link.tsx"; import { ErrorText, Page } from "../components/page.tsx"; import { listStyles, Panel, PanelNote } from "../components/panel.tsx"; -import { Release } from "../components/release.tsx"; +import { Release, ShortId } from "../components/release.tsx"; import { Status } from "../components/status.tsx"; import { errorMessage } from "../lib/client.ts"; -import { environmentStatus } from "../lib/format.ts"; -import { useEnvironments, useProject } from "../lib/queries.ts"; -import { colors, fonts, fontSizes, lineHeights } from "../tokens.stylex.ts"; +import { environmentStatus, formatBytes, timeAgo } from "../lib/format.ts"; +import { useAssetRevisions, useEnvironments, useProject } from "../lib/queries.ts"; +import { colors, fonts, fontSizes, lineHeights, space } from "../tokens.stylex.ts"; const styles = stylex.create({ title: { fontSize: fontSizes.xl, lineHeight: lineHeights.xl, fontWeight: 600, letterSpacing: "-0.025em" }, @@ -38,13 +38,26 @@ const styles = stylex.create({ lineHeight: lineHeights.xs, }, status: { width: "7.7rem", textAlign: "right" }, + revision: { + display: "flex", + alignItems: "center", + gap: space.s4, + paddingInline: space.s5, + paddingBlock: space.s3, + fontSize: fontSizes.xs, + lineHeight: lineHeights.xs, + }, + revisionId: { flex: 1, minWidth: 0 }, + muted: { flexShrink: 0, color: colors.mutedForeground }, + head: { flexShrink: 0, fontWeight: 500 }, }); export function Project() { const { project: projectId } = useParams({ from: "/p/$project" }); const project = useProject(projectId); const environments = useEnvironments(projectId); - const error = project.error ?? environments.error; + const assets = useAssetRevisions(projectId); + const error = project.error ?? environments.error ?? assets.error; return ( @@ -74,6 +87,24 @@ export function Project() { )} + + {assets.data?.revisions.length === 0 ? ( + No assets yet. Run chunk assets push to publish them. + ) : ( +
      + {assets.data?.revisions.map((revision) => ( +
    • + + + + {revision.id === assets.data.headId && Head} + {formatBytes(revision.sizeBytes)} + +
    • + ))} +
    + )} +
    ); } diff --git a/packages/management/README.md b/packages/management/README.md index 1d680894..1bcebd65 100644 --- a/packages/management/README.md +++ b/packages/management/README.md @@ -100,6 +100,13 @@ Environments fetch blobs at `GET /blobs/` with their bearer token, and p serve only blobs of revisions the environment's deployments pin, packs only for the pack URL, streaming them from disk or redirecting to a presigned URL of the release bucket. +Each `SetAssetHead` collects the project's unused revisions. A revision is kept while it is the head, one of the +project's 10 newest ready revisions, less than an hour old (as long as its upload URLs last), or pinned by an +environment's active deployment, an unfinished one or one of its 10 newest. The rest are deleted, and with them every +blob no remaining revision holds, from the database and the release store; a rollback to an older deployment fails once +its revision is gone. Collection holds a per-project lock that uploads and deployments share, so it never deletes a blob +a revision is being declared with or a revision a deployment is pinning. + ## Release storage Release archives stay on management's disk under `CHUNK_DATA_DIR` unless `CHUNK_RELEASE_STORE_BUCKET` names an diff --git a/packages/management/src/assets/retention.ts b/packages/management/src/assets/retention.ts new file mode 100644 index 00000000..52361195 --- /dev/null +++ b/packages/management/src/assets/retention.ts @@ -0,0 +1,117 @@ +import { and, eq, notExists, sql } from "drizzle-orm"; + +import { type Db, fetchRows } from "../db.ts"; +import type { Deps } from "../deps.ts"; +import { AssetRevisionState } from "../gen/chunk/management/v1/assets_pb.ts"; +import { DeploymentState } from "../gen/chunk/management/v1/common_pb.ts"; +import { blobKey } from "../releases/store.ts"; +import { assetBlobs, assetRevisionBlobs } from "../schema.ts"; +import { eachLimited } from "./store.ts"; + +/** How many of a project's newest ready revisions are kept, deployed or not. */ +export const keptRecentRevisions = 10; +/** How many of each environment's newest deployments keep the revisions they pin, so they can be rolled back to. */ +export const keptRecentDeployments = 10; +/** How long a revision is kept after an upload of it was declared: as long as its upload URLs last. */ +export const uploadLifetimeMs = 60 * 60 * 1000; +/** How many blobs are removed from the release store at once. */ +const deleteConcurrency = 16; + +/** + * Serializes collection, which takes the lock `exclusive`ly, with what makes a project's revisions or blobs used: + * declaring a revision's blobs and pinning a revision in a deployment. Held until the transaction `tx` ends. + */ +export async function lockAssets(tx: Db, projectId: string, mode: "shared" | "exclusive"): Promise { + const key = `assets/${projectId}`; + await tx.execute( + mode === "shared" + ? sql`select pg_advisory_xact_lock_shared(hashtext(${key}))` + : sql`select pg_advisory_xact_lock(hashtext(${key}))`, + ); +} + +/** + * Deletes the project's revisions that nothing keeps, and then the blobs no remaining revision holds, from the database + * and the release store. A revision is kept while it is the project's head, one of its `keptRecentRevisions` newest + * ready revisions, younger than `uploadLifetimeMs`, pinned by the active, an unfinished or one of the + * `keptRecentDeployments` newest deployments of any of its environments. Returns how many revisions and blobs it + * deleted; a blob the store fails to delete is logged and stays there unreferenced. + */ +export async function collectAssets( + { db, releases }: Deps, + projectId: string, +): Promise<{ revisions: number; blobs: number }> { + const { revisions, orphans } = await db.transaction(async (tx) => { + await lockAssets(tx, projectId, "exclusive"); + const served = [DeploymentState.PENDING, DeploymentState.IN_PROGRESS, DeploymentState.ACTIVE]; + // Every CTE reads the snapshot from before the deletion, so `blobs` holds the deleted revisions' blobs. + const [deleted] = await fetchRows<{ revisions: number; blobs: string[] | null }>( + tx, + sql` + with deployed as ( + select d.asset_revision_id, d.state, d.id = e.active_deployment_id as active, + row_number() over (partition by d.environment_id order by d.seq desc) as recency + from deployments d join environments e on e.id = d.environment_id + where e.project_id = ${projectId} + ), + kept as ( + select asset_head_id as id from projects where id = ${projectId} + union + select id from ( + select id from asset_revisions where project_id = ${projectId} and state = ${AssetRevisionState.READY} + order by seq desc limit ${keptRecentRevisions} + ) recent + union + select asset_revision_id from deployed + where active or recency <= ${keptRecentDeployments} or state in (${sql.join(served, sql`, `)}) + ), + doomed as ( + select r.id from asset_revisions r + where r.project_id = ${projectId} + and r.create_time < now() - make_interval(secs => ${uploadLifetimeMs / 1000}) + and r.id not in (select id from kept) + ), + blobs as ( + select distinct b.sha256 from asset_revision_blobs b join doomed on doomed.id = b.revision_id + where b.project_id = ${projectId} + ), + gone as ( + delete from asset_revisions r using doomed where r.project_id = ${projectId} and r.id = doomed.id + returning r.id + ) + select (select count(*) from gone)::int as revisions, (select array_agg(sha256) from blobs) as blobs`, + ); + const unverified = await tx + .delete(assetBlobs) + .where( + and( + eq(assetBlobs.project_id, projectId), + notExists( + tx + .select({ one: sql`1` }) + .from(assetRevisionBlobs) + .where( + and(eq(assetRevisionBlobs.project_id, projectId), eq(assetRevisionBlobs.sha256, assetBlobs.sha256)), + ), + ), + ), + ) + .returning({ sha256: assetBlobs.sha256 }); + const orphans = new Set([...(deleted?.blobs ?? []), ...unverified.map((blob) => blob.sha256)]); + if (orphans.size > 0) { + const held = await tx + .selectDistinct({ sha256: assetRevisionBlobs.sha256 }) + .from(assetRevisionBlobs) + .where(eq(assetRevisionBlobs.project_id, projectId)); + for (const { sha256 } of held) orphans.delete(sha256); + } + // Inside the transaction, so no revision declares one of these blobs again before it is gone. + await eachLimited([...orphans], deleteConcurrency, (sha256) => + releases.delete(blobKey(projectId, sha256)).catch((error: unknown) => { + console.error(`deleting asset blob ${sha256} of project ${projectId} failed:`, error); + }), + ); + return { revisions: deleted?.revisions ?? 0, orphans }; + }); + return { revisions, blobs: orphans.size }; +} diff --git a/packages/management/src/assets/service.ts b/packages/management/src/assets/service.ts index 04cc91a9..06aab347 100644 --- a/packages/management/src/assets/service.ts +++ b/packages/management/src/assets/service.ts @@ -3,7 +3,7 @@ import { createHash } from "node:crypto"; import { create } from "@bufbuild/protobuf"; import { timestampFromDate } from "@bufbuild/protobuf/wkt"; import { Code, ConnectError, type ServiceImpl } from "@connectrpc/connect"; -import { and, desc, eq, lt } from "drizzle-orm"; +import { and, desc, eq, lt, sql } from "drizzle-orm"; import type { Deps } from "../deps.ts"; import { @@ -17,17 +17,18 @@ import { blobKey } from "../releases/store.ts"; import { callerOf } from "../rpc/caller.ts"; import { failedPrecondition, invalid, notFound, page, pageOf, required, seqAfter } from "../rpc/validate.ts"; import { assetBlobs, assetRevisionBlobs, assetRevisions, projects } from "../schema.ts"; +import { collectAssets, lockAssets, uploadLifetimeMs } from "./retention.ts"; import { decodeRevision, revisionBlobs, sha256Hex } from "./revision.ts"; -import { findRevision, heldBlobs, loadReadyRevision, toAssetRevision } from "./store.ts"; +import { eachLimited, findRevision, heldBlobs, loadReadyRevision, toAssetRevision } from "./store.ts"; -const uploadLifetimeMs = 60 * 60 * 1000; const downloadLifetimeMs = 60 * 60 * 1000; /** How many blobs completion verifies at once. */ const verifyConcurrency = 8; /** How many blob existence checks run at once; a revision holds at most 4,096 blobs. */ const existsConcurrency = 64; -export function assetService({ db, releases }: Deps): Partial> { +export function assetService(deps: Deps): Partial> { + const { db, releases } = deps; /** The project's verified blobs among `digests` that the blob store still holds. */ async function storedBlobs(projectId: string, digests: string[]) { const held = await heldBlobs(db, projectId, digests); @@ -50,6 +51,7 @@ export function assetService({ db, releases }: Deps): Partial total + BigInt(blob.size), 0n); await db.transaction(async (tx) => { + await lockAssets(tx, project.id, "shared"); const declared = await tx .insert(assetRevisions) .values({ @@ -61,7 +63,21 @@ export function assetService({ db, releases }: Deps): Partial ({ project_id: project.id, revision_id: id, sha256, pack }))); @@ -200,6 +216,10 @@ export function assetService({ db, releases }: Deps): Partial { + console.error(`collecting project ${project.id}'s unused asset revisions failed:`, error); + }); return { previousHeadId: request.expectedHeadId }; }, }; @@ -220,12 +240,3 @@ async function verifyBlob(blob: ReadableStream, sha256: string, size } return sha1Hash.digest("hex"); } - -/** Runs `run` on every item, at most `limit` at once. */ -async function eachLimited(items: T[], limit: number, run: (item: T) => Promise): Promise { - let next = 0; - const worker = async () => { - while (next < items.length) await run(items[next++] as T); - }; - await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)); -} diff --git a/packages/management/src/assets/store.ts b/packages/management/src/assets/store.ts index 43e40067..9a0d0736 100644 --- a/packages/management/src/assets/store.ts +++ b/packages/management/src/assets/store.ts @@ -48,3 +48,12 @@ export async function heldBlobs( .where(and(eq(assetBlobs.project_id, projectId), inArray(assetBlobs.sha256, digests))); return new Map(rows.map(({ sha256, ...blob }) => [sha256, blob])); } + +/** Runs `run` on every item, at most `limit` at once. */ +export async function eachLimited(items: T[], limit: number, run: (item: T) => Promise): Promise { + let next = 0; + const worker = async () => { + while (next < items.length) await run(items[next++] as T); + }; + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)); +} diff --git a/packages/management/src/deployments/store.ts b/packages/management/src/deployments/store.ts index cdc6fe51..58be514e 100644 --- a/packages/management/src/deployments/store.ts +++ b/packages/management/src/deployments/store.ts @@ -2,6 +2,7 @@ import { create } from "@bufbuild/protobuf"; import { ConnectError } from "@connectrpc/connect"; import { and, eq, inArray, ne, sql } from "drizzle-orm"; +import { lockAssets } from "../assets/retention.ts"; import { checkContract, decodeRevision } from "../assets/revision.ts"; import { loadReadyRevision } from "../assets/store.ts"; import { newId } from "../crypto.ts"; @@ -94,6 +95,7 @@ export async function createDeployment( throw failedPrecondition("the release has not finished uploading"); } requireJvmImage(release.manifest, jvmImageTemplate); + await lockAssets(db, environment.project_id, "shared"); const assets = await loadReadyRevision(db, environment.project_id, assetRevisionId); try { checkContract(release.manifest.assets, decodeRevision(assets.manifest)); diff --git a/packages/management/src/releases/local-store.ts b/packages/management/src/releases/local-store.ts index d5b9ae1d..7ceccce8 100644 --- a/packages/management/src/releases/local-store.ts +++ b/packages/management/src/releases/local-store.ts @@ -103,6 +103,11 @@ export async function localReleaseStore({ exists: async (key) => (await stored(key)) !== undefined, + async delete(key) { + if (!keyPattern.test(key)) throw new Error(`invalid release store key ${key}`); + await rm(join(root, key), { force: true }); + }, + async complete(key, _expected, verify) { // Uploads were checked against the key's digest as they were stored, so the stored object is the upload. const file = await stored(key); diff --git a/packages/management/src/releases/s3-store.ts b/packages/management/src/releases/s3-store.ts index 23c4d3f3..8c2f0f8e 100644 --- a/packages/management/src/releases/s3-store.ts +++ b/packages/management/src/releases/s3-store.ts @@ -39,6 +39,12 @@ export function s3ReleaseStore(bucket: ReleaseBucket): ReleaseStore { exists: (key) => exists(archive(key)), + async delete(key) { + await s3Request("release store deletion", () => + Promise.all([client.delete(archive(key)), client.delete(upload(key))]), + ); + }, + async complete(key, expected, verify) { if (!(await exists(upload(key)))) { return (await exists(archive(key))) ? verify(read(archive(key))) : undefined; diff --git a/packages/management/src/releases/store.ts b/packages/management/src/releases/store.ts index 7dda8ac7..fdeeba90 100644 --- a/packages/management/src/releases/store.ts +++ b/packages/management/src/releases/store.ts @@ -29,6 +29,8 @@ export interface ReleaseStore { serve(key: string, audience: Audience): Promise; /** Whether an object is stored under the key. */ exists(key: string): Promise; + /** Removes the object stored under the key, and any upload of it; resolves once neither exists. */ + delete(key: string): Promise; /** * Streams the object uploaded under the key, or else the one stored there, through `verify`, and stores an uploaded * one that matches `expected`. Resolves to what `verify` resolves to, or undefined when there is no object; rejects diff --git a/packages/management/test/assets.test.ts b/packages/management/test/assets.test.ts index c13afea3..5e4a00cf 100644 --- a/packages/management/test/assets.test.ts +++ b/packages/management/test/assets.test.ts @@ -2,6 +2,7 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import { Code } from "@connectrpc/connect"; +import { keptRecentRevisions } from "../src/assets/retention.ts"; import { decodeRevision, sha256Hex } from "../src/assets/revision.ts"; import { issueEnvironmentToken } from "../src/auth/tokens.ts"; import { desiredState } from "../src/environments/desired.ts"; @@ -180,4 +181,40 @@ describe.skipIf(!databaseUrl)("AssetService", () => { const otherPrefix = (await desiredState(h.deps, otherId)).message.packUrlPrefix; expect((await fetch(`${otherPrefix}${pinnedPack}`)).status).toBe(404); }); + + test("collection keeps the head, recent and deployed revisions, and deletes the rest with their blobs", async () => { + const { projectId, environmentId } = await createEnvironment(h); + const assets = h.client(AssetService); + await uploadRelease(h, projectId, "r1"); + const deployed = assetRevision({ shared: { "d.txt": "deployed", "f.txt": "r0" } }); + await uploadAssets(h, projectId, deployed); + await h.client(DeploymentService).deploy({ + requestId: crypto.randomUUID(), + environmentId, + releaseId: "r1", + assetRevisionId: deployed.id, + }); + const recent = Array.from({ length: keptRecentRevisions + 2 }, (_, i) => + assetRevision({ shared: { "f.txt": `r${i}` } }), + ); + for (const revision of recent) await uploadAssets(h, projectId, revision); + const unfinished = assetRevision({ shared: { "u.txt": "unfinished" } }); + const [upload] = (await assets.uploadAssets({ projectId, manifest: unfinished.manifest })).uploads; + await fetch(upload?.upload?.url ?? "", { method: "PUT", body: unfinished.blobs.get(upload?.sha256 ?? "") }); + await h.sql`update asset_revisions set create_time = now() - interval '2 hours' where project_id = ${projectId}`; + const head = recent[0]?.id ?? ""; + await assets.setAssetHead({ projectId, revisionId: head, expectedHeadId: "" }); + + const remaining = await h.sql<{ id: string }[]>`select id from asset_revisions where project_id = ${projectId}`; + const kept = [deployed, ...recent.slice(0, 1), ...recent.slice(2)].map((revision) => revision.id); + expect(remaining.map((row) => row.id).sort()).toEqual(kept.sort()); + const stored = (revision: typeof deployed) => + Promise.all([...revision.blobs.keys()].map((sha256) => h.releases.exists(blobKey(projectId, sha256)))); + expect(await stored(recent[0] ?? deployed)).toEqual([true]); + expect(await stored(recent[1] ?? deployed)).toEqual([false]); + expect(await stored(unfinished)).toEqual([false]); + expect(await assets.uploadAssets({ projectId, manifest: recent[1]?.manifest ?? new Uint8Array() })).toMatchObject({ + uploads: [{ sha256: [...(recent[1]?.blobs.keys() ?? [])][0] }], + }); + }); }); From 05f34cb7b4ad6f1fe63e9aee3c7946e90a3a36ed Mon Sep 17 00:00:00 2001 From: oglassdev <23040613+oglassdev@users.noreply.github.com> Date: Fri, 20 Nov 2026 15:19:44 -0500 Subject: [PATCH 2/5] fix(assets): keep draining deployments' revisions, delete blobs in two phases and refresh protection on repeated uploads - management: collection also keeps revisions pinned by a superseded deployment within its environment's drain deadline. - SetAssetHead checks the revision and moves the head under the shared asset lock, in one transaction. - Collection deletes rows and records blobs in asset_deletions in one transaction, then deletes the store objects of unreferenced keys in a second, retrying earlier failures. - uploadAssets sets protected_until on the revision it declares again, ready or not, replacing the create_time rule. - core records the kept set only after a prune fully succeeds, and invalidates it whenever a load finishes. Refs #328 --- crates/chunk-environment/src/managed.rs | 33 +- .../chunk-environment/src/managed/retire.rs | 23 +- .../src/managed/tests/assets.rs | 28 + packages/management/README.md | 13 +- .../migrations/0004_asset_deletions.sql | 9 + .../migrations/meta/0004_snapshot.json | 1948 +++++++++++++++++ .../management/migrations/meta/_journal.json | 9 +- packages/management/src/assets/retention.ts | 179 +- packages/management/src/assets/service.ts | 49 +- packages/management/src/assets/store.ts | 4 +- packages/management/src/schema.ts | 15 + packages/management/test/assets.test.ts | 68 +- 12 files changed, 2259 insertions(+), 119 deletions(-) create mode 100644 packages/management/migrations/0004_asset_deletions.sql create mode 100644 packages/management/migrations/meta/0004_snapshot.json diff --git a/crates/chunk-environment/src/managed.rs b/crates/chunk-environment/src/managed.rs index 4af9bbe0..8b885c39 100644 --- a/crates/chunk-environment/src/managed.rs +++ b/crates/chunk-environment/src/managed.rs @@ -31,7 +31,7 @@ use std::{ pin::Pin, sync::{ Arc, Mutex, MutexGuard, OnceLock, PoisonError, - atomic::{AtomicBool, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, }, time::{Duration, Instant}, }; @@ -81,9 +81,11 @@ pub(crate) struct Managed<'a> { releases: release::Store, /// Where each deployment's asset blobs are downloaded to. assets: chunk_build::assets::Store, - /// Held shared while a load fills `assets`, and exclusively while unused revisions are removed from it. Holds the - /// revisions the last removal kept. - asset_loads: tokio::sync::RwLock>>, + /// Held shared while a load fills `assets`, and exclusively while unused revisions are removed from it. Holds how + /// many loads had finished, and which revisions were kept, when the last removal fully succeeded. + asset_loads: tokio::sync::RwLock)>>, + /// How many loads into `assets` have finished, whether they succeeded, failed or were abandoned. + asset_loads_finished: AtomicU64, /// Where the activation management has not yet accepted is recorded. activation: PathBuf, core: &'a Core, @@ -175,6 +177,18 @@ struct Work<'a> { deployments: &'a Mutex, } +/// A load filling the asset store, which removal of unused revisions waits for. +struct AssetLoad<'a> { + _shared: tokio::sync::RwLockReadGuard<'a, Option<(u64, BTreeSet)>>, + finished: &'a AtomicU64, +} + +impl Drop for AssetLoad<'_> { + fn drop(&mut self) { + self.finished.fetch_add(1, Ordering::Relaxed); + } +} + impl Drop for Work<'_> { fn drop(&mut self) { let mut deployments = lock(self.deployments); @@ -214,6 +228,7 @@ impl<'a> Managed<'a> { releases: release::Store::new(state, core.archives().clone()), assets: core.asset_store(), asset_loads: tokio::sync::RwLock::default(), + asset_loads_finished: AtomicU64::default(), activation: state.join("managed.json"), core, gateway, @@ -437,6 +452,12 @@ impl<'a> Managed<'a> { } } + /// Keeps unused revisions from being removed from `assets` until the returned guard drops, which also records that + /// a load finished, as it may have left blobs behind that no revision holds. + async fn begin_asset_load(&self) -> AssetLoad<'_> { + AssetLoad { _shared: self.asset_loads.read().await, finished: &self.asset_loads_finished } + } + fn start(&self, desired: v1::AttachResponse) -> Work<'_> { let mut deployments = lock(&self.deployments); deployments.loading = Some(desired.deployment_id.clone()); @@ -519,7 +540,7 @@ impl<'a> Managed<'a> { if backend.check_deployment(id).await.is_err() { return false; } - let _loading = self.asset_loads.read().await; + let _loading = self.begin_asset_load().await; match assets::restore(&self.client, &self.assets, desired.assets.as_ref(), cancel).await { Ok(restored) => restored.is_some(), Err(error) => { @@ -537,7 +558,7 @@ impl<'a> Managed<'a> { let Some(loaded) = release::load(&self.client, &self.releases, artifact, cancel).await? else { return Ok(false); }; - let loading = self.asset_loads.read().await; + let loading = self.begin_asset_load().await; let revision = assets::load(&self.client, &self.assets, desired.assets.as_ref(), loaded.assets(), cancel); let Some(revision) = revision.await? else { return Ok(false); diff --git a/crates/chunk-environment/src/managed/retire.rs b/crates/chunk-environment/src/managed/retire.rs index 5c45fcc4..0d362185 100644 --- a/crates/chunk-environment/src/managed/retire.rs +++ b/crates/chunk-environment/src/managed/retire.rs @@ -7,7 +7,7 @@ use chunk_management::v1; use std::{ collections::BTreeSet, convert::Infallible, - sync::{Arc, Mutex, PoisonError}, + sync::{Arc, Mutex, PoisonError, atomic::Ordering}, time::Duration, }; use tokio::task::JoinSet; @@ -198,9 +198,10 @@ impl Managed<'_> { } /// Removes the asset revisions and blobs that neither a deployment resident in the backend, whose JVMs read them, - /// nor the one loading pins, whenever that set changes. Skipped while a load fills the store, and loads wait - /// meanwhile, so none reuses a blob being removed. - async fn remove_unused_assets(&self) { + /// nor the one loading pins, unless the last removal that fully succeeded kept the same set and no load has + /// finished since. Skipped while a load fills the store, and loads wait meanwhile, so none reuses a blob being + /// removed. + pub(super) async fn remove_unused_assets(&self) { let (Ok(control), Some(backend)) = (self.core.control(), self.core.backend()) else { return }; let Ok(mut last_kept) = self.asset_loads.try_write() else { return }; // Read before control, so a load that finishes meanwhile has made its deployment control's by then. @@ -216,15 +217,19 @@ impl Managed<'_> { } } kept.remove(""); - if last_kept.as_ref() == Some(&kept) { + // No load runs while the write lock is held, so the count stays the same until the removal ends. + let finished = self.asset_loads_finished.load(Ordering::Relaxed); + if matches!(&*last_kept, Some((count, last)) if *count == finished && *last == kept) { return; } match super::assets::prune(&self.assets, kept.clone()).await { - Ok(None) => return, - Ok(Some(0)) => {} - Ok(Some(removed)) => tracing::info!(removed, kept = kept.len(), "unused asset revisions removed"), + Ok(None) => {} + Ok(Some(0)) => *last_kept = Some((finished, kept)), + Ok(Some(removed)) => { + tracing::info!(removed, kept = kept.len(), "unused asset revisions removed"); + *last_kept = Some((finished, kept)); + } Err(error) => tracing::warn!(%error, "unused asset revisions not removed"), } - *last_kept = Some(kept); } } diff --git a/crates/chunk-environment/src/managed/tests/assets.rs b/crates/chunk-environment/src/managed/tests/assets.rs index 0396e124..28198c44 100644 --- a/crates/chunk-environment/src/managed/tests/assets.rs +++ b/crates/chunk-environment/src/managed/tests/assets.rs @@ -55,3 +55,31 @@ async fn pruning_keeps_everything_until_the_store_holds_every_kept_revision() { assert_eq!(assets::prune(&store, BTreeSet::from([loading.id()])).await.unwrap(), Some(2)); assert!(!store.contains(&blob(b"old").sha256).unwrap()); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_tick_after_a_failed_load_prunes_the_blobs_it_left_behind() { + let harness = Harness::new().await; + let core = crate::Core::start(harness.core(), || {}).await.unwrap(); + let (gateway, lease) = (std::sync::OnceLock::new(), watch::Sender::new(crate::managed::Lease::Waiting)); + let managed = crate::managed::Managed::new( + &harness.management_config(), + lease, + crate::managed::Registration::local("env_test"), + &harness.state(), + &core, + &gateway, + None, + ); + let store = core.asset_store(); + let blob = |bytes: &[u8]| format!("{:x}", Sha256::digest(bytes)); + + store.insert(&blob(b"first"), b"first").unwrap(); + managed.remove_unused_assets().await; + assert!(!store.contains(&blob(b"first")).unwrap()); + + // A load that fails after downloading leaves its blob behind, and the kept set stays the same. + store.insert(&blob(b"second"), b"second").unwrap(); + drop(managed.begin_asset_load().await); + managed.remove_unused_assets().await; + assert!(!store.contains(&blob(b"second")).unwrap()); +} diff --git a/packages/management/README.md b/packages/management/README.md index 1bcebd65..ee0a24a7 100644 --- a/packages/management/README.md +++ b/packages/management/README.md @@ -101,11 +101,14 @@ serve only blobs of revisions the environment's deployments pin, packs only for or redirecting to a presigned URL of the release bucket. Each `SetAssetHead` collects the project's unused revisions. A revision is kept while it is the head, one of the -project's 10 newest ready revisions, less than an hour old (as long as its upload URLs last), or pinned by an -environment's active deployment, an unfinished one or one of its 10 newest. The rest are deleted, and with them every -blob no remaining revision holds, from the database and the release store; a rollback to an older deployment fails once -its revision is gone. Collection holds a per-project lock that uploads and deployments share, so it never deletes a blob -a revision is being declared with or a revision a deployment is pinning. +project's 10 newest ready revisions, declared by an upload within the last hour (as long as its upload URLs last), +pinned by an environment's active deployment, an unfinished one or one of its 10 newest, or pinned by a superseded +deployment still within its environment's drain deadline. The rest are deleted, and with them every blob no remaining +revision holds, from the database; their keys are recorded in `asset_deletions` and the release store's objects are +deleted in a second transaction that skips a key a new upload has claimed again. A delete the store fails stays recorded +and is retried by the next collection. A rollback to an older deployment fails once its revision is gone. Collection +holds a per-project lock that uploads, deployments and head moves share, so it never deletes a blob a revision is being +declared with or a revision a deployment or the head is pinning. ## Release storage diff --git a/packages/management/migrations/0004_asset_deletions.sql b/packages/management/migrations/0004_asset_deletions.sql new file mode 100644 index 00000000..5738af41 --- /dev/null +++ b/packages/management/migrations/0004_asset_deletions.sql @@ -0,0 +1,9 @@ +CREATE TABLE "asset_deletions" ( + "project_id" text NOT NULL, + "sha256" text NOT NULL, + "create_time" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "asset_deletions_project_id_sha256_pk" PRIMARY KEY("project_id","sha256") +); +--> statement-breakpoint +ALTER TABLE "asset_revisions" ADD COLUMN "protected_until" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint +ALTER TABLE "asset_deletions" ADD CONSTRAINT "asset_deletions_project_id_projects_id_fk" FOREIGN KEY ("project_id") REFERENCES "public"."projects"("id") ON DELETE cascade ON UPDATE no action; \ No newline at end of file diff --git a/packages/management/migrations/meta/0004_snapshot.json b/packages/management/migrations/meta/0004_snapshot.json new file mode 100644 index 00000000..fc1505e5 --- /dev/null +++ b/packages/management/migrations/meta/0004_snapshot.json @@ -0,0 +1,1948 @@ +{ + "id": "36c1afcc-4b11-42d9-898f-63e998d7b7dc", + "prevId": "8d0e1855-6b63-4984-a43f-c2e76ed93d82", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.api_tokens": { + "name": "api_tokens", + "schema": "", + "columns": { + "seq": { + "name": "seq", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret_hash": { + "name": "secret_hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expire_time": { + "name": "expire_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoke_time": { + "name": "revoke_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'person'" + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "renews": { + "name": "renews", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "api_tokens_principal": { + "name": "api_tokens_principal", + "columns": [ + { + "expression": "principal_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_tokens_environment": { + "name": "api_tokens_environment", + "columns": [ + { + "expression": "environment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"api_tokens\".\"environment_id\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "api_tokens_project_id_projects_id_fk": { + "name": "api_tokens_project_id_projects_id_fk", + "tableFrom": "api_tokens", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "api_tokens_environment_id_environments_id_fk": { + "name": "api_tokens_environment_id_environments_id_fk", + "tableFrom": "api_tokens", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_tokens_seq_unique": { + "name": "api_tokens_seq_unique", + "nullsNotDistinct": false, + "columns": [ + "seq" + ] + }, + "api_tokens_secret_hash_unique": { + "name": "api_tokens_secret_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "secret_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.asset_blobs": { + "name": "asset_blobs", + "schema": "", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "sha1": { + "name": "sha1", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "asset_blobs_project_id_projects_id_fk": { + "name": "asset_blobs_project_id_projects_id_fk", + "tableFrom": "asset_blobs", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "asset_blobs_project_id_sha256_pk": { + "name": "asset_blobs_project_id_sha256_pk", + "columns": [ + "project_id", + "sha256" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.asset_deletions": { + "name": "asset_deletions", + "schema": "", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "asset_deletions_project_id_projects_id_fk": { + "name": "asset_deletions_project_id_projects_id_fk", + "tableFrom": "asset_deletions", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "asset_deletions_project_id_sha256_pk": { + "name": "asset_deletions_project_id_sha256_pk", + "columns": [ + "project_id", + "sha256" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.asset_revision_blobs": { + "name": "asset_revision_blobs", + "schema": "", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision_id": { + "name": "revision_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pack": { + "name": "pack", + "type": "boolean", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "asset_revision_blobs_project_id_revision_id_asset_revisions_project_id_id_fk": { + "name": "asset_revision_blobs_project_id_revision_id_asset_revisions_project_id_id_fk", + "tableFrom": "asset_revision_blobs", + "tableTo": "asset_revisions", + "columnsFrom": [ + "project_id", + "revision_id" + ], + "columnsTo": [ + "project_id", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "asset_revision_blobs_project_id_revision_id_sha256_pk": { + "name": "asset_revision_blobs_project_id_revision_id_sha256_pk", + "columns": [ + "project_id", + "revision_id", + "sha256" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.asset_revisions": { + "name": "asset_revisions", + "schema": "", + "columns": { + "seq": { + "name": "seq", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "protected_until": { + "name": "protected_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "asset_revisions_project": { + "name": "asset_revisions_project", + "columns": [ + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "asset_revisions_project_id_projects_id_fk": { + "name": "asset_revisions_project_id_projects_id_fk", + "tableFrom": "asset_revisions", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "asset_revisions_project_id_id_pk": { + "name": "asset_revisions_project_id_id_pk", + "columns": [ + "project_id", + "id" + ] + } + }, + "uniqueConstraints": { + "asset_revisions_seq_unique": { + "name": "asset_revisions_seq_unique", + "nullsNotDistinct": false, + "columns": [ + "seq" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blocked_addresses": { + "name": "blocked_addresses", + "schema": "", + "columns": { + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expire_time": { + "name": "expire_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "blocked_addresses_environment_id_environments_id_fk": { + "name": "blocked_addresses_environment_id_environments_id_fk", + "tableFrom": "blocked_addresses", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "blocked_addresses_environment_id_address_pk": { + "name": "blocked_addresses_environment_id_address_pk", + "columns": [ + "environment_id", + "address" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.capacity_requests": { + "name": "capacity_requests", + "schema": "", + "columns": { + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workload": { + "name": "workload", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "machine_profile": { + "name": "machine_profile", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "release_id": { + "name": "release_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "memory_mib": { + "name": "memory_mib", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "java_version": { + "name": "java_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "machine_id": { + "name": "machine_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "machine_addresses": { + "name": "machine_addresses", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "torn_down": { + "name": "torn_down", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "started": { + "name": "started", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "resuming": { + "name": "resuming", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "credential": { + "name": "credential", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "credential_digest": { + "name": "credential_digest", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "owner_instance_id": { + "name": "owner_instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "capacity_requests_environment_id_environments_id_fk": { + "name": "capacity_requests_environment_id_environments_id_fk", + "tableFrom": "capacity_requests", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "capacity_requests_environment_id_request_id_pk": { + "name": "capacity_requests_environment_id_request_id_pk", + "columns": [ + "environment_id", + "request_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployments": { + "name": "deployments", + "schema": "", + "columns": { + "seq": { + "name": "seq", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "release_id": { + "name": "release_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "trigger": { + "name": "trigger", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "update_time": { + "name": "update_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "activate_time": { + "name": "activate_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "stop_previous": { + "name": "stop_previous", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "asset_revision_id": { + "name": "asset_revision_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "deployments_environment": { + "name": "deployments_environment", + "columns": [ + { + "expression": "environment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployments_environment_id_environments_id_fk": { + "name": "deployments_environment_id_environments_id_fk", + "tableFrom": "deployments", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "deployments_seq_unique": { + "name": "deployments_seq_unique", + "nullsNotDistinct": false, + "columns": [ + "seq" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.domains": { + "name": "domains", + "schema": "", + "columns": { + "seq": { + "name": "seq", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hostname": { + "name": "hostname", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "challenge": { + "name": "challenge", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "domains_verified_hostname": { + "name": "domains_verified_hostname", + "columns": [ + { + "expression": "hostname", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"domains\".\"state\" = 2", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "domains_environment_id_environments_id_fk": { + "name": "domains_environment_id_environments_id_fk", + "tableFrom": "domains", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "domains_seq_unique": { + "name": "domains_seq_unique", + "nullsNotDistinct": false, + "columns": [ + "seq" + ] + }, + "domains_environment_id_hostname_unique": { + "name": "domains_environment_id_hostname_unique", + "nullsNotDistinct": false, + "columns": [ + "environment_id", + "hostname" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.environments": { + "name": "environments", + "schema": "", + "columns": { + "seq": { + "name": "seq", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "active_deployment_id": { + "name": "active_deployment_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "hostname": { + "name": "hostname", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "sleeping_ping": { + "name": "sleeping_ping", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "online_players": { + "name": "online_players", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "forked_from_environment_id": { + "name": "forked_from_environment_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "forked_from_snapshot_id": { + "name": "forked_from_snapshot_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revision": { + "name": "revision", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "1" + }, + "lease": { + "name": "lease", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "epoch": { + "name": "epoch", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "owner_instance_id": { + "name": "owner_instance_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "owner_since": { + "name": "owner_since", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "gateway_addresses": { + "name": "gateway_addresses", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "pings": { + "name": "pings", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "report_sequence": { + "name": "report_sequence", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "report_desired_revision": { + "name": "report_desired_revision", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "ready_to_suspend": { + "name": "ready_to_suspend", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "report_logins": { + "name": "report_logins", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "alarm_epoch": { + "name": "alarm_epoch", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "alarm_generation": { + "name": "alarm_generation", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "alarm_due_seconds": { + "name": "alarm_due_seconds", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "alarm_due_nanos": { + "name": "alarm_due_nanos", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "alarm_fired": { + "name": "alarm_fired", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "machine_id": { + "name": "machine_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "machine_addresses": { + "name": "machine_addresses", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "machine_token": { + "name": "machine_token", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "wake_window_start": { + "name": "wake_window_start", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "wake_count": { + "name": "wake_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "wake_logins_pending": { + "name": "wake_logins_pending", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "drain_max_age_seconds": { + "name": "drain_max_age_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10800 + }, + "drain_deadline_seconds": { + "name": "drain_deadline_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 14400 + }, + "pack_token": { + "name": "pack_token", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "replace(gen_random_uuid()::text || gen_random_uuid()::text, '-', '')" + } + }, + "indexes": {}, + "foreignKeys": { + "environments_project_id_projects_id_fk": { + "name": "environments_project_id_projects_id_fk", + "tableFrom": "environments", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "environments_seq_unique": { + "name": "environments_seq_unique", + "nullsNotDistinct": false, + "columns": [ + "seq" + ] + }, + "environments_pack_token_unique": { + "name": "environments_pack_token_unique", + "nullsNotDistinct": false, + "columns": [ + "pack_token" + ] + }, + "environments_project_id_name_unique": { + "name": "environments_project_id_name_unique", + "nullsNotDistinct": false, + "columns": [ + "project_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": { + "environments_drain_max_age_seconds_check": { + "name": "environments_drain_max_age_seconds_check", + "value": "\"environments\".\"drain_max_age_seconds\" > 0" + }, + "environments_drain_deadline_seconds_check": { + "name": "environments_drain_deadline_seconds_check", + "value": "\"environments\".\"drain_deadline_seconds\" > 0" + }, + "drain_deadline_after_max_age": { + "name": "drain_deadline_after_max_age", + "value": "\"environments\".\"drain_deadline_seconds\" >= \"environments\".\"drain_max_age_seconds\"" + } + }, + "isRLSEnabled": false + }, + "public.idempotent_requests": { + "name": "idempotent_requests", + "schema": "", + "columns": { + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprint": { + "name": "fingerprint", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "response": { + "name": "response", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "idempotent_requests_scope_request_id_pk": { + "name": "idempotent_requests_scope_request_id_pk", + "columns": [ + "scope", + "request_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installation": { + "name": "installation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.log_entries": { + "name": "log_entries", + "schema": "", + "columns": { + "seq": { + "name": "seq", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sequence": { + "name": "sequence", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "time": { + "name": "time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_id": { + "name": "deployment_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "log_entries_environment": { + "name": "log_entries_environment", + "columns": [ + { + "expression": "environment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "log_entries_environment_id_environments_id_fk": { + "name": "log_entries_environment_id_environments_id_fk", + "tableFrom": "log_entries", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "log_entries_environment_id_instance_id_sequence_unique": { + "name": "log_entries_environment_id_instance_id_sequence_unique", + "nullsNotDistinct": false, + "columns": [ + "environment_id", + "instance_id", + "sequence" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.logins": { + "name": "logins", + "schema": "", + "columns": { + "id_hash": { + "name": "id_hash", + "type": "bytea", + "primaryKey": true, + "notNull": true + }, + "user_code": { + "name": "user_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_name": { + "name": "client_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expire_time": { + "name": "expire_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_id": { + "name": "token_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_secret": { + "name": "token_secret", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "logins_user_code_unique": { + "name": "logins_user_code_unique", + "nullsNotDistinct": false, + "columns": [ + "user_code" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.metric_samples": { + "name": "metric_samples", + "schema": "", + "columns": { + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "labels": { + "name": "labels", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "time": { + "name": "time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "double precision", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "metric_samples_environment_id_environments_id_fk": { + "name": "metric_samples_environment_id_environments_id_fk", + "tableFrom": "metric_samples", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "metric_samples_environment_id_instance_id_name_labels_pk": { + "name": "metric_samples_environment_id_instance_id_name_labels_pk", + "columns": [ + "environment_id", + "instance_id", + "name", + "labels" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.projects": { + "name": "projects", + "schema": "", + "columns": { + "seq": { + "name": "seq", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "asset_head_id": { + "name": "asset_head_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "projects_seq_unique": { + "name": "projects_seq_unique", + "nullsNotDistinct": false, + "columns": [ + "seq" + ] + }, + "projects_owner_id_name_unique": { + "name": "projects_owner_id_name_unique", + "nullsNotDistinct": false, + "columns": [ + "owner_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.reconciler_leader": { + "name": "reconciler_leader", + "schema": "", + "columns": { + "epoch": { + "name": "epoch", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.releases": { + "name": "releases", + "schema": "", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "archive_sha256": { + "name": "archive_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "archive_size_bytes": { + "name": "archive_size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "releases_project_id_projects_id_fk": { + "name": "releases_project_id_projects_id_fk", + "tableFrom": "releases", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "releases_project_id_id_pk": { + "name": "releases_project_id_id_pk", + "columns": [ + "project_id", + "id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.secrets": { + "name": "secrets", + "schema": "", + "columns": { + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "update_time": { + "name": "update_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "secrets_environment_id_environments_id_fk": { + "name": "secrets_environment_id_environments_id_fk", + "tableFrom": "secrets", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "secrets_environment_id_name_pk": { + "name": "secrets_environment_id_name_pk", + "columns": [ + "environment_id", + "name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.superseded_instances": { + "name": "superseded_instances", + "schema": "", + "columns": { + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owned_since": { + "name": "owned_since", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "superseded_time": { + "name": "superseded_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "superseded_instances_environment_id_environments_id_fk": { + "name": "superseded_instances_environment_id_environments_id_fk", + "tableFrom": "superseded_instances", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "superseded_instances_environment_id_instance_id_pk": { + "name": "superseded_instances_environment_id_instance_id_pk", + "columns": [ + "environment_id", + "instance_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_records": { + "name": "usage_records", + "schema": "", + "columns": { + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "start_time": { + "name": "start_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "end_time": { + "name": "end_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "player_seconds": { + "name": "player_seconds", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "usage_records_environment_id_environments_id_fk": { + "name": "usage_records_environment_id_environments_id_fk", + "tableFrom": "usage_records", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "usage_records_environment_id_id_pk": { + "name": "usage_records_environment_id_id_pk", + "columns": [ + "environment_id", + "id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/management/migrations/meta/_journal.json b/packages/management/migrations/meta/_journal.json index b2aab603..81d097e5 100644 --- a/packages/management/migrations/meta/_journal.json +++ b/packages/management/migrations/meta/_journal.json @@ -29,6 +29,13 @@ "when": 1790953833076, "tag": "0003_assets", "breakpoints": true + }, + { + "idx": 4, + "version": "7", + "when": 1795205857037, + "tag": "0004_asset_deletions", + "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/packages/management/src/assets/retention.ts b/packages/management/src/assets/retention.ts index 52361195..1c3bf83a 100644 --- a/packages/management/src/assets/retention.ts +++ b/packages/management/src/assets/retention.ts @@ -1,11 +1,11 @@ -import { and, eq, notExists, sql } from "drizzle-orm"; +import { and, eq, inArray, notExists, sql } from "drizzle-orm"; import { type Db, fetchRows } from "../db.ts"; import type { Deps } from "../deps.ts"; import { AssetRevisionState } from "../gen/chunk/management/v1/assets_pb.ts"; import { DeploymentState } from "../gen/chunk/management/v1/common_pb.ts"; import { blobKey } from "../releases/store.ts"; -import { assetBlobs, assetRevisionBlobs } from "../schema.ts"; +import { assetBlobs, assetDeletions, assetRevisionBlobs } from "../schema.ts"; import { eachLimited } from "./store.ts"; /** How many of a project's newest ready revisions are kept, deployed or not. */ @@ -14,6 +14,8 @@ export const keptRecentRevisions = 10; export const keptRecentDeployments = 10; /** How long a revision is kept after an upload of it was declared: as long as its upload URLs last. */ export const uploadLifetimeMs = 60 * 60 * 1000; +/** The time until which an upload declared now protects its revision from collection. */ +export const protectedUntil = () => sql`now() + make_interval(secs => ${uploadLifetimeMs / 1000})`; /** How many blobs are removed from the release store at once. */ const deleteConcurrency = 16; @@ -33,85 +35,120 @@ export async function lockAssets(tx: Db, projectId: string, mode: "shared" | "ex /** * Deletes the project's revisions that nothing keeps, and then the blobs no remaining revision holds, from the database * and the release store. A revision is kept while it is the project's head, one of its `keptRecentRevisions` newest - * ready revisions, younger than `uploadLifetimeMs`, pinned by the active, an unfinished or one of the - * `keptRecentDeployments` newest deployments of any of its environments. Returns how many revisions and blobs it - * deleted; a blob the store fails to delete is logged and stays there unreferenced. + * ready revisions, protected by an upload declared within `uploadLifetimeMs`, pinned by the active, an unfinished or one + * of the `keptRecentDeployments` newest deployments of any of its environments, or pinned by a deployment that + * is still draining. Returns how many revisions it deleted and how many blobs it removed from the release store. + * + * The database rows are deleted first, recording the blobs' keys in `asset_deletions`; a later transaction deletes the + * bytes of each key no row has claimed again. A store delete that fails leaves its key recorded, so the next + * collection retries it. */ export async function collectAssets( { db, releases }: Deps, projectId: string, ): Promise<{ revisions: number; blobs: number }> { - const { revisions, orphans } = await db.transaction(async (tx) => { + const revisions = await db.transaction((tx) => deleteUnusedRows(tx, projectId)); + const blobs = await db.transaction(async (tx) => { await lockAssets(tx, projectId, "exclusive"); - const served = [DeploymentState.PENDING, DeploymentState.IN_PROGRESS, DeploymentState.ACTIVE]; - // Every CTE reads the snapshot from before the deletion, so `blobs` holds the deleted revisions' blobs. - const [deleted] = await fetchRows<{ revisions: number; blobs: string[] | null }>( + const pending = await fetchRows<{ sha256: string; referenced: boolean }>( tx, sql` - with deployed as ( - select d.asset_revision_id, d.state, d.id = e.active_deployment_id as active, - row_number() over (partition by d.environment_id order by d.seq desc) as recency - from deployments d join environments e on e.id = d.environment_id - where e.project_id = ${projectId} - ), - kept as ( - select asset_head_id as id from projects where id = ${projectId} - union - select id from ( - select id from asset_revisions where project_id = ${projectId} and state = ${AssetRevisionState.READY} - order by seq desc limit ${keptRecentRevisions} - ) recent - union - select asset_revision_id from deployed - where active or recency <= ${keptRecentDeployments} or state in (${sql.join(served, sql`, `)}) - ), - doomed as ( - select r.id from asset_revisions r - where r.project_id = ${projectId} - and r.create_time < now() - make_interval(secs => ${uploadLifetimeMs / 1000}) - and r.id not in (select id from kept) - ), - blobs as ( - select distinct b.sha256 from asset_revision_blobs b join doomed on doomed.id = b.revision_id - where b.project_id = ${projectId} + select d.sha256, + exists (select 1 from asset_blobs b where b.project_id = d.project_id and b.sha256 = d.sha256) + or exists (select 1 from asset_revision_blobs b where b.project_id = d.project_id and b.sha256 = d.sha256) + as referenced + from asset_deletions d where d.project_id = ${projectId}`, + ); + const done = pending.filter(({ referenced }) => referenced).map(({ sha256 }) => sha256); + const unreferenced = pending.filter(({ referenced }) => !referenced).map(({ sha256 }) => sha256); + const deleted: string[] = []; + await eachLimited(unreferenced, deleteConcurrency, (sha256) => + releases.delete(blobKey(projectId, sha256)).then( + () => void deleted.push(sha256), + (error: unknown) => console.error(`deleting asset blob ${sha256} of project ${projectId} failed:`, error), ), - gone as ( - delete from asset_revisions r using doomed where r.project_id = ${projectId} and r.id = doomed.id - returning r.id - ) - select (select count(*) from gone)::int as revisions, (select array_agg(sha256) from blobs) as blobs`, ); - const unverified = await tx - .delete(assetBlobs) - .where( - and( - eq(assetBlobs.project_id, projectId), - notExists( - tx - .select({ one: sql`1` }) - .from(assetRevisionBlobs) - .where( - and(eq(assetRevisionBlobs.project_id, projectId), eq(assetRevisionBlobs.sha256, assetBlobs.sha256)), - ), - ), - ), - ) - .returning({ sha256: assetBlobs.sha256 }); - const orphans = new Set([...(deleted?.blobs ?? []), ...unverified.map((blob) => blob.sha256)]); - if (orphans.size > 0) { - const held = await tx - .selectDistinct({ sha256: assetRevisionBlobs.sha256 }) - .from(assetRevisionBlobs) - .where(eq(assetRevisionBlobs.project_id, projectId)); - for (const { sha256 } of held) orphans.delete(sha256); + const settled = [...done, ...deleted]; + if (settled.length > 0) { + await tx + .delete(assetDeletions) + .where(and(eq(assetDeletions.project_id, projectId), inArray(assetDeletions.sha256, settled))); } - // Inside the transaction, so no revision declares one of these blobs again before it is gone. - await eachLimited([...orphans], deleteConcurrency, (sha256) => - releases.delete(blobKey(projectId, sha256)).catch((error: unknown) => { - console.error(`deleting asset blob ${sha256} of project ${projectId} failed:`, error); - }), - ); - return { revisions: deleted?.revisions ?? 0, orphans }; + return deleted.length; }); - return { revisions, blobs: orphans.size }; + return { revisions, blobs }; +} + +/** Deletes the revisions nothing keeps and then the blobs none holds, recording the blobs to delete from the store. */ +async function deleteUnusedRows(tx: Db, projectId: string): Promise { + await lockAssets(tx, projectId, "exclusive"); + const served = [DeploymentState.PENDING, DeploymentState.IN_PROGRESS, DeploymentState.ACTIVE]; + // Every CTE reads the snapshot from before the deletion, so `blobs` holds the deleted revisions' blobs. + const [deleted] = await fetchRows<{ revisions: number; blobs: string[] | null }>( + tx, + sql` + with deployed as ( + select d.asset_revision_id, d.state, d.id = e.active_deployment_id as active, + d.state = ${DeploymentState.SUPERSEDED} + and d.update_time + make_interval(secs => e.drain_deadline_seconds) > now() as draining, + row_number() over (partition by d.environment_id order by d.seq desc) as recency + from deployments d join environments e on e.id = d.environment_id + where e.project_id = ${projectId} + ), + kept as ( + select asset_head_id as id from projects where id = ${projectId} + union + select id from ( + select id from asset_revisions where project_id = ${projectId} and state = ${AssetRevisionState.READY} + order by seq desc limit ${keptRecentRevisions} + ) recent + union + select id from asset_revisions where project_id = ${projectId} and protected_until > now() + union + select asset_revision_id from deployed + where active or draining or recency <= ${keptRecentDeployments} or state in (${sql.join(served, sql`, `)}) + ), + doomed as ( + select r.id from asset_revisions r + where r.project_id = ${projectId} and r.id not in (select id from kept) + ), + blobs as ( + select distinct b.sha256 from asset_revision_blobs b join doomed on doomed.id = b.revision_id + where b.project_id = ${projectId} + ), + gone as ( + delete from asset_revisions r using doomed where r.project_id = ${projectId} and r.id = doomed.id + returning r.id + ) + select (select count(*) from gone)::int as revisions, (select array_agg(sha256) from blobs) as blobs`, + ); + const unverified = await tx + .delete(assetBlobs) + .where( + and( + eq(assetBlobs.project_id, projectId), + notExists( + tx + .select({ one: sql`1` }) + .from(assetRevisionBlobs) + .where(and(eq(assetRevisionBlobs.project_id, projectId), eq(assetRevisionBlobs.sha256, assetBlobs.sha256))), + ), + ), + ) + .returning({ sha256: assetBlobs.sha256 }); + const orphans = new Set([...(deleted?.blobs ?? []), ...unverified.map((blob) => blob.sha256)]); + if (orphans.size > 0) { + const held = await tx + .selectDistinct({ sha256: assetRevisionBlobs.sha256 }) + .from(assetRevisionBlobs) + .where(eq(assetRevisionBlobs.project_id, projectId)); + for (const { sha256 } of held) orphans.delete(sha256); + } + if (orphans.size > 0) { + await tx + .insert(assetDeletions) + .values([...orphans].map((sha256) => ({ project_id: projectId, sha256 }))) + .onConflictDoNothing(); + } + return deleted?.revisions ?? 0; } diff --git a/packages/management/src/assets/service.ts b/packages/management/src/assets/service.ts index 06aab347..282a40cc 100644 --- a/packages/management/src/assets/service.ts +++ b/packages/management/src/assets/service.ts @@ -3,7 +3,7 @@ import { createHash } from "node:crypto"; import { create } from "@bufbuild/protobuf"; import { timestampFromDate } from "@bufbuild/protobuf/wkt"; import { Code, ConnectError, type ServiceImpl } from "@connectrpc/connect"; -import { and, desc, eq, lt, sql } from "drizzle-orm"; +import { and, desc, eq, lt } from "drizzle-orm"; import type { Deps } from "../deps.ts"; import { @@ -17,7 +17,7 @@ import { blobKey } from "../releases/store.ts"; import { callerOf } from "../rpc/caller.ts"; import { failedPrecondition, invalid, notFound, page, pageOf, required, seqAfter } from "../rpc/validate.ts"; import { assetBlobs, assetRevisionBlobs, assetRevisions, projects } from "../schema.ts"; -import { collectAssets, lockAssets, uploadLifetimeMs } from "./retention.ts"; +import { collectAssets, lockAssets, protectedUntil, uploadLifetimeMs } from "./retention.ts"; import { decodeRevision, revisionBlobs, sha256Hex } from "./revision.ts"; import { eachLimited, findRevision, heldBlobs, loadReadyRevision, toAssetRevision } from "./store.ts"; @@ -60,21 +60,17 @@ export function assetService(deps: Deps): Partial { + await lockAssets(tx, project.id, "shared"); + const revision = await loadReadyRevision(tx, project.id, revisionId); + const [moved] = await tx + .update(projects) + .set({ asset_head_id: revision.id }) + .where(and(eq(projects.id, project.id), eq(projects.asset_head_id, request.expectedHeadId))) + .returning({ id: projects.id }); + if (!moved) { + const [current] = await tx + .select({ head: projects.asset_head_id }) + .from(projects) + .where(eq(projects.id, project.id)); + throw new ConnectError(`the asset head moved to ${current?.head || "nothing"}`, Code.Aborted); + } + }); // Each push moves the head, so the revisions it leaves behind are collected here. await collectAssets(deps, project.id).catch((error: unknown) => { console.error(`collecting project ${project.id}'s unused asset revisions failed:`, error); diff --git a/packages/management/src/assets/store.ts b/packages/management/src/assets/store.ts index 9a0d0736..c5b2a949 100644 --- a/packages/management/src/assets/store.ts +++ b/packages/management/src/assets/store.ts @@ -8,7 +8,9 @@ import { assetBlobs, assetRevisions } from "../schema.ts"; export type AssetRevisionRow = typeof assetRevisions.$inferSelect; -export function toAssetRevision(row: Omit & { manifest?: Uint8Array }): AssetRevision { +export function toAssetRevision( + row: Omit & { manifest?: Uint8Array }, +): AssetRevision { return create(AssetRevisionSchema, { id: row.id, projectId: row.project_id, diff --git a/packages/management/src/schema.ts b/packages/management/src/schema.ts index e612c553..851db7a7 100644 --- a/packages/management/src/schema.ts +++ b/packages/management/src/schema.ts @@ -222,6 +222,8 @@ export const assetRevisions = pgTable( /** The distinct bytes of its blobs. */ size_bytes: int8().notNull(), create_time: createTime(), + /** Collection keeps the revision until then, as long as the upload URLs of its latest declaration last. */ + protected_until: time().notNull().defaultNow(), }, (t) => [primaryKey({ columns: [t.project_id, t.id] }), index("asset_revisions_project").on(t.project_id, t.seq)], ); @@ -259,6 +261,19 @@ export const assetBlobs = pgTable( (t) => [primaryKey({ columns: [t.project_id, t.sha256] })], ); +/** Blobs whose database rows are gone and whose release store objects are yet to be deleted. */ +export const assetDeletions = pgTable( + "asset_deletions", + { + project_id: text() + .notNull() + .references(() => projects.id, { onDelete: "cascade" }), + sha256: text().notNull(), + create_time: createTime(), + }, + (t) => [primaryKey({ columns: [t.project_id, t.sha256] })], +); + export const deployments = pgTable( "deployments", { diff --git a/packages/management/test/assets.test.ts b/packages/management/test/assets.test.ts index 5e4a00cf..e16717bc 100644 --- a/packages/management/test/assets.test.ts +++ b/packages/management/test/assets.test.ts @@ -2,7 +2,7 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import { Code } from "@connectrpc/connect"; -import { keptRecentRevisions } from "../src/assets/retention.ts"; +import { collectAssets, keptRecentRevisions } from "../src/assets/retention.ts"; import { decodeRevision, sha256Hex } from "../src/assets/revision.ts"; import { issueEnvironmentToken } from "../src/auth/tokens.ts"; import { desiredState } from "../src/environments/desired.ts"; @@ -201,7 +201,7 @@ describe.skipIf(!databaseUrl)("AssetService", () => { const unfinished = assetRevision({ shared: { "u.txt": "unfinished" } }); const [upload] = (await assets.uploadAssets({ projectId, manifest: unfinished.manifest })).uploads; await fetch(upload?.upload?.url ?? "", { method: "PUT", body: unfinished.blobs.get(upload?.sha256 ?? "") }); - await h.sql`update asset_revisions set create_time = now() - interval '2 hours' where project_id = ${projectId}`; + await h.sql`update asset_revisions set protected_until = now() - interval '1 minute' where project_id = ${projectId}`; const head = recent[0]?.id ?? ""; await assets.setAssetHead({ projectId, revisionId: head, expectedHeadId: "" }); @@ -217,4 +217,68 @@ describe.skipIf(!databaseUrl)("AssetService", () => { uploads: [{ sha256: [...(recent[1]?.blobs.keys() ?? [])][0] }], }); }); + + test("a deployment keeps its revision while it drains, and not past its environment's drain deadline", async () => { + const { projectId, environmentId } = await createEnvironment(h); + const deploy = h.client(DeploymentService); + await uploadRelease(h, projectId, "r1"); + const draining = assetRevision({ shared: { "f.txt": "draining" } }); + await uploadAssets(h, projectId, draining); + const fillers = Array.from({ length: keptRecentRevisions + 1 }, (_, i) => + assetRevision({ shared: { "f.txt": `filler${i}` } }), + ); + for (const filler of fillers) await uploadAssets(h, projectId, filler); + const pin = (assetRevisionId: string) => + deploy.deploy({ requestId: crypto.randomUUID(), environmentId, releaseId: "r1", assetRevisionId }); + await pin(draining.id); + for (let i = 0; i < 11; i++) await pin(fillers[0]?.id ?? ""); + await h.sql`update asset_revisions set protected_until = now() - interval '1 minute' where project_id = ${projectId}`; + await h.sql`update projects set asset_head_id = ${fillers[1]?.id ?? ""} where id = ${projectId}`; + const stored = async () => + (await h.sql<{ id: string }[]>`select id from asset_revisions where project_id = ${projectId}`).some( + (row) => row.id === draining.id, + ); + + await collectAssets(h.deps, projectId); + expect(await stored()).toBe(true); + await h.sql`update deployments set update_time = now() - interval '5 hours' where asset_revision_id = ${draining.id}`; + await collectAssets(h.deps, projectId); + expect(await stored()).toBe(false); + }); + + test("a recorded deletion is dropped when its blob is referenced again, and retried when the store fails", async () => { + const { projectId } = await createEnvironment(h); + const [referenced, lost, flaky] = [1, 2, 3].map((i) => assetRevision({ shared: { "f.txt": `blob${i}` } })); + for (const revision of [referenced, lost, flaky]) await uploadAssets(h, projectId, revision); + const digest = (revision?: { blobs: Map }) => [...(revision?.blobs.keys() ?? [])][0] ?? ""; + for (const revision of [lost, flaky]) { + await h.sql`delete from asset_revisions where project_id = ${projectId} and id = ${revision?.id ?? ""}`; + await h.sql`delete from asset_blobs where project_id = ${projectId} and sha256 = ${digest(revision)}`; + } + for (const revision of [referenced, lost, flaky]) { + await h.sql`insert into asset_deletions (project_id, sha256) values (${projectId}, ${digest(revision)})`; + } + const recorded = async () => + (await h.sql<{ sha256: string }[]>`select sha256 from asset_deletions where project_id = ${projectId}`).map( + (row) => row.sha256, + ); + const present = (revision?: { blobs: Map }) => + h.releases.exists(blobKey(projectId, digest(revision))); + const { delete: remove } = h.deps.releases; + const flakyKey = blobKey(projectId, digest(flaky)); + h.deps.releases.delete = async (key) => { + if (key === flakyKey) throw new Error("store unavailable"); + await remove(key); + }; + try { + await collectAssets(h.deps, projectId); + expect(await recorded()).toEqual([digest(flaky)]); + expect([await present(referenced), await present(lost), await present(flaky)]).toEqual([true, false, true]); + } finally { + h.deps.releases.delete = remove; + } + await collectAssets(h.deps, projectId); + expect(await recorded()).toEqual([]); + expect(await present(flaky)).toBe(false); + }); }); From 91b9e6981d8d3e6197a2c62feb62602962d8efae Mon Sep 17 00:00:00 2001 From: oglassdev <23040613+oglassdev@users.noreply.github.com> Date: Fri, 20 Nov 2026 15:35:08 -0500 Subject: [PATCH 3/5] fix(assets): complete uploads under the asset lock, keep superseded revisions for the deadline they were drained under, and count loads once their inserts finish - completeAssetUpload runs in one transaction holding the shared project lock, so collection can't delete the revision or its blobs between verification and the READY transition. eachLimited now waits for every started run before it rejects. - A deployment records drain_until when it is superseded; collection keeps its revision until the later of that and update_time plus the environment's current deadline, so shortening the setting removes nothing early. - Migration 0004 protects existing revisions for an hour, and 0005 backfills drain_until. - core's asset load accounting waits for the blocking inserts a cancelled or failed load left running. - management README notes the remaining window of a store delete outliving its lock. Refs #328 --- crates/chunk-environment/src/managed.rs | 32 +- .../chunk-environment/src/managed/assets.rs | 42 +- .../src/managed/tests/assets.rs | 9 +- packages/management/README.md | 14 +- .../migrations/0004_asset_deletions.sql | 1 + .../migrations/0005_drain_until.sql | 3 + .../migrations/meta/0005_snapshot.json | 1954 +++++++++++++++++ .../management/migrations/meta/_journal.json | 9 +- packages/management/src/assets/retention.ts | 7 +- packages/management/src/assets/service.ts | 102 +- packages/management/src/assets/store.ts | 15 +- packages/management/src/deployments/store.ts | 12 +- packages/management/src/schema.ts | 2 + packages/management/test/assets.test.ts | 50 + 14 files changed, 2166 insertions(+), 86 deletions(-) create mode 100644 packages/management/migrations/0005_drain_until.sql create mode 100644 packages/management/migrations/meta/0005_snapshot.json diff --git a/crates/chunk-environment/src/managed.rs b/crates/chunk-environment/src/managed.rs index 8b885c39..68b132b3 100644 --- a/crates/chunk-environment/src/managed.rs +++ b/crates/chunk-environment/src/managed.rs @@ -85,7 +85,7 @@ pub(crate) struct Managed<'a> { /// many loads had finished, and which revisions were kept, when the last removal fully succeeded. asset_loads: tokio::sync::RwLock)>>, /// How many loads into `assets` have finished, whether they succeeded, failed or were abandoned. - asset_loads_finished: AtomicU64, + asset_loads_finished: Arc, /// Where the activation management has not yet accepted is recorded. activation: PathBuf, core: &'a Core, @@ -180,12 +180,17 @@ struct Work<'a> { /// A load filling the asset store, which removal of unused revisions waits for. struct AssetLoad<'a> { _shared: tokio::sync::RwLockReadGuard<'a, Option<(u64, BTreeSet)>>, - finished: &'a AtomicU64, + done: Arc, } -impl Drop for AssetLoad<'_> { +/// Counts a load as finished once dropped. The load shares it with every blocking insert it starts, which keep running +/// when the load is cancelled or fails, so the count advances only after the last of them is done. +#[derive(Default)] +struct LoadDone(Arc); + +impl Drop for LoadDone { fn drop(&mut self) { - self.finished.fetch_add(1, Ordering::Relaxed); + self.0.fetch_add(1, Ordering::Relaxed); } } @@ -228,7 +233,7 @@ impl<'a> Managed<'a> { releases: release::Store::new(state, core.archives().clone()), assets: core.asset_store(), asset_loads: tokio::sync::RwLock::default(), - asset_loads_finished: AtomicU64::default(), + asset_loads_finished: Arc::default(), activation: state.join("managed.json"), core, gateway, @@ -452,10 +457,14 @@ impl<'a> Managed<'a> { } } - /// Keeps unused revisions from being removed from `assets` until the returned guard drops, which also records that - /// a load finished, as it may have left blobs behind that no revision holds. + /// Keeps unused revisions from being removed from `assets` until the returned guard drops. The load is counted as + /// finished once that and the clones of `done` are dropped, as it may have left blobs behind that no revision + /// holds. async fn begin_asset_load(&self) -> AssetLoad<'_> { - AssetLoad { _shared: self.asset_loads.read().await, finished: &self.asset_loads_finished } + AssetLoad { + _shared: self.asset_loads.read().await, + done: Arc::new(LoadDone(Arc::clone(&self.asset_loads_finished))), + } } fn start(&self, desired: v1::AttachResponse) -> Work<'_> { @@ -540,8 +549,8 @@ impl<'a> Managed<'a> { if backend.check_deployment(id).await.is_err() { return false; } - let _loading = self.begin_asset_load().await; - match assets::restore(&self.client, &self.assets, desired.assets.as_ref(), cancel).await { + let loading = self.begin_asset_load().await; + match assets::restore(&self.client, &self.assets, desired.assets.as_ref(), &loading.done, cancel).await { Ok(restored) => restored.is_some(), Err(error) => { tracing::warn!(%error, deployment = %desired.deployment_id, "cannot restore the deployment's assets"); @@ -559,7 +568,8 @@ impl<'a> Managed<'a> { return Ok(false); }; let loading = self.begin_asset_load().await; - let revision = assets::load(&self.client, &self.assets, desired.assets.as_ref(), loaded.assets(), cancel); + let revision = + assets::load(&self.client, &self.assets, desired.assets.as_ref(), loaded.assets(), &loading.done, cancel); let Some(revision) = revision.await? else { return Ok(false); }; diff --git a/crates/chunk-environment/src/managed/assets.rs b/crates/chunk-environment/src/managed/assets.rs index d9739498..6d079c58 100644 --- a/crates/chunk-environment/src/managed/assets.rs +++ b/crates/chunk-environment/src/managed/assets.rs @@ -1,7 +1,10 @@ //! Loading the asset revision a deployment pins into core's asset store before the deployment activates, and removing //! the revisions no deployment pins any more. -use super::release::{self, Staged}; +use super::{ + LoadDone, + release::{self, Staged}, +}; use chunk_build::assets::Store; use chunk_contract::{ASSET_REVISION_VERSION, AssetContract, AssetRevision}; use chunk_management::{Client, v1}; @@ -9,6 +12,7 @@ use futures_util::TryStreamExt as _; use std::{ collections::{BTreeMap, BTreeSet}, fs, io, + sync::Arc, }; use tokio_util::sync::CancellationToken; @@ -17,12 +21,14 @@ const CONCURRENT_DOWNLOADS: usize = 4; /// Checks the revision `artifact` names against the release's declarations in `contract`, then downloads into `store` /// each blob of it that JVMs read and `store` lacks, verified, and the revision itself. Without an artifact, the -/// deployment pins the empty revision. Returns `None` once `cancel` stops it before its downloads finish. +/// deployment pins the empty revision. Returns `None` once `cancel` stops it before its downloads finish. Blocking +/// inserts it started may outlive it, each holding a clone of `done`. pub(super) async fn load( client: &Client, store: &Store, artifact: Option<&v1::AssetArtifact>, contract: &AssetContract, + done: &Arc, cancel: &CancellationToken, ) -> io::Result> { let Some(artifact) = artifact else { @@ -33,15 +39,17 @@ pub(super) async fn load( }; let revision = decode(artifact)?; contract.check(&revision).map_err(io::Error::other)?; - download(client, store, artifact, revision, cancel).await + download(client, store, artifact, revision, done, cancel).await } /// Downloads into `store` whatever of the revision `artifact` names it lost since core activated it, such as to a crash -/// before the store's writes were durable. Returns `None` once `cancel` stops it before its downloads finish. +/// before the store's writes were durable. Returns `None` once `cancel` stops it before its downloads finish. Blocking +/// inserts it started may outlive it, each holding a clone of `done`. pub(super) async fn restore( client: &Client, store: &Store, artifact: Option<&v1::AssetArtifact>, + done: &Arc, cancel: &CancellationToken, ) -> io::Result> { let Some(artifact) = artifact else { @@ -49,7 +57,7 @@ pub(super) async fn restore( store.write_revision(&revision)?; return Ok(Some(revision)); }; - download(client, store, artifact, decode(artifact)?, cancel).await + download(client, store, artifact, decode(artifact)?, done, cancel).await } fn decode(artifact: &v1::AssetArtifact) -> io::Result { @@ -65,6 +73,7 @@ async fn download( store: &Store, artifact: &v1::AssetArtifact, revision: AssetRevision, + done: &Arc, cancel: &CancellationToken, ) -> io::Result> { fs::create_dir_all(store.root())?; @@ -76,7 +85,7 @@ async fn download( } let fetched = futures_util::stream::iter(missing) .try_for_each_concurrent(CONCURRENT_DOWNLOADS, |(sha256, size)| { - fetch_blob(client, store, &artifact.blob_url_prefix, sha256, size) + fetch_blob(client, store, &artifact.blob_url_prefix, sha256, size, done) }); tokio::select! { fetched = fetched => fetched?, @@ -86,13 +95,26 @@ async fn download( Ok(Some(revision)) } -/// Downloads the blob into a hidden file beside the store, then inserts it, which verifies its SHA-256. -async fn fetch_blob(client: &Client, store: &Store, url_prefix: &str, sha256: &str, size: u64) -> io::Result<()> { +/// Downloads the blob into a hidden file beside the store, then inserts it, which verifies its SHA-256. The insert runs +/// to its end even when this future is dropped, holding `done` until then. +async fn fetch_blob( + client: &Client, + store: &Store, + url_prefix: &str, + sha256: &str, + size: u64, + done: &Arc, +) -> io::Result<()> { let staged = Staged(store.root().join(format!(".{}.blob", uuid::Uuid::new_v4()))); let url = format!("{url_prefix}{sha256}"); release::fetch(|| client.download_blob(&url), size, "asset blob", &staged.0).await?; - let (store, sha256) = (store.clone(), sha256.to_owned()); - tokio::task::spawn_blocking(move || store.insert_file(&sha256, &staged.0)).await.map_err(io::Error::other)? + let (store, sha256, done) = (store.clone(), sha256.to_owned(), Arc::clone(done)); + tokio::task::spawn_blocking(move || { + let _done = done; + store.insert_file(&sha256, &staged.0) + }) + .await + .map_err(io::Error::other)? } /// Every blob some app's JVMs read: each app's worlds and files, and the shared files. Players' clients download packs diff --git a/crates/chunk-environment/src/managed/tests/assets.rs b/crates/chunk-environment/src/managed/tests/assets.rs index 28198c44..e198f515 100644 --- a/crates/chunk-environment/src/managed/tests/assets.rs +++ b/crates/chunk-environment/src/managed/tests/assets.rs @@ -1,7 +1,7 @@ //! Loading the asset revision a deployment pins before it activates. use super::*; -use crate::managed::assets; +use crate::managed::{LoadDone, assets}; use chunk_build::assets::Store; use chunk_contract::{AssetBlob, AssetContract, AssetRevision}; use chunk_management::v1::AssetArtifact; @@ -27,15 +27,16 @@ async fn a_revision_and_its_missing_blobs_load_unless_it_lacks_a_declared_world( ..AssetContract::default() }; let cancel = CancellationToken::new(); + let done = Arc::::default(); - let loaded = assets::load(&client, &store, Some(&artifact), &declared("koth"), &cancel).await.unwrap(); + let loaded = assets::load(&client, &store, Some(&artifact), &declared("koth"), &done, &cancel).await.unwrap(); assert!(store.contains(&blob(missing).sha256).unwrap()); assert_eq!(store.read_revision(&revision.id()).unwrap(), revision); assert_eq!(loaded, Some(revision)); - let error = assets::load(&client, &store, Some(&artifact), &declared("arena"), &cancel).await.unwrap_err(); + let error = assets::load(&client, &store, Some(&artifact), &declared("arena"), &done, &cancel).await.unwrap_err(); assert!(error.to_string().contains("no world arena for app lobby"), "{error}"); let renamed = AssetArtifact { revision_id: "0".repeat(64), ..artifact }; - assert!(assets::load(&client, &store, Some(&renamed), &declared("koth"), &cancel).await.is_err()); + assert!(assets::load(&client, &store, Some(&renamed), &declared("koth"), &done, &cancel).await.is_err()); } #[tokio::test] diff --git a/packages/management/README.md b/packages/management/README.md index ee0a24a7..9e8c4a27 100644 --- a/packages/management/README.md +++ b/packages/management/README.md @@ -103,12 +103,14 @@ or redirecting to a presigned URL of the release bucket. Each `SetAssetHead` collects the project's unused revisions. A revision is kept while it is the head, one of the project's 10 newest ready revisions, declared by an upload within the last hour (as long as its upload URLs last), pinned by an environment's active deployment, an unfinished one or one of its 10 newest, or pinned by a superseded -deployment still within its environment's drain deadline. The rest are deleted, and with them every blob no remaining -revision holds, from the database; their keys are recorded in `asset_deletions` and the release store's objects are -deleted in a second transaction that skips a key a new upload has claimed again. A delete the store fails stays recorded -and is retried by the next collection. A rollback to an older deployment fails once its revision is gone. Collection -holds a per-project lock that uploads, deployments and head moves share, so it never deletes a blob a revision is being -declared with or a revision a deployment or the head is pinning. +deployment still within its environment's drain deadline, as it was when the deployment was superseded or is now, +whichever is later. The rest are deleted, and with them every blob no remaining revision holds, from the database; their +keys are recorded in `asset_deletions` and the release store's objects are deleted in a second transaction that skips a +key a new upload has claimed again. A delete the store fails stays recorded and is retried by the next collection. A +store delete still running when management loses its lock, such as to a crash, can land after the same content is +uploaded again, which pushing the assets again repairs. A rollback to an older deployment fails once its revision is +gone. Collection holds a per-project lock that uploads, deployments and head moves share, so it never deletes a blob a +revision is being declared with or a revision a deployment or the head is pinning. ## Release storage diff --git a/packages/management/migrations/0004_asset_deletions.sql b/packages/management/migrations/0004_asset_deletions.sql index 5738af41..1c758fba 100644 --- a/packages/management/migrations/0004_asset_deletions.sql +++ b/packages/management/migrations/0004_asset_deletions.sql @@ -6,4 +6,5 @@ CREATE TABLE "asset_deletions" ( ); --> statement-breakpoint ALTER TABLE "asset_revisions" ADD COLUMN "protected_until" timestamp with time zone DEFAULT now() NOT NULL;--> statement-breakpoint +UPDATE "asset_revisions" SET "protected_until" = now() + interval '1 hour';--> statement-breakpoint ALTER TABLE "asset_deletions" ADD CONSTRAINT "asset_deletions_project_id_projects_id_fk" FOREIGN KEY ("project_id") REFERENCES "public"."projects"("id") ON DELETE cascade ON UPDATE no action; \ No newline at end of file diff --git a/packages/management/migrations/0005_drain_until.sql b/packages/management/migrations/0005_drain_until.sql new file mode 100644 index 00000000..ad53fad4 --- /dev/null +++ b/packages/management/migrations/0005_drain_until.sql @@ -0,0 +1,3 @@ +ALTER TABLE "deployments" ADD COLUMN "drain_until" timestamp with time zone; +--> statement-breakpoint +UPDATE "deployments" SET "drain_until" = "deployments"."update_time" + make_interval(secs => (SELECT "drain_deadline_seconds" FROM "environments" WHERE "environments"."id" = "deployments"."environment_id")) WHERE "state" = 5; diff --git a/packages/management/migrations/meta/0005_snapshot.json b/packages/management/migrations/meta/0005_snapshot.json new file mode 100644 index 00000000..e93f0c66 --- /dev/null +++ b/packages/management/migrations/meta/0005_snapshot.json @@ -0,0 +1,1954 @@ +{ + "id": "60010118-6540-48a6-a4c3-05428489739b", + "prevId": "36c1afcc-4b11-42d9-898f-63e998d7b7dc", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.api_tokens": { + "name": "api_tokens", + "schema": "", + "columns": { + "seq": { + "name": "seq", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret_hash": { + "name": "secret_hash", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expire_time": { + "name": "expire_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoke_time": { + "name": "revoke_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'person'" + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "renews": { + "name": "renews", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + } + }, + "indexes": { + "api_tokens_principal": { + "name": "api_tokens_principal", + "columns": [ + { + "expression": "principal_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_tokens_environment": { + "name": "api_tokens_environment", + "columns": [ + { + "expression": "environment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"api_tokens\".\"environment_id\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "api_tokens_project_id_projects_id_fk": { + "name": "api_tokens_project_id_projects_id_fk", + "tableFrom": "api_tokens", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "api_tokens_environment_id_environments_id_fk": { + "name": "api_tokens_environment_id_environments_id_fk", + "tableFrom": "api_tokens", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_tokens_seq_unique": { + "name": "api_tokens_seq_unique", + "nullsNotDistinct": false, + "columns": [ + "seq" + ] + }, + "api_tokens_secret_hash_unique": { + "name": "api_tokens_secret_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "secret_hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.asset_blobs": { + "name": "asset_blobs", + "schema": "", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "sha1": { + "name": "sha1", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "asset_blobs_project_id_projects_id_fk": { + "name": "asset_blobs_project_id_projects_id_fk", + "tableFrom": "asset_blobs", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "asset_blobs_project_id_sha256_pk": { + "name": "asset_blobs_project_id_sha256_pk", + "columns": [ + "project_id", + "sha256" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.asset_deletions": { + "name": "asset_deletions", + "schema": "", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "asset_deletions_project_id_projects_id_fk": { + "name": "asset_deletions_project_id_projects_id_fk", + "tableFrom": "asset_deletions", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "asset_deletions_project_id_sha256_pk": { + "name": "asset_deletions_project_id_sha256_pk", + "columns": [ + "project_id", + "sha256" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.asset_revision_blobs": { + "name": "asset_revision_blobs", + "schema": "", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revision_id": { + "name": "revision_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pack": { + "name": "pack", + "type": "boolean", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "asset_revision_blobs_project_id_revision_id_asset_revisions_project_id_id_fk": { + "name": "asset_revision_blobs_project_id_revision_id_asset_revisions_project_id_id_fk", + "tableFrom": "asset_revision_blobs", + "tableTo": "asset_revisions", + "columnsFrom": [ + "project_id", + "revision_id" + ], + "columnsTo": [ + "project_id", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "asset_revision_blobs_project_id_revision_id_sha256_pk": { + "name": "asset_revision_blobs_project_id_revision_id_sha256_pk", + "columns": [ + "project_id", + "revision_id", + "sha256" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.asset_revisions": { + "name": "asset_revisions", + "schema": "", + "columns": { + "seq": { + "name": "seq", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "size_bytes": { + "name": "size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "protected_until": { + "name": "protected_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "asset_revisions_project": { + "name": "asset_revisions_project", + "columns": [ + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "asset_revisions_project_id_projects_id_fk": { + "name": "asset_revisions_project_id_projects_id_fk", + "tableFrom": "asset_revisions", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "asset_revisions_project_id_id_pk": { + "name": "asset_revisions_project_id_id_pk", + "columns": [ + "project_id", + "id" + ] + } + }, + "uniqueConstraints": { + "asset_revisions_seq_unique": { + "name": "asset_revisions_seq_unique", + "nullsNotDistinct": false, + "columns": [ + "seq" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blocked_addresses": { + "name": "blocked_addresses", + "schema": "", + "columns": { + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expire_time": { + "name": "expire_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "blocked_addresses_environment_id_environments_id_fk": { + "name": "blocked_addresses_environment_id_environments_id_fk", + "tableFrom": "blocked_addresses", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "blocked_addresses_environment_id_address_pk": { + "name": "blocked_addresses_environment_id_address_pk", + "columns": [ + "environment_id", + "address" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.capacity_requests": { + "name": "capacity_requests", + "schema": "", + "columns": { + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workload": { + "name": "workload", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "machine_profile": { + "name": "machine_profile", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "release_id": { + "name": "release_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "memory_mib": { + "name": "memory_mib", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "java_version": { + "name": "java_version", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "state": { + "name": "state", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "machine_id": { + "name": "machine_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "machine_addresses": { + "name": "machine_addresses", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "torn_down": { + "name": "torn_down", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "started": { + "name": "started", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "resuming": { + "name": "resuming", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "credential": { + "name": "credential", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "credential_digest": { + "name": "credential_digest", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "owner_instance_id": { + "name": "owner_instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "capacity_requests_environment_id_environments_id_fk": { + "name": "capacity_requests_environment_id_environments_id_fk", + "tableFrom": "capacity_requests", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "capacity_requests_environment_id_request_id_pk": { + "name": "capacity_requests_environment_id_request_id_pk", + "columns": [ + "environment_id", + "request_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployments": { + "name": "deployments", + "schema": "", + "columns": { + "seq": { + "name": "seq", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "release_id": { + "name": "release_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "trigger": { + "name": "trigger", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "update_time": { + "name": "update_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "activate_time": { + "name": "activate_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "stop_previous": { + "name": "stop_previous", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "asset_revision_id": { + "name": "asset_revision_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "drain_until": { + "name": "drain_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "deployments_environment": { + "name": "deployments_environment", + "columns": [ + { + "expression": "environment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployments_environment_id_environments_id_fk": { + "name": "deployments_environment_id_environments_id_fk", + "tableFrom": "deployments", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "deployments_seq_unique": { + "name": "deployments_seq_unique", + "nullsNotDistinct": false, + "columns": [ + "seq" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.domains": { + "name": "domains", + "schema": "", + "columns": { + "seq": { + "name": "seq", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hostname": { + "name": "hostname", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "challenge": { + "name": "challenge", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "domains_verified_hostname": { + "name": "domains_verified_hostname", + "columns": [ + { + "expression": "hostname", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"domains\".\"state\" = 2", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "domains_environment_id_environments_id_fk": { + "name": "domains_environment_id_environments_id_fk", + "tableFrom": "domains", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "domains_seq_unique": { + "name": "domains_seq_unique", + "nullsNotDistinct": false, + "columns": [ + "seq" + ] + }, + "domains_environment_id_hostname_unique": { + "name": "domains_environment_id_hostname_unique", + "nullsNotDistinct": false, + "columns": [ + "environment_id", + "hostname" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.environments": { + "name": "environments", + "schema": "", + "columns": { + "seq": { + "name": "seq", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "active_deployment_id": { + "name": "active_deployment_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "hostname": { + "name": "hostname", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "sleeping_ping": { + "name": "sleeping_ping", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "online_players": { + "name": "online_players", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "forked_from_environment_id": { + "name": "forked_from_environment_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "forked_from_snapshot_id": { + "name": "forked_from_snapshot_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "revision": { + "name": "revision", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "1" + }, + "lease": { + "name": "lease", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "epoch": { + "name": "epoch", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "owner_instance_id": { + "name": "owner_instance_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "owner_since": { + "name": "owner_since", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "gateway_addresses": { + "name": "gateway_addresses", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "pings": { + "name": "pings", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "report_sequence": { + "name": "report_sequence", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "report_desired_revision": { + "name": "report_desired_revision", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "ready_to_suspend": { + "name": "ready_to_suspend", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "report_logins": { + "name": "report_logins", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "alarm_epoch": { + "name": "alarm_epoch", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "alarm_generation": { + "name": "alarm_generation", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "alarm_due_seconds": { + "name": "alarm_due_seconds", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "alarm_due_nanos": { + "name": "alarm_due_nanos", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "alarm_fired": { + "name": "alarm_fired", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "machine_id": { + "name": "machine_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "machine_addresses": { + "name": "machine_addresses", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "machine_token": { + "name": "machine_token", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "wake_window_start": { + "name": "wake_window_start", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "wake_count": { + "name": "wake_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "wake_logins_pending": { + "name": "wake_logins_pending", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "drain_max_age_seconds": { + "name": "drain_max_age_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 10800 + }, + "drain_deadline_seconds": { + "name": "drain_deadline_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 14400 + }, + "pack_token": { + "name": "pack_token", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "replace(gen_random_uuid()::text || gen_random_uuid()::text, '-', '')" + } + }, + "indexes": {}, + "foreignKeys": { + "environments_project_id_projects_id_fk": { + "name": "environments_project_id_projects_id_fk", + "tableFrom": "environments", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "environments_seq_unique": { + "name": "environments_seq_unique", + "nullsNotDistinct": false, + "columns": [ + "seq" + ] + }, + "environments_pack_token_unique": { + "name": "environments_pack_token_unique", + "nullsNotDistinct": false, + "columns": [ + "pack_token" + ] + }, + "environments_project_id_name_unique": { + "name": "environments_project_id_name_unique", + "nullsNotDistinct": false, + "columns": [ + "project_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": { + "environments_drain_max_age_seconds_check": { + "name": "environments_drain_max_age_seconds_check", + "value": "\"environments\".\"drain_max_age_seconds\" > 0" + }, + "environments_drain_deadline_seconds_check": { + "name": "environments_drain_deadline_seconds_check", + "value": "\"environments\".\"drain_deadline_seconds\" > 0" + }, + "drain_deadline_after_max_age": { + "name": "drain_deadline_after_max_age", + "value": "\"environments\".\"drain_deadline_seconds\" >= \"environments\".\"drain_max_age_seconds\"" + } + }, + "isRLSEnabled": false + }, + "public.idempotent_requests": { + "name": "idempotent_requests", + "schema": "", + "columns": { + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "request_id": { + "name": "request_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "method": { + "name": "method", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprint": { + "name": "fingerprint", + "type": "bytea", + "primaryKey": false, + "notNull": true + }, + "response": { + "name": "response", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "idempotent_requests_scope_request_id_pk": { + "name": "idempotent_requests_scope_request_id_pk", + "columns": [ + "scope", + "request_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.installation": { + "name": "installation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.log_entries": { + "name": "log_entries", + "schema": "", + "columns": { + "seq": { + "name": "seq", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sequence": { + "name": "sequence", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "time": { + "name": "time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_id": { + "name": "deployment_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "log_entries_environment": { + "name": "log_entries_environment", + "columns": [ + { + "expression": "environment_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "seq", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "log_entries_environment_id_environments_id_fk": { + "name": "log_entries_environment_id_environments_id_fk", + "tableFrom": "log_entries", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "log_entries_environment_id_instance_id_sequence_unique": { + "name": "log_entries_environment_id_instance_id_sequence_unique", + "nullsNotDistinct": false, + "columns": [ + "environment_id", + "instance_id", + "sequence" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.logins": { + "name": "logins", + "schema": "", + "columns": { + "id_hash": { + "name": "id_hash", + "type": "bytea", + "primaryKey": true, + "notNull": true + }, + "user_code": { + "name": "user_code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_name": { + "name": "client_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expire_time": { + "name": "expire_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "principal_id": { + "name": "principal_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_id": { + "name": "token_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_secret": { + "name": "token_secret", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "logins_user_code_unique": { + "name": "logins_user_code_unique", + "nullsNotDistinct": false, + "columns": [ + "user_code" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.metric_samples": { + "name": "metric_samples", + "schema": "", + "columns": { + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "labels": { + "name": "labels", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "time": { + "name": "time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "double precision", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "metric_samples_environment_id_environments_id_fk": { + "name": "metric_samples_environment_id_environments_id_fk", + "tableFrom": "metric_samples", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "metric_samples_environment_id_instance_id_name_labels_pk": { + "name": "metric_samples_environment_id_instance_id_name_labels_pk", + "columns": [ + "environment_id", + "instance_id", + "name", + "labels" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.projects": { + "name": "projects", + "schema": "", + "columns": { + "seq": { + "name": "seq", + "type": "bigserial", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "asset_head_id": { + "name": "asset_head_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "projects_seq_unique": { + "name": "projects_seq_unique", + "nullsNotDistinct": false, + "columns": [ + "seq" + ] + }, + "projects_owner_id_name_unique": { + "name": "projects_owner_id_name_unique", + "nullsNotDistinct": false, + "columns": [ + "owner_id", + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.reconciler_leader": { + "name": "reconciler_leader", + "schema": "", + "columns": { + "epoch": { + "name": "epoch", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.releases": { + "name": "releases", + "schema": "", + "columns": { + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "smallint", + "primaryKey": false, + "notNull": true + }, + "archive_sha256": { + "name": "archive_sha256", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "archive_size_bytes": { + "name": "archive_size_bytes", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "manifest": { + "name": "manifest", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "create_time": { + "name": "create_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "releases_project_id_projects_id_fk": { + "name": "releases_project_id_projects_id_fk", + "tableFrom": "releases", + "tableTo": "projects", + "columnsFrom": [ + "project_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "releases_project_id_id_pk": { + "name": "releases_project_id_id_pk", + "columns": [ + "project_id", + "id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.secrets": { + "name": "secrets", + "schema": "", + "columns": { + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "ciphertext": { + "name": "ciphertext", + "type": "bytea", + "primaryKey": false, + "notNull": false + }, + "update_time": { + "name": "update_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "secrets_environment_id_environments_id_fk": { + "name": "secrets_environment_id_environments_id_fk", + "tableFrom": "secrets", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "secrets_environment_id_name_pk": { + "name": "secrets_environment_id_name_pk", + "columns": [ + "environment_id", + "name" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.superseded_instances": { + "name": "superseded_instances", + "schema": "", + "columns": { + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owned_since": { + "name": "owned_since", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "superseded_time": { + "name": "superseded_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "superseded_instances_environment_id_environments_id_fk": { + "name": "superseded_instances_environment_id_environments_id_fk", + "tableFrom": "superseded_instances", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "superseded_instances_environment_id_instance_id_pk": { + "name": "superseded_instances_environment_id_instance_id_pk", + "columns": [ + "environment_id", + "instance_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.usage_records": { + "name": "usage_records", + "schema": "", + "columns": { + "environment_id": { + "name": "environment_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "start_time": { + "name": "start_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "end_time": { + "name": "end_time", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "player_seconds": { + "name": "player_seconds", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "usage_records_environment_id_environments_id_fk": { + "name": "usage_records_environment_id_environments_id_fk", + "tableFrom": "usage_records", + "tableTo": "environments", + "columnsFrom": [ + "environment_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "usage_records_environment_id_id_pk": { + "name": "usage_records_environment_id_id_pk", + "columns": [ + "environment_id", + "id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/management/migrations/meta/_journal.json b/packages/management/migrations/meta/_journal.json index 81d097e5..c12ed58b 100644 --- a/packages/management/migrations/meta/_journal.json +++ b/packages/management/migrations/meta/_journal.json @@ -36,6 +36,13 @@ "when": 1795205857037, "tag": "0004_asset_deletions", "breakpoints": true + }, + { + "idx": 5, + "version": "7", + "when": 1795206793110, + "tag": "0005_drain_until", + "breakpoints": true } ] -} \ No newline at end of file +} diff --git a/packages/management/src/assets/retention.ts b/packages/management/src/assets/retention.ts index 1c3bf83a..7a1c5e09 100644 --- a/packages/management/src/assets/retention.ts +++ b/packages/management/src/assets/retention.ts @@ -36,8 +36,9 @@ export async function lockAssets(tx: Db, projectId: string, mode: "shared" | "ex * Deletes the project's revisions that nothing keeps, and then the blobs no remaining revision holds, from the database * and the release store. A revision is kept while it is the project's head, one of its `keptRecentRevisions` newest * ready revisions, protected by an upload declared within `uploadLifetimeMs`, pinned by the active, an unfinished or one - * of the `keptRecentDeployments` newest deployments of any of its environments, or pinned by a deployment that - * is still draining. Returns how many revisions it deleted and how many blobs it removed from the release store. + * of the `keptRecentDeployments` newest deployments of any of its environments, or pinned by a superseded deployment + * that is still draining: until the later of the deadline its environment had when it was superseded and the one it has + * now, so shortening the deadline never removes what core may still serve. Returns how many revisions it deleted and how many blobs it removed from the release store. * * The database rows are deleted first, recording the blobs' keys in `asset_deletions`; a later transaction deletes the * bytes of each key no row has claimed again. A store delete that fails leaves its key recorded, so the next @@ -90,7 +91,7 @@ async function deleteUnusedRows(tx: Db, projectId: string): Promise { with deployed as ( select d.asset_revision_id, d.state, d.id = e.active_deployment_id as active, d.state = ${DeploymentState.SUPERSEDED} - and d.update_time + make_interval(secs => e.drain_deadline_seconds) > now() as draining, + and now() < greatest(d.drain_until, d.update_time + make_interval(secs => e.drain_deadline_seconds)) as draining, row_number() over (partition by d.environment_id order by d.seq desc) as recency from deployments d join environments e on e.id = d.environment_id where e.project_id = ${projectId} diff --git a/packages/management/src/assets/service.ts b/packages/management/src/assets/service.ts index 282a40cc..ea8319a6 100644 --- a/packages/management/src/assets/service.ts +++ b/packages/management/src/assets/service.ts @@ -5,6 +5,7 @@ import { timestampFromDate } from "@bufbuild/protobuf/wkt"; import { Code, ConnectError, type ServiceImpl } from "@connectrpc/connect"; import { and, desc, eq, lt } from "drizzle-orm"; +import type { Db } from "../db.ts"; import type { Deps } from "../deps.ts"; import { AssetRevisionState, @@ -30,8 +31,8 @@ const existsConcurrency = 64; export function assetService(deps: Deps): Partial> { const { db, releases } = deps; /** The project's verified blobs among `digests` that the blob store still holds. */ - async function storedBlobs(projectId: string, digests: string[]) { - const held = await heldBlobs(db, projectId, digests); + async function storedBlobs(conn: Db, projectId: string, digests: string[]) { + const held = await heldBlobs(conn, projectId, digests); await eachLimited([...held.keys()], existsConcurrency, async (sha256) => { if (!(await releases.exists(blobKey(projectId, sha256)))) held.delete(sha256); }); @@ -80,7 +81,7 @@ export function assetService(deps: Deps): Partial !held.has(sha256)); - await eachLimited(unverified, verifyConcurrency, async ([sha256, { size }]) => { - const sizeBytes = BigInt(size); - const sha1 = await releases.complete(blobKey(project.id, sha256), { sha256, sizeBytes }, (blob) => - verifyBlob(blob, sha256, sizeBytes).catch((error: unknown) => { - throw failedPrecondition(`blob ${sha256} is invalid: ${(error as Error).message}`); - }), - ); - if (sha1 === undefined) { - missing.push(sha256); - return; + const revisionId = required(request.revisionId, "revision_id"); + // The shared lock is taken before the revision is read and held until it is READY, so collection can't delete the + // revision or a blob it was checked against in between, and a later declaration of the same manifest can't be + // marked READY by an earlier completion. + return db.transaction(async (tx) => { + await lockAssets(tx, project.id, "shared"); + const row = await findRevision(tx, project.id, revisionId); + if (!row) throw notFound("asset revision"); + const revision = decodeRevision(row.manifest); + const blobs = revisionBlobs(revision); + const held = await storedBlobs(tx, project.id, [...blobs.keys()]); + const missing: string[] = []; + const unverified = [...blobs].filter(([sha256]) => !held.has(sha256)); + await eachLimited(unverified, verifyConcurrency, async ([sha256, { size }]) => { + const sizeBytes = BigInt(size); + const sha1 = await releases.complete(blobKey(project.id, sha256), { sha256, sizeBytes }, (blob) => + verifyBlob(blob, sha256, sizeBytes).catch((error: unknown) => { + throw failedPrecondition(`blob ${sha256} is invalid: ${(error as Error).message}`); + }), + ); + if (sha1 === undefined) { + missing.push(sha256); + return; + } + await tx + .insert(assetBlobs) + .values({ project_id: project.id, sha256, size_bytes: sizeBytes, sha1 }) + .onConflictDoNothing(); + held.set(sha256, { size_bytes: sizeBytes, sha1 }); + }); + if (missing.length > 0) { + throw failedPrecondition( + `${missing.length} of the revision's blobs are not uploaded, such as ${missing.sort()[0]}`, + ); } - await db - .insert(assetBlobs) - .values({ project_id: project.id, sha256, size_bytes: sizeBytes, sha1 }) - .onConflictDoNothing(); - held.set(sha256, { size_bytes: sizeBytes, sha1 }); - }); - if (missing.length > 0) { - throw failedPrecondition( - `${missing.length} of the revision's blobs are not uploaded, such as ${missing.sort()[0]}`, - ); - } - for (const [sha256, { size }] of blobs) { - if (held.get(sha256)?.size_bytes !== BigInt(size)) { - throw failedPrecondition(`blob ${sha256} is not ${size} bytes, as the revision declares`); + for (const [sha256, { size }] of blobs) { + if (held.get(sha256)?.size_bytes !== BigInt(size)) { + throw failedPrecondition(`blob ${sha256} is not ${size} bytes, as the revision declares`); + } } - } - for (const [name, pack] of revision.packs) { - if (held.get(pack.sha256)?.sha1 !== pack.sha1) { - throw failedPrecondition(`pack ${name}'s SHA-1 does not match its bytes`); + for (const [name, pack] of revision.packs) { + if (held.get(pack.sha256)?.sha1 !== pack.sha1) { + throw failedPrecondition(`pack ${name}'s SHA-1 does not match its bytes`); + } } - } - if (row.state === AssetRevisionState.READY) return { revision: toAssetRevision(row) }; - const [ready] = await db - .update(assetRevisions) - .set({ state: AssetRevisionState.READY }) - .where(and(eq(assetRevisions.project_id, project.id), eq(assetRevisions.id, row.id))) - .returning(); - if (!ready) throw notFound("asset revision"); - return { revision: toAssetRevision(ready) }; + if (row.state === AssetRevisionState.READY) return { revision: toAssetRevision(row) }; + const [ready] = await tx + .update(assetRevisions) + .set({ state: AssetRevisionState.READY }) + .where(and(eq(assetRevisions.project_id, project.id), eq(assetRevisions.id, row.id))) + .returning(); + if (!ready) throw notFound("asset revision"); + return { revision: toAssetRevision(ready) }; + }); }, async getAssetRevision(request, context) { diff --git a/packages/management/src/assets/store.ts b/packages/management/src/assets/store.ts index c5b2a949..224795c6 100644 --- a/packages/management/src/assets/store.ts +++ b/packages/management/src/assets/store.ts @@ -51,11 +51,22 @@ export async function heldBlobs( return new Map(rows.map(({ sha256, ...blob }) => [sha256, blob])); } -/** Runs `run` on every item, at most `limit` at once. */ +/** + * Runs `run` on every item, at most `limit` at once. Once a run fails, it starts no more, waits for those in flight and + * rejects with the first failure, so nothing it started is still running when it settles. + */ export async function eachLimited(items: T[], limit: number, run: (item: T) => Promise): Promise { let next = 0; + let failure: { error: unknown } | undefined; const worker = async () => { - while (next < items.length) await run(items[next++] as T); + while (!failure && next < items.length) { + try { + await run(items[next++] as T); + } catch (error) { + failure ??= { error }; + } + } }; await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker)); + if (failure) throw failure.error; } diff --git a/packages/management/src/deployments/store.ts b/packages/management/src/deployments/store.ts index 58be514e..eae79911 100644 --- a/packages/management/src/deployments/store.ts +++ b/packages/management/src/deployments/store.ts @@ -69,6 +69,14 @@ export function requireJvmImage(manifest: ReleaseManifest, template: string | un const unfinished = [DeploymentState.PENDING, DeploymentState.IN_PROGRESS]; +/** The values a deployment takes when superseded; its environment's drain deadline, as of now, bounds its draining. */ +const superseded = { + state: DeploymentState.SUPERSEDED, + update_time: sql`now()`, + drain_until: sql`now() + make_interval(secs => ( + select e.drain_deadline_seconds from environments e where e.id = ${deployments.environment_id}))`, +}; + /** What a deployment pins: a release and an asset revision of one project. */ export interface Deployable { release_id: string; @@ -104,7 +112,7 @@ export async function createDeployment( } await db .update(deployments) - .set({ state: DeploymentState.SUPERSEDED, update_time: sql`now()` }) + .set(superseded) .where(and(eq(deployments.environment_id, environment.id), inArray(deployments.state, unfinished))); const [row] = await db .insert(deployments) @@ -188,7 +196,7 @@ export async function activateDeployment(db: Db, id: string): Promise { if (!row) return; await db .update(deployments) - .set({ state: DeploymentState.SUPERSEDED, update_time: sql`now()` }) + .set(superseded) .where( and( eq(deployments.environment_id, row.environment_id), diff --git a/packages/management/src/schema.ts b/packages/management/src/schema.ts index 851db7a7..31eb9213 100644 --- a/packages/management/src/schema.ts +++ b/packages/management/src/schema.ts @@ -290,6 +290,8 @@ export const deployments = pgTable( /** Stop the deployments this one replaces at once instead of draining them. */ stop_previous: boolean().notNull().default(false), asset_revision_id: text().notNull(), + /** When a superseded deployment stops draining: its environment's drain deadline after it was superseded. */ + drain_until: time(), }, (t) => [index("deployments_environment").on(t.environment_id, t.seq)], ); diff --git a/packages/management/test/assets.test.ts b/packages/management/test/assets.test.ts index e16717bc..a67690c5 100644 --- a/packages/management/test/assets.test.ts +++ b/packages/management/test/assets.test.ts @@ -241,11 +241,61 @@ describe.skipIf(!databaseUrl)("AssetService", () => { await collectAssets(h.deps, projectId); expect(await stored()).toBe(true); + // Shortening the deadline doesn't release what the deployment's own deadline, set when it was superseded, keeps. + await h.sql`update environments set drain_max_age_seconds = 60, drain_deadline_seconds = 60 where id = ${environmentId}`; await h.sql`update deployments set update_time = now() - interval '5 hours' where asset_revision_id = ${draining.id}`; await collectAssets(h.deps, projectId); + expect(await stored()).toBe(true); + await h.sql`update deployments set drain_until = now() - interval '1 minute' where asset_revision_id = ${draining.id}`; + await collectAssets(h.deps, projectId); expect(await stored()).toBe(false); }); + test("a completion paused after verifying its blobs holds off collection, so the revision does not become ready without them", async () => { + const { projectId } = await createEnvironment(h); + const assets = h.client(AssetService); + const revision = assetRevision({ shared: { "a.txt": "a" } }); + const [upload] = (await assets.uploadAssets({ projectId, manifest: revision.manifest })).uploads; + await fetch(upload?.upload?.url ?? "", { method: "PUT", body: revision.blobs.get(upload?.sha256 ?? "") }); + await h.sql`update asset_revisions set protected_until = now() - interval '1 minute' where project_id = ${projectId}`; + + const { complete } = h.deps.releases; + let verified: () => void = () => {}; + const verifiedSignal = new Promise((resolve) => (verified = resolve)); + let resume: () => void = () => {}; + const gate = new Promise((resolve) => (resume = resolve)); + h.deps.releases.complete = async (key, expected, verify) => { + const result = await complete(key, expected, verify); + verified(); + await gate; + return result; + }; + try { + const completing = assets.completeAssetUpload({ projectId, revisionId: revision.id }); + await verifiedSignal; + const state = { collected: false }; + const collecting = collectAssets(h.deps, projectId).finally(() => (state.collected = true)); + // Collection is blocked behind the completion's shared lock. + while (!state.collected) { + const [row] = await h.sql<{ waiting: number }[]>` + select count(*)::int as waiting from pg_locks where locktype = 'advisory' and not granted`; + if (row && row.waiting > 0) break; + await Bun.sleep(10); + } + resume(); + expect((await completing).revision?.state).toBe(AssetRevisionState.READY); + expect(await collecting).toEqual({ revisions: 0, blobs: 0 }); + } finally { + resume(); + h.deps.releases.complete = complete; + } + const sha256 = [...revision.blobs.keys()][0] ?? ""; + expect(await h.releases.exists(blobKey(projectId, sha256))).toBe(true); + expect((await assets.getAssetRevision({ projectId, revisionId: revision.id })).revision?.state).toBe( + AssetRevisionState.READY, + ); + }); + test("a recorded deletion is dropped when its blob is referenced again, and retried when the store fails", async () => { const { projectId } = await createEnvironment(h); const [referenced, lost, flaky] = [1, 2, 3].map((i) => assetRevision({ shared: { "f.txt": `blob${i}` } })); From 736485da94ffe74d5ec3e5ee862b1e3d026b9ff4 Mon Sep 17 00:00:00 2001 From: oglassdev <23040613+oglassdev@users.noreply.github.com> Date: Fri, 20 Nov 2026 15:45:40 -0500 Subject: [PATCH 4/5] fix(assets): keep what core can still serve by its reported resident set, wait out store deletes and register blobs in digest order - core reports every deployment resident in its backend, including one loading, in ReportStatusRequest.resident_deployment_ids; management stores the set of each accepted report on the environment. - collection keeps the revisions of superseded deployments that can still be activated and of deployments in the reported resident set, replacing the drain deadline rules and drain_until (migration 0005 now adds the resident set). - activateDeployment takes the shared asset lock, so activation and collection don't interleave. - the S3 store's delete waits for both of its deletes to settle before it fails. - completions verify blobs concurrently and register them in one insert sorted by digest, so concurrent completions can't deadlock. Refs #328 --- crates/chunk-environment/README.md | 12 +- crates/chunk-environment/src/managed.rs | 14 +- .../chunk-environment/src/managed/status.rs | 23 +- crates/chunk-environment/src/managed/tests.rs | 3 +- .../gen/chunk/management/v1/environment_pb.ts | 13 +- packages/management/README.md | 22 +- .../migrations/0005_drain_until.sql | 3 - .../migrations/0005_resident_deployments.sql | 1 + .../migrations/meta/0004_snapshot.json | 260 ++++------------- .../migrations/meta/0005_snapshot.json | 275 +++++------------- .../management/migrations/meta/_journal.json | 4 +- packages/management/src/assets/retention.ts | 20 +- packages/management/src/assets/service.ts | 17 +- packages/management/src/deployments/store.ts | 19 +- .../management/src/environments/service.ts | 1 + .../gen/chunk/management/v1/environment_pb.ts | 13 +- packages/management/src/releases/s3-store.ts | 10 +- packages/management/src/schema.ts | 4 +- packages/management/test/assets.test.ts | 117 ++++++-- proto/chunk/management/v1/environment.proto | 6 + 20 files changed, 356 insertions(+), 481 deletions(-) delete mode 100644 packages/management/migrations/0005_drain_until.sql create mode 100644 packages/management/migrations/0005_resident_deployments.sql diff --git a/crates/chunk-environment/README.md b/crates/chunk-environment/README.md index a181f94b..a4f89c5a 100644 --- a/crates/chunk-environment/README.md +++ b/crates/chunk-environment/README.md @@ -65,11 +65,13 @@ non-zero if stopped before management answered. It downloads each release archiv deployment pins against the worlds and packs the release declares, and downloads each blob that JVMs read and `$CHUNK_STATE/assets/` lacks, four at a time, verified against its digest, then writes the revision there; packs reach players' clients from management, never through core. Once no deployment resident in the backend, nor the one loading, -pins a revision, core removes it and the blobs no kept revision holds. Only then does it make the deployment the -backend's and control's current release, and report it `ACTIVE`. A deployment it rejects is reported `FAILED`, and the -previous deployment keeps serving. The gateway starts with the first active deployment. Core retires the versions it no -longer needs, stopping their JVMs first, and removes their unpacked releases. JVMs run on machines core asks management -for (`EnsureCapacity`), each running [`chunk-jvm`](../chunk-jvm/README.md). A core that management fences stops. +pins a revision, core removes it and the blobs no kept revision holds. Each status report lists those resident +deployments and the one loading, which management keeps the asset revisions of. Only then does it make the deployment +the backend's and control's current release, and report it `ACTIVE`. A deployment it rejects is reported `FAILED`, and +the previous deployment keeps serving. The gateway starts with the first active deployment. Core retires the versions it +no longer needs, stopping their JVMs first, and removes their unpacked releases. JVMs run on machines core asks +management for (`EnsureCapacity`), each running [`chunk-jvm`](../chunk-jvm/README.md). A core that management fences +stops. Core reports its status about every 15 seconds, and at once when it changes. Each report carries the server-list status the gateway last answered for each hostname, which the [edge](../chunk-edge/README.md) answers pings with while the diff --git a/crates/chunk-environment/src/managed.rs b/crates/chunk-environment/src/managed.rs index 68b132b3..e0489de4 100644 --- a/crates/chunk-environment/src/managed.rs +++ b/crates/chunk-environment/src/managed.rs @@ -693,6 +693,7 @@ impl<'a> Managed<'a> { logins: self.logins(), ready_to_suspend: false, pings: self.pings(), + resident_deployments: self.reporter.resident(None), } } @@ -701,7 +702,8 @@ impl<'a> Managed<'a> { let gateway = self.gateway.get()?; self.private_address.gateway(gateway.address()).await }; - let (address, ready_to_suspend) = tokio::join!(address, self.ready_to_suspend(revision)); + let (address, ready_to_suspend, resident) = + tokio::join!(address, self.ready_to_suspend(revision), self.resident_deployments()); let gateway_addresses = address.as_ref().map(SocketAddr::to_string).into_iter().collect(); let online_players = self.online_players().unwrap_or_else(|error| { tracing::warn!(%error, "online players unknown"); @@ -715,9 +717,19 @@ impl<'a> Managed<'a> { logins: self.logins(), ready_to_suspend, pings: self.pings(), + resident_deployments: self.reporter.resident(resident), } } + /// The deployments resident in the backend and the one loading, unless the backend can't be read within + /// [`READ_WAIT`]. The loading one is read first, so one that becomes resident meanwhile is counted either way. + async fn resident_deployments(&self) -> Option> { + let loading = lock(&self.deployments).loading.clone(); + let resident = read_within(self.core.backend()?.deployments()).await?; + let ids: BTreeSet = resident.iter().map(|id| id.as_str().to_owned()).chain(loading).collect(); + Some(ids.into_iter().collect()) + } + fn online_players(&self) -> io::Result { let online = self.core.control()?.online_players().map_err(io::Error::other)?; Ok(u32::try_from(online).unwrap_or(u32::MAX)) diff --git a/crates/chunk-environment/src/managed/status.rs b/crates/chunk-environment/src/managed/status.rs index 2e42bb9d..05d0b892 100644 --- a/crates/chunk-environment/src/managed/status.rs +++ b/crates/chunk-environment/src/managed/status.rs @@ -42,6 +42,8 @@ pub(super) struct Observed { pub ready_to_suspend: bool, /// The status the gateway last answered for each hostname, which edges answer pings with while core sleeps. pub pings: Vec, + /// Every deployment resident in the backend, including one loading, which management keeps the assets of. + pub resident_deployments: Vec, } impl Observed { @@ -73,6 +75,8 @@ struct Sent { lease: u64, /// The gateway addresses of the latest report management accepted, under this lease or an earlier one. addresses: Vec, + /// The resident deployments last read, under this lease or an earlier one. + resident: Vec, sequence: u64, /// What the latest report started said. latest: Option, @@ -171,8 +175,12 @@ impl Reporter { return Ok(false); } if observed.lease > sent.lease { - *sent = - Sent { lease: observed.lease, addresses: std::mem::take(&mut sent.addresses), ..Sent::default() }; + *sent = Sent { + lease: observed.lease, + addresses: std::mem::take(&mut sent.addresses), + resident: std::mem::take(&mut sent.resident), + ..Sent::default() + }; } sent.sequence += 1; let sequence = sent.sequence; @@ -189,6 +197,7 @@ impl Reporter { online_players: observed.online_players, logins: observed.logins, pings: observed.pings, + resident_deployment_ids: observed.resident_deployments, deployment: deployment.or_else(|| sent.progress.as_ref().map(|(_, progress)| progress.clone())), ready_to_suspend: observed.ready_to_suspend, lease: observed.lease, @@ -206,6 +215,16 @@ impl Reporter { result.map(|_| true) } + /// The deployments resident in the backend: `read` when it could be read, which is remembered, and else the ones + /// last read. + pub(super) fn resident(&self, read: Option>) -> Vec { + let mut sent = lock(&self.sent); + if let Some(read) = read { + sent.resident = read; + } + sent.resident.clone() + } + /// The gateway addresses of the latest report management accepted. pub(super) fn accepted_addresses(&self) -> Vec { lock(&self.sent).addresses.clone() diff --git a/crates/chunk-environment/src/managed/tests.rs b/crates/chunk-environment/src/managed/tests.rs index c554467f..342449b9 100644 --- a/crates/chunk-environment/src/managed/tests.rs +++ b/crates/chunk-environment/src/managed/tests.rs @@ -609,7 +609,8 @@ async fn a_stop_previous_activation_stops_every_resident_deployment_even_across_ harness.expect(1, "dep_a", DeploymentState::Active).await; harness.deploy("dep_b", harness.valid()); harness.expect(2, "dep_b", DeploymentState::InProgress).await; - harness.expect(2, "dep_b", DeploymentState::Active).await; + let active = harness.expect(2, "dep_b", DeploymentState::Active).await; + assert!(active.resident_deployment_ids.contains(&"dep_b".to_owned()), "{:?}", active.resident_deployment_ids); // dep_c stops dep_a and dep_b, but management doesn't accept it, so dep_b stays as its fallback. Then dep_d // arrives before dep_b stops. diff --git a/packages/dashboard/src/gen/chunk/management/v1/environment_pb.ts b/packages/dashboard/src/gen/chunk/management/v1/environment_pb.ts index e43b62a7..18b90eb7 100644 --- a/packages/dashboard/src/gen/chunk/management/v1/environment_pb.ts +++ b/packages/dashboard/src/gen/chunk/management/v1/environment_pb.ts @@ -15,7 +15,7 @@ import { file_chunk_management_v1_common } from "./common_pb.ts"; * Describes the file chunk/management/v1/environment.proto. */ export const file_chunk_management_v1_environment: GenFile /*@__PURE__*/ = fileDesc( - "CiVjaHVuay9tYW5hZ2VtZW50L3YxL2Vudmlyb25tZW50LnByb3RvEhNjaHVuay5tYW5hZ2VtZW50LnYxInIKDUF0dGFjaFJlcXVlc3QSEwoLaW5zdGFuY2VfaWQYASABKAkSDwoHdmVyc2lvbhgCIAEoCRIMCgRjb3JlGAMgASgIEg0KBWVwb2NoGAQgASgEEh4KFnJlc3RvcmVkX2RlcGxveW1lbnRfaWQYBSABKAki8wMKDkF0dGFjaFJlc3BvbnNlEhAKCHJldmlzaW9uGAEgASgEEhYKDmVudmlyb25tZW50X2lkGAIgASgJEhIKCnByb2plY3RfaWQYAyABKAkSFQoNZGVwbG95bWVudF9pZBgEIAEoCRI1CgdyZWxlYXNlGAUgASgLMiQuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlQXJ0aWZhY3QSMQoHc2VjcmV0cxgGIAMoCzIgLmNodW5rLm1hbmFnZW1lbnQudjEuU2VjcmV0VmFsdWUSMwoJbG9nX3N0b3JlGAcgASgLMiAuY2h1bmsubWFuYWdlbWVudC52MS5PYmplY3RTdG9yZRItCgdyZXN0b3JlGAggASgLMhwuY2h1bmsubWFuYWdlbWVudC52MS5SZXN0b3JlEg0KBWxlYXNlGAkgASgEEhgKEGVudmlyb25tZW50X25hbWUYCiABKAkSMQoFZHJhaW4YCyABKAsyIi5jaHVuay5tYW5hZ2VtZW50LnYxLkRyYWluU2V0dGluZ3MSFQoNc3RvcF9wcmV2aW91cxgMIAEoCBIyCgZhc3NldHMYDSABKAsyIi5jaHVuay5tYW5hZ2VtZW50LnYxLkFzc2V0QXJ0aWZhY3QSFwoPcGFja191cmxfcHJlZml4GA4gASgJIlYKD1JlbGVhc2VBcnRpZmFjdBISCgpyZWxlYXNlX2lkGAEgASgJEgsKA3VybBgCIAEoCRIOCgZzaGEyNTYYAyABKAkSEgoKc2l6ZV9ieXRlcxgEIAEoBCJPCg1Bc3NldEFydGlmYWN0EhMKC3JldmlzaW9uX2lkGAEgASgJEhAKCG1hbmlmZXN0GAIgASgMEhcKD2Jsb2JfdXJsX3ByZWZpeBgDIAEoCSI7CgtTZWNyZXRWYWx1ZRIMCgRuYW1lGAEgASgJEg8KB3ZlcnNpb24YAiABKAQSDQoFdmFsdWUYAyABKAwiyQEKC09iamVjdFN0b3JlEhAKCGVuZHBvaW50GAEgASgJEg4KBnJlZ2lvbhgCIAEoCRIOCgZidWNrZXQYAyABKAkSDgoGcHJlZml4GAQgASgJEhUKDWFjY2Vzc19rZXlfaWQYBSABKAkSGQoRc2VjcmV0X2FjY2Vzc19rZXkYBiABKAkSFQoNc2Vzc2lvbl90b2tlbhgHIAEoCRIvCgtleHBpcmVfdGltZRgIIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAibwoHUmVzdG9yZRIwCgZzb3VyY2UYASABKAsyIC5jaHVuay5tYW5hZ2VtZW50LnYxLk9iamVjdFN0b3JlEhMKC3NuYXBzaG90X2lkGAIgASgJEh0KFXNvdXJjZV9lbnZpcm9ubWVudF9pZBgDIAEoCSLMAgoTUmVwb3J0U3RhdHVzUmVxdWVzdBIwCgxvYnNlcnZlX3RpbWUYASABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEhkKEWdhdGV3YXlfYWRkcmVzc2VzGAIgAygJEhYKDm9ubGluZV9wbGF5ZXJzGAMgASgNEi4KBXBpbmdzGAQgAygLMh8uY2h1bmsubWFuYWdlbWVudC52MS5QaW5nU3RhdHVzEjsKCmRlcGxveW1lbnQYBSABKAsyJy5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveW1lbnRQcm9ncmVzcxIYChByZWFkeV90b19zdXNwZW5kGAYgASgIEg0KBWxlYXNlGAcgASgEEhAKCHNlcXVlbmNlGAggASgEEhgKEGRlc2lyZWRfcmV2aXNpb24YCSABKAQSDgoGbG9naW5zGAogASgEIjMKClBpbmdTdGF0dXMSEAoIaG9zdG5hbWUYASABKAkSEwoLc3RhdHVzX2pzb24YAiABKAkicQoSRGVwbG95bWVudFByb2dyZXNzEhUKDWRlcGxveW1lbnRfaWQYASABKAkSMwoFc3RhdGUYAiABKA4yJC5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveW1lbnRTdGF0ZRIPCgdtZXNzYWdlGAMgASgJIhYKFFJlcG9ydFN0YXR1c1Jlc3BvbnNlInUKE1NldFdha2VBbGFybVJlcXVlc3QSEgoKZ2VuZXJhdGlvbhgBIAEoBBIsCghkdWVfdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASDQoFZXBvY2gYAyABKAQSDQoFbGVhc2UYBCABKAQiZwoUU2V0V2FrZUFsYXJtUmVzcG9uc2USEgoKZ2VuZXJhdGlvbhgBIAEoBBIsCghkdWVfdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASDQoFZXBvY2gYAyABKAQivAEKFUVuc3VyZUNhcGFjaXR5UmVxdWVzdBISCgpyZXF1ZXN0X2lkGAEgASgJEi8KCHdvcmtsb2FkGAIgASgOMh0uY2h1bmsubWFuYWdlbWVudC52MS5Xb3JrbG9hZBIXCg9tYWNoaW5lX3Byb2ZpbGUYAyABKAkSEgoKcmVsZWFzZV9pZBgEIAEoCRIOCgZhcHBfaWQYBSABKAkSDQoFbGVhc2UYBiABKAQSEgoKY3JlZGVudGlhbBgHIAEoCSJJChZFbnN1cmVDYXBhY2l0eVJlc3BvbnNlEi8KCGNhcGFjaXR5GAEgASgLMh0uY2h1bmsubWFuYWdlbWVudC52MS5DYXBhY2l0eSJ2CghDYXBhY2l0eRISCgpyZXF1ZXN0X2lkGAEgASgJEjEKBXN0YXRlGAIgASgOMiIuY2h1bmsubWFuYWdlbWVudC52MS5DYXBhY2l0eVN0YXRlEhIKCm1hY2hpbmVfaWQYAyABKAkSDwoHbWVzc2FnZRgEIAEoCSI7ChZSZWxlYXNlQ2FwYWNpdHlSZXF1ZXN0EhIKCnJlcXVlc3RfaWQYASABKAkSDQoFbGVhc2UYAiABKAQiSgoXUmVsZWFzZUNhcGFjaXR5UmVzcG9uc2USLwoIY2FwYWNpdHkYASABKAsyHS5jaHVuay5tYW5hZ2VtZW50LnYxLkNhcGFjaXR5IkcKElJlcG9ydFVzYWdlUmVxdWVzdBIxCgdyZWNvcmRzGAEgAygLMiAuY2h1bmsubWFuYWdlbWVudC52MS5Vc2FnZVJlY29yZCKkAQoLVXNhZ2VSZWNvcmQSCgoCaWQYASABKAkSLgoKc3RhcnRfdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASLAoIZW5kX3RpbWUYAyABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEhYKDnBsYXllcl9zZWNvbmRzGAQgASgEEhMKC2luc3RhbmNlX2lkGAUgASgJIhUKE1JlcG9ydFVzYWdlUmVzcG9uc2UiQwoRUmVwb3J0TG9nc1JlcXVlc3QSLgoHZW50cmllcxgBIAMoCzIdLmNodW5rLm1hbmFnZW1lbnQudjEuTG9nRW50cnkiFAoSUmVwb3J0TG9nc1Jlc3BvbnNlIkoKFFJlcG9ydE1ldHJpY3NSZXF1ZXN0EjIKB3NhbXBsZXMYASADKAsyIS5jaHVuay5tYW5hZ2VtZW50LnYxLk1ldHJpY1NhbXBsZSLYAQoMTWV0cmljU2FtcGxlEgwKBG5hbWUYASABKAkSPQoGbGFiZWxzGAIgAygLMi0uY2h1bmsubWFuYWdlbWVudC52MS5NZXRyaWNTYW1wbGUuTGFiZWxzRW50cnkSKAoEdGltZRgDIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASDQoFdmFsdWUYBCABKAESEwoLaW5zdGFuY2VfaWQYBSABKAkaLQoLTGFiZWxzRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4ASIXChVSZXBvcnRNZXRyaWNzUmVzcG9uc2UiTAoXUmVwb3J0RmFpbGVkQXV0aFJlcXVlc3QSMQoIZmFpbHVyZXMYASADKAsyHy5jaHVuay5tYW5hZ2VtZW50LnYxLkZhaWxlZEF1dGgiTgoKRmFpbGVkQXV0aBIWCg5jbGllbnRfYWRkcmVzcxgBIAEoCRIoCgR0aW1lGAIgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcCIaChhSZXBvcnRGYWlsZWRBdXRoUmVzcG9uc2UqYQoIV29ya2xvYWQSGAoUV09SS0xPQURfVU5TUEVDSUZJRUQQABIQCgxXT1JLTE9BRF9KVk0QARIUChBXT1JLTE9BRF9HQVRFV0FZEAIiBAgDEAMqDVdPUktMT0FEX0VYRUMqwAEKDUNhcGFjaXR5U3RhdGUSHgoaQ0FQQUNJVFlfU1RBVEVfVU5TUEVDSUZJRUQQABIfChtDQVBBQ0lUWV9TVEFURV9QUk9WSVNJT05JTkcQARIYChRDQVBBQ0lUWV9TVEFURV9SRUFEWRACEhkKFUNBUEFDSVRZX1NUQVRFX0ZBSUxFRBADEhsKF0NBUEFDSVRZX1NUQVRFX1JFTEVBU0VEEAQSHAoYQ0FQQUNJVFlfU1RBVEVfUkVMRUFTSU5HEAUypgcKEkVudmlyb25tZW50U2VydmljZRJTCgZBdHRhY2gSIi5jaHVuay5tYW5hZ2VtZW50LnYxLkF0dGFjaFJlcXVlc3QaIy5jaHVuay5tYW5hZ2VtZW50LnYxLkF0dGFjaFJlc3BvbnNlMAESYwoMUmVwb3J0U3RhdHVzEiguY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRTdGF0dXNSZXF1ZXN0GikuY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRTdGF0dXNSZXNwb25zZRJjCgxTZXRXYWtlQWxhcm0SKC5jaHVuay5tYW5hZ2VtZW50LnYxLlNldFdha2VBbGFybVJlcXVlc3QaKS5jaHVuay5tYW5hZ2VtZW50LnYxLlNldFdha2VBbGFybVJlc3BvbnNlEmkKDkVuc3VyZUNhcGFjaXR5EiouY2h1bmsubWFuYWdlbWVudC52MS5FbnN1cmVDYXBhY2l0eVJlcXVlc3QaKy5jaHVuay5tYW5hZ2VtZW50LnYxLkVuc3VyZUNhcGFjaXR5UmVzcG9uc2USbAoPUmVsZWFzZUNhcGFjaXR5EisuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlQ2FwYWNpdHlSZXF1ZXN0GiwuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlQ2FwYWNpdHlSZXNwb25zZRJgCgtSZXBvcnRVc2FnZRInLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0VXNhZ2VSZXF1ZXN0GiguY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRVc2FnZVJlc3BvbnNlEl0KClJlcG9ydExvZ3MSJi5jaHVuay5tYW5hZ2VtZW50LnYxLlJlcG9ydExvZ3NSZXF1ZXN0GicuY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRMb2dzUmVzcG9uc2USZgoNUmVwb3J0TWV0cmljcxIpLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0TWV0cmljc1JlcXVlc3QaKi5jaHVuay5tYW5hZ2VtZW50LnYxLlJlcG9ydE1ldHJpY3NSZXNwb25zZRJvChBSZXBvcnRGYWlsZWRBdXRoEiwuY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRGYWlsZWRBdXRoUmVxdWVzdBotLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0RmFpbGVkQXV0aFJlc3BvbnNlYgZwcm90bzM", + "CiVjaHVuay9tYW5hZ2VtZW50L3YxL2Vudmlyb25tZW50LnByb3RvEhNjaHVuay5tYW5hZ2VtZW50LnYxInIKDUF0dGFjaFJlcXVlc3QSEwoLaW5zdGFuY2VfaWQYASABKAkSDwoHdmVyc2lvbhgCIAEoCRIMCgRjb3JlGAMgASgIEg0KBWVwb2NoGAQgASgEEh4KFnJlc3RvcmVkX2RlcGxveW1lbnRfaWQYBSABKAki8wMKDkF0dGFjaFJlc3BvbnNlEhAKCHJldmlzaW9uGAEgASgEEhYKDmVudmlyb25tZW50X2lkGAIgASgJEhIKCnByb2plY3RfaWQYAyABKAkSFQoNZGVwbG95bWVudF9pZBgEIAEoCRI1CgdyZWxlYXNlGAUgASgLMiQuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlQXJ0aWZhY3QSMQoHc2VjcmV0cxgGIAMoCzIgLmNodW5rLm1hbmFnZW1lbnQudjEuU2VjcmV0VmFsdWUSMwoJbG9nX3N0b3JlGAcgASgLMiAuY2h1bmsubWFuYWdlbWVudC52MS5PYmplY3RTdG9yZRItCgdyZXN0b3JlGAggASgLMhwuY2h1bmsubWFuYWdlbWVudC52MS5SZXN0b3JlEg0KBWxlYXNlGAkgASgEEhgKEGVudmlyb25tZW50X25hbWUYCiABKAkSMQoFZHJhaW4YCyABKAsyIi5jaHVuay5tYW5hZ2VtZW50LnYxLkRyYWluU2V0dGluZ3MSFQoNc3RvcF9wcmV2aW91cxgMIAEoCBIyCgZhc3NldHMYDSABKAsyIi5jaHVuay5tYW5hZ2VtZW50LnYxLkFzc2V0QXJ0aWZhY3QSFwoPcGFja191cmxfcHJlZml4GA4gASgJIlYKD1JlbGVhc2VBcnRpZmFjdBISCgpyZWxlYXNlX2lkGAEgASgJEgsKA3VybBgCIAEoCRIOCgZzaGEyNTYYAyABKAkSEgoKc2l6ZV9ieXRlcxgEIAEoBCJPCg1Bc3NldEFydGlmYWN0EhMKC3JldmlzaW9uX2lkGAEgASgJEhAKCG1hbmlmZXN0GAIgASgMEhcKD2Jsb2JfdXJsX3ByZWZpeBgDIAEoCSI7CgtTZWNyZXRWYWx1ZRIMCgRuYW1lGAEgASgJEg8KB3ZlcnNpb24YAiABKAQSDQoFdmFsdWUYAyABKAwiyQEKC09iamVjdFN0b3JlEhAKCGVuZHBvaW50GAEgASgJEg4KBnJlZ2lvbhgCIAEoCRIOCgZidWNrZXQYAyABKAkSDgoGcHJlZml4GAQgASgJEhUKDWFjY2Vzc19rZXlfaWQYBSABKAkSGQoRc2VjcmV0X2FjY2Vzc19rZXkYBiABKAkSFQoNc2Vzc2lvbl90b2tlbhgHIAEoCRIvCgtleHBpcmVfdGltZRgIIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAibwoHUmVzdG9yZRIwCgZzb3VyY2UYASABKAsyIC5jaHVuay5tYW5hZ2VtZW50LnYxLk9iamVjdFN0b3JlEhMKC3NuYXBzaG90X2lkGAIgASgJEh0KFXNvdXJjZV9lbnZpcm9ubWVudF9pZBgDIAEoCSLtAgoTUmVwb3J0U3RhdHVzUmVxdWVzdBIwCgxvYnNlcnZlX3RpbWUYASABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEhkKEWdhdGV3YXlfYWRkcmVzc2VzGAIgAygJEhYKDm9ubGluZV9wbGF5ZXJzGAMgASgNEi4KBXBpbmdzGAQgAygLMh8uY2h1bmsubWFuYWdlbWVudC52MS5QaW5nU3RhdHVzEjsKCmRlcGxveW1lbnQYBSABKAsyJy5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveW1lbnRQcm9ncmVzcxIYChByZWFkeV90b19zdXNwZW5kGAYgASgIEg0KBWxlYXNlGAcgASgEEhAKCHNlcXVlbmNlGAggASgEEhgKEGRlc2lyZWRfcmV2aXNpb24YCSABKAQSDgoGbG9naW5zGAogASgEEh8KF3Jlc2lkZW50X2RlcGxveW1lbnRfaWRzGAsgAygJIjMKClBpbmdTdGF0dXMSEAoIaG9zdG5hbWUYASABKAkSEwoLc3RhdHVzX2pzb24YAiABKAkicQoSRGVwbG95bWVudFByb2dyZXNzEhUKDWRlcGxveW1lbnRfaWQYASABKAkSMwoFc3RhdGUYAiABKA4yJC5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveW1lbnRTdGF0ZRIPCgdtZXNzYWdlGAMgASgJIhYKFFJlcG9ydFN0YXR1c1Jlc3BvbnNlInUKE1NldFdha2VBbGFybVJlcXVlc3QSEgoKZ2VuZXJhdGlvbhgBIAEoBBIsCghkdWVfdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASDQoFZXBvY2gYAyABKAQSDQoFbGVhc2UYBCABKAQiZwoUU2V0V2FrZUFsYXJtUmVzcG9uc2USEgoKZ2VuZXJhdGlvbhgBIAEoBBIsCghkdWVfdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASDQoFZXBvY2gYAyABKAQivAEKFUVuc3VyZUNhcGFjaXR5UmVxdWVzdBISCgpyZXF1ZXN0X2lkGAEgASgJEi8KCHdvcmtsb2FkGAIgASgOMh0uY2h1bmsubWFuYWdlbWVudC52MS5Xb3JrbG9hZBIXCg9tYWNoaW5lX3Byb2ZpbGUYAyABKAkSEgoKcmVsZWFzZV9pZBgEIAEoCRIOCgZhcHBfaWQYBSABKAkSDQoFbGVhc2UYBiABKAQSEgoKY3JlZGVudGlhbBgHIAEoCSJJChZFbnN1cmVDYXBhY2l0eVJlc3BvbnNlEi8KCGNhcGFjaXR5GAEgASgLMh0uY2h1bmsubWFuYWdlbWVudC52MS5DYXBhY2l0eSJ2CghDYXBhY2l0eRISCgpyZXF1ZXN0X2lkGAEgASgJEjEKBXN0YXRlGAIgASgOMiIuY2h1bmsubWFuYWdlbWVudC52MS5DYXBhY2l0eVN0YXRlEhIKCm1hY2hpbmVfaWQYAyABKAkSDwoHbWVzc2FnZRgEIAEoCSI7ChZSZWxlYXNlQ2FwYWNpdHlSZXF1ZXN0EhIKCnJlcXVlc3RfaWQYASABKAkSDQoFbGVhc2UYAiABKAQiSgoXUmVsZWFzZUNhcGFjaXR5UmVzcG9uc2USLwoIY2FwYWNpdHkYASABKAsyHS5jaHVuay5tYW5hZ2VtZW50LnYxLkNhcGFjaXR5IkcKElJlcG9ydFVzYWdlUmVxdWVzdBIxCgdyZWNvcmRzGAEgAygLMiAuY2h1bmsubWFuYWdlbWVudC52MS5Vc2FnZVJlY29yZCKkAQoLVXNhZ2VSZWNvcmQSCgoCaWQYASABKAkSLgoKc3RhcnRfdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASLAoIZW5kX3RpbWUYAyABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEhYKDnBsYXllcl9zZWNvbmRzGAQgASgEEhMKC2luc3RhbmNlX2lkGAUgASgJIhUKE1JlcG9ydFVzYWdlUmVzcG9uc2UiQwoRUmVwb3J0TG9nc1JlcXVlc3QSLgoHZW50cmllcxgBIAMoCzIdLmNodW5rLm1hbmFnZW1lbnQudjEuTG9nRW50cnkiFAoSUmVwb3J0TG9nc1Jlc3BvbnNlIkoKFFJlcG9ydE1ldHJpY3NSZXF1ZXN0EjIKB3NhbXBsZXMYASADKAsyIS5jaHVuay5tYW5hZ2VtZW50LnYxLk1ldHJpY1NhbXBsZSLYAQoMTWV0cmljU2FtcGxlEgwKBG5hbWUYASABKAkSPQoGbGFiZWxzGAIgAygLMi0uY2h1bmsubWFuYWdlbWVudC52MS5NZXRyaWNTYW1wbGUuTGFiZWxzRW50cnkSKAoEdGltZRgDIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASDQoFdmFsdWUYBCABKAESEwoLaW5zdGFuY2VfaWQYBSABKAkaLQoLTGFiZWxzRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4ASIXChVSZXBvcnRNZXRyaWNzUmVzcG9uc2UiTAoXUmVwb3J0RmFpbGVkQXV0aFJlcXVlc3QSMQoIZmFpbHVyZXMYASADKAsyHy5jaHVuay5tYW5hZ2VtZW50LnYxLkZhaWxlZEF1dGgiTgoKRmFpbGVkQXV0aBIWCg5jbGllbnRfYWRkcmVzcxgBIAEoCRIoCgR0aW1lGAIgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcCIaChhSZXBvcnRGYWlsZWRBdXRoUmVzcG9uc2UqYQoIV29ya2xvYWQSGAoUV09SS0xPQURfVU5TUEVDSUZJRUQQABIQCgxXT1JLTE9BRF9KVk0QARIUChBXT1JLTE9BRF9HQVRFV0FZEAIiBAgDEAMqDVdPUktMT0FEX0VYRUMqwAEKDUNhcGFjaXR5U3RhdGUSHgoaQ0FQQUNJVFlfU1RBVEVfVU5TUEVDSUZJRUQQABIfChtDQVBBQ0lUWV9TVEFURV9QUk9WSVNJT05JTkcQARIYChRDQVBBQ0lUWV9TVEFURV9SRUFEWRACEhkKFUNBUEFDSVRZX1NUQVRFX0ZBSUxFRBADEhsKF0NBUEFDSVRZX1NUQVRFX1JFTEVBU0VEEAQSHAoYQ0FQQUNJVFlfU1RBVEVfUkVMRUFTSU5HEAUypgcKEkVudmlyb25tZW50U2VydmljZRJTCgZBdHRhY2gSIi5jaHVuay5tYW5hZ2VtZW50LnYxLkF0dGFjaFJlcXVlc3QaIy5jaHVuay5tYW5hZ2VtZW50LnYxLkF0dGFjaFJlc3BvbnNlMAESYwoMUmVwb3J0U3RhdHVzEiguY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRTdGF0dXNSZXF1ZXN0GikuY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRTdGF0dXNSZXNwb25zZRJjCgxTZXRXYWtlQWxhcm0SKC5jaHVuay5tYW5hZ2VtZW50LnYxLlNldFdha2VBbGFybVJlcXVlc3QaKS5jaHVuay5tYW5hZ2VtZW50LnYxLlNldFdha2VBbGFybVJlc3BvbnNlEmkKDkVuc3VyZUNhcGFjaXR5EiouY2h1bmsubWFuYWdlbWVudC52MS5FbnN1cmVDYXBhY2l0eVJlcXVlc3QaKy5jaHVuay5tYW5hZ2VtZW50LnYxLkVuc3VyZUNhcGFjaXR5UmVzcG9uc2USbAoPUmVsZWFzZUNhcGFjaXR5EisuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlQ2FwYWNpdHlSZXF1ZXN0GiwuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlQ2FwYWNpdHlSZXNwb25zZRJgCgtSZXBvcnRVc2FnZRInLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0VXNhZ2VSZXF1ZXN0GiguY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRVc2FnZVJlc3BvbnNlEl0KClJlcG9ydExvZ3MSJi5jaHVuay5tYW5hZ2VtZW50LnYxLlJlcG9ydExvZ3NSZXF1ZXN0GicuY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRMb2dzUmVzcG9uc2USZgoNUmVwb3J0TWV0cmljcxIpLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0TWV0cmljc1JlcXVlc3QaKi5jaHVuay5tYW5hZ2VtZW50LnYxLlJlcG9ydE1ldHJpY3NSZXNwb25zZRJvChBSZXBvcnRGYWlsZWRBdXRoEiwuY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRGYWlsZWRBdXRoUmVxdWVzdBotLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0RmFpbGVkQXV0aFJlc3BvbnNlYgZwcm90bzM", [file_chunk_management_v1_common, file_google_protobuf_timestamp], ); @@ -476,6 +476,17 @@ export type ReportStatusRequest = Message<"chunk.management.v1.ReportStatusReque * @generated from field: uint64 logins = 10; */ logins: bigint; + + /** + * The IDs of every deployment resident in core's backend, including one that + * is loading. The management plane keeps the asset revisions of these + * deployments, which core's JVMs may still read, whatever their state there. + * It stores the set of the latest accepted report, so each report carries the + * whole set. + * + * @generated from field: repeated string resident_deployment_ids = 11; + */ + residentDeploymentIds: string[]; }; /** diff --git a/packages/management/README.md b/packages/management/README.md index 9e8c4a27..227b9991 100644 --- a/packages/management/README.md +++ b/packages/management/README.md @@ -101,16 +101,18 @@ serve only blobs of revisions the environment's deployments pin, packs only for or redirecting to a presigned URL of the release bucket. Each `SetAssetHead` collects the project's unused revisions. A revision is kept while it is the head, one of the -project's 10 newest ready revisions, declared by an upload within the last hour (as long as its upload URLs last), -pinned by an environment's active deployment, an unfinished one or one of its 10 newest, or pinned by a superseded -deployment still within its environment's drain deadline, as it was when the deployment was superseded or is now, -whichever is later. The rest are deleted, and with them every blob no remaining revision holds, from the database; their -keys are recorded in `asset_deletions` and the release store's objects are deleted in a second transaction that skips a -key a new upload has claimed again. A delete the store fails stays recorded and is retried by the next collection. A -store delete still running when management loses its lock, such as to a crash, can land after the same content is -uploaded again, which pushing the assets again repairs. A rollback to an older deployment fails once its revision is -gone. Collection holds a per-project lock that uploads, deployments and head moves share, so it never deletes a blob a -revision is being declared with or a revision a deployment or the head is pinning. +project's 10 newest ready revisions, or declared by an upload within the last hour (as long as its upload URLs last). It +is also kept while a deployment of one of the project's environments pins it and is active or unfinished, one of the +environment's 10 newest (so a rollback to it works), superseded but newer than the environment's active deployment (core +may still activate it), or in the set of deployments resident in core's backend that the environment's latest accepted +status report listed. Core decides what stays resident, so no clock or setting of management's can lag it. The rest are +deleted, and with them every blob no remaining revision holds, from the database; their keys are recorded in +`asset_deletions` and the release store's objects are deleted in a second transaction that skips a key a new upload has +claimed again. A delete the store fails stays recorded and is retried by the next collection. A store delete still +running when management loses its lock, such as to a crash, can land after the same content is uploaded again, which +pushing the assets again repairs. A rollback to a deployment older than the 10 newest fails once its revision is gone. +Collection holds a per-project lock that uploads, deployments, activations and head moves share, so it never deletes a +blob a revision is being declared with or a revision a deployment or the head is pinning. ## Release storage diff --git a/packages/management/migrations/0005_drain_until.sql b/packages/management/migrations/0005_drain_until.sql deleted file mode 100644 index ad53fad4..00000000 --- a/packages/management/migrations/0005_drain_until.sql +++ /dev/null @@ -1,3 +0,0 @@ -ALTER TABLE "deployments" ADD COLUMN "drain_until" timestamp with time zone; ---> statement-breakpoint -UPDATE "deployments" SET "drain_until" = "deployments"."update_time" + make_interval(secs => (SELECT "drain_deadline_seconds" FROM "environments" WHERE "environments"."id" = "deployments"."environment_id")) WHERE "state" = 5; diff --git a/packages/management/migrations/0005_resident_deployments.sql b/packages/management/migrations/0005_resident_deployments.sql new file mode 100644 index 00000000..a1c5c8d1 --- /dev/null +++ b/packages/management/migrations/0005_resident_deployments.sql @@ -0,0 +1 @@ +ALTER TABLE "environments" ADD COLUMN "resident_deployment_ids" text[] DEFAULT '{}' NOT NULL; \ No newline at end of file diff --git a/packages/management/migrations/meta/0004_snapshot.json b/packages/management/migrations/meta/0004_snapshot.json index fc1505e5..2c827e35 100644 --- a/packages/management/migrations/meta/0004_snapshot.json +++ b/packages/management/migrations/meta/0004_snapshot.json @@ -128,12 +128,8 @@ "name": "api_tokens_project_id_projects_id_fk", "tableFrom": "api_tokens", "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["project_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -141,12 +137,8 @@ "name": "api_tokens_environment_id_environments_id_fk", "tableFrom": "api_tokens", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -156,16 +148,12 @@ "api_tokens_seq_unique": { "name": "api_tokens_seq_unique", "nullsNotDistinct": false, - "columns": [ - "seq" - ] + "columns": ["seq"] }, "api_tokens_secret_hash_unique": { "name": "api_tokens_secret_hash_unique", "nullsNotDistinct": false, - "columns": [ - "secret_hash" - ] + "columns": ["secret_hash"] } }, "policies": {}, @@ -214,12 +202,8 @@ "name": "asset_blobs_project_id_projects_id_fk", "tableFrom": "asset_blobs", "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["project_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -227,10 +211,7 @@ "compositePrimaryKeys": { "asset_blobs_project_id_sha256_pk": { "name": "asset_blobs_project_id_sha256_pk", - "columns": [ - "project_id", - "sha256" - ] + "columns": ["project_id", "sha256"] } }, "uniqueConstraints": {}, @@ -268,12 +249,8 @@ "name": "asset_deletions_project_id_projects_id_fk", "tableFrom": "asset_deletions", "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["project_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -281,10 +258,7 @@ "compositePrimaryKeys": { "asset_deletions_project_id_sha256_pk": { "name": "asset_deletions_project_id_sha256_pk", - "columns": [ - "project_id", - "sha256" - ] + "columns": ["project_id", "sha256"] } }, "uniqueConstraints": {}, @@ -327,14 +301,8 @@ "name": "asset_revision_blobs_project_id_revision_id_asset_revisions_project_id_id_fk", "tableFrom": "asset_revision_blobs", "tableTo": "asset_revisions", - "columnsFrom": [ - "project_id", - "revision_id" - ], - "columnsTo": [ - "project_id", - "id" - ], + "columnsFrom": ["project_id", "revision_id"], + "columnsTo": ["project_id", "id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -342,11 +310,7 @@ "compositePrimaryKeys": { "asset_revision_blobs_project_id_revision_id_sha256_pk": { "name": "asset_revision_blobs_project_id_revision_id_sha256_pk", - "columns": [ - "project_id", - "revision_id", - "sha256" - ] + "columns": ["project_id", "revision_id", "sha256"] } }, "uniqueConstraints": {}, @@ -437,12 +401,8 @@ "name": "asset_revisions_project_id_projects_id_fk", "tableFrom": "asset_revisions", "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["project_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -450,19 +410,14 @@ "compositePrimaryKeys": { "asset_revisions_project_id_id_pk": { "name": "asset_revisions_project_id_id_pk", - "columns": [ - "project_id", - "id" - ] + "columns": ["project_id", "id"] } }, "uniqueConstraints": { "asset_revisions_seq_unique": { "name": "asset_revisions_seq_unique", "nullsNotDistinct": false, - "columns": [ - "seq" - ] + "columns": ["seq"] } }, "policies": {}, @@ -498,12 +453,8 @@ "name": "blocked_addresses_environment_id_environments_id_fk", "tableFrom": "blocked_addresses", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -511,10 +462,7 @@ "compositePrimaryKeys": { "blocked_addresses_environment_id_address_pk": { "name": "blocked_addresses_environment_id_address_pk", - "columns": [ - "environment_id", - "address" - ] + "columns": ["environment_id", "address"] } }, "uniqueConstraints": {}, @@ -654,12 +602,8 @@ "name": "capacity_requests_environment_id_environments_id_fk", "tableFrom": "capacity_requests", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -667,10 +611,7 @@ "compositePrimaryKeys": { "capacity_requests_environment_id_request_id_pk": { "name": "capacity_requests_environment_id_request_id_pk", - "columns": [ - "environment_id", - "request_id" - ] + "columns": ["environment_id", "request_id"] } }, "uniqueConstraints": {}, @@ -787,12 +728,8 @@ "name": "deployments_environment_id_environments_id_fk", "tableFrom": "deployments", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -802,9 +739,7 @@ "deployments_seq_unique": { "name": "deployments_seq_unique", "nullsNotDistinct": false, - "columns": [ - "seq" - ] + "columns": ["seq"] } }, "policies": {}, @@ -882,12 +817,8 @@ "name": "domains_environment_id_environments_id_fk", "tableFrom": "domains", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -897,17 +828,12 @@ "domains_seq_unique": { "name": "domains_seq_unique", "nullsNotDistinct": false, - "columns": [ - "seq" - ] + "columns": ["seq"] }, "domains_environment_id_hostname_unique": { "name": "domains_environment_id_hostname_unique", "nullsNotDistinct": false, - "columns": [ - "environment_id", - "hostname" - ] + "columns": ["environment_id", "hostname"] } }, "policies": {}, @@ -1174,12 +1100,8 @@ "name": "environments_project_id_projects_id_fk", "tableFrom": "environments", "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["project_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1189,24 +1111,17 @@ "environments_seq_unique": { "name": "environments_seq_unique", "nullsNotDistinct": false, - "columns": [ - "seq" - ] + "columns": ["seq"] }, "environments_pack_token_unique": { "name": "environments_pack_token_unique", "nullsNotDistinct": false, - "columns": [ - "pack_token" - ] + "columns": ["pack_token"] }, "environments_project_id_name_unique": { "name": "environments_project_id_name_unique", "nullsNotDistinct": false, - "columns": [ - "project_id", - "name" - ] + "columns": ["project_id", "name"] } }, "policies": {}, @@ -1279,10 +1194,7 @@ "compositePrimaryKeys": { "idempotent_requests_scope_request_id_pk": { "name": "idempotent_requests_scope_request_id_pk", - "columns": [ - "scope", - "request_id" - ] + "columns": ["scope", "request_id"] } }, "uniqueConstraints": {}, @@ -1402,12 +1314,8 @@ "name": "log_entries_environment_id_environments_id_fk", "tableFrom": "log_entries", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1417,11 +1325,7 @@ "log_entries_environment_id_instance_id_sequence_unique": { "name": "log_entries_environment_id_instance_id_sequence_unique", "nullsNotDistinct": false, - "columns": [ - "environment_id", - "instance_id", - "sequence" - ] + "columns": ["environment_id", "instance_id", "sequence"] } }, "policies": {}, @@ -1489,9 +1393,7 @@ "logins_user_code_unique": { "name": "logins_user_code_unique", "nullsNotDistinct": false, - "columns": [ - "user_code" - ] + "columns": ["user_code"] } }, "policies": {}, @@ -1545,12 +1447,8 @@ "name": "metric_samples_environment_id_environments_id_fk", "tableFrom": "metric_samples", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1558,12 +1456,7 @@ "compositePrimaryKeys": { "metric_samples_environment_id_instance_id_name_labels_pk": { "name": "metric_samples_environment_id_instance_id_name_labels_pk", - "columns": [ - "environment_id", - "instance_id", - "name", - "labels" - ] + "columns": ["environment_id", "instance_id", "name", "labels"] } }, "uniqueConstraints": {}, @@ -1621,17 +1514,12 @@ "projects_seq_unique": { "name": "projects_seq_unique", "nullsNotDistinct": false, - "columns": [ - "seq" - ] + "columns": ["seq"] }, "projects_owner_id_name_unique": { "name": "projects_owner_id_name_unique", "nullsNotDistinct": false, - "columns": [ - "owner_id", - "name" - ] + "columns": ["owner_id", "name"] } }, "policies": {}, @@ -1711,12 +1599,8 @@ "name": "releases_project_id_projects_id_fk", "tableFrom": "releases", "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["project_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1724,10 +1608,7 @@ "compositePrimaryKeys": { "releases_project_id_id_pk": { "name": "releases_project_id_id_pk", - "columns": [ - "project_id", - "id" - ] + "columns": ["project_id", "id"] } }, "uniqueConstraints": {}, @@ -1777,12 +1658,8 @@ "name": "secrets_environment_id_environments_id_fk", "tableFrom": "secrets", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1790,10 +1667,7 @@ "compositePrimaryKeys": { "secrets_environment_id_name_pk": { "name": "secrets_environment_id_name_pk", - "columns": [ - "environment_id", - "name" - ] + "columns": ["environment_id", "name"] } }, "uniqueConstraints": {}, @@ -1837,12 +1711,8 @@ "name": "superseded_instances_environment_id_environments_id_fk", "tableFrom": "superseded_instances", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1850,10 +1720,7 @@ "compositePrimaryKeys": { "superseded_instances_environment_id_instance_id_pk": { "name": "superseded_instances_environment_id_instance_id_pk", - "columns": [ - "environment_id", - "instance_id" - ] + "columns": ["environment_id", "instance_id"] } }, "uniqueConstraints": {}, @@ -1909,12 +1776,8 @@ "name": "usage_records_environment_id_environments_id_fk", "tableFrom": "usage_records", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1922,10 +1785,7 @@ "compositePrimaryKeys": { "usage_records_environment_id_id_pk": { "name": "usage_records_environment_id_id_pk", - "columns": [ - "environment_id", - "id" - ] + "columns": ["environment_id", "id"] } }, "uniqueConstraints": {}, @@ -1945,4 +1805,4 @@ "schemas": {}, "tables": {} } -} \ No newline at end of file +} diff --git a/packages/management/migrations/meta/0005_snapshot.json b/packages/management/migrations/meta/0005_snapshot.json index e93f0c66..52ebae79 100644 --- a/packages/management/migrations/meta/0005_snapshot.json +++ b/packages/management/migrations/meta/0005_snapshot.json @@ -1,5 +1,5 @@ { - "id": "60010118-6540-48a6-a4c3-05428489739b", + "id": "bf9cf375-edd0-4e43-a3e0-abd64104c853", "prevId": "36c1afcc-4b11-42d9-898f-63e998d7b7dc", "version": "7", "dialect": "postgresql", @@ -128,12 +128,8 @@ "name": "api_tokens_project_id_projects_id_fk", "tableFrom": "api_tokens", "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["project_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -141,12 +137,8 @@ "name": "api_tokens_environment_id_environments_id_fk", "tableFrom": "api_tokens", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -156,16 +148,12 @@ "api_tokens_seq_unique": { "name": "api_tokens_seq_unique", "nullsNotDistinct": false, - "columns": [ - "seq" - ] + "columns": ["seq"] }, "api_tokens_secret_hash_unique": { "name": "api_tokens_secret_hash_unique", "nullsNotDistinct": false, - "columns": [ - "secret_hash" - ] + "columns": ["secret_hash"] } }, "policies": {}, @@ -214,12 +202,8 @@ "name": "asset_blobs_project_id_projects_id_fk", "tableFrom": "asset_blobs", "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["project_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -227,10 +211,7 @@ "compositePrimaryKeys": { "asset_blobs_project_id_sha256_pk": { "name": "asset_blobs_project_id_sha256_pk", - "columns": [ - "project_id", - "sha256" - ] + "columns": ["project_id", "sha256"] } }, "uniqueConstraints": {}, @@ -268,12 +249,8 @@ "name": "asset_deletions_project_id_projects_id_fk", "tableFrom": "asset_deletions", "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["project_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -281,10 +258,7 @@ "compositePrimaryKeys": { "asset_deletions_project_id_sha256_pk": { "name": "asset_deletions_project_id_sha256_pk", - "columns": [ - "project_id", - "sha256" - ] + "columns": ["project_id", "sha256"] } }, "uniqueConstraints": {}, @@ -327,14 +301,8 @@ "name": "asset_revision_blobs_project_id_revision_id_asset_revisions_project_id_id_fk", "tableFrom": "asset_revision_blobs", "tableTo": "asset_revisions", - "columnsFrom": [ - "project_id", - "revision_id" - ], - "columnsTo": [ - "project_id", - "id" - ], + "columnsFrom": ["project_id", "revision_id"], + "columnsTo": ["project_id", "id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -342,11 +310,7 @@ "compositePrimaryKeys": { "asset_revision_blobs_project_id_revision_id_sha256_pk": { "name": "asset_revision_blobs_project_id_revision_id_sha256_pk", - "columns": [ - "project_id", - "revision_id", - "sha256" - ] + "columns": ["project_id", "revision_id", "sha256"] } }, "uniqueConstraints": {}, @@ -437,12 +401,8 @@ "name": "asset_revisions_project_id_projects_id_fk", "tableFrom": "asset_revisions", "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["project_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -450,19 +410,14 @@ "compositePrimaryKeys": { "asset_revisions_project_id_id_pk": { "name": "asset_revisions_project_id_id_pk", - "columns": [ - "project_id", - "id" - ] + "columns": ["project_id", "id"] } }, "uniqueConstraints": { "asset_revisions_seq_unique": { "name": "asset_revisions_seq_unique", "nullsNotDistinct": false, - "columns": [ - "seq" - ] + "columns": ["seq"] } }, "policies": {}, @@ -498,12 +453,8 @@ "name": "blocked_addresses_environment_id_environments_id_fk", "tableFrom": "blocked_addresses", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -511,10 +462,7 @@ "compositePrimaryKeys": { "blocked_addresses_environment_id_address_pk": { "name": "blocked_addresses_environment_id_address_pk", - "columns": [ - "environment_id", - "address" - ] + "columns": ["environment_id", "address"] } }, "uniqueConstraints": {}, @@ -654,12 +602,8 @@ "name": "capacity_requests_environment_id_environments_id_fk", "tableFrom": "capacity_requests", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -667,10 +611,7 @@ "compositePrimaryKeys": { "capacity_requests_environment_id_request_id_pk": { "name": "capacity_requests_environment_id_request_id_pk", - "columns": [ - "environment_id", - "request_id" - ] + "columns": ["environment_id", "request_id"] } }, "uniqueConstraints": {}, @@ -757,12 +698,6 @@ "type": "text", "primaryKey": false, "notNull": true - }, - "drain_until": { - "name": "drain_until", - "type": "timestamp with time zone", - "primaryKey": false, - "notNull": false } }, "indexes": { @@ -793,12 +728,8 @@ "name": "deployments_environment_id_environments_id_fk", "tableFrom": "deployments", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -808,9 +739,7 @@ "deployments_seq_unique": { "name": "deployments_seq_unique", "nullsNotDistinct": false, - "columns": [ - "seq" - ] + "columns": ["seq"] } }, "policies": {}, @@ -888,12 +817,8 @@ "name": "domains_environment_id_environments_id_fk", "tableFrom": "domains", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -903,17 +828,12 @@ "domains_seq_unique": { "name": "domains_seq_unique", "nullsNotDistinct": false, - "columns": [ - "seq" - ] + "columns": ["seq"] }, "domains_environment_id_hostname_unique": { "name": "domains_environment_id_hostname_unique", "nullsNotDistinct": false, - "columns": [ - "environment_id", - "hostname" - ] + "columns": ["environment_id", "hostname"] } }, "policies": {}, @@ -1043,6 +963,13 @@ "notNull": true, "default": "'{}'" }, + "resident_deployment_ids": { + "name": "resident_deployment_ids", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, "pings": { "name": "pings", "type": "jsonb", @@ -1180,12 +1107,8 @@ "name": "environments_project_id_projects_id_fk", "tableFrom": "environments", "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["project_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1195,24 +1118,17 @@ "environments_seq_unique": { "name": "environments_seq_unique", "nullsNotDistinct": false, - "columns": [ - "seq" - ] + "columns": ["seq"] }, "environments_pack_token_unique": { "name": "environments_pack_token_unique", "nullsNotDistinct": false, - "columns": [ - "pack_token" - ] + "columns": ["pack_token"] }, "environments_project_id_name_unique": { "name": "environments_project_id_name_unique", "nullsNotDistinct": false, - "columns": [ - "project_id", - "name" - ] + "columns": ["project_id", "name"] } }, "policies": {}, @@ -1285,10 +1201,7 @@ "compositePrimaryKeys": { "idempotent_requests_scope_request_id_pk": { "name": "idempotent_requests_scope_request_id_pk", - "columns": [ - "scope", - "request_id" - ] + "columns": ["scope", "request_id"] } }, "uniqueConstraints": {}, @@ -1408,12 +1321,8 @@ "name": "log_entries_environment_id_environments_id_fk", "tableFrom": "log_entries", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1423,11 +1332,7 @@ "log_entries_environment_id_instance_id_sequence_unique": { "name": "log_entries_environment_id_instance_id_sequence_unique", "nullsNotDistinct": false, - "columns": [ - "environment_id", - "instance_id", - "sequence" - ] + "columns": ["environment_id", "instance_id", "sequence"] } }, "policies": {}, @@ -1495,9 +1400,7 @@ "logins_user_code_unique": { "name": "logins_user_code_unique", "nullsNotDistinct": false, - "columns": [ - "user_code" - ] + "columns": ["user_code"] } }, "policies": {}, @@ -1551,12 +1454,8 @@ "name": "metric_samples_environment_id_environments_id_fk", "tableFrom": "metric_samples", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1564,12 +1463,7 @@ "compositePrimaryKeys": { "metric_samples_environment_id_instance_id_name_labels_pk": { "name": "metric_samples_environment_id_instance_id_name_labels_pk", - "columns": [ - "environment_id", - "instance_id", - "name", - "labels" - ] + "columns": ["environment_id", "instance_id", "name", "labels"] } }, "uniqueConstraints": {}, @@ -1627,17 +1521,12 @@ "projects_seq_unique": { "name": "projects_seq_unique", "nullsNotDistinct": false, - "columns": [ - "seq" - ] + "columns": ["seq"] }, "projects_owner_id_name_unique": { "name": "projects_owner_id_name_unique", "nullsNotDistinct": false, - "columns": [ - "owner_id", - "name" - ] + "columns": ["owner_id", "name"] } }, "policies": {}, @@ -1717,12 +1606,8 @@ "name": "releases_project_id_projects_id_fk", "tableFrom": "releases", "tableTo": "projects", - "columnsFrom": [ - "project_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["project_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1730,10 +1615,7 @@ "compositePrimaryKeys": { "releases_project_id_id_pk": { "name": "releases_project_id_id_pk", - "columns": [ - "project_id", - "id" - ] + "columns": ["project_id", "id"] } }, "uniqueConstraints": {}, @@ -1783,12 +1665,8 @@ "name": "secrets_environment_id_environments_id_fk", "tableFrom": "secrets", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1796,10 +1674,7 @@ "compositePrimaryKeys": { "secrets_environment_id_name_pk": { "name": "secrets_environment_id_name_pk", - "columns": [ - "environment_id", - "name" - ] + "columns": ["environment_id", "name"] } }, "uniqueConstraints": {}, @@ -1843,12 +1718,8 @@ "name": "superseded_instances_environment_id_environments_id_fk", "tableFrom": "superseded_instances", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1856,10 +1727,7 @@ "compositePrimaryKeys": { "superseded_instances_environment_id_instance_id_pk": { "name": "superseded_instances_environment_id_instance_id_pk", - "columns": [ - "environment_id", - "instance_id" - ] + "columns": ["environment_id", "instance_id"] } }, "uniqueConstraints": {}, @@ -1915,12 +1783,8 @@ "name": "usage_records_environment_id_environments_id_fk", "tableFrom": "usage_records", "tableTo": "environments", - "columnsFrom": [ - "environment_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["environment_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1928,10 +1792,7 @@ "compositePrimaryKeys": { "usage_records_environment_id_id_pk": { "name": "usage_records_environment_id_id_pk", - "columns": [ - "environment_id", - "id" - ] + "columns": ["environment_id", "id"] } }, "uniqueConstraints": {}, @@ -1951,4 +1812,4 @@ "schemas": {}, "tables": {} } -} \ No newline at end of file +} diff --git a/packages/management/migrations/meta/_journal.json b/packages/management/migrations/meta/_journal.json index c12ed58b..47594981 100644 --- a/packages/management/migrations/meta/_journal.json +++ b/packages/management/migrations/meta/_journal.json @@ -40,8 +40,8 @@ { "idx": 5, "version": "7", - "when": 1795206793110, - "tag": "0005_drain_until", + "when": 1795207330303, + "tag": "0005_resident_deployments", "breakpoints": true } ] diff --git a/packages/management/src/assets/retention.ts b/packages/management/src/assets/retention.ts index 7a1c5e09..00e89d0a 100644 --- a/packages/management/src/assets/retention.ts +++ b/packages/management/src/assets/retention.ts @@ -35,10 +35,11 @@ export async function lockAssets(tx: Db, projectId: string, mode: "shared" | "ex /** * Deletes the project's revisions that nothing keeps, and then the blobs no remaining revision holds, from the database * and the release store. A revision is kept while it is the project's head, one of its `keptRecentRevisions` newest - * ready revisions, protected by an upload declared within `uploadLifetimeMs`, pinned by the active, an unfinished or one - * of the `keptRecentDeployments` newest deployments of any of its environments, or pinned by a superseded deployment - * that is still draining: until the later of the deadline its environment had when it was superseded and the one it has - * now, so shortening the deadline never removes what core may still serve. Returns how many revisions it deleted and how many blobs it removed from the release store. + * ready revisions, or protected by an upload declared within `uploadLifetimeMs`. It is also kept while an environment + * has a deployment that pins it and is active, unfinished, one of the environment's `keptRecentDeployments` newest, + * superseded but still activatable (newer than the active one, as `activateDeployment` accepts), or resident in core's + * backend as of the latest status report management accepted. Core decides what stays resident, so no clock or setting + * of management's can lag it. Returns how many revisions it deleted and how many blobs it removed from the release store. * * The database rows are deleted first, recording the blobs' keys in `asset_deletions`; a later transaction deletes the * bytes of each key no row has claimed again. A store delete that fails leaves its key recorded, so the next @@ -89,9 +90,12 @@ async function deleteUnusedRows(tx: Db, projectId: string): Promise { tx, sql` with deployed as ( - select d.asset_revision_id, d.state, d.id = e.active_deployment_id as active, - d.state = ${DeploymentState.SUPERSEDED} - and now() < greatest(d.drain_until, d.update_time + make_interval(secs => e.drain_deadline_seconds)) as draining, + select d.asset_revision_id, d.state, + d.state = ${DeploymentState.SUPERSEDED} and d.seq > coalesce(( + select max(a.seq) from deployments a + where a.environment_id = d.environment_id and a.state = ${DeploymentState.ACTIVE} + ), 0) as activatable, + d.id = any(e.resident_deployment_ids) as resident, row_number() over (partition by d.environment_id order by d.seq desc) as recency from deployments d join environments e on e.id = d.environment_id where e.project_id = ${projectId} @@ -107,7 +111,7 @@ async function deleteUnusedRows(tx: Db, projectId: string): Promise { select id from asset_revisions where project_id = ${projectId} and protected_until > now() union select asset_revision_id from deployed - where active or draining or recency <= ${keptRecentDeployments} or state in (${sql.join(served, sql`, `)}) + where activatable or resident or recency <= ${keptRecentDeployments} or state in (${sql.join(served, sql`, `)}) ), doomed as ( select r.id from asset_revisions r diff --git a/packages/management/src/assets/service.ts b/packages/management/src/assets/service.ts index ea8319a6..df554673 100644 --- a/packages/management/src/assets/service.ts +++ b/packages/management/src/assets/service.ts @@ -115,6 +115,7 @@ export function assetService(deps: Deps): Partial(); const unverified = [...blobs].filter(([sha256]) => !held.has(sha256)); await eachLimited(unverified, verifyConcurrency, async ([sha256, { size }]) => { const sizeBytes = BigInt(size); @@ -127,10 +128,7 @@ export function assetService(deps: Deps): Partial 0) { @@ -148,6 +146,17 @@ export function assetService(deps: Deps): Partial 0) { + await tx + .insert(assetBlobs) + .values( + [...verified] + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([sha256, blob]) => ({ project_id: project.id, sha256, ...blob })), + ) + .onConflictDoNothing(); + } if (row.state === AssetRevisionState.READY) return { revision: toAssetRevision(row) }; const [ready] = await tx .update(assetRevisions) diff --git a/packages/management/src/deployments/store.ts b/packages/management/src/deployments/store.ts index eae79911..5016e51e 100644 --- a/packages/management/src/deployments/store.ts +++ b/packages/management/src/deployments/store.ts @@ -69,13 +69,8 @@ export function requireJvmImage(manifest: ReleaseManifest, template: string | un const unfinished = [DeploymentState.PENDING, DeploymentState.IN_PROGRESS]; -/** The values a deployment takes when superseded; its environment's drain deadline, as of now, bounds its draining. */ -const superseded = { - state: DeploymentState.SUPERSEDED, - update_time: sql`now()`, - drain_until: sql`now() + make_interval(secs => ( - select e.drain_deadline_seconds from environments e where e.id = ${deployments.environment_id}))`, -}; +/** The values a deployment takes when superseded. */ +const superseded = { state: DeploymentState.SUPERSEDED, update_time: sql`now()` }; /** What a deployment pins: a release and an asset revision of one project. */ export interface Deployable { @@ -178,8 +173,18 @@ export async function deployRestored( /** * Records that the environment serves a deployment, superseding the one it replaced. A superseded deployment the * environment activated anyway counts when it is newer than the active one; newer unfinished deployments stay desired. + * Holds the project's shared asset lock until `db`'s transaction ends. */ export async function activateDeployment(db: Db, id: string): Promise { + // Shared with uploads and createDeployment, so a collection never interleaves with the activation of a deployment + // whose revision it may delete. + const [owner] = await db + .select({ project_id: environments.project_id }) + .from(deployments) + .innerJoin(environments, eq(environments.id, deployments.environment_id)) + .where(eq(deployments.id, id)); + if (!owner) return; + await lockAssets(db, owner.project_id, "shared"); const [row] = await fetchRows<{ environment_id: string }>( db, sql` diff --git a/packages/management/src/environments/service.ts b/packages/management/src/environments/service.ts index 351e0623..8151f4da 100644 --- a/packages/management/src/environments/service.ts +++ b/packages/management/src/environments/service.ts @@ -105,6 +105,7 @@ export function environmentService(deps: Deps): Partial exists(archive(key)), async delete(key) { - await s3Request("release store deletion", () => - Promise.all([client.delete(archive(key)), client.delete(upload(key))]), - ); + // Both deletes settle before this resolves or rejects, so neither lands once the caller's lock is released. + await s3Request("release store deletion", async () => { + const results = await Promise.allSettled([client.delete(archive(key)), client.delete(upload(key))]); + for (const result of results) { + if (result.status === "rejected") throw result.reason; + } + }); }, async complete(key, expected, verify) { diff --git a/packages/management/src/schema.ts b/packages/management/src/schema.ts index 31eb9213..5b640075 100644 --- a/packages/management/src/schema.ts +++ b/packages/management/src/schema.ts @@ -141,6 +141,8 @@ export const environments = pgTable( owner_since: time(), // The latest accepted status report under the current lease. gateway_addresses: text().array().notNull().default([]), + /** The deployments resident in core's backend, loading or not, as that report said. */ + resident_deployment_ids: text().array().notNull().default([]), pings: jsonb>() .notNull() .default(sql`'{}'::jsonb`), @@ -290,8 +292,6 @@ export const deployments = pgTable( /** Stop the deployments this one replaces at once instead of draining them. */ stop_previous: boolean().notNull().default(false), asset_revision_id: text().notNull(), - /** When a superseded deployment stops draining: its environment's drain deadline after it was superseded. */ - drain_until: time(), }, (t) => [index("deployments_environment").on(t.environment_id, t.seq)], ); diff --git a/packages/management/test/assets.test.ts b/packages/management/test/assets.test.ts index a67690c5..3c279e2f 100644 --- a/packages/management/test/assets.test.ts +++ b/packages/management/test/assets.test.ts @@ -2,12 +2,14 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import { Code } from "@connectrpc/connect"; -import { collectAssets, keptRecentRevisions } from "../src/assets/retention.ts"; +import { collectAssets, keptRecentDeployments, keptRecentRevisions } from "../src/assets/retention.ts"; import { decodeRevision, sha256Hex } from "../src/assets/revision.ts"; import { issueEnvironmentToken } from "../src/auth/tokens.ts"; import { desiredState } from "../src/environments/desired.ts"; import { AssetRevisionState, AssetService } from "../src/gen/chunk/management/v1/assets_pb.ts"; +import { DeploymentState } from "../src/gen/chunk/management/v1/common_pb.ts"; import { DeploymentService } from "../src/gen/chunk/management/v1/deployments_pb.ts"; +import { EnvironmentService } from "../src/gen/chunk/management/v1/environment_pb.ts"; import { ProjectService } from "../src/gen/chunk/management/v1/projects_pb.ts"; import { blobKey } from "../src/releases/store.ts"; import { assetRevision } from "./fixtures.ts"; @@ -218,37 +220,104 @@ describe.skipIf(!databaseUrl)("AssetService", () => { }); }); - test("a deployment keeps its revision while it drains, and not past its environment's drain deadline", async () => { + /** + * An environment whose project holds an old revision, `old`, and enough newer ones and unfinished deployments that + * neither the recent-revision nor the recent-deployment rule keeps anything of `old`, with a core attached to it. + */ + async function aged() { const { projectId, environmentId } = await createEnvironment(h); const deploy = h.client(DeploymentService); await uploadRelease(h, projectId, "r1"); - const draining = assetRevision({ shared: { "f.txt": "draining" } }); - await uploadAssets(h, projectId, draining); + const [first, second] = [ + assetRevision({ shared: { "f.txt": "first" } }), + assetRevision({ shared: { "f.txt": "second" } }), + ]; + for (const revision of [first, second]) await uploadAssets(h, projectId, revision); const fillers = Array.from({ length: keptRecentRevisions + 1 }, (_, i) => assetRevision({ shared: { "f.txt": `filler${i}` } }), ); for (const filler of fillers) await uploadAssets(h, projectId, filler); - const pin = (assetRevisionId: string) => - deploy.deploy({ requestId: crypto.randomUUID(), environmentId, releaseId: "r1", assetRevisionId }); - await pin(draining.id); - for (let i = 0; i < 11; i++) await pin(fillers[0]?.id ?? ""); - await h.sql`update asset_revisions set protected_until = now() - interval '1 minute' where project_id = ${projectId}`; - await h.sql`update projects set asset_head_id = ${fillers[1]?.id ?? ""} where id = ${projectId}`; - const stored = async () => - (await h.sql<{ id: string }[]>`select id from asset_revisions where project_id = ${projectId}`).some( - (row) => row.id === draining.id, - ); + await h.sql`update projects set asset_head_id = ${fillers[0]?.id ?? ""} where id = ${projectId}`; + const pin = async (assetRevisionId: string) => + (await deploy.deploy({ requestId: crypto.randomUUID(), environmentId, releaseId: "r1", assetRevisionId })) + .deployment?.id ?? ""; + const client = h.client(EnvironmentService, await issueEnvironmentToken(h.db, environmentId)); + const abort = new AbortController(); + const stream = client.attach({ instanceId: crypto.randomUUID(), core: true, epoch: 1n }, { signal: abort.signal }); + const { lease, revision } = (await stream[Symbol.asyncIterator]().next()).value; + let sequence = 0n; + return { + projectId, + first, + second, + fillers, + pin, + /** Pins `fillers[0]` enough times that the deployments before them are none of the newest. */ + bury: async () => { + for (let i = 0; i < keptRecentDeployments + 1; i++) await pin(fillers[0]?.id ?? ""); + await h.sql`update asset_revisions set protected_until = now() - interval '1 minute' where project_id = ${projectId}`; + }, + report: (resident: string[], deployment?: { deploymentId: string; state: DeploymentState }) => + client.reportStatus({ + lease, + sequence: ++sequence, + desiredRevision: revision, + residentDeploymentIds: resident, + ...(deployment ? { deployment } : {}), + }), + /** A report sent now whose arrival is delayed until the returned function is called. */ + delayed: (resident: string[], deployment: { deploymentId: string; state: DeploymentState }) => { + const delayedSequence = ++sequence; + return () => + client.reportStatus({ + lease, + sequence: delayedSequence, + desiredRevision: revision, + residentDeploymentIds: resident, + deployment, + }); + }, + stored: async (id: string) => + (await h.sql<{ id: string }[]>`select id from asset_revisions where project_id = ${projectId} and id = ${id}`) + .length > 0, + close: () => abort.abort(), + }; + } - await collectAssets(h.deps, projectId); - expect(await stored()).toBe(true); - // Shortening the deadline doesn't release what the deployment's own deadline, set when it was superseded, keeps. - await h.sql`update environments set drain_max_age_seconds = 60, drain_deadline_seconds = 60 where id = ${environmentId}`; - await h.sql`update deployments set update_time = now() - interval '5 hours' where asset_revision_id = ${draining.id}`; - await collectAssets(h.deps, projectId); - expect(await stored()).toBe(true); - await h.sql`update deployments set drain_until = now() - interval '1 minute' where asset_revision_id = ${draining.id}`; - await collectAssets(h.deps, projectId); - expect(await stored()).toBe(false); + test("a deployment core activated, whose report is delayed past newer deployments, activates with its revision present", async () => { + const env = await aged(); + const active = await env.pin(env.first.id); + await env.report([active], { deploymentId: active, state: DeploymentState.ACTIVE }); + const activated = await env.pin(env.second.id); + const lateReport = env.delayed([active, activated], { deploymentId: activated, state: DeploymentState.ACTIVE }); + await env.bury(); + + await collectAssets(h.deps, env.projectId); + expect(await env.stored(env.second.id)).toBe(true); + + await lateReport(); + const { deployment } = await h.client(DeploymentService).getDeployment({ deploymentId: activated }); + expect(deployment?.state).toBe(DeploymentState.ACTIVE); + expect(await env.stored(deployment?.assetRevisionId ?? "")).toBe(true); + env.close(); + }); + + test("a superseded deployment in core's reported resident set keeps its revision until a report omits it", async () => { + const env = await aged(); + const old = await env.pin(env.first.id); + await env.report([old], { deploymentId: old, state: DeploymentState.ACTIVE }); + const current = await env.pin(env.second.id); + await env.report([old, current], { deploymentId: current, state: DeploymentState.ACTIVE }); + await env.bury(); + + await collectAssets(h.deps, env.projectId); + expect(await env.stored(env.first.id)).toBe(true); + + await env.report([current]); + await collectAssets(h.deps, env.projectId); + expect(await env.stored(env.first.id)).toBe(false); + expect(await env.stored(env.second.id)).toBe(true); + env.close(); }); test("a completion paused after verifying its blobs holds off collection, so the revision does not become ready without them", async () => { diff --git a/proto/chunk/management/v1/environment.proto b/proto/chunk/management/v1/environment.proto index ce1e1784..89968ba5 100644 --- a/proto/chunk/management/v1/environment.proto +++ b/proto/chunk/management/v1/environment.proto @@ -218,6 +218,12 @@ message ReportStatusRequest { // so a report carries every login since the last one, even players who have // already left. The management plane refunds pending login wakes from it. uint64 logins = 10; + // The IDs of every deployment resident in core's backend, including one that + // is loading. The management plane keeps the asset revisions of these + // deployments, which core's JVMs may still read, whatever their state there. + // It stores the set of the latest accepted report, so each report carries the + // whole set. + repeated string resident_deployment_ids = 11; } message PingStatus { From 2442616f9c8ec08a74a03ac093cf8ab57b849714 Mon Sep 17 00:00:00 2001 From: oglassdev <23040613+oglassdev@users.noreply.github.com> Date: Fri, 20 Nov 2026 15:54:37 -0500 Subject: [PATCH 5/5] fix(assets): keep every deployment's assets until core has read its resident set An unknown resident set looked like an empty one: a fresh core whose first read timed out reported nothing, and the next collection deleted the revision of a draining deployment. - ReportStatusRequest.resident_deployments_known is true only when the observation read the resident set. Core no longer repeats a cached set. - Management stores the IDs of reports that read them and records the flag on the environment, resetting it on each new lease. - Collection keeps the revisions of all of an environment's deployments while the flag is false. Refs #328 --- crates/chunk-environment/README.md | 14 ++--- crates/chunk-environment/src/managed.rs | 4 +- .../chunk-environment/src/managed/status.rs | 28 +++------ crates/chunk-environment/src/managed/tests.rs | 1 + .../gen/chunk/management/v1/environment_pb.ts | 18 +++++- packages/management/README.md | 17 +++--- .../migrations/0005_resident_deployments.sql | 3 +- .../migrations/meta/0005_snapshot.json | 7 +++ packages/management/src/assets/retention.ts | 7 ++- .../management/src/environments/service.ts | 3 +- packages/management/src/environments/store.ts | 1 + .../gen/chunk/management/v1/environment_pb.ts | 18 +++++- packages/management/src/schema.ts | 2 + packages/management/test/assets.test.ts | 59 ++++++++++++++++++- proto/chunk/management/v1/environment.proto | 11 +++- 15 files changed, 140 insertions(+), 53 deletions(-) diff --git a/crates/chunk-environment/README.md b/crates/chunk-environment/README.md index a4f89c5a..448769d2 100644 --- a/crates/chunk-environment/README.md +++ b/crates/chunk-environment/README.md @@ -65,13 +65,13 @@ non-zero if stopped before management answered. It downloads each release archiv deployment pins against the worlds and packs the release declares, and downloads each blob that JVMs read and `$CHUNK_STATE/assets/` lacks, four at a time, verified against its digest, then writes the revision there; packs reach players' clients from management, never through core. Once no deployment resident in the backend, nor the one loading, -pins a revision, core removes it and the blobs no kept revision holds. Each status report lists those resident -deployments and the one loading, which management keeps the asset revisions of. Only then does it make the deployment -the backend's and control's current release, and report it `ACTIVE`. A deployment it rejects is reported `FAILED`, and -the previous deployment keeps serving. The gateway starts with the first active deployment. Core retires the versions it -no longer needs, stopping their JVMs first, and removes their unpacked releases. JVMs run on machines core asks -management for (`EnsureCapacity`), each running [`chunk-jvm`](../chunk-jvm/README.md). A core that management fences -stops. +pins a revision, core removes it and the blobs no kept revision holds. A status report lists those resident deployments +and the one loading, which management keeps the asset revisions of, only when it read them; otherwise it says so, and +management keeps the assets of every deployment. Only then does it make the deployment the backend's and control's +current release, and report it `ACTIVE`. A deployment it rejects is reported `FAILED`, and the previous deployment keeps +serving. The gateway starts with the first active deployment. Core retires the versions it no longer needs, stopping +their JVMs first, and removes their unpacked releases. JVMs run on machines core asks management for (`EnsureCapacity`), +each running [`chunk-jvm`](../chunk-jvm/README.md). A core that management fences stops. Core reports its status about every 15 seconds, and at once when it changes. Each report carries the server-list status the gateway last answered for each hostname, which the [edge](../chunk-edge/README.md) answers pings with while the diff --git a/crates/chunk-environment/src/managed.rs b/crates/chunk-environment/src/managed.rs index e0489de4..8d80f218 100644 --- a/crates/chunk-environment/src/managed.rs +++ b/crates/chunk-environment/src/managed.rs @@ -693,7 +693,7 @@ impl<'a> Managed<'a> { logins: self.logins(), ready_to_suspend: false, pings: self.pings(), - resident_deployments: self.reporter.resident(None), + resident_deployments: None, } } @@ -717,7 +717,7 @@ impl<'a> Managed<'a> { logins: self.logins(), ready_to_suspend, pings: self.pings(), - resident_deployments: self.reporter.resident(resident), + resident_deployments: resident, } } diff --git a/crates/chunk-environment/src/managed/status.rs b/crates/chunk-environment/src/managed/status.rs index 05d0b892..fdde9632 100644 --- a/crates/chunk-environment/src/managed/status.rs +++ b/crates/chunk-environment/src/managed/status.rs @@ -42,8 +42,9 @@ pub(super) struct Observed { pub ready_to_suspend: bool, /// The status the gateway last answered for each hostname, which edges answer pings with while core sleeps. pub pings: Vec, - /// Every deployment resident in the backend, including one loading, which management keeps the assets of. - pub resident_deployments: Vec, + /// Every deployment resident in the backend, including one loading, which management keeps the assets of. `None` + /// when this observation couldn't read them. + pub resident_deployments: Option>, } impl Observed { @@ -75,8 +76,6 @@ struct Sent { lease: u64, /// The gateway addresses of the latest report management accepted, under this lease or an earlier one. addresses: Vec, - /// The resident deployments last read, under this lease or an earlier one. - resident: Vec, sequence: u64, /// What the latest report started said. latest: Option, @@ -175,12 +174,8 @@ impl Reporter { return Ok(false); } if observed.lease > sent.lease { - *sent = Sent { - lease: observed.lease, - addresses: std::mem::take(&mut sent.addresses), - resident: std::mem::take(&mut sent.resident), - ..Sent::default() - }; + *sent = + Sent { lease: observed.lease, addresses: std::mem::take(&mut sent.addresses), ..Sent::default() }; } sent.sequence += 1; let sequence = sent.sequence; @@ -197,7 +192,8 @@ impl Reporter { online_players: observed.online_players, logins: observed.logins, pings: observed.pings, - resident_deployment_ids: observed.resident_deployments, + resident_deployments_known: observed.resident_deployments.is_some(), + resident_deployment_ids: observed.resident_deployments.unwrap_or_default(), deployment: deployment.or_else(|| sent.progress.as_ref().map(|(_, progress)| progress.clone())), ready_to_suspend: observed.ready_to_suspend, lease: observed.lease, @@ -215,16 +211,6 @@ impl Reporter { result.map(|_| true) } - /// The deployments resident in the backend: `read` when it could be read, which is remembered, and else the ones - /// last read. - pub(super) fn resident(&self, read: Option>) -> Vec { - let mut sent = lock(&self.sent); - if let Some(read) = read { - sent.resident = read; - } - sent.resident.clone() - } - /// The gateway addresses of the latest report management accepted. pub(super) fn accepted_addresses(&self) -> Vec { lock(&self.sent).addresses.clone() diff --git a/crates/chunk-environment/src/managed/tests.rs b/crates/chunk-environment/src/managed/tests.rs index 342449b9..9d0e8d48 100644 --- a/crates/chunk-environment/src/managed/tests.rs +++ b/crates/chunk-environment/src/managed/tests.rs @@ -611,6 +611,7 @@ async fn a_stop_previous_activation_stops_every_resident_deployment_even_across_ harness.expect(2, "dep_b", DeploymentState::InProgress).await; let active = harness.expect(2, "dep_b", DeploymentState::Active).await; assert!(active.resident_deployment_ids.contains(&"dep_b".to_owned()), "{:?}", active.resident_deployment_ids); + assert!(active.resident_deployments_known); // dep_c stops dep_a and dep_b, but management doesn't accept it, so dep_b stays as its fallback. Then dep_d // arrives before dep_b stops. diff --git a/packages/dashboard/src/gen/chunk/management/v1/environment_pb.ts b/packages/dashboard/src/gen/chunk/management/v1/environment_pb.ts index 18b90eb7..22f05177 100644 --- a/packages/dashboard/src/gen/chunk/management/v1/environment_pb.ts +++ b/packages/dashboard/src/gen/chunk/management/v1/environment_pb.ts @@ -15,7 +15,7 @@ import { file_chunk_management_v1_common } from "./common_pb.ts"; * Describes the file chunk/management/v1/environment.proto. */ export const file_chunk_management_v1_environment: GenFile /*@__PURE__*/ = fileDesc( - "CiVjaHVuay9tYW5hZ2VtZW50L3YxL2Vudmlyb25tZW50LnByb3RvEhNjaHVuay5tYW5hZ2VtZW50LnYxInIKDUF0dGFjaFJlcXVlc3QSEwoLaW5zdGFuY2VfaWQYASABKAkSDwoHdmVyc2lvbhgCIAEoCRIMCgRjb3JlGAMgASgIEg0KBWVwb2NoGAQgASgEEh4KFnJlc3RvcmVkX2RlcGxveW1lbnRfaWQYBSABKAki8wMKDkF0dGFjaFJlc3BvbnNlEhAKCHJldmlzaW9uGAEgASgEEhYKDmVudmlyb25tZW50X2lkGAIgASgJEhIKCnByb2plY3RfaWQYAyABKAkSFQoNZGVwbG95bWVudF9pZBgEIAEoCRI1CgdyZWxlYXNlGAUgASgLMiQuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlQXJ0aWZhY3QSMQoHc2VjcmV0cxgGIAMoCzIgLmNodW5rLm1hbmFnZW1lbnQudjEuU2VjcmV0VmFsdWUSMwoJbG9nX3N0b3JlGAcgASgLMiAuY2h1bmsubWFuYWdlbWVudC52MS5PYmplY3RTdG9yZRItCgdyZXN0b3JlGAggASgLMhwuY2h1bmsubWFuYWdlbWVudC52MS5SZXN0b3JlEg0KBWxlYXNlGAkgASgEEhgKEGVudmlyb25tZW50X25hbWUYCiABKAkSMQoFZHJhaW4YCyABKAsyIi5jaHVuay5tYW5hZ2VtZW50LnYxLkRyYWluU2V0dGluZ3MSFQoNc3RvcF9wcmV2aW91cxgMIAEoCBIyCgZhc3NldHMYDSABKAsyIi5jaHVuay5tYW5hZ2VtZW50LnYxLkFzc2V0QXJ0aWZhY3QSFwoPcGFja191cmxfcHJlZml4GA4gASgJIlYKD1JlbGVhc2VBcnRpZmFjdBISCgpyZWxlYXNlX2lkGAEgASgJEgsKA3VybBgCIAEoCRIOCgZzaGEyNTYYAyABKAkSEgoKc2l6ZV9ieXRlcxgEIAEoBCJPCg1Bc3NldEFydGlmYWN0EhMKC3JldmlzaW9uX2lkGAEgASgJEhAKCG1hbmlmZXN0GAIgASgMEhcKD2Jsb2JfdXJsX3ByZWZpeBgDIAEoCSI7CgtTZWNyZXRWYWx1ZRIMCgRuYW1lGAEgASgJEg8KB3ZlcnNpb24YAiABKAQSDQoFdmFsdWUYAyABKAwiyQEKC09iamVjdFN0b3JlEhAKCGVuZHBvaW50GAEgASgJEg4KBnJlZ2lvbhgCIAEoCRIOCgZidWNrZXQYAyABKAkSDgoGcHJlZml4GAQgASgJEhUKDWFjY2Vzc19rZXlfaWQYBSABKAkSGQoRc2VjcmV0X2FjY2Vzc19rZXkYBiABKAkSFQoNc2Vzc2lvbl90b2tlbhgHIAEoCRIvCgtleHBpcmVfdGltZRgIIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAibwoHUmVzdG9yZRIwCgZzb3VyY2UYASABKAsyIC5jaHVuay5tYW5hZ2VtZW50LnYxLk9iamVjdFN0b3JlEhMKC3NuYXBzaG90X2lkGAIgASgJEh0KFXNvdXJjZV9lbnZpcm9ubWVudF9pZBgDIAEoCSLtAgoTUmVwb3J0U3RhdHVzUmVxdWVzdBIwCgxvYnNlcnZlX3RpbWUYASABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEhkKEWdhdGV3YXlfYWRkcmVzc2VzGAIgAygJEhYKDm9ubGluZV9wbGF5ZXJzGAMgASgNEi4KBXBpbmdzGAQgAygLMh8uY2h1bmsubWFuYWdlbWVudC52MS5QaW5nU3RhdHVzEjsKCmRlcGxveW1lbnQYBSABKAsyJy5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveW1lbnRQcm9ncmVzcxIYChByZWFkeV90b19zdXNwZW5kGAYgASgIEg0KBWxlYXNlGAcgASgEEhAKCHNlcXVlbmNlGAggASgEEhgKEGRlc2lyZWRfcmV2aXNpb24YCSABKAQSDgoGbG9naW5zGAogASgEEh8KF3Jlc2lkZW50X2RlcGxveW1lbnRfaWRzGAsgAygJIjMKClBpbmdTdGF0dXMSEAoIaG9zdG5hbWUYASABKAkSEwoLc3RhdHVzX2pzb24YAiABKAkicQoSRGVwbG95bWVudFByb2dyZXNzEhUKDWRlcGxveW1lbnRfaWQYASABKAkSMwoFc3RhdGUYAiABKA4yJC5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveW1lbnRTdGF0ZRIPCgdtZXNzYWdlGAMgASgJIhYKFFJlcG9ydFN0YXR1c1Jlc3BvbnNlInUKE1NldFdha2VBbGFybVJlcXVlc3QSEgoKZ2VuZXJhdGlvbhgBIAEoBBIsCghkdWVfdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASDQoFZXBvY2gYAyABKAQSDQoFbGVhc2UYBCABKAQiZwoUU2V0V2FrZUFsYXJtUmVzcG9uc2USEgoKZ2VuZXJhdGlvbhgBIAEoBBIsCghkdWVfdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASDQoFZXBvY2gYAyABKAQivAEKFUVuc3VyZUNhcGFjaXR5UmVxdWVzdBISCgpyZXF1ZXN0X2lkGAEgASgJEi8KCHdvcmtsb2FkGAIgASgOMh0uY2h1bmsubWFuYWdlbWVudC52MS5Xb3JrbG9hZBIXCg9tYWNoaW5lX3Byb2ZpbGUYAyABKAkSEgoKcmVsZWFzZV9pZBgEIAEoCRIOCgZhcHBfaWQYBSABKAkSDQoFbGVhc2UYBiABKAQSEgoKY3JlZGVudGlhbBgHIAEoCSJJChZFbnN1cmVDYXBhY2l0eVJlc3BvbnNlEi8KCGNhcGFjaXR5GAEgASgLMh0uY2h1bmsubWFuYWdlbWVudC52MS5DYXBhY2l0eSJ2CghDYXBhY2l0eRISCgpyZXF1ZXN0X2lkGAEgASgJEjEKBXN0YXRlGAIgASgOMiIuY2h1bmsubWFuYWdlbWVudC52MS5DYXBhY2l0eVN0YXRlEhIKCm1hY2hpbmVfaWQYAyABKAkSDwoHbWVzc2FnZRgEIAEoCSI7ChZSZWxlYXNlQ2FwYWNpdHlSZXF1ZXN0EhIKCnJlcXVlc3RfaWQYASABKAkSDQoFbGVhc2UYAiABKAQiSgoXUmVsZWFzZUNhcGFjaXR5UmVzcG9uc2USLwoIY2FwYWNpdHkYASABKAsyHS5jaHVuay5tYW5hZ2VtZW50LnYxLkNhcGFjaXR5IkcKElJlcG9ydFVzYWdlUmVxdWVzdBIxCgdyZWNvcmRzGAEgAygLMiAuY2h1bmsubWFuYWdlbWVudC52MS5Vc2FnZVJlY29yZCKkAQoLVXNhZ2VSZWNvcmQSCgoCaWQYASABKAkSLgoKc3RhcnRfdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASLAoIZW5kX3RpbWUYAyABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEhYKDnBsYXllcl9zZWNvbmRzGAQgASgEEhMKC2luc3RhbmNlX2lkGAUgASgJIhUKE1JlcG9ydFVzYWdlUmVzcG9uc2UiQwoRUmVwb3J0TG9nc1JlcXVlc3QSLgoHZW50cmllcxgBIAMoCzIdLmNodW5rLm1hbmFnZW1lbnQudjEuTG9nRW50cnkiFAoSUmVwb3J0TG9nc1Jlc3BvbnNlIkoKFFJlcG9ydE1ldHJpY3NSZXF1ZXN0EjIKB3NhbXBsZXMYASADKAsyIS5jaHVuay5tYW5hZ2VtZW50LnYxLk1ldHJpY1NhbXBsZSLYAQoMTWV0cmljU2FtcGxlEgwKBG5hbWUYASABKAkSPQoGbGFiZWxzGAIgAygLMi0uY2h1bmsubWFuYWdlbWVudC52MS5NZXRyaWNTYW1wbGUuTGFiZWxzRW50cnkSKAoEdGltZRgDIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASDQoFdmFsdWUYBCABKAESEwoLaW5zdGFuY2VfaWQYBSABKAkaLQoLTGFiZWxzRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4ASIXChVSZXBvcnRNZXRyaWNzUmVzcG9uc2UiTAoXUmVwb3J0RmFpbGVkQXV0aFJlcXVlc3QSMQoIZmFpbHVyZXMYASADKAsyHy5jaHVuay5tYW5hZ2VtZW50LnYxLkZhaWxlZEF1dGgiTgoKRmFpbGVkQXV0aBIWCg5jbGllbnRfYWRkcmVzcxgBIAEoCRIoCgR0aW1lGAIgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcCIaChhSZXBvcnRGYWlsZWRBdXRoUmVzcG9uc2UqYQoIV29ya2xvYWQSGAoUV09SS0xPQURfVU5TUEVDSUZJRUQQABIQCgxXT1JLTE9BRF9KVk0QARIUChBXT1JLTE9BRF9HQVRFV0FZEAIiBAgDEAMqDVdPUktMT0FEX0VYRUMqwAEKDUNhcGFjaXR5U3RhdGUSHgoaQ0FQQUNJVFlfU1RBVEVfVU5TUEVDSUZJRUQQABIfChtDQVBBQ0lUWV9TVEFURV9QUk9WSVNJT05JTkcQARIYChRDQVBBQ0lUWV9TVEFURV9SRUFEWRACEhkKFUNBUEFDSVRZX1NUQVRFX0ZBSUxFRBADEhsKF0NBUEFDSVRZX1NUQVRFX1JFTEVBU0VEEAQSHAoYQ0FQQUNJVFlfU1RBVEVfUkVMRUFTSU5HEAUypgcKEkVudmlyb25tZW50U2VydmljZRJTCgZBdHRhY2gSIi5jaHVuay5tYW5hZ2VtZW50LnYxLkF0dGFjaFJlcXVlc3QaIy5jaHVuay5tYW5hZ2VtZW50LnYxLkF0dGFjaFJlc3BvbnNlMAESYwoMUmVwb3J0U3RhdHVzEiguY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRTdGF0dXNSZXF1ZXN0GikuY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRTdGF0dXNSZXNwb25zZRJjCgxTZXRXYWtlQWxhcm0SKC5jaHVuay5tYW5hZ2VtZW50LnYxLlNldFdha2VBbGFybVJlcXVlc3QaKS5jaHVuay5tYW5hZ2VtZW50LnYxLlNldFdha2VBbGFybVJlc3BvbnNlEmkKDkVuc3VyZUNhcGFjaXR5EiouY2h1bmsubWFuYWdlbWVudC52MS5FbnN1cmVDYXBhY2l0eVJlcXVlc3QaKy5jaHVuay5tYW5hZ2VtZW50LnYxLkVuc3VyZUNhcGFjaXR5UmVzcG9uc2USbAoPUmVsZWFzZUNhcGFjaXR5EisuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlQ2FwYWNpdHlSZXF1ZXN0GiwuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlQ2FwYWNpdHlSZXNwb25zZRJgCgtSZXBvcnRVc2FnZRInLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0VXNhZ2VSZXF1ZXN0GiguY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRVc2FnZVJlc3BvbnNlEl0KClJlcG9ydExvZ3MSJi5jaHVuay5tYW5hZ2VtZW50LnYxLlJlcG9ydExvZ3NSZXF1ZXN0GicuY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRMb2dzUmVzcG9uc2USZgoNUmVwb3J0TWV0cmljcxIpLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0TWV0cmljc1JlcXVlc3QaKi5jaHVuay5tYW5hZ2VtZW50LnYxLlJlcG9ydE1ldHJpY3NSZXNwb25zZRJvChBSZXBvcnRGYWlsZWRBdXRoEiwuY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRGYWlsZWRBdXRoUmVxdWVzdBotLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0RmFpbGVkQXV0aFJlc3BvbnNlYgZwcm90bzM", + "CiVjaHVuay9tYW5hZ2VtZW50L3YxL2Vudmlyb25tZW50LnByb3RvEhNjaHVuay5tYW5hZ2VtZW50LnYxInIKDUF0dGFjaFJlcXVlc3QSEwoLaW5zdGFuY2VfaWQYASABKAkSDwoHdmVyc2lvbhgCIAEoCRIMCgRjb3JlGAMgASgIEg0KBWVwb2NoGAQgASgEEh4KFnJlc3RvcmVkX2RlcGxveW1lbnRfaWQYBSABKAki8wMKDkF0dGFjaFJlc3BvbnNlEhAKCHJldmlzaW9uGAEgASgEEhYKDmVudmlyb25tZW50X2lkGAIgASgJEhIKCnByb2plY3RfaWQYAyABKAkSFQoNZGVwbG95bWVudF9pZBgEIAEoCRI1CgdyZWxlYXNlGAUgASgLMiQuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlQXJ0aWZhY3QSMQoHc2VjcmV0cxgGIAMoCzIgLmNodW5rLm1hbmFnZW1lbnQudjEuU2VjcmV0VmFsdWUSMwoJbG9nX3N0b3JlGAcgASgLMiAuY2h1bmsubWFuYWdlbWVudC52MS5PYmplY3RTdG9yZRItCgdyZXN0b3JlGAggASgLMhwuY2h1bmsubWFuYWdlbWVudC52MS5SZXN0b3JlEg0KBWxlYXNlGAkgASgEEhgKEGVudmlyb25tZW50X25hbWUYCiABKAkSMQoFZHJhaW4YCyABKAsyIi5jaHVuay5tYW5hZ2VtZW50LnYxLkRyYWluU2V0dGluZ3MSFQoNc3RvcF9wcmV2aW91cxgMIAEoCBIyCgZhc3NldHMYDSABKAsyIi5jaHVuay5tYW5hZ2VtZW50LnYxLkFzc2V0QXJ0aWZhY3QSFwoPcGFja191cmxfcHJlZml4GA4gASgJIlYKD1JlbGVhc2VBcnRpZmFjdBISCgpyZWxlYXNlX2lkGAEgASgJEgsKA3VybBgCIAEoCRIOCgZzaGEyNTYYAyABKAkSEgoKc2l6ZV9ieXRlcxgEIAEoBCJPCg1Bc3NldEFydGlmYWN0EhMKC3JldmlzaW9uX2lkGAEgASgJEhAKCG1hbmlmZXN0GAIgASgMEhcKD2Jsb2JfdXJsX3ByZWZpeBgDIAEoCSI7CgtTZWNyZXRWYWx1ZRIMCgRuYW1lGAEgASgJEg8KB3ZlcnNpb24YAiABKAQSDQoFdmFsdWUYAyABKAwiyQEKC09iamVjdFN0b3JlEhAKCGVuZHBvaW50GAEgASgJEg4KBnJlZ2lvbhgCIAEoCRIOCgZidWNrZXQYAyABKAkSDgoGcHJlZml4GAQgASgJEhUKDWFjY2Vzc19rZXlfaWQYBSABKAkSGQoRc2VjcmV0X2FjY2Vzc19rZXkYBiABKAkSFQoNc2Vzc2lvbl90b2tlbhgHIAEoCRIvCgtleHBpcmVfdGltZRgIIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAibwoHUmVzdG9yZRIwCgZzb3VyY2UYASABKAsyIC5jaHVuay5tYW5hZ2VtZW50LnYxLk9iamVjdFN0b3JlEhMKC3NuYXBzaG90X2lkGAIgASgJEh0KFXNvdXJjZV9lbnZpcm9ubWVudF9pZBgDIAEoCSKRAwoTUmVwb3J0U3RhdHVzUmVxdWVzdBIwCgxvYnNlcnZlX3RpbWUYASABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEhkKEWdhdGV3YXlfYWRkcmVzc2VzGAIgAygJEhYKDm9ubGluZV9wbGF5ZXJzGAMgASgNEi4KBXBpbmdzGAQgAygLMh8uY2h1bmsubWFuYWdlbWVudC52MS5QaW5nU3RhdHVzEjsKCmRlcGxveW1lbnQYBSABKAsyJy5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveW1lbnRQcm9ncmVzcxIYChByZWFkeV90b19zdXNwZW5kGAYgASgIEg0KBWxlYXNlGAcgASgEEhAKCHNlcXVlbmNlGAggASgEEhgKEGRlc2lyZWRfcmV2aXNpb24YCSABKAQSDgoGbG9naW5zGAogASgEEh8KF3Jlc2lkZW50X2RlcGxveW1lbnRfaWRzGAsgAygJEiIKGnJlc2lkZW50X2RlcGxveW1lbnRzX2tub3duGAwgASgIIjMKClBpbmdTdGF0dXMSEAoIaG9zdG5hbWUYASABKAkSEwoLc3RhdHVzX2pzb24YAiABKAkicQoSRGVwbG95bWVudFByb2dyZXNzEhUKDWRlcGxveW1lbnRfaWQYASABKAkSMwoFc3RhdGUYAiABKA4yJC5jaHVuay5tYW5hZ2VtZW50LnYxLkRlcGxveW1lbnRTdGF0ZRIPCgdtZXNzYWdlGAMgASgJIhYKFFJlcG9ydFN0YXR1c1Jlc3BvbnNlInUKE1NldFdha2VBbGFybVJlcXVlc3QSEgoKZ2VuZXJhdGlvbhgBIAEoBBIsCghkdWVfdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASDQoFZXBvY2gYAyABKAQSDQoFbGVhc2UYBCABKAQiZwoUU2V0V2FrZUFsYXJtUmVzcG9uc2USEgoKZ2VuZXJhdGlvbhgBIAEoBBIsCghkdWVfdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASDQoFZXBvY2gYAyABKAQivAEKFUVuc3VyZUNhcGFjaXR5UmVxdWVzdBISCgpyZXF1ZXN0X2lkGAEgASgJEi8KCHdvcmtsb2FkGAIgASgOMh0uY2h1bmsubWFuYWdlbWVudC52MS5Xb3JrbG9hZBIXCg9tYWNoaW5lX3Byb2ZpbGUYAyABKAkSEgoKcmVsZWFzZV9pZBgEIAEoCRIOCgZhcHBfaWQYBSABKAkSDQoFbGVhc2UYBiABKAQSEgoKY3JlZGVudGlhbBgHIAEoCSJJChZFbnN1cmVDYXBhY2l0eVJlc3BvbnNlEi8KCGNhcGFjaXR5GAEgASgLMh0uY2h1bmsubWFuYWdlbWVudC52MS5DYXBhY2l0eSJ2CghDYXBhY2l0eRISCgpyZXF1ZXN0X2lkGAEgASgJEjEKBXN0YXRlGAIgASgOMiIuY2h1bmsubWFuYWdlbWVudC52MS5DYXBhY2l0eVN0YXRlEhIKCm1hY2hpbmVfaWQYAyABKAkSDwoHbWVzc2FnZRgEIAEoCSI7ChZSZWxlYXNlQ2FwYWNpdHlSZXF1ZXN0EhIKCnJlcXVlc3RfaWQYASABKAkSDQoFbGVhc2UYAiABKAQiSgoXUmVsZWFzZUNhcGFjaXR5UmVzcG9uc2USLwoIY2FwYWNpdHkYASABKAsyHS5jaHVuay5tYW5hZ2VtZW50LnYxLkNhcGFjaXR5IkcKElJlcG9ydFVzYWdlUmVxdWVzdBIxCgdyZWNvcmRzGAEgAygLMiAuY2h1bmsubWFuYWdlbWVudC52MS5Vc2FnZVJlY29yZCKkAQoLVXNhZ2VSZWNvcmQSCgoCaWQYASABKAkSLgoKc3RhcnRfdGltZRgCIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASLAoIZW5kX3RpbWUYAyABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEhYKDnBsYXllcl9zZWNvbmRzGAQgASgEEhMKC2luc3RhbmNlX2lkGAUgASgJIhUKE1JlcG9ydFVzYWdlUmVzcG9uc2UiQwoRUmVwb3J0TG9nc1JlcXVlc3QSLgoHZW50cmllcxgBIAMoCzIdLmNodW5rLm1hbmFnZW1lbnQudjEuTG9nRW50cnkiFAoSUmVwb3J0TG9nc1Jlc3BvbnNlIkoKFFJlcG9ydE1ldHJpY3NSZXF1ZXN0EjIKB3NhbXBsZXMYASADKAsyIS5jaHVuay5tYW5hZ2VtZW50LnYxLk1ldHJpY1NhbXBsZSLYAQoMTWV0cmljU2FtcGxlEgwKBG5hbWUYASABKAkSPQoGbGFiZWxzGAIgAygLMi0uY2h1bmsubWFuYWdlbWVudC52MS5NZXRyaWNTYW1wbGUuTGFiZWxzRW50cnkSKAoEdGltZRgDIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXASDQoFdmFsdWUYBCABKAESEwoLaW5zdGFuY2VfaWQYBSABKAkaLQoLTGFiZWxzRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgJOgI4ASIXChVSZXBvcnRNZXRyaWNzUmVzcG9uc2UiTAoXUmVwb3J0RmFpbGVkQXV0aFJlcXVlc3QSMQoIZmFpbHVyZXMYASADKAsyHy5jaHVuay5tYW5hZ2VtZW50LnYxLkZhaWxlZEF1dGgiTgoKRmFpbGVkQXV0aBIWCg5jbGllbnRfYWRkcmVzcxgBIAEoCRIoCgR0aW1lGAIgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcCIaChhSZXBvcnRGYWlsZWRBdXRoUmVzcG9uc2UqYQoIV29ya2xvYWQSGAoUV09SS0xPQURfVU5TUEVDSUZJRUQQABIQCgxXT1JLTE9BRF9KVk0QARIUChBXT1JLTE9BRF9HQVRFV0FZEAIiBAgDEAMqDVdPUktMT0FEX0VYRUMqwAEKDUNhcGFjaXR5U3RhdGUSHgoaQ0FQQUNJVFlfU1RBVEVfVU5TUEVDSUZJRUQQABIfChtDQVBBQ0lUWV9TVEFURV9QUk9WSVNJT05JTkcQARIYChRDQVBBQ0lUWV9TVEFURV9SRUFEWRACEhkKFUNBUEFDSVRZX1NUQVRFX0ZBSUxFRBADEhsKF0NBUEFDSVRZX1NUQVRFX1JFTEVBU0VEEAQSHAoYQ0FQQUNJVFlfU1RBVEVfUkVMRUFTSU5HEAUypgcKEkVudmlyb25tZW50U2VydmljZRJTCgZBdHRhY2gSIi5jaHVuay5tYW5hZ2VtZW50LnYxLkF0dGFjaFJlcXVlc3QaIy5jaHVuay5tYW5hZ2VtZW50LnYxLkF0dGFjaFJlc3BvbnNlMAESYwoMUmVwb3J0U3RhdHVzEiguY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRTdGF0dXNSZXF1ZXN0GikuY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRTdGF0dXNSZXNwb25zZRJjCgxTZXRXYWtlQWxhcm0SKC5jaHVuay5tYW5hZ2VtZW50LnYxLlNldFdha2VBbGFybVJlcXVlc3QaKS5jaHVuay5tYW5hZ2VtZW50LnYxLlNldFdha2VBbGFybVJlc3BvbnNlEmkKDkVuc3VyZUNhcGFjaXR5EiouY2h1bmsubWFuYWdlbWVudC52MS5FbnN1cmVDYXBhY2l0eVJlcXVlc3QaKy5jaHVuay5tYW5hZ2VtZW50LnYxLkVuc3VyZUNhcGFjaXR5UmVzcG9uc2USbAoPUmVsZWFzZUNhcGFjaXR5EisuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlQ2FwYWNpdHlSZXF1ZXN0GiwuY2h1bmsubWFuYWdlbWVudC52MS5SZWxlYXNlQ2FwYWNpdHlSZXNwb25zZRJgCgtSZXBvcnRVc2FnZRInLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0VXNhZ2VSZXF1ZXN0GiguY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRVc2FnZVJlc3BvbnNlEl0KClJlcG9ydExvZ3MSJi5jaHVuay5tYW5hZ2VtZW50LnYxLlJlcG9ydExvZ3NSZXF1ZXN0GicuY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRMb2dzUmVzcG9uc2USZgoNUmVwb3J0TWV0cmljcxIpLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0TWV0cmljc1JlcXVlc3QaKi5jaHVuay5tYW5hZ2VtZW50LnYxLlJlcG9ydE1ldHJpY3NSZXNwb25zZRJvChBSZXBvcnRGYWlsZWRBdXRoEiwuY2h1bmsubWFuYWdlbWVudC52MS5SZXBvcnRGYWlsZWRBdXRoUmVxdWVzdBotLmNodW5rLm1hbmFnZW1lbnQudjEuUmVwb3J0RmFpbGVkQXV0aFJlc3BvbnNlYgZwcm90bzM", [file_chunk_management_v1_common, file_google_protobuf_timestamp], ); @@ -481,12 +481,24 @@ export type ReportStatusRequest = Message<"chunk.management.v1.ReportStatusReque * The IDs of every deployment resident in core's backend, including one that * is loading. The management plane keeps the asset revisions of these * deployments, which core's JVMs may still read, whatever their state there. - * It stores the set of the latest accepted report, so each report carries the - * whole set. + * It stores the set of the latest accepted report that read it, so each such + * report carries the whole set. Meaningful only when + * resident_deployments_known is true. * * @generated from field: repeated string resident_deployment_ids = 11; */ residentDeploymentIds: string[]; + + /** + * True only when this observation read core's resident set successfully. A + * report that repeats an earlier read or couldn't read the set leaves it + * false, and the management plane then keeps the stored IDs but also keeps + * the assets of every deployment of the environment, as it does after a new + * lease until a report reads the set. + * + * @generated from field: bool resident_deployments_known = 12; + */ + residentDeploymentsKnown: boolean; }; /** diff --git a/packages/management/README.md b/packages/management/README.md index 227b9991..2af78d5a 100644 --- a/packages/management/README.md +++ b/packages/management/README.md @@ -105,14 +105,15 @@ project's 10 newest ready revisions, or declared by an upload within the last ho is also kept while a deployment of one of the project's environments pins it and is active or unfinished, one of the environment's 10 newest (so a rollback to it works), superseded but newer than the environment's active deployment (core may still activate it), or in the set of deployments resident in core's backend that the environment's latest accepted -status report listed. Core decides what stays resident, so no clock or setting of management's can lag it. The rest are -deleted, and with them every blob no remaining revision holds, from the database; their keys are recorded in -`asset_deletions` and the release store's objects are deleted in a second transaction that skips a key a new upload has -claimed again. A delete the store fails stays recorded and is retried by the next collection. A store delete still -running when management loses its lock, such as to a crash, can land after the same content is uploaded again, which -pushing the assets again repairs. A rollback to a deployment older than the 10 newest fails once its revision is gone. -Collection holds a per-project lock that uploads, deployments, activations and head moves share, so it never deletes a -blob a revision is being declared with or a revision a deployment or the head is pinning. +status report listed. While that report didn't read the set, or none has been accepted since core's latest attach, every +deployment of the environment is kept. Core decides what stays resident, so no clock or setting of management's can lag +it. The rest are deleted, and with them every blob no remaining revision holds, from the database; their keys are +recorded in `asset_deletions` and the release store's objects are deleted in a second transaction that skips a key a new +upload has claimed again. A delete the store fails stays recorded and is retried by the next collection. A store delete +still running when management loses its lock, such as to a crash, can land after the same content is uploaded again, +which pushing the assets again repairs. A rollback to a deployment older than the 10 newest fails once its revision is +gone. Collection holds a per-project lock that uploads, deployments, activations and head moves share, so it never +deletes a blob a revision is being declared with or a revision a deployment or the head is pinning. ## Release storage diff --git a/packages/management/migrations/0005_resident_deployments.sql b/packages/management/migrations/0005_resident_deployments.sql index a1c5c8d1..022ceda7 100644 --- a/packages/management/migrations/0005_resident_deployments.sql +++ b/packages/management/migrations/0005_resident_deployments.sql @@ -1 +1,2 @@ -ALTER TABLE "environments" ADD COLUMN "resident_deployment_ids" text[] DEFAULT '{}' NOT NULL; \ No newline at end of file +ALTER TABLE "environments" ADD COLUMN "resident_deployment_ids" text[] DEFAULT '{}' NOT NULL;--> statement-breakpoint +ALTER TABLE "environments" ADD COLUMN "resident_deployments_known" boolean DEFAULT false NOT NULL; \ No newline at end of file diff --git a/packages/management/migrations/meta/0005_snapshot.json b/packages/management/migrations/meta/0005_snapshot.json index 52ebae79..dbb22917 100644 --- a/packages/management/migrations/meta/0005_snapshot.json +++ b/packages/management/migrations/meta/0005_snapshot.json @@ -970,6 +970,13 @@ "notNull": true, "default": "'{}'" }, + "resident_deployments_known": { + "name": "resident_deployments_known", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, "pings": { "name": "pings", "type": "jsonb", diff --git a/packages/management/src/assets/retention.ts b/packages/management/src/assets/retention.ts index 00e89d0a..23838e09 100644 --- a/packages/management/src/assets/retention.ts +++ b/packages/management/src/assets/retention.ts @@ -38,8 +38,9 @@ export async function lockAssets(tx: Db, projectId: string, mode: "shared" | "ex * ready revisions, or protected by an upload declared within `uploadLifetimeMs`. It is also kept while an environment * has a deployment that pins it and is active, unfinished, one of the environment's `keptRecentDeployments` newest, * superseded but still activatable (newer than the active one, as `activateDeployment` accepts), or resident in core's - * backend as of the latest status report management accepted. Core decides what stays resident, so no clock or setting - * of management's can lag it. Returns how many revisions it deleted and how many blobs it removed from the release store. + * backend as of the latest status report management accepted that read it, or whenever the environment's latest + * accepted report didn't read it or none has been accepted under the current lease. Core decides what stays resident, + * so no clock or setting of management's can lag it. Returns how many revisions it deleted and how many blobs it removed from the release store. * * The database rows are deleted first, recording the blobs' keys in `asset_deletions`; a later transaction deletes the * bytes of each key no row has claimed again. A store delete that fails leaves its key recorded, so the next @@ -95,7 +96,7 @@ async function deleteUnusedRows(tx: Db, projectId: string): Promise { select max(a.seq) from deployments a where a.environment_id = d.environment_id and a.state = ${DeploymentState.ACTIVE} ), 0) as activatable, - d.id = any(e.resident_deployment_ids) as resident, + not e.resident_deployments_known or d.id = any(e.resident_deployment_ids) as resident, row_number() over (partition by d.environment_id order by d.seq desc) as recency from deployments d join environments e on e.id = d.environment_id where e.project_id = ${projectId} diff --git a/packages/management/src/environments/service.ts b/packages/management/src/environments/service.ts index 8151f4da..576d68d3 100644 --- a/packages/management/src/environments/service.ts +++ b/packages/management/src/environments/service.ts @@ -105,7 +105,8 @@ export function environmentService(deps: Deps): Partial>() .notNull() .default(sql`'{}'::jsonb`), diff --git a/packages/management/test/assets.test.ts b/packages/management/test/assets.test.ts index 3c279e2f..a5ecc1db 100644 --- a/packages/management/test/assets.test.ts +++ b/packages/management/test/assets.test.ts @@ -244,10 +244,12 @@ describe.skipIf(!databaseUrl)("AssetService", () => { const client = h.client(EnvironmentService, await issueEnvironmentToken(h.db, environmentId)); const abort = new AbortController(); const stream = client.attach({ instanceId: crypto.randomUUID(), core: true, epoch: 1n }, { signal: abort.signal }); - const { lease, revision } = (await stream[Symbol.asyncIterator]().next()).value; + const { lease: attachedLease, revision } = (await stream[Symbol.asyncIterator]().next()).value; let sequence = 0n; + let lease = attachedLease; return { projectId, + environmentId, first, second, fillers, @@ -257,14 +259,25 @@ describe.skipIf(!databaseUrl)("AssetService", () => { for (let i = 0; i < keptRecentDeployments + 1; i++) await pin(fillers[0]?.id ?? ""); await h.sql`update asset_revisions set protected_until = now() - interval '1 minute' where project_id = ${projectId}`; }, - report: (resident: string[], deployment?: { deploymentId: string; state: DeploymentState }) => + report: (resident: string[], deployment?: { deploymentId: string; state: DeploymentState }, known = true) => client.reportStatus({ lease, sequence: ++sequence, desiredRevision: revision, residentDeploymentIds: resident, + residentDeploymentsKnown: known, ...(deployment ? { deployment } : {}), }), + /** Attaches the core again, which starts a new lease whose reports haven't read the resident set. */ + reattach: async () => { + const next = client.attach( + { instanceId: crypto.randomUUID(), core: true, epoch: 2n }, + { signal: abort.signal }, + ); + const first = (await next[Symbol.asyncIterator]().next()).value; + lease = first.lease; + sequence = 0n; + }, /** A report sent now whose arrival is delayed until the returned function is called. */ delayed: (resident: string[], deployment: { deploymentId: string; state: DeploymentState }) => { const delayedSequence = ++sequence; @@ -274,6 +287,7 @@ describe.skipIf(!databaseUrl)("AssetService", () => { sequence: delayedSequence, desiredRevision: revision, residentDeploymentIds: resident, + residentDeploymentsKnown: true, deployment, }); }, @@ -320,6 +334,47 @@ describe.skipIf(!databaseUrl)("AssetService", () => { env.close(); }); + test("a report that didn't read the resident set keeps the stored set and every revision of the environment", async () => { + const env = await aged(); + const old = await env.pin(env.first.id); + await env.report([old], { deploymentId: old, state: DeploymentState.ACTIVE }); + const current = await env.pin(env.second.id); + await env.report([current], { deploymentId: current, state: DeploymentState.ACTIVE }); + await env.bury(); + + await env.report([], undefined, false); + await collectAssets(h.deps, env.projectId); + expect(await env.stored(env.first.id)).toBe(true); + expect( + await h.sql< + { resident_deployment_ids: string[] }[] + >`select resident_deployment_ids from environments where id = ${env.environmentId}`, + ).toEqual([{ resident_deployment_ids: [current] }]); + + await env.report([current]); + await collectAssets(h.deps, env.projectId); + expect(await env.stored(env.first.id)).toBe(false); + env.close(); + }); + + test("a new lease keeps every revision of the environment until a report reads the resident set", async () => { + const env = await aged(); + const old = await env.pin(env.first.id); + await env.report([old], { deploymentId: old, state: DeploymentState.ACTIVE }); + const current = await env.pin(env.second.id); + await env.report([current], { deploymentId: current, state: DeploymentState.ACTIVE }); + await env.bury(); + + await env.reattach(); + await collectAssets(h.deps, env.projectId); + expect(await env.stored(env.first.id)).toBe(true); + + await env.report([current]); + await collectAssets(h.deps, env.projectId); + expect(await env.stored(env.first.id)).toBe(false); + env.close(); + }); + test("a completion paused after verifying its blobs holds off collection, so the revision does not become ready without them", async () => { const { projectId } = await createEnvironment(h); const assets = h.client(AssetService); diff --git a/proto/chunk/management/v1/environment.proto b/proto/chunk/management/v1/environment.proto index 89968ba5..7e774b1a 100644 --- a/proto/chunk/management/v1/environment.proto +++ b/proto/chunk/management/v1/environment.proto @@ -221,9 +221,16 @@ message ReportStatusRequest { // The IDs of every deployment resident in core's backend, including one that // is loading. The management plane keeps the asset revisions of these // deployments, which core's JVMs may still read, whatever their state there. - // It stores the set of the latest accepted report, so each report carries the - // whole set. + // It stores the set of the latest accepted report that read it, so each such + // report carries the whole set. Meaningful only when + // resident_deployments_known is true. repeated string resident_deployment_ids = 11; + // True only when this observation read core's resident set successfully. A + // report that repeats an earlier read or couldn't read the set leaves it + // false, and the management plane then keeps the stored IDs but also keeps + // the assets of every deployment of the environment, as it does after a new + // lease until a report reads the set. + bool resident_deployments_known = 12; } message PingStatus {