Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

40 changes: 40 additions & 0 deletions crates/chunk-build/src/assets.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
//! assets from.

use std::{
collections::BTreeSet,
fs,
io::{self, Read, Write},
path::{Path, PathBuf},
Expand Down Expand Up @@ -113,6 +114,20 @@ impl Store {
Ok(revision)
}

/// Removes every revision but `kept`, their materialized directories, and the blobs none of `kept` holds. Hidden
/// entries, such as writes in progress, stay. Directories materialized from removed blobs keep their hard links.
/// Returns how many entries it removed.
/// # Errors
/// Reports filesystem failures; whatever it removed before one stays removed.
pub fn retain(&self, kept: &[&AssetRevision]) -> io::Result<usize> {
let ids: BTreeSet<String> = kept.iter().map(|revision| revision.id()).collect();
let blobs: BTreeSet<&str> = kept.iter().flat_map(|revision| revision.blobs().into_keys()).collect();
let revision = |name: &str| name.strip_suffix(".json").is_some_and(|id| ids.contains(id));
Ok(remove_unless(&self.root.join("blobs"), |name| blobs.contains(name))?
+ remove_unless(&self.root.join("revisions"), revision)?
+ remove_unless(&self.root.join("apps"), |name| ids.contains(name))?)
}

fn blob(&self, sha256: &str) -> io::Result<PathBuf> {
check_digest(sha256)?;
Ok(self.root.join("blobs").join(sha256))
Expand Down Expand Up @@ -245,6 +260,31 @@ fn sync_parent(path: &Path) -> io::Result<()> {
Ok(())
}

/// Removes the entries of `directory` that are not hidden and whose names `keep` rejects, returning how many.
fn remove_unless(directory: &Path, keep: impl Fn(&str) -> bool) -> io::Result<usize> {
let entries = match fs::read_dir(directory) {
Ok(entries) => entries,
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(0),
Err(error) => return Err(error),
};
let mut removed = 0;
for entry in entries {
let entry = entry?;
let name = entry.file_name();
if name.to_str().is_some_and(|name| name.starts_with('.') || keep(name)) {
continue;
}
let path = entry.path();
let result = if entry.file_type()?.is_dir() { fs::remove_dir_all(&path) } else { fs::remove_file(&path) };
match result {
Ok(()) => removed += 1,
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
Err(error) => return Err(error),
}
}
Ok(removed)
}

fn check_digest(value: &str) -> io::Result<()> {
if value.len() != 64 || !value.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) {
return Err(io::Error::new(io::ErrorKind::InvalidInput, format!("invalid SHA-256 {value:?}")));
Expand Down
22 changes: 22 additions & 0 deletions crates/chunk-build/src/assets/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -117,3 +117,25 @@ fn incomplete_materialized_directories_are_rebuilt() {
});
assert_eq!(fs::read(directory.join("worlds/lobby.polar")).unwrap(), POLAR);
}

#[test]
fn retaining_revisions_removes_every_other_revision_and_blob() {
let project_directory = tempfile::tempdir().unwrap();
let root = project_directory.path();
project(root);
let store = Store::new(root.join("store"));
let old = build_revision(root, &store).unwrap();
materialize(&store, &old, "arena").unwrap();
write(root, "assets/config/game.json", b"changed");
let new = build_revision(root, &store).unwrap();
let unfinished = store.root().join("blobs/.write-unfinished");
fs::write(&unfinished, b"").unwrap();

assert_eq!(store.retain(&[&new]).unwrap(), 3);
assert!(new.blobs().keys().all(|sha256| store.contains(sha256).unwrap()));
let changed = &old.shared["config/game.json"].sha256;
assert!(!store.contains(changed).unwrap());
assert!(store.read_revision(&old.id()).is_err() && !store.root().join("apps").join(old.id()).exists());
assert_eq!(store.read_revision(&new.id()).unwrap(), new);
assert!(unfinished.exists());
}
1 change: 1 addition & 0 deletions crates/chunk-cli/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ chunk-environment = { path = "../chunk-environment", default-features = false }
chunk-management = { path = "../chunk-management" }
chunk-proto = { path = "../chunk-proto" }
clap = { version = "4", features = ["derive", "env"] }
futures-util = "0.3"
url = { version = "2", features = ["serde"] }
dirs = "6"
serde = { version = "1", features = ["derive"] }
Expand Down
20 changes: 12 additions & 8 deletions crates/chunk-cli/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,10 @@ State lives in `PROJECT/.chunk/local`, or `--state DIR`: the backend's data, pub
`control/nodes/`, and `control.json`, which `chunk players` and `chunk nodes` read to reach control. One `chunk dev`
runs per project at a time. `--java PATH` overrides the Java executable Gradle selected; it must satisfy the release's
Java version. `--offline-logins` admits players without Mojang authentication, under offline-mode UUIDs; use it only for
local testing. `--connection-timeout-seconds` (default 10, from 1 to 3600) sets the gateway's login deadline.
local testing. `--connection-timeout-seconds` (default 10, from 1 to 3600) sets the gateway's login deadline. Players'
clients download resource packs over HTTP from an ephemeral loopback port, or from `--pack-bind ADDR`; when they reach
it at another address, such as from other machines, `--pack-url URL` names it (required when `--pack-bind` listens on
every address, such as `0.0.0.0:25568`).

Backend functions and the apps' generated `Vars` read `chunk.toml`'s top-level `[vars]`, or with `--env NAME` its
`[env.NAME.vars]` over them, key by key. Actions read their secrets from `.dev.vars` in the project root: `NAME=value`
Expand Down Expand Up @@ -203,13 +206,14 @@ environment holds up to 256 secrets. `chunk secrets list` shows names and versio

An asset revision is an immutable manifest of the project's worlds, resource packs and files by content digest, which
every build writes beside its release; a deployment pins one release and one revision, so assets change without
rebuilding the release. `chunk assets push` builds the project as `chunk dev` does, uploads the blobs the project lacks,
and moves the project's head to the new revision. It records the revision as the base in `.chunk/assets.json`, and
refuses when someone else moved the head since, until you pull; `--force` replaces the head anyway. `chunk assets pull`
fetches the head and updates each local file whose source is one file (plain files, `.polar` worlds and `.zip` packs): a
file you haven't changed since the base is replaced or deleted, and a file you changed is left as it is and reported as
a conflict, which your next push publishes. Pack directories are listed but never pulled. `chunk assets deploy --env E`
deploys the environment's active release with the head revision, or the one `--revision ID` names.
rebuilding the release. `chunk assets push` builds only the project's assets, without Gradle, into the store `chunk dev`
uses, uploads the blobs the project lacks, and moves the project's head to the new revision. It records the revision as
the base in `.chunk/assets.json`, and refuses when someone else moved the head since, until you pull; `--force` replaces
the head anyway. `chunk assets pull` fetches the head and updates each local file whose source is one file (plain files,
`.polar` worlds and `.zip` packs): a file you haven't changed since the base is replaced or deleted, and a file you
changed is left as it is and reported as a conflict, which your next push publishes. Pack directories are listed but
never pulled. `chunk assets deploy --env E` deploys the environment's active release with the head revision, or the one
`--revision ID` names.

`chunk domains add HOSTNAME --env E` claims a custom hostname and prints its state and the DNS records to create: a TXT
ownership proof and an SRV record that routes players to the environment. A hostname routes only once it is verified:
Expand Down
9 changes: 8 additions & 1 deletion crates/chunk-cli/src/local.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,13 @@ pub(crate) struct Options {
/// Control address, shared by every release.
#[arg(long, default_value = "127.0.0.1:25567")]
control_bind: SocketAddr,
/// Where the resource pack server listens (defaults to an ephemeral loopback port).
#[arg(long, default_value = "127.0.0.1:0")]
pack_bind: SocketAddr,
/// The URL players' clients download packs from, such as `http://192.168.1.20:25568`, when they can't reach
/// --pack-bind's address; required when it listens on every address.
#[arg(long, value_name = "URL")]
pack_url: Option<url::Url>,
/// Print plain progress lines instead of the terminal UI (automatic when stdout is not a terminal).
#[arg(long)]
plain: bool,
Expand Down Expand Up @@ -160,7 +167,7 @@ async fn serve(
let required = built.java.version;
reporter.running("Java", format!("Checking Java {required}+"));
let assets = built.asset_store.clone();
let packs = packs::Packs::start(assets.clone()).await?;
let packs = packs::Packs::start(assets.clone(), options.pack_bind, options.pack_url.as_ref()).await?;
let pack_url_prefix = Some(packs.url_prefix().to_owned());
let staged = stage(&project, built, options.java.as_deref(), pack_url_prefix.as_deref(), &stop).await?;
packs.serve(&staged.control.assets);
Expand Down
34 changes: 28 additions & 6 deletions crates/chunk-cli/src/local/packs.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
//! Serves the resource packs of the releases dev staged to its players' clients, over plain HTTP/1.1 on loopback.
//! Serves the resource packs of the releases dev staged to its players' clients, over plain HTTP/1.1.

use std::{
collections::BTreeSet,
convert::Infallible,
io,
net::Ipv4Addr,
net::SocketAddr,
path::PathBuf,
sync::{Arc, Mutex, PoisonError},
time::Duration,
Expand All @@ -23,10 +23,13 @@ pub(super) struct Packs {
}

impl Packs {
/// Starts serving packs from the asset store at `store` on an ephemeral loopback port.
pub async fn start(store: PathBuf) -> io::Result<Self> {
let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).await?;
let url_prefix = format!("http://{}/packs/", listener.local_addr()?);
/// Starts serving packs from the asset store at `store` on `bind`, to clients that reach it at `public`, or else
/// at the address it listens on.
pub async fn start(store: PathBuf, bind: SocketAddr, public: Option<&url::Url>) -> io::Result<Self> {
let listener = TcpListener::bind(bind)
.await
.map_err(|error| io::Error::new(error.kind(), format!("cannot serve packs on {bind}: {error}")))?;
let url_prefix = url_prefix(listener.local_addr()?, public)?;
let allowed = Arc::default();
let server = tokio::spawn(serve(listener, store.join("blobs"), Arc::clone(&allowed))).abort_handle();
Ok(Self { url_prefix, allowed, server })
Expand All @@ -49,6 +52,25 @@ impl Drop for Packs {
}
}

/// Where clients download packs: `public`'s path followed by `packs/`, or else the listening address's.
pub(super) fn url_prefix(listening: SocketAddr, public: Option<&url::Url>) -> io::Result<String> {
let Some(public) = public else {
if listening.ip().is_unspecified() {
return Err(io::Error::other(format!(
"--pack-bind {listening} listens on every address; set --pack-url to the one players' clients reach"
)));
}
return Ok(format!("http://{listening}/packs/"));
};
if !["http", "https"].contains(&public.scheme()) || public.host().is_none() {
return Err(io::Error::other(format!("--pack-url {public} must be an http or https URL with a host")));
}
if public.query().is_some() || public.fragment().is_some() {
return Err(io::Error::other(format!("--pack-url {public} cannot have a query or fragment")));
}
Ok(format!("{}/packs/", public.as_str().trim_end_matches('/')))
}

async fn serve(listener: TcpListener, blobs: PathBuf, allowed: Arc<Mutex<BTreeSet<String>>>) {
let mut connections = JoinSet::new();
loop {
Expand Down
14 changes: 12 additions & 2 deletions crates/chunk-cli/src/local/packs/tests.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use sha2::{Digest, Sha256};
use tokio::io::{AsyncReadExt, AsyncWriteExt};

use super::Packs;
use super::{Packs, url_prefix};

#[tokio::test]
async fn only_the_packs_of_staged_releases_are_served() {
Expand All @@ -11,7 +11,7 @@ async fn only_the_packs_of_staged_releases_are_served() {
let (pack, other): (&[u8], &[u8]) = (b"PK pack", b"a world");
store.insert(&sha256(pack), pack).unwrap();
store.insert(&sha256(other), other).unwrap();
let packs = Packs::start(directory.path().into()).await.unwrap();
let packs = Packs::start(directory.path().into(), "127.0.0.1:0".parse().unwrap(), None).await.unwrap();
let blob = chunk_contract::PackBlob { sha256: sha256(pack), sha1: "0".repeat(40), size: 7 };
let assets = chunk_control::DeploymentAssets {
packs: [("base".into(), blob)].into(),
Expand All @@ -36,3 +36,13 @@ async fn only_the_packs_of_staged_releases_are_served() {
assert!(served.starts_with("HTTP/1.1 200") && served.ends_with("PK pack"), "{served}");
assert!(get(sha256(other)).await.starts_with("HTTP/1.1 404"));
}

#[test]
fn packs_are_served_at_the_public_url_or_else_the_listening_address() {
let listening = "127.0.0.1:4000".parse().unwrap();
assert_eq!(url_prefix(listening, None).unwrap(), "http://127.0.0.1:4000/packs/");
let public = url::Url::parse("https://packs.example.com/dev/").unwrap();
assert_eq!(url_prefix(listening, Some(&public)).unwrap(), "https://packs.example.com/dev/packs/");
assert!(url_prefix("0.0.0.0:4000".parse().unwrap(), None).is_err());
assert!(url_prefix(listening, Some(&url::Url::parse("ftp://host/").unwrap())).is_err());
}
54 changes: 35 additions & 19 deletions crates/chunk-cli/src/platform/assets.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
//! runs an environment's release against a revision.

use std::{
collections::BTreeMap,
collections::{BTreeMap, BTreeSet},
fs,
io::{self, Read, Write as _},
path::{Component, Path, PathBuf},
Expand All @@ -19,6 +19,7 @@ use chunk_management::{
},
};
use clap::{Args, Subcommand};
use futures_util::TryStreamExt as _;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use tokio_util::sync::CancellationToken;
Expand All @@ -28,11 +29,13 @@ use super::{
deploy::{self, mebibytes},
resources::{table, time},
};
use crate::building::{self, BuildMode, progress::Progress};
use plan::{Entry, Pull, Push};

mod plan;

/// How many blobs a pull downloads at once.
const CONCURRENT_DOWNLOADS: usize = 4;

#[derive(Args)]
pub(crate) struct Assets {
#[command(subcommand)]
Expand Down Expand Up @@ -192,19 +195,25 @@ pub(super) async fn upload_revision(

async fn push(session: &Session, path: PathBuf, selector: &ProjectArg, force: bool) -> io::Result<()> {
let project = session.project(selector).await?;
let root = path.canonicalize()?;
chunk_service::run(|stop| async move {
let local = building::prepare(&building::Options { project: path, output: None, frozen: false })?;
cliclack::log::info("Building assets…")?;
let built = building::execute(&local, BuildMode::Dev, stop.clone(), Progress::default()).await?;
let store = dev_store(&root);
let (project_root, target) = (root.clone(), store.clone());
let built = tokio::task::spawn_blocking(move || chunk_build::assets::build_revision(&project_root, &target));
let revision = &tokio::select! {
built = built => built.map_err(io::Error::other)??,
() = stop.cancelled() => return Err(io::Error::new(io::ErrorKind::Interrupted, "push stopped")),
};
let client = &session.client;
let (revision, id) = (&built.assets, built.assets.id());
let id = revision.id();
let head = head_id(client, &project.id).await?;
let base = read_base(&local.root, &project.id)?;
let base = read_base(&root, &project.id)?;
match plan::check_push(&head, base.as_deref(), &id, force) {
Push::UpToDate => {
// Uploads whatever of the head the platform lost; normally nothing.
upload_revision(client, &project.id, revision, &built.asset_store).await?;
write_base(&local.root, &project.id, &id)?;
upload_revision(client, &project.id, revision, store.root()).await?;
write_base(&root, &project.id, &id)?;
return cliclack::log::info(format!("Nothing to push; the head is already revision {id}"));
}
Push::Moved => {
Expand All @@ -219,11 +228,11 @@ async fn push(session: &Session, path: PathBuf, selector: &ProjectArg, force: bo
let previous = if head.is_empty() {
AssetRevision::default()
} else {
manifest(client, &Store::new(&built.asset_store), &project.id, &head).await?
manifest(client, &store, &project.id, &head).await?
};
let moved = tokio::select! {
moved = async {
upload_revision(client, &project.id, revision, &built.asset_store).await?;
upload_revision(client, &project.id, revision, store.root()).await?;
let request = SetAssetHeadRequest {
project_id: project.id.clone(),
revision_id: id.clone(),
Expand All @@ -242,7 +251,7 @@ async fn push(session: &Session, path: PathBuf, selector: &ProjectArg, force: bo
() = stop.cancelled() => return Err(io::Error::new(io::ErrorKind::Interrupted, "push stopped")),
};
moved?;
write_base(&local.root, &project.id, &id)?;
write_base(&root, &project.id, &id)?;
let mut message = format!("Pushed assets revision {id}");
for line in plan::summary(&plan::entries(&previous), &plan::entries(revision)) {
message.push_str("\n ");
Expand Down Expand Up @@ -298,7 +307,7 @@ async fn pull_head(
let mut changes = Vec::new();
let mut conflicts = Vec::new();
let mut skipped = Vec::new();
for entry in base.keys().chain(heads.keys()).collect::<std::collections::BTreeSet<_>>() {
for entry in base.keys().chain(heads.keys()).collect::<BTreeSet<_>>() {
let (before, after) = (base.get(entry).copied(), heads.get(entry).copied());
if before == after {
continue;
Expand All @@ -321,16 +330,23 @@ async fn pull_head(
}
}
}
let blobs: BTreeMap<_, _> =
response.downloads.iter().map(|download| (download.sha256.as_str(), download)).collect();
let urls: BTreeMap<_, _> =
response.downloads.iter().map(|download| (download.sha256.as_str(), download.url.as_str())).collect();
let (sizes, mut missing) = (head.blobs(), BTreeSet::new());
for sha256 in changes.iter().filter_map(|change| change.write) {
if store.contains(sha256)? {
continue;
if !store.contains(sha256)? {
missing.insert(sha256);
}
let url =
blobs.get(sha256).ok_or_else(|| io::Error::other(format!("The platform gave no download for {sha256}")))?;
download(client, &store, sha256, head.blobs()[sha256], &url.url).await?;
}
futures_util::stream::iter(missing.into_iter().map(Ok))
.try_for_each_concurrent(CONCURRENT_DOWNLOADS, |sha256| {
let (url, size, store) = (urls.get(sha256).copied(), sizes[sha256], &store);
async move {
let url = url.ok_or_else(|| io::Error::other(format!("The platform gave no download for {sha256}")))?;
download(client, store, sha256, size, url).await
}
})
.await?;
let (mut updated, mut deleted) = (Vec::new(), Vec::new());
for change in changes {
let shown = format!("{}: {}", change.entry, change.path.strip_prefix(root).unwrap_or(&change.path).display());
Expand Down
1 change: 1 addition & 0 deletions crates/chunk-environment/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ chunk-proto = { path = "../chunk-proto" }
chunk-proxy = { path = "../chunk-proxy", default-features = false }
chunk-service = { path = "../chunk-service" }
chunk-store = { path = "../chunk-store" }
futures-util = "0.3"
http-body = "1"
prost = "0.14.3"
serde = { version = "1", features = ["derive"] }
Expand Down
Loading