From 9d323f7d8163399504e7538cf90b4ae6e7db0f5a Mon Sep 17 00:00:00 2001 From: GeiserX <9169332+GeiserX@users.noreply.github.com> Date: Sun, 22 Nov 2026 01:27:34 +0100 Subject: [PATCH 1/5] feat(live): log in, keep sessions and take run messages in og live serve og live serve now does what the Python server does behind its password: the PBKDF2-SHA256 hash of bin/owngenome_live.py hash-password (an existing site keeps its login), sessions with their hour and idle limits, the limit on failed logins, --allowed-host, --trust-proxy, HTTPS with --tls-cert and --tls-key, --cookie-secure, and the ingest listener that cuts and stores each message. /metrics needs a login or its bearer token once a password is set. This is the parity step; the D12 refusals follow. Bead: D1 Bead: D14 --- rust/Cargo.lock | 170 ++++- rust/crates/og-live/Cargo.toml | 4 + rust/crates/og-live/src/auth.rs | 706 +++++++++++++++++- rust/crates/og-live/src/ingest.rs | 383 +++++++++- rust/crates/og-live/src/plugins.rs | 6 +- rust/crates/og-live/src/server.rs | 428 +++++++---- .../og-live/tests/port_live_auth_ingest.rs | 643 ++++++++++++++++ .../tests/port_live_auth_ingest_browser.rs | 63 ++ rust/crates/og-live/tests/port_live_server.rs | 7 +- 9 files changed, 2268 insertions(+), 142 deletions(-) create mode 100644 rust/crates/og-live/tests/port_live_auth_ingest.rs create mode 100644 rust/crates/og-live/tests/port_live_auth_ingest_browser.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 307997f19..fe0b987aa 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -233,7 +233,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -291,6 +291,17 @@ dependencies = [ "quick-error", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -513,7 +524,7 @@ version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" dependencies = [ - "getrandom", + "getrandom 0.4.3", "libc", ] @@ -857,6 +868,8 @@ dependencies = [ "og-summary", "pycompat", "regex", + "rustls", + "sha2", ] [[package]] @@ -1034,6 +1047,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1149,6 +1168,20 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rust-htslib" version = "1.0.2" @@ -1196,7 +1229,41 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "102f35f7de7b06d8c61f1cff6592f5253e0dea431c74867aad5dfd70f44663fb" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a0760a887c14823fca5a1c970b01d47c328c696c0c64829d1df50e8f853fd5" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -1294,6 +1361,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.119" @@ -1389,6 +1462,12 @@ version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -1422,12 +1501,27 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -1437,6 +1531,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "writeable" version = "0.6.4" @@ -1487,6 +1645,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ccff8a4805aba1eaa4cd4afcd49f7557dd2cf5fd79ffc37f8959466effc5aa" + [[package]] name = "zerotrie" version = "0.2.5" diff --git a/rust/crates/og-live/Cargo.toml b/rust/crates/og-live/Cargo.toml index 779ae27d6..7c224667d 100644 --- a/rust/crates/og-live/Cargo.toml +++ b/rust/crates/og-live/Cargo.toml @@ -25,6 +25,10 @@ og-bloodtype.workspace = true # og live serve: the summary the step headlines read while summary.json is # older than the steps that are done (og summary collect, in-process). og-summary.workspace = true +# og live serve: the password hash (PBKDF2-SHA256), the session ids and +# HTTPS (--tls-cert), with ring as its only crypto, built from source. +sha2 = "0.11" +rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } [dev-dependencies] regex.workspace = true diff --git a/rust/crates/og-live/src/auth.rs b/rust/crates/og-live/src/auth.rs index 9123ad3d5..c19593cb8 100644 --- a/rust/crates/og-live/src/auth.rs +++ b/rust/crates/og-live/src/auth.rs @@ -1 +1,705 @@ -//! Sign-in for the live site. Empty until its unit of the 1.0.0 plan lands. +//! The live site's login: the password hash, the sessions, the limit on +//! failed logins, the client behind a trusted proxy, and HTTPS. +//! +//! The password is never stored. `og live serve hash-password` prints a +//! PBKDF2-SHA256 hash (`pbkdf2_sha256$$$`, the format +//! of `bin/owngenome_live.py hash-password`, so a site keeps its login), and +//! the server reads only that hash, from `--password-file` or +//! `OWNGENOME_LIVE_PASSWORD_HASH`. A login sets a random cookie; the server +//! keeps only the SHA-256 of it, and the access log only the first 8 +//! characters of that. + +use crate::server::{COOKIE, Req, quote}; +use crate::state::{lock, now}; +use crate::static_files as sf; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::fs::File; +use std::io::{self, BufRead, Read, Write}; +use std::net::{IpAddr, TcpStream}; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +/// The rounds `hash-password` writes. +pub const ITERATIONS: u32 = 600_000; +/// The window failed logins are counted in. +pub const FAIL_WINDOW_SECONDS: f64 = 15.0 * 60.0; +/// How long a client may take to finish the TLS handshake. +pub const TLS_HANDSHAKE: Duration = Duration::from_secs(10); +/// The largest login form read. +const MAX_FORM: i64 = 4096; + +// --- the password --------------------------------------------------------------------------- + +/// The two SHA-256 states of HMAC with `key`: after the inner and after +/// the outer padded key. +fn hmac_keys(key: &[u8]) -> (Sha256, Sha256) { + let mut k = [0u8; 64]; + if key.len() > 64 { + k[..32].copy_from_slice(&Sha256::digest(key)); + } else { + k[..key.len()].copy_from_slice(key); + } + let mut inner = Sha256::new(); + inner.update(k.map(|b| b ^ 0x36)); + let mut outer = Sha256::new(); + outer.update(k.map(|b| b ^ 0x5c)); + (inner, outer) +} + +/// HMAC-SHA256 of `data` under the key of `keys`. +fn hmac(keys: &(Sha256, Sha256), data: &[u8]) -> [u8; 32] { + let mut inner = keys.0.clone(); + inner.update(data); + let mut outer = keys.1.clone(); + outer.update(inner.finalize()); + let mut out = [0u8; 32]; + out.copy_from_slice(&outer.finalize()); + out +} + +/// PBKDF2-HMAC-SHA256 of `password` and `salt`, one 32-byte block +/// (`hashlib.pbkdf2_hmac("sha256", ...)`). +pub fn pbkdf2_sha256(password: &[u8], salt: &[u8], iterations: u32) -> [u8; 32] { + let keys = hmac_keys(password); + let mut first = salt.to_vec(); + first.extend_from_slice(&1u32.to_be_bytes()); + let mut u = hmac(&keys, &first); + let mut out = u; + for _ in 1..iterations { + u = hmac(&keys, &u); + for (o, x) in out.iter_mut().zip(u) { + *o ^= x; + } + } + out +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// `n` random bytes from the system. +pub fn random_bytes(n: usize) -> io::Result> { + let mut b = vec![0u8; n]; + File::open("/dev/urandom")?.read_exact(&mut b)?; + Ok(b) +} + +/// URL-safe base64 without padding (`secrets.token_urlsafe`). +pub fn base64url(bytes: &[u8]) -> String { + const ABC: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4); + for chunk in bytes.chunks(3) { + let n = chunk + .iter() + .enumerate() + .fold(0u32, |n, (i, b)| n | (u32::from(*b) << (16 - 8 * i))); + for i in 0..=chunk.len() { + out.push(char::from(ABC[(n >> (18 - 6 * i)) as usize & 63])); + } + } + out +} + +/// A random token of 32 bytes, as 43 URL-safe characters. +pub fn new_token() -> io::Result { + random_bytes(32).map(|b| base64url(&b)) +} + +/// The hash line of `password` with a new random salt. +pub fn hash_password(password: &str, iterations: u32) -> io::Result { + let salt = hex(&random_bytes(16)?); + let dk = pbkdf2_sha256(password.as_bytes(), salt.as_bytes(), iterations); + Ok(format!("pbkdf2_sha256${iterations}${salt}${}", hex(&dk))) +} + +/// (iterations, salt, hex digest) of a hash line, or why it is not one. +pub fn parse_hash(encoded: &str) -> Result<(u32, &str, &str), String> { + let bad = || { + "not a og live serve hash-password line (pbkdf2_sha256$$$)" + .to_owned() + }; + let parts: Vec<&str> = encoded.trim().split('$').collect(); + let [kind, rounds, salt, digest] = parts[..] else { + return Err(bad()); + }; + let rounds = (!rounds.is_empty() && rounds.bytes().all(|b| b.is_ascii_digit())) + .then(|| rounds.parse::().ok()) + .flatten() + .filter(|n| *n > 0); + let digest_ok = digest.len() == 64 + && digest + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)); + match rounds { + Some(n) if kind == "pbkdf2_sha256" && !salt.is_empty() && digest_ok => { + Ok((n, salt, digest)) + } + _ => Err(bad()), + } +} + +/// Equal bytes, in time that does not depend on where they differ. +pub fn same_secret(a: &[u8], b: &[u8]) -> bool { + a.len() == b.len() && a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)) == 0 +} + +/// Whether `password` is the one `encoded` hashes; false for a line that is +/// not a hash. +pub fn verify_password(password: &str, encoded: &str) -> bool { + let Ok((rounds, salt, digest)) = parse_hash(encoded) else { + return false; + }; + let dk = pbkdf2_sha256(password.as_bytes(), salt.as_bytes(), rounds); + same_secret(hex(&dk).as_bytes(), digest.as_bytes()) +} + +/// The hash from `--password-file` (else `env`, the value of +/// `OWNGENOME_LIVE_PASSWORD_HASH`): `None` for none set, an error for a file +/// that cannot be read or a value that is not a hash line. +pub fn read_password_hash( + file: Option<&Path>, + env: Option<&str>, +) -> Result, String> { + let h = match file { + Some(p) => std::fs::read_to_string(p) + .map_err(|e| format!("{}: {e}", p.display()))? + .trim() + .to_owned(), + None => env.unwrap_or("").trim().to_owned(), + }; + if h.is_empty() { + return Ok(None); + } + parse_hash(&h)?; + Ok(Some(h)) +} + +/// Sets the terminal's echo on or off; nothing when stdin is no terminal. +fn echo(on: bool) { + let _ = Command::new("stty") + .arg(if on { "echo" } else { "-echo" }) + .stdin(Stdio::inherit()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); +} + +/// One line of `input` without its line break. +fn read_line(input: &mut dyn BufRead) -> String { + let mut line = String::new(); + let _ = input.read_line(&mut line); + line.trim_end_matches(['\r', '\n']).to_owned() +} + +/// `og live serve hash-password`: reads a password (twice, without echo, +/// from a terminal; else one line of `input`) and prints its hash. +pub fn hash_password_cli( + args: &[String], + input: &mut dyn BufRead, + tty: bool, + out: &mut dyn Write, + err: &mut dyn Write, +) -> u8 { + if args.iter().any(|a| a == "-h" || a == "--help") { + let _ = writeln!( + out, + "usage: og live serve hash-password\n\nPrint the hash of a password for --password-file." + ); + return 0; + } + if let Some(a) = args.first() { + let _ = writeln!( + err, + "og live serve hash-password: error: unrecognized arguments: {a}" + ); + return 2; + } + let pw = if tty { + let ask = |err: &mut dyn Write, input: &mut dyn BufRead, prompt: &str| { + let _ = write!(err, "{prompt}"); + let _ = err.flush(); + echo(false); + let line = read_line(input); + echo(true); + let _ = writeln!(err); + line + }; + let first = ask(err, input, "Password: "); + if first != ask(err, input, "Again: ") { + let _ = writeln!(err, "ERROR: the two passwords differ"); + return 2; + } + first + } else { + read_line(input) + }; + if pw.chars().count() < 8 { + let _ = writeln!(err, "ERROR: use a password of 8 characters or more"); + return 2; + } + match hash_password(&pw, ITERATIONS) { + Ok(h) => { + let _ = writeln!(out, "{h}"); + 0 + } + Err(e) => { + let _ = writeln!(err, "ERROR: no random salt: {e}"); + 1 + } + } +} + +// --- sessions and failed logins --------------------------------------------------------------- + +/// The SHA-256 of a cookie, in hex: the only form of it the server keeps. +pub fn session_id(token: &str) -> String { + hex(&Sha256::digest(token.as_bytes())) +} + +#[derive(Clone, Copy)] +struct Session { + created: f64, + seen: f64, +} + +/// The password and the logins of a running site. +pub struct Auth { + /// The hash logins are checked against; `None`: no login. + pub hash: Option, + /// A login lasts at most this long. + pub session_hours: f64, + /// A login unused this long ends. + pub session_idle_minutes: f64, + /// Failed logins per client in [`FAIL_WINDOW_SECONDS`]. + pub login_max_failures: usize, + sessions: Mutex>, + failures: Mutex>>, +} + +impl Default for Auth { + fn default() -> Auth { + Auth::new(None) + } +} + +impl Auth { + /// A site with password hash `hash` and the default limits. + pub fn new(hash: Option) -> Auth { + Auth { + hash, + session_hours: 12.0, + session_idle_minutes: 120.0, + login_max_failures: 5, + sessions: Mutex::new(HashMap::new()), + failures: Mutex::new(HashMap::new()), + } + } + + fn expired(&self, s: Session, t: f64) -> bool { + t - s.created > self.session_hours * 3600.0 || t - s.seen > self.session_idle_minutes * 60.0 + } + + /// A new session: its cookie. + pub fn new_session(&self) -> io::Result { + let token = new_token()?; + let t = now(); + lock(&self.sessions).insert( + session_id(&token), + Session { + created: t, + seen: t, + }, + ); + Ok(token) + } + + /// The session id of cookie `token` while it is live (this counts as a + /// visit); an expired one is dropped. + pub fn session(&self, token: Option<&str>) -> Option { + let sid = session_id(token.filter(|t| !t.is_empty())?); + let t = now(); + let mut all = lock(&self.sessions); + let s = all.get_mut(&sid)?; + if self.expired(*s, t) { + all.remove(&sid); + return None; + } + s.seen = t; + Some(sid) + } + + /// Whether cookie `token`'s session is still live, without counting + /// this as a visit: an open event stream must not keep an idle session + /// alive. + pub fn alive(&self, token: Option<&str>) -> bool { + let Some(token) = token.filter(|t| !t.is_empty()) else { + return false; + }; + let t = now(); + lock(&self.sessions) + .get(&session_id(token)) + .is_some_and(|s| !self.expired(*s, t)) + } + + /// Ends session `sid` (a logout). + pub fn end_session(&self, sid: &str) { + lock(&self.sessions).remove(sid); + } + + /// False while `client` is blocked; else counts this attempt now, under + /// the same lock, so parallel wrong guesses cannot all pass the check (a + /// right password removes the count). + pub fn reserve_attempt(&self, client: &str) -> bool { + let t = now(); + let mut all = lock(&self.failures); + let f = all.entry(client.to_owned()).or_default(); + f.retain(|x| t - x < FAIL_WINDOW_SECONDS); + if f.len() >= self.login_max_failures { + return false; + } + f.push(t); + true + } + + /// Forgets `client`'s failed logins (after a right password). + pub fn clear_failures(&self, client: &str) { + lock(&self.failures).remove(client); + } +} + +/// The session cookie line: `value` for `max_age` seconds, `Secure` when +/// the browser reaches the site over HTTPS. +pub fn cookie(value: &str, max_age: i64, secure: bool) -> String { + let s = if secure { "; Secure" } else { "" }; + format!("{COOKIE}={value}; Path=/; HttpOnly; SameSite=Strict; Max-Age={max_age}{s}") +} + +/// The value of the site's cookie in a `Cookie` header. +pub fn cookie_token(header: Option<&str>) -> Option { + header?.split(';').find_map(|part| { + let (k, v) = part.trim().split_once('=').unwrap_or((part.trim(), "")); + (k == COOKIE).then(|| v.to_owned()) + }) +} + +// --- the client behind a proxy -------------------------------------------------------------- + +/// An address block of `--trust-proxy`: an address, or `ADDR/PREFIX` +/// (host bits ignored, like `ipaddress.ip_network(strict=False)`). +#[derive(Debug, Clone, PartialEq)] +pub struct Net { + addr: IpAddr, + prefix: u8, +} + +impl Net { + pub fn parse(text: &str) -> Result { + let bad = || format!("--trust-proxy {text}: not an address or ADDR/PREFIX"); + let (a, p) = text.trim().split_once('/').unwrap_or((text.trim(), "")); + let addr: IpAddr = a + .trim_matches(|c| c == '[' || c == ']') + .parse() + .map_err(|_| bad())?; + let max = if addr.is_ipv4() { 32 } else { 128 }; + let prefix = if p.is_empty() { + max + } else { + p.parse::().ok().filter(|n| *n <= max).ok_or_else(bad)? + }; + Ok(Net { addr, prefix }) + } + + pub fn contains(&self, ip: IpAddr) -> bool { + let bits = |ip: IpAddr| -> Option<(u128, u32)> { + match ip { + IpAddr::V4(v) => Some((u128::from(u32::from(v)), 32)), + IpAddr::V6(v) => Some((u128::from(v), 128)), + } + }; + let (Some((a, w)), Some((b, w2))) = (bits(self.addr), bits(ip)) else { + return false; + }; + if w != w2 { + return false; + } + let host = w - u32::from(self.prefix); + host >= 128 || (a >> host) == (b >> host) + } +} + +/// Who a request comes from: the peer, or the last `X-Forwarded-For` entry +/// when the peer is a proxy of `--trust-proxy`. +pub fn client(peer: &str, forwarded_for: Option<&str>, trust: &[Net]) -> String { + if let Ok(ip) = peer.parse::() + && trust.iter().any(|n| n.contains(ip)) + && let Some(last) = forwarded_for + .unwrap_or("") + .split(',') + .map(str::trim) + .rfind(|x| !x.is_empty()) + { + return last.to_owned(); + } + peer.to_owned() +} + +// --- the routes --------------------------------------------------------------------------- + +/// Reads the request's session: sets its id and the 8 characters the +/// access log keeps. Whether the request may see the runs: a live session, +/// or a site with no password. +pub fn begin(req: &mut Req<'_>) -> bool { + let auth = Arc::clone(&req.live.cfg.auth); + if auth.hash.is_none() { + return true; + } + req.sid = auth.session(cookie_token(req.header("cookie")).as_deref()); + req.session8 = req.sid.as_deref().map_or("-".into(), |s| s[..8].to_owned()); + req.sid.is_some() +} + +/// What a request without a session gets: a page (`/` or a path ending in +/// `/`) goes to the login form, anything else is 401. +pub fn refuse(req: &mut Req<'_>) -> io::Result<()> { + let page = req.path == "/" || req.path.ends_with('/'); + req.log("no-session", "-", "-", if page { 303 } else { 401 }); + if page { + let to = format!("/login?next={}", quote(&req.path)); + return req.redirect(303, &to); + } + req.deny(401, None) +} + +/// Whether the browser reaches the site over HTTPS (TLS here, or at a +/// proxy that says so with `--cookie-secure`). +fn secure(req: &Req<'_>) -> bool { + req.live.cfg.cookie_secure || req.live.cfg.tls.is_some() +} + +fn login_page( + req: &mut Req<'_>, + code: u16, + message: &str, + extra: &[(&str, String)], +) -> io::Result<()> { + let next = sf::query_value(&req.query, "next").unwrap_or_else(|| "/".into()); + match sf::login_page(&req.live.cfg.assets, message, &next) { + Ok(page) => req.send( + code, + "text/html; charset=utf-8", + page.as_bytes(), + extra, + crate::csp::CSP, + ), + Err(_) => req.deny(500, None), + } +} + +/// Where a login goes next: a path of this site, else `/`. +fn next_ok(next: &str) -> bool { + next.starts_with('/') + && !next.starts_with("//") + && next[1..].bytes().all(|b| { + b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'~' | b'/' | b'%' | b'-') + }) +} + +/// `/login`: the form; a POST checks the password. +pub fn login(req: &mut Req<'_>) -> io::Result<()> { + if req.method != "POST" { + return login_page(req, 200, "", &[]); + } + let auth = Arc::clone(&req.live.cfg.auth); + let client = req.client.clone(); + let Some(hash) = auth.hash.clone() else { + return req.redirect(303, "/"); + }; + if !auth.reserve_attempt(&client) { + req.log("login-blocked", "-", "-", 429); + return login_page( + req, + 429, + "Too many failed logins. Wait 15 minutes.", + &[("Retry-After", "900".into())], + ); + } + let n = match req.header("content-length") { + None => 0, + Some(v) => v.trim().parse::().unwrap_or(-1), + }; + if !(0..=MAX_FORM).contains(&n) { + return req.deny(413, Some("too large")); + } + let body = req.read_body(n as usize)?; + let form = String::from_utf8_lossy(&body).into_owned(); + let pw = sf::query_value(&form, "password").unwrap_or_default(); + let next = sf::query_value(&form, "next") + .filter(|n| next_ok(n)) + .unwrap_or_else(|| "/".into()); + if !verify_password(&pw, &hash) { + req.log("login-fail", "-", "-", 401); + return login_page(req, 401, "Wrong password.", &[]); + } + auth.clear_failures(&client); + let token = auth.new_session()?; + req.session8 = session_id(&token)[..8].to_owned(); + req.log("login", "-", "-", 303); + let max_age = (auth.session_hours * 3600.0) as i64; + let set = cookie(&token, max_age, secure(req)); + req.send( + 303, + "text/plain", + b"", + &[("Location", next), ("Set-Cookie", set)], + crate::csp::CSP, + ) +} + +/// `POST /logout`: ends the session and clears the cookie. +pub fn logout(req: &mut Req<'_>) -> io::Result<()> { + if let Some(sid) = req.sid.clone() { + req.live.cfg.auth.end_session(&sid); + } + req.log("logout", "-", "-", 303); + let clear = cookie("", 0, secure(req)); + req.send( + 303, + "text/plain", + b"", + &[("Location", "/login".into()), ("Set-Cookie", clear)], + crate::csp::CSP, + ) +} + +// --- HTTPS -------------------------------------------------------------------------------- + +/// The TLS settings of `--tls-cert` (a PEM chain) and `--tls-key`. +pub fn tls_config(cert: &Path, key: &Path) -> Result, String> { + use rustls::pki_types::pem::PemObject; + use rustls::pki_types::{CertificateDer, PrivateKeyDer}; + let certs = CertificateDer::pem_file_iter(cert) + .and_then(Iterator::collect::, _>>) + .map_err(|e| format!("--tls-cert {}: {e}", cert.display()))?; + if certs.is_empty() { + return Err(format!("--tls-cert {}: no certificate", cert.display())); + } + let key = PrivateKeyDer::from_pem_file(key) + .map_err(|e| format!("--tls-key {}: {e}", key.display()))?; + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let cfg = rustls::ServerConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .map_err(|e| e.to_string())? + .with_no_client_auth() + .with_single_cert(certs, key) + .map_err(|e| format!("--tls-cert and --tls-key: {e}"))?; + Ok(Arc::new(cfg)) +} + +/// A connection after its TLS handshake, done in the connection's own +/// thread within [`TLS_HANDSHAKE`], so a client that connects and sends +/// nothing holds up no other; `None` when the handshake fails. +pub fn accept_tls( + cfg: &Arc, + mut tcp: TcpStream, +) -> Option> { + let mut conn = rustls::ServerConnection::new(Arc::clone(cfg)).ok()?; + tcp.set_read_timeout(Some(TLS_HANDSHAKE)).ok()?; + tcp.set_write_timeout(Some(TLS_HANDSHAKE)).ok()?; + while conn.is_handshaking() { + conn.complete_io(&mut tcp).ok()?; + } + Some(rustls::StreamOwned::new(conn, tcp)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pbkdf2_matches_the_rfc_7914_vector() { + // RFC 7914 section 11: P="passwd", S="salt", c=1, dkLen=64 (the + // first 32 bytes). + assert_eq!( + hex(&pbkdf2_sha256(b"passwd", b"salt", 1)), + "55ac046e56e3089fec1691c22544b605f94185216dde0465e68b9d57c20dacbc" + ); + // A key longer than a SHA-256 block is hashed first. + let long = [b'k'; 100]; + assert_eq!(pbkdf2_sha256(&long, b"s", 2).len(), 32); + } + + #[test] + fn a_hash_verifies_its_password_and_no_other() { + let h = hash_password("synthetic test password 7", 1000).unwrap(); + assert!(h.starts_with("pbkdf2_sha256$1000$")); + assert!(verify_password("synthetic test password 7", &h)); + assert!(!verify_password("synthetic test password 7x", &h)); + assert!(!verify_password("x", "plain password")); + for bad in [ + "", + "pbkdf2_sha256$x$s$aa", + "pbkdf2_sha256$0$s$", + "md5$1$s$00", + ] { + assert!(parse_hash(bad).is_err(), "{bad}"); + } + } + + #[test] + fn tokens_are_43_url_safe_characters() { + assert_eq!(base64url(b"\xfb\xff"), "-_8"); + assert_eq!(base64url(b"any carnal pleas"), "YW55IGNhcm5hbCBwbGVhcw"); + let t = new_token().unwrap(); + assert_eq!(t.len(), 43); + assert!( + t.bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') + ); + assert_ne!(t, new_token().unwrap()); + } + + #[test] + fn trusted_proxies_and_the_forwarded_client() { + let trust = [ + Net::parse("127.0.0.1").unwrap(), + Net::parse("10.1.2.3/8").unwrap(), + ]; + assert_eq!( + client("127.0.0.1", Some("192.0.2.1, 198.51.100.7"), &trust), + "198.51.100.7" + ); + assert_eq!(client("10.9.9.9", Some("192.0.2.1"), &trust), "192.0.2.1"); + assert_eq!( + client("192.0.2.5", Some("198.51.100.7"), &trust), + "192.0.2.5" + ); + assert_eq!(client("127.0.0.1", None, &trust), "127.0.0.1"); + assert!(Net::parse("::1").unwrap().contains("::1".parse().unwrap())); + assert!( + !Net::parse("::1") + .unwrap() + .contains("127.0.0.1".parse().unwrap()) + ); + assert!( + Net::parse("0.0.0.0/0") + .unwrap() + .contains("192.0.2.1".parse().unwrap()) + ); + for bad in ["", "nope", "10.0.0.0/33", "10.0.0.0/x"] { + assert!(Net::parse(bad).is_err(), "{bad}"); + } + } + + #[test] + fn the_cookie_is_read_from_its_header() { + let h = "a=1; owngenome_live=tok; b=2"; + assert_eq!(cookie_token(Some(h)).as_deref(), Some("tok")); + assert_eq!(cookie_token(Some("owngenome_livex=1")), None); + assert_eq!(cookie_token(None), None); + assert!(cookie("t", 5, true).ends_with("Max-Age=5; Secure")); + } +} diff --git a/rust/crates/og-live/src/ingest.rs b/rust/crates/og-live/src/ingest.rs index 24f2b8b32..100cd9744 100644 --- a/rust/crates/og-live/src/ingest.rs +++ b/rust/crates/og-live/src/ingest.rs @@ -1,2 +1,381 @@ -//! Runs taken into the live site. Empty until its unit of the 1.0.0 plan -//! lands. +//! The ingest listener: the run's own processes post their progress here +//! (`POST /ingest/`): a Nextflow weblog message, or a `pgp_step` / +//! `pgp_run_finished` message from `scripts/run-all.sh`. Each message is cut +//! to the fields a progress view needs ([`cut`]) and appended to +//! `STATE_DIR/runs//events.jsonl`, which the browser side reads. It +//! serves nothing. + +use crate::runs::valid_run; +use crate::server::{REQUEST_SECONDS, read_head}; +use crate::state::{Live, dumps_compact, lock}; +use crate::viewer::unquote; +use indexmap::IndexMap; +use pycompat::json::{self, Value}; +use std::fs::{self, OpenOptions}; +use std::io::{self, BufReader, Read, Write}; +use std::net::{Shutdown, TcpListener, TcpStream}; +use std::sync::atomic::Ordering; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::Duration; + +/// The weblog's task fields kept. The work directory and the parameters +/// other than [`PARAM_KEYS`] are paths or personal: they never reach a page. +pub const TRACE_KEYS: [&str; 18] = [ + "task_id", + "hash", + "name", + "process", + "tag", + "status", + "exit", + "attempt", + "submit", + "start", + "complete", + "realtime", + "cpus", + "memory", + "peak_rss", + "peak_vmem", + "rchar", + "wchar", +]; +/// The run parameters kept. +pub const PARAM_KEYS: [&str; 3] = ["outdir", "max_memory", "max_cpus"]; +/// The states of a `pgp_step` message. +pub const STEP_STATES: [&str; 3] = ["started", "ok", "failed"]; + +/// The first `n` characters of `s`. +fn cut_str(s: &str, n: usize) -> String { + s.chars().take(n).collect() +} + +/// `str(v)` of a scalar, as Python writes it. +fn py_str(v: &Value) -> String { + match v { + Value::Null => "None".into(), + Value::Bool(b) => if *b { "True" } else { "False" }.into(), + Value::Int(i) => i.to_string(), + Value::Float(f) => pycompat::fmt::repr(*f), + Value::Str(s) => s.clone(), + other => dumps_compact(other), + } +} + +/// A scalar as kept: null, a flag and a number as they are, anything else +/// as text cut at `n` characters. +fn scalar(v: &Value, n: usize) -> Value { + match v { + Value::Null | Value::Bool(_) | Value::Int(_) | Value::Float(_) => v.clone(), + other => Value::Str(cut_str(&py_str(other), n)), + } +} + +fn is_nested(v: &Value) -> bool { + matches!(v, Value::Object(_) | Value::Array(_)) +} + +/// Whether `v` holds a number that is not finite (`NaN`, `Infinity`), which +/// would make events.jsonl something other than plain JSON. +fn has_non_finite(v: &Value) -> bool { + match v { + Value::Float(f) => !f.is_finite(), + Value::Array(a) => a.iter().any(has_non_finite), + Value::Object(o) => o.values().any(has_non_finite), + _ => false, + } +} + +/// The fields of one message the site keeps, or why it is not a message. +pub fn cut(ev: &Value) -> Result { + let Value::Object(ev) = ev else { + return Err("not a JSON object".into()); + }; + let kind = match ev.get("event") { + Some(Value::Str(k)) + if (1..=40).contains(&k.len()) + && k.bytes().all(|b| b.is_ascii_lowercase() || b == b'_') => + { + k.clone() + } + _ => return Err("no 'event' name".into()), + }; + let mut out: IndexMap = IndexMap::new(); + out.insert("event".into(), Value::Str(kind.clone())); + for k in ["runName", "runId", "utcTime"] { + if let Some(Value::Str(s)) = ev.get(k) { + out.insert(k.into(), Value::Str(cut_str(s, 100))); + } + } + if let Some(Value::Object(tr)) = ev.get("trace") { + let mut t: IndexMap = IndexMap::new(); + for k in TRACE_KEYS { + if let Some(v) = tr.get(k).filter(|v| !is_nested(v)) { + t.insert(k.into(), scalar(v, 200)); + } + } + // Read as text later: a number there would not be. + for k in ["name", "process", "tag", "status", "hash"] { + if let Some(v) = t.get_mut(k).filter(|v| !matches!(v, Value::Null)) { + *v = Value::Str(cut_str(&py_str(v), 200)); + } + } + out.insert("trace".into(), Value::Object(t)); + } + if let Some(Value::Object(md)) = ev.get("metadata") { + let mut keep: IndexMap = IndexMap::new(); + if let Some(Value::Object(params)) = md.get("parameters") { + let mut p: IndexMap = IndexMap::new(); + for k in PARAM_KEYS { + if let Some(v) = params.get(k).filter(|v| !is_nested(v)) { + p.insert(k.into(), scalar(v, 1000)); + } + } + if !p.is_empty() { + keep.insert("parameters".into(), Value::Object(p)); + } + } + if let Some(Value::Object(wf)) = md.get("workflow") { + if let Some(Value::Str(w)) = wf.get("workDir") { + let mut o = IndexMap::new(); + o.insert("workDir".to_owned(), Value::Str(cut_str(w, 1000))); + keep.insert("workflow".into(), Value::Object(o)); + } + if let Some(Value::Str(m)) = wf.get("errorMessage") { + out.insert("errorMessage".into(), Value::Str(cut_str(m, 500))); + } + } + if !keep.is_empty() { + out.insert("metadata".into(), Value::Object(keep)); + } + } + if let Some(Value::Str(m)) = ev.get("errorMessage") { + out.insert("errorMessage".into(), Value::Str(cut_str(m, 500))); + } + if kind == "pgp_step" { + let step = match ev.get("step") { + Some(Value::Str(s)) if step_ok(s) => s.clone(), + _ => return Err(STEP_NEEDS.into()), + }; + let state = match ev.get("state") { + Some(Value::Str(s)) if STEP_STATES.contains(&s.as_str()) => s.clone(), + _ => return Err(STEP_NEEDS.into()), + }; + out.insert("step".into(), Value::Str(step)); + out.insert("state".into(), Value::Str(state)); + } else if kind == "pgp_run_finished" { + match ev.get("exit") { + Some(Value::Int(n)) => { + out.insert("exit".into(), Value::Int(*n)); + } + _ => return Err("pgp_run_finished needs an integer 'exit'".into()), + } + } + Ok(Value::Object(out)) +} + +const STEP_NEEDS: &str = + "pgp_step needs 'step' (like 14 or 04b) and 'state' (started, ok or failed)"; + +/// Two digits and an optional lower-case letter (`14`, `04b`). +fn step_ok(s: &str) -> bool { + let b = s.as_bytes(); + (b.len() == 2 || (b.len() == 3 && b[2].is_ascii_lowercase())) + && b[0].is_ascii_digit() + && b[1].is_ascii_digit() +} + +/// The message a body holds, cut ([`cut`]), or why it is not one. +pub fn parse(body: &[u8]) -> Result { + let text = std::str::from_utf8(body).map_err(|e| e.to_string())?; + let v = json::loads(text).map_err(|e| format!("{e:?}"))?; + if has_non_finite(&v) { + return Err("NaN and Infinity are not JSON".into()); + } + cut(&v) +} + +/// One writer at a time for every events file. +static APPEND: Mutex<()> = Mutex::new(()); + +/// Appends message `ev` to `run`'s events file. +pub fn append(live: &Live, run: &str, ev: &Value) -> io::Result<()> { + let p = live.store.path(run); + let _g = lock(&APPEND); + if let Some(d) = p.parent() { + fs::create_dir_all(d)?; + } + let mut f = OpenOptions::new().create(true).append(true).open(p)?; + f.write_all(format!("{}\n", dumps_compact(ev)).as_bytes()) +} + +/// The answer: `code` and one line of text. +fn reply(out: &mut dyn Write, code: u16, text: &str) -> io::Result<()> { + let reason = match code { + 200 => "OK", + 400 => "Bad Request", + 403 => "Forbidden", + 404 => "Not Found", + 411 => "Length Required", + 413 => "Request Entity Too Large", + _ => "Not Implemented", + }; + let body = format!("{text}\n"); + let date = jiff::Timestamp::now().strftime("%a, %d %b %Y %H:%M:%S GMT"); + write!( + out, + "HTTP/1.1 {code} {reason}\r\nServer: owngenome-live-ingest\r\nDate: {date}\r\n\ + Content-Type: text/plain; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + )?; + out.flush() +} + +/// The status and text for one request. +fn answer( + live: &Live, + method: &str, + target: &str, + headers: &[(String, String)], + body: &mut dyn Read, +) -> (u16, String) { + let header = |n: &str| { + headers + .iter() + .find(|(k, _)| k == n) + .map(|(_, v)| v.as_str()) + }; + if method != "POST" { + return if method == "GET" { + (404, "not found".into()) + } else { + (501, "unsupported method".into()) + }; + } + let path = target.split(['?', '#']).next().unwrap_or(""); + // A OWNGENOME_LIVE_URL ending in '/' gives '//ingest/...'. + let path = format!("/{}", path.trim_start_matches('/')); + let Some(run) = path.strip_prefix("/ingest/") else { + return (404, "not found".into()); + }; + if header("origin").is_some() { + // Nextflow and curl send no Origin. + return (403, "a browser cannot post here".into()); + } + let run = unquote(run); + if !valid_run(&run) { + return (400, "bad run name".into()); + } + let n = match header("content-length").map(|v| v.trim().parse::()) { + None | Some(Err(_)) => return (411, "Content-Length required".into()), + Some(Ok(n)) => n, + }; + if n < 0 || n > live.cfg.max_body_kb as i64 * 1024 { + return (413, "too large".into()); + } + let mut raw = Vec::with_capacity(n as usize); + if body.take(n as u64).read_to_end(&mut raw).is_err() { + return (400, "not a message: the body was cut off".into()); + } + let ev = match parse(&raw) { + Ok(ev) => ev, + Err(e) => return (400, cut_str(&format!("not a message: {e}"), 200)), + }; + match append(live, &run, &ev) { + Ok(()) => (200, "ok".into()), + Err(e) => { + eprintln!( + "og live serve: ingest: cannot store a message of {run}: {}", + e.kind() + ); + (500, "not stored".into()) + } + } +} + +fn serve_conn(live: &Live, stream: TcpStream) { + let timeout = Some(Duration::from_secs(REQUEST_SECONDS)); + let _ = stream.set_read_timeout(timeout); + let _ = stream.set_write_timeout(timeout); + let Ok(read_half) = stream.try_clone() else { + return; + }; + let mut reader = BufReader::new(read_half); + let mut out = io::BufWriter::new(&stream); + let (code, text) = match read_head(&mut reader) { + None => (400, "bad request".to_owned()), + Some((method, target, headers)) => answer(live, &method, &target, &headers, &mut reader), + }; + let _ = reply(&mut out, code, &text); + drop(out); + let _ = stream.shutdown(Shutdown::Write); +} + +/// Serves `listener` on threads until `live` is stopping. +pub fn serve(live: Arc, listener: TcpListener) -> thread::JoinHandle<()> { + thread::spawn(move || { + for conn in listener.incoming() { + if live.stopping.load(Ordering::SeqCst) { + break; + } + let Ok(stream) = conn else { continue }; + let l = Arc::clone(&live); + thread::spawn(move || serve_conn(&l, stream)); + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn obj(text: &str) -> Value { + json::loads(text).unwrap() + } + + #[test] + fn a_message_keeps_only_what_a_progress_view_needs() { + let ev = obj( + r#"{"event":"process_started","runName":"r","trace":{"name":123,"process":"P","tag":null, + "status":true,"workdir":"/w","script":"bcftools roh","exit":0,"env":{"A":1}}, + "metadata":{"parameters":{"outdir":"/data/genome","hpo":"HP:0001250","vcf":"x.vcf.gz"}, + "workflow":{"workDir":"/w","commandLine":"nextflow run","errorMessage":"boom"}}}"#, + ); + let got = dumps_compact(&cut(&ev).unwrap()); + assert_eq!( + got, + r#"{"event":"process_started","runName":"r","trace":{"name":"123","process":"P","tag":null,"status":"True","exit":0},"errorMessage":"boom","metadata":{"parameters":{"outdir":"/data/genome"},"workflow":{"workDir":"/w"}}}"# + ); + } + + #[test] + fn script_messages_need_their_fields() { + let ok = cut(&obj(r#"{"event":"pgp_step","step":"04b","state":"ok"}"#)).unwrap(); + assert_eq!( + dumps_compact(&ok), + r#"{"event":"pgp_step","step":"04b","state":"ok"}"# + ); + for bad in [ + r#"{"event":"pgp_step","step":"4","state":"ok"}"#, + r#"{"event":"pgp_step","step":"04","state":"done"}"#, + r#"{"event":"pgp_run_finished","exit":true}"#, + r#"{"event":"pgp_run_finished","exit":"0"}"#, + r#"{"event":"Bad"}"#, + r#"[1]"#, + ] { + assert!(cut(&obj(bad)).is_err(), "{bad}"); + } + assert!(cut(&obj(r#"{"event":"pgp_run_finished","exit":3}"#)).is_ok()); + } + + #[test] + fn nan_infinity_and_text_that_is_not_json_are_refused() { + assert!(parse(br#"{"event":"started","trace":{"attempt":NaN}}"#).is_err()); + assert!(parse(br#"{"event":"started","trace":{"exit":Infinity}}"#).is_err()); + assert!(parse(br#"{"event":"started","x":[1e400]}"#).is_err()); + assert!(parse(b"not json").is_err()); + assert!(parse(b"\xff").is_err()); + assert!(parse(br#"{"event":"started"}"#).is_ok()); + } +} diff --git a/rust/crates/og-live/src/plugins.rs b/rust/crates/og-live/src/plugins.rs index eac6b131b..401a0f53b 100644 --- a/rust/crates/og-live/src/plugins.rs +++ b/rust/crates/og-live/src/plugins.rs @@ -436,11 +436,11 @@ fn family_page(live: &Live, name: &str, d: &Path) -> Result { /// `GET /metrics`: every run's health in Prometheus' text format. Without a /// password the site is loopback only and `/metrics` is open, like every -/// page. +/// page; with one it needs a login or `--metrics-token-file`'s bearer token. fn metrics(req: &mut Req<'_>) -> io::Result<()> { let who = Asker { - login_required: false, - logged_in: false, + login_required: req.live.cfg.auth.hash.is_some(), + logged_in: req.sid.is_some(), authorization: req.header("authorization"), }; let token = req.live.cfg.metrics_token_file.clone(); diff --git a/rust/crates/og-live/src/server.rs b/rust/crates/og-live/src/server.rs index 5d65e5f58..4d6ae4914 100644 --- a/rust/crates/og-live/src/server.rs +++ b/rust/crates/og-live/src/server.rs @@ -20,15 +20,15 @@ //! and the plugins' routes ([`crate::plugins`]). The plugins are compiled //! in: there is no plugin folder. //! -//! Not here yet: the password, the sessions, TLS, the proxy options and the -//! ingest listener. Until they land, `og live serve` listens on loopback -//! only, takes no password, and refuses the flags that would make a site -//! less safe if they were ignored (exit 2). It reads the messages another -//! process stores under `STATE_DIR/runs//events.jsonl` (the Python -//! server's ingest listener today), so both can run beside each other on one -//! state folder. The access log (`STATE_DIR/access.log`, TSV) records who -//! opened what, never a query string, a request body or a result. - +//! With a password hash every route but `/login`, `/assets/` and +//! `/metrics` (which takes its own bearer token) needs a session +//! ([`crate::auth`]). The ingest listener ([`crate::ingest`]) stores the +//! run's messages under `STATE_DIR/runs//events.jsonl`, the file the +//! pages read. The access log (`STATE_DIR/access.log`, TSV) records who +//! opened what, never a query string, a request body, a result or the +//! password. + +use crate::auth::{self, Auth, Net}; use crate::csp::{self, CSP, DENY, RUN_HTML_CSP}; use crate::family::{inside, realpath}; use crate::ops::OpsConfig; @@ -37,11 +37,13 @@ use crate::runs::{self, Run}; use crate::state::{self, Live, dumps, lock}; use crate::static_files as sf; use crate::viewer::unquote; +use std::cell::RefCell; use std::collections::BTreeMap; use std::fs::{self, OpenOptions}; use std::io::{self, BufRead, BufReader, Read, Write}; use std::net::{IpAddr, Shutdown, SocketAddr, TcpListener, TcpStream, ToSocketAddrs}; use std::path::{Path, PathBuf}; +use std::rc::Rc; use std::sync::atomic::Ordering; use std::sync::{Arc, Mutex}; use std::thread::{self, JoinHandle}; @@ -54,17 +56,21 @@ pub const REQUEST_SECONDS: u64 = 60; pub const HEARTBEAT: Duration = Duration::from_secs(15); /// The largest request head read. const MAX_HEAD: usize = 64 * 1024; -/// The cookie the login sets (`og live serve` sets none yet; logout clears -/// it). +/// The cookie a login sets. pub const COOKIE: &str = "owngenome_live"; const USAGE: &str = "usage: og live serve --root GENOME_DIR [--state-dir DIR] [--bind ADDR] [--port N] - [--max-streams N] [--access-log FILE] [--access-log-max-mb N] - [--config FILE] [--quiet] + [--ingest-bind ADDR] [--ingest-port N] [--password-file FILE] + [--allowed-host NAME ...] [--trust-proxy CIDR ...] + [--tls-cert FILE --tls-key FILE] [--cookie-secure] + [--session-hours H] [--session-idle-minutes M] [--login-max-failures N] + [--max-streams N] [--max-body-kb N] [--access-log FILE] + [--access-log-max-mb N] [--config FILE] [--quiet] [--viewer-fasta FILE --viewer-igv-js FILE] [--disk-warn-percent N] [--mem-warn-percent N] [--stall-minutes N] [--metrics-token-file FILE] [--eta-box NAME] [--eta-cores N] + og live serve hash-password "; /// What the site is started with. @@ -106,6 +112,21 @@ pub struct Config { pub results: Option, /// How often a quiet event stream writes. pub heartbeat: Duration, + /// The password and the sessions. + pub auth: Arc, + /// Host names the browser may use besides loopback and the bind + /// address. + pub allowed_host: Vec, + /// The proxies whose `X-Forwarded-For` names the client. + pub trust_proxy: Vec, + /// The certificate chain and key of HTTPS. + pub tls: Option<(PathBuf, PathBuf)>, + /// Mark the cookie `Secure` (TLS ends at a proxy). + pub cookie_secure: bool, + /// Where the ingest listener listens; `None`: none is started. + pub ingest: Option<(String, u16)>, + /// The largest message the ingest listener takes. + pub max_body_kb: usize, } /// The first ancestor of this binary that holds `rel`. @@ -143,6 +164,13 @@ impl Config { og: std::env::current_exe().unwrap_or_else(|_| PathBuf::from("og")), results: None, heartbeat: HEARTBEAT, + auth: Arc::new(Auth::default()), + allowed_host: Vec::new(), + trust_proxy: Vec::new(), + tls: None, + cookie_secure: false, + ingest: None, + max_body_kb: 1024, } } } @@ -232,7 +260,13 @@ pub struct Req<'a> { pub path: String, pub query: String, headers: Vec<(String, String)>, + /// The peer, or the client a trusted proxy names. pub client: String, + /// The session id (SHA-256 of the cookie) when it is live. + pub sid: Option, + /// Its first 8 characters for the access log, else `-`. + pub session8: String, + input: &'a mut dyn BufRead, out: &'a mut dyn Write, } @@ -364,9 +398,21 @@ impl Req<'_> { /// One access-log row for this request. pub fn log(&self, action: &str, run: &str, section: &str, status: u16) { - self.live - .access - .write(&[&self.client, "-", action, run, section, &status.to_string()]); + self.live.access.write(&[ + &self.client, + &self.session8, + action, + run, + section, + &status.to_string(), + ]); + } + + /// The first `n` bytes of the request's body. + pub fn read_body(&mut self, n: usize) -> io::Result> { + let mut b = Vec::with_capacity(n); + Read::take(&mut *self.input, n as u64).read_to_end(&mut b)?; + Ok(b) } /// The folder of run `name`, when the site knows it and it has one. @@ -454,26 +500,26 @@ pub fn handle(req: &mut Req<'_>) -> io::Result<()> { req.log("origin-refused", "-", "-", 403); return req.deny(403, None); } + let authed = auth::begin(req); let path = req.path.clone(); if path == "/login" { - return login(req); + return auth::login(req); } if path == "/logout" && req.method == "POST" { - req.log("logout", "-", "-", 303); - let clear = format!("{COOKIE}=; Path=/; HttpOnly; SameSite=Strict; Max-Age=0"); - return req.send( - 303, - "text/plain", - b"", - &[("Location", "/login".into()), ("Set-Cookie", clear)], - CSP, - ); + return auth::logout(req); } if req.method == "GET" && let Some(name) = path.strip_prefix("/assets/") { return asset(req, name); } + // /metrics takes a bearer token of its own (crate::plugins). + if !authed && path != "/metrics" { + if req.method != "GET" { + return req.deny(404, None); + } + return auth::refuse(req); + } if plugins::route(req, &path)? { return Ok(()); } @@ -511,18 +557,6 @@ pub fn handle(req: &mut Req<'_>) -> io::Result<()> { } } -/// `/login`: the login page; a POST goes home (no password is set). -fn login(req: &mut Req<'_>) -> io::Result<()> { - if req.method == "POST" { - return req.redirect(303, "/"); - } - let next = sf::query_value(&req.query, "next").unwrap_or_else(|| "/".into()); - match sf::login_page(&req.live.cfg.assets, "", &next) { - Ok(page) => req.send_body(200, "text/html; charset=utf-8", page), - Err(_) => req.deny(500, None), - } -} - fn asset(req: &mut Req<'_>, name: &str) -> io::Result<()> { let top = realpath(&req.live.cfg.assets); match sf::asset(&top, name) { @@ -667,6 +701,7 @@ fn events(req: &mut Req<'_>, run: &Run) -> io::Result<()> { if req.head_only { return req.out().flush(); } + let token = auth::cookie_token(req.header("cookie")); let out = req.out(); out.write_all(b"retry: 3000\n\n")?; out.flush()?; @@ -675,6 +710,9 @@ fn events(req: &mut Req<'_>, run: &Run) -> io::Result<()> { let new = live .store .wait(name, sent, live.cfg.heartbeat, &live.stopping); + if live.cfg.auth.hash.is_some() && !live.cfg.auth.alive(token.as_deref()) { + break; // logged out or expired: an open stream ends with its session + } if new.is_empty() { out.write_all(b": heartbeat\n\n")?; } else { @@ -700,13 +738,11 @@ type Headers = Vec<(String, String)>; /// The head of one request: method, target and headers (names lowered); /// `None` for one that cannot be read. -fn read_head(r: &mut impl BufRead) -> Option<(String, String, Headers)> { +pub(crate) fn read_head(r: &mut dyn BufRead) -> Option<(String, String, Headers)> { let mut head = Vec::new(); loop { let mut line = Vec::new(); - let n = r - .by_ref() - .take((MAX_HEAD - head.len()) as u64 + 1) + let n = Read::take(&mut *r, (MAX_HEAD - head.len()) as u64 + 1) .read_until(b'\n', &mut line) .ok()?; if n == 0 { @@ -742,20 +778,68 @@ fn read_head(r: &mut impl BufRead) -> Option<(String, String, Headers)> { Some((method.to_owned(), target.to_owned(), headers)) } -fn serve_conn(live: &Live, stream: TcpStream) { - let timeout = Some(Duration::from_secs(REQUEST_SECONDS)); - let _ = stream.set_read_timeout(timeout); - let _ = stream.set_write_timeout(timeout); - let client = stream +/// One TLS connection, read and written by turns from the one thread that +/// serves it. +struct Shared(Rc>>); + +impl Read for Shared { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.0.borrow_mut().read(buf) + } +} + +impl Write for Shared { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.0.borrow_mut().write(buf) + } + fn flush(&mut self) -> io::Result<()> { + self.0.borrow_mut().flush() + } +} + +fn serve_conn(live: &Live, stream: TcpStream, tls: Option<&Arc>) { + let peer = stream .peer_addr() .map(|a| a.ip().to_string()) .unwrap_or_else(|_| "-".into()); - let Ok(read_half) = stream.try_clone() else { + let timeout = Some(Duration::from_secs(REQUEST_SECONDS)); + let Some(cfg) = tls else { + let _ = stream.set_read_timeout(timeout); + let _ = stream.set_write_timeout(timeout); + let Ok(read_half) = stream.try_clone() else { + return; + }; + let mut reader = BufReader::new(read_half); + let mut out = io::BufWriter::new(&stream); + serve_io(live, &peer, &mut reader, &mut out); + drop(out); + let _ = stream.shutdown(Shutdown::Write); + // What the client still sends is read and dropped once the answer + // is out, so closing does not reset the connection before the + // client has read it. + let _ = stream.set_read_timeout(Some(Duration::from_secs(1))); + let _ = io::copy(&mut reader.take(MAX_HEAD as u64), &mut io::sink()); return; }; - let mut reader = BufReader::new(read_half); - let mut out = io::BufWriter::new(&stream); - let Some((method, target, headers)) = read_head(&mut reader) else { + let Some(tls) = auth::accept_tls(cfg, stream) else { + return; + }; + let _ = tls.sock.set_read_timeout(timeout); + let _ = tls.sock.set_write_timeout(timeout); + let tls = Rc::new(RefCell::new(tls)); + let mut reader = BufReader::new(Shared(Rc::clone(&tls))); + let mut out = io::BufWriter::new(Shared(Rc::clone(&tls))); + serve_io(live, &peer, &mut reader, &mut out); + drop(out); + let mut t = tls.borrow_mut(); + t.conn.send_close_notify(); + let _ = t.flush(); + let _ = t.sock.shutdown(Shutdown::Write); +} + +/// Reads one request from `reader` and answers it on `out`. +fn serve_io(live: &Live, peer: &str, reader: &mut dyn BufRead, out: &mut dyn Write) { + let Some((method, target, headers)) = read_head(reader) else { let _ = out.write_all( b"HTTP/1.1 400 Bad Request\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Length: 12\r\n\ Connection: close\r\n\r\nbad request\n", @@ -782,6 +866,11 @@ fn serve_conn(live: &Live, stream: TcpStream) { return; } }; + let forwarded_for = headers + .iter() + .find(|(k, _)| k == "x-forwarded-for") + .map(|(_, v)| v.as_str()); + let client = auth::client(peer, forwarded_for, &live.cfg.trust_proxy); let mut req = Req { live, method: m.into(), @@ -790,17 +879,13 @@ fn serve_conn(live: &Live, stream: TcpStream) { query: query.split('#').next().unwrap_or("").to_owned(), headers, client, - out: &mut out, + sid: None, + session8: "-".into(), + input: reader, + out, }; let _ = handle(&mut req); - let _ = out.flush(); - drop(out); - let _ = stream.shutdown(Shutdown::Write); - // No route reads a body: what the client still sends is read and - // dropped once the answer is out, so closing does not reset the - // connection before the client has read it. - let _ = stream.set_read_timeout(Some(Duration::from_secs(1))); - let _ = io::copy(&mut reader.take(MAX_HEAD as u64), &mut io::sink()); + let _ = req.out.flush(); } /// A running site: [`Server::start`] binds and serves on threads, dropping @@ -808,23 +893,53 @@ fn serve_conn(live: &Live, stream: TcpStream) { pub struct Server { live: Arc, addr: SocketAddr, + ingest_addr: Option, accept: Option>, + ingest: Option>, +} + +/// Binds `bind:port` (a bracketed IPv6 address too). +fn listen(bind: &str, port: u16, what: &str) -> Result { + let host = bind.trim_matches(|c| c == '[' || c == ']'); + TcpListener::bind((host, port)) + .map_err(|e| format!("cannot listen{what} on {bind}:{port}: {e}")) } impl Server { /// Reads steps.tsv, opens the access log, binds `cfg.bind:cfg.port` (0: - /// any free port) and serves. + /// any free port) and the ingest listener of `cfg.ingest`, and serves. pub fn start(cfg: Config) -> Result { let steps = og_runlog::steps::read_steps(&cfg.steps).map_err(|e| format!("steps: {e}"))?; fs::create_dir_all(&cfg.state_dir) .map_err(|e| format!("--state-dir {}: {e}", cfg.state_dir.display()))?; let access = AccessLog::new(&cfg.access_log, cfg.access_log_max_mb) .map_err(|e| format!("--access-log {}: {e}", cfg.access_log.display()))?; - let host = cfg.bind.trim_matches(|c| c == '[' || c == ']').to_owned(); - let listener = TcpListener::bind((host.as_str(), cfg.port)) - .map_err(|e| format!("cannot listen on {}:{}: {e}", cfg.bind, cfg.port))?; + let tls = match &cfg.tls { + Some((cert, key)) => Some(auth::tls_config(cert, key)?), + None => None, + }; + let listener = listen(&cfg.bind, cfg.port, "")?; let addr = listener.local_addr().map_err(|e| e.to_string())?; - let live = Arc::new(Live::new(cfg, steps, access)); + let ingest_listener = match &cfg.ingest { + Some((bind, port)) => Some(listen(bind, *port, " for ingest")?), + None => None, + }; + let mut live = Live::new(cfg, steps, access); + let more: Vec = live + .cfg + .allowed_host + .iter() + .map(|h| h.to_lowercase()) + .collect(); + live.allowed.extend(more); + let live = Arc::new(live); + let (ingest_addr, ingest) = match ingest_listener { + Some(l) => ( + Some(l.local_addr().map_err(|e| e.to_string())?), + Some(crate::ingest::serve(Arc::clone(&live), l)), + ), + None => (None, None), + }; let l2 = Arc::clone(&live); let accept = thread::spawn(move || { for conn in listener.incoming() { @@ -833,13 +948,16 @@ impl Server { } let Ok(stream) = conn else { continue }; let l3 = Arc::clone(&l2); - thread::spawn(move || serve_conn(&l3, stream)); + let tls = tls.clone(); + thread::spawn(move || serve_conn(&l3, stream, tls.as_ref())); } }); Ok(Server { live, addr, + ingest_addr, accept: Some(accept), + ingest, }) } @@ -848,6 +966,11 @@ impl Server { self.addr.port() } + /// The port of the ingest listener, when it has one. + pub fn ingest_port(&self) -> Option { + self.ingest_addr.map(|a| a.port()) + } + /// The state it serves. pub fn live(&self) -> &Live { &self.live @@ -864,10 +987,15 @@ impl Server { impl Drop for Server { fn drop(&mut self) { self.live.stopping.store(true, Ordering::SeqCst); - // Wakes the accept loop so it sees the flag. - let _ = TcpStream::connect_timeout(&self.addr, Duration::from_secs(1)); - if let Some(h) = self.accept.take() { - let _ = h.join(); + // Wakes each accept loop so it sees the flag. + for (a, h) in [ + (Some(self.addr), self.accept.take()), + (self.ingest_addr, self.ingest.take()), + ] { + if let (Some(a), Some(h)) = (a, h) { + let _ = TcpStream::connect_timeout(&a, Duration::from_secs(1)); + let _ = h.join(); + } } } } @@ -896,29 +1024,6 @@ pub fn is_loopback(addr: &str) -> bool { /// The flags that take no value. const SWITCHES: [&str; 2] = ["quiet", "cookie_secure"]; -/// The flags of a site with a password, TLS or a proxy: refused until they -/// are ported, because a site that ignored them would be less safe than -/// its owner asked for. -const NOT_YET: [&str; 6] = [ - "password_file", - "allowed_host", - "trust_proxy", - "tls_cert", - "tls_key", - "cookie_secure", -]; - -/// Flags read and not used yet: the ingest listener's and the sessions'. -/// `og live serve` starts neither, so they change nothing. -const UNUSED: [&str; 6] = [ - "ingest_bind", - "ingest_port", - "max_body_kb", - "session_hours", - "session_idle_minutes", - "login_max_failures", -]; - /// Every flag, by its name with `_`. const FLAGS: [&str; 26] = [ "root", @@ -1074,35 +1179,17 @@ fn number(f: &Flags, k: &str, kind: &str) -> Result